From af53a968f6e31f932b9ca2a54a981e40fc5b9901 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 26 Jan 2024 00:52:56 +0530 Subject: [PATCH] feat: Record time difference in tests --- .github/workflows/e2e.yml | 48 +++++++++++++++++++ .gitignore | 3 +- cSpell.json | 1 + cypress.config.ts | 15 ++++++ cypress/helpers/util.ts | 8 ++++ cypress/platform/viewer.js | 2 + scripts/runTime.ts | 95 ++++++++++++++++++++++++++++++++++++++ 7 files changed, 171 insertions(+), 1 deletion(-) create mode 100644 scripts/runTime.ts diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index e9a4966b40..03a1f1c8e8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -69,6 +69,15 @@ jobs: start: pnpm run dev wait-on: 'http://localhost:9000' browser: chrome + spec: | + cypress/integration/rendering/classDiagram.spec.js + cypress/integration/rendering/flowchart-v2.spec.js + + - name: Move runtime data + if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }} + run: | + mkdir -p cypress/snapshots/runtimes + mv cypress/runtimes cypress/snapshots/runtimes/base e2e: runs-on: ubuntu-latest @@ -140,6 +149,9 @@ jobs: # e.g. if this action was run from a fork record: ${{ secrets.CYPRESS_RECORD_KEY != '' }} parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }} + spec: | + cypress/integration/rendering/classDiagram.spec.js + cypress/integration/rendering/flowchart-v2.spec.js env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} VITEST_COVERAGE: true @@ -171,6 +183,16 @@ jobs: runs-on: ubuntu-latest if: ${{ always() }} steps: + - uses: actions/checkout@v4 + + - uses: pnpm/action-setup@v2 + # uses version from "packageManager" field in package.json + + - name: Setup Node.js 18.x + uses: actions/setup-node@v4 + with: + node-version: 18.x + # Download all snapshot artifacts and merge them into a single folder - name: Download All Artifacts uses: actions/download-artifact@v4 @@ -179,6 +201,32 @@ jobs: pattern: snapshots-* merge-multiple: true + - name: Build + id: runtime + if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }} + run: | + ls -l cypress/snapshots/runtimes + mv cypress/snapshots/runtimes cypress/snapshots/runtimes/head + ls -l cypress/snapshots/runtimes + ls -l cypress/snapshots/runtimes/base + ls -l cypress/snapshots/runtimes/head + tree cypress/snapshots/runtimes + npm config set ignore-scripts true + pnpm install --frozen-lockfile + { + echo 'runtime_diff<> "$GITHUB_OUTPUT" + + - name: Comment PR runtime difference + if: ${{ github.event_name == 'pull_request' }} + uses: thollander/actions-comment-pull-request@v2 + with: + message: | + ${{ steps.runtime.outputs.runtime_diff }} + comment_tag: size-diff + # For successful push events, we save the snapshots cache - name: Save snapshots cache id: cache-upload diff --git a/.gitignore b/.gitignore index a0fd1c50b8..68399dbc27 100644 --- a/.gitignore +++ b/.gitignore @@ -29,6 +29,7 @@ Gemfile.lock cypress/screenshots/ cypress/snapshots/ +cypress/runtimes/ # eslint --cache file .eslintcache @@ -50,4 +51,4 @@ demos/dev/** tsx-0/** # autogenereated by langium-cli -generated/ \ No newline at end of file +generated/ diff --git a/cSpell.json b/cSpell.json index 5566b673fe..1b3a2a6570 100644 --- a/cSpell.json +++ b/cSpell.json @@ -122,6 +122,7 @@ "rehype", "roledescription", "rozhkov", + "runtimes", "sandboxed", "sankey", "setupgraphviewbox", diff --git a/cypress.config.ts b/cypress.config.ts index 4182d92a87..9578630604 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -1,4 +1,6 @@ import { defineConfig } from 'cypress'; +import fs from 'fs'; +import path from 'path'; import { addMatchImageSnapshotPlugin } from 'cypress-image-snapshot/plugin'; import coverage from '@cypress/code-coverage/task'; import eyesPlugin from '@applitools/eyes-cypress'; @@ -17,6 +19,19 @@ export default eyesPlugin( } return launchOptions; }); + on('task', { + recordRenderTime({ fileName, testName, timeTaken }) { + const resultsPath = path.join('cypress', 'runtimes'); + if (!fs.existsSync(resultsPath)) { + fs.mkdirSync(resultsPath, { recursive: true }); + } + fs.appendFileSync( + path.join(resultsPath, `${fileName}.csv`), + `${testName},${timeTaken}\n` + ); + return true; + }, + }); addMatchImageSnapshotPlugin(on, config); // copy any needed variables from process.env to config.env config.env.useAppli = process.env.USE_APPLI ? true : false; diff --git a/cypress/helpers/util.ts b/cypress/helpers/util.ts index aed5d7973c..d195628ebe 100644 --- a/cypress/helpers/util.ts +++ b/cypress/helpers/util.ts @@ -110,6 +110,14 @@ export const openURLAndVerifyRendering = ( cy.visit(url); cy.window().should('have.property', 'rendered', true); + cy.window().then((win) => { + cy.task('recordRenderTime', { + fileName: Cypress.spec.name, + testName: name, + // @ts-ignore Dynamically added property. + timeTaken: win.renderTime, + }); + }); cy.get('svg').should('be.visible'); if (validation) { diff --git a/cypress/platform/viewer.js b/cypress/platform/viewer.js index 39f456c230..5c08cff60f 100644 --- a/cypress/platform/viewer.js +++ b/cypress/platform/viewer.js @@ -8,6 +8,7 @@ function b64ToUtf8(str) { // Adds a rendered flag to window when rendering is done, so cypress can wait for it. function markRendered() { + window.renderTime = Date.now() - window.loadTime; if (window.Cypress) { window.rendered = true; } @@ -131,6 +132,7 @@ if (typeof document !== 'undefined') { window.addEventListener( 'load', function () { + this.window.loadTime = Date.now(); if (this.location.href.match('xss.html')) { this.console.log('Using api'); void contentLoadedApi().finally(markRendered); diff --git a/scripts/runTime.ts b/scripts/runTime.ts new file mode 100644 index 0000000000..f7c9822b21 --- /dev/null +++ b/scripts/runTime.ts @@ -0,0 +1,95 @@ +/* eslint-disable no-console */ +import { readFile } from 'fs/promises'; +import { globby } from 'globby'; +import { markdownTable } from 'markdown-table'; + +interface RunTimes { + [key: string]: number; +} +interface TestResult { + [key: string]: RunTimes; +} + +const getRuntimes = (csv: string): RunTimes => { + const lines = csv.split('\n'); + const runtimes: RunTimes = {}; + for (const line of lines) { + const [testName, timeTaken] = line.split(','); + if (testName && timeTaken) { + runtimes[testName] = Number(timeTaken); + } + } + return runtimes; +}; + +const readStats = async (path: string): Promise => { + const files = await globby(path); + const contents = await Promise.all( + files.map(async (file) => [file, await readFile(file, 'utf-8')]) + ); + const sizes = contents.map(([file, content]) => [file.split('/').pop(), getRuntimes(content)]); + return Object.fromEntries(sizes); +}; + +const percentChangeThreshold = 5; +const percentageDifference = ( + oldValue: number, + newValue: number +): { change: string; crossedThreshold: boolean } => { + const difference = Math.abs(newValue - oldValue); + const avg = (newValue + oldValue) / 2; + const percentage = (difference / avg) * 100; + const roundedPercentage = percentage.toFixed(2); // Round to two decimal places + if (roundedPercentage === '0.00') { + return { change: '0.00%', crossedThreshold: false }; + } + const sign = newValue > oldValue ? '+' : '-'; + return { + change: `${sign}${roundedPercentage}%`, + crossedThreshold: percentage > percentChangeThreshold, + }; +}; + +const main = async () => { + const oldStats = await readStats('./cypress/snapshots/runtimes/base/**/*.csv'); + const newStats = await readStats('./cypress/snapshots/runtimes/head/**/*.csv'); + const fullData: string[][] = []; + const changed: string[][] = []; + for (const [fileName, runtimes] of Object.entries(newStats)) { + const oldStat = oldStats[fileName]; + if (!oldStat) { + continue; + } + for (const [testName, timeTaken] of Object.entries(runtimes)) { + const oldTimeTaken = oldStat[testName]; + if (!oldTimeTaken) { + continue; + } + const delta = timeTaken - oldTimeTaken; + const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken); + const out = [ + fileName, + testName, + oldTimeTaken.toString(), + timeTaken.toString(), + change, + `${delta.toString()}ms`, + ]; + if (crossedThreshold) { + changed.push(out); + console.warn(`${testName} (${fileName}): ${timeTaken}ms (${delta}ms, ${change})`); + } + fullData.push(out); + } + } + const headers = ['File', 'Test', 'Old Time', 'New Time', '% Change', 'Difference']; + console.log(markdownTable([headers, ...changed])); + console.log(` +
+ Full Data + ${markdownTable([headers, ...fullData])} +
+`); +}; + +void main().catch((e) => console.error(e));