diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 03a1f1c8e8..2613f33b25 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -69,15 +69,11 @@ jobs: start: pnpm run dev wait-on: 'http://localhost:9000' browser: chrome - spec: | - cypress/integration/rendering/classDiagram.spec.js - cypress/integration/rendering/flowchart-v2.spec.js - name: Move runtime data if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }} run: | - mkdir -p cypress/snapshots/runtimes - mv cypress/runtimes cypress/snapshots/runtimes/base + mv cypress/snapshots/runtimes/current cypress/snapshots/runtimes/base e2e: runs-on: ubuntu-latest @@ -149,9 +145,6 @@ jobs: # e.g. if this action was run from a fork record: ${{ secrets.CYPRESS_RECORD_KEY != '' }} parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }} - spec: | - cypress/integration/rendering/classDiagram.spec.js - cypress/integration/rendering/flowchart-v2.spec.js env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} VITEST_COVERAGE: true @@ -205,17 +198,12 @@ jobs: id: runtime if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }} run: | - ls -l cypress/snapshots/runtimes - mv cypress/snapshots/runtimes cypress/snapshots/runtimes/head - ls -l cypress/snapshots/runtimes - ls -l cypress/snapshots/runtimes/base - ls -l cypress/snapshots/runtimes/head - tree cypress/snapshots/runtimes + mv ./snapshots/runtimes/current ./snapshots/runtimes/head npm config set ignore-scripts true pnpm install --frozen-lockfile { echo 'runtime_diff<> "$GITHUB_OUTPUT" diff --git a/cypress.config.ts b/cypress.config.ts index 9578630604..2ee4ca7886 100644 --- a/cypress.config.ts +++ b/cypress.config.ts @@ -21,7 +21,7 @@ export default eyesPlugin( }); on('task', { recordRenderTime({ fileName, testName, timeTaken }) { - const resultsPath = path.join('cypress', 'runtimes'); + const resultsPath = path.join('cypress', 'snapshots', 'runtimes', 'current'); if (!fs.existsSync(resultsPath)) { fs.mkdirSync(resultsPath, { recursive: true }); } diff --git a/scripts/runTime.ts b/scripts/runTime.ts index f7c9822b21..d8da3203d1 100644 --- a/scripts/runTime.ts +++ b/scripts/runTime.ts @@ -1,7 +1,6 @@ /* eslint-disable no-console */ import { readFile } from 'fs/promises'; import { globby } from 'globby'; -import { markdownTable } from 'markdown-table'; interface RunTimes { [key: string]: number; @@ -51,10 +50,14 @@ const percentageDifference = ( }; const main = async () => { - const oldStats = await readStats('./cypress/snapshots/runtimes/base/**/*.csv'); - const newStats = await readStats('./cypress/snapshots/runtimes/head/**/*.csv'); + const base = process.argv[2] || './cypress/snapshots'; + const oldStats = await readStats(`${base}/runtimes/base/**/*.csv`); + const newStats = await readStats(`${base}/runtimes/head/**/*.csv`); const fullData: string[][] = []; const changed: string[][] = []; + let oldRuntimeSum = 0; + let newRuntimeSum = 0; + let testCount = 0; for (const [fileName, runtimes] of Object.entries(newStats)) { const oldStat = oldStats[fileName]; if (!oldStat) { @@ -65,31 +68,68 @@ const main = async () => { if (!oldTimeTaken) { continue; } + oldRuntimeSum += oldTimeTaken; + newRuntimeSum += timeTaken; + testCount++; const delta = timeTaken - oldTimeTaken; + const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken); const out = [ fileName, - testName, - oldTimeTaken.toString(), - timeTaken.toString(), - change, - `${delta.toString()}ms`, + testName.replace('#', ''), + `${oldTimeTaken}/${timeTaken}`, + `${delta.toString()}ms ${change}`, ]; - if (crossedThreshold) { + if (crossedThreshold && Math.abs(delta) > 25) { changed.push(out); - console.warn(`${testName} (${fileName}): ${timeTaken}ms (${delta}ms, ${change})`); } fullData.push(out); } } - const headers = ['File', 'Test', 'Old Time', 'New Time', '% Change', 'Difference']; - console.log(markdownTable([headers, ...changed])); + const oldAverage = oldRuntimeSum / testCount; + const newAverage = newRuntimeSum / testCount; + const { change, crossedThreshold } = percentageDifference(oldAverage, newAverage); + + const headers = ['File', 'Test', 'Time Old/New', 'Change (%)']; + console.log(`## Runtime Changes +Old runtime average: ${oldAverage.toFixed(2)}ms +New runtime average: ${newAverage.toFixed(2)}ms +Change: ${change} ${crossedThreshold ? '⚠️' : ''} + `); + console.log(` +
+ Changed tests + ${htmlTable([headers, ...changed])} +
+`); console.log(`
Full Data - ${markdownTable([headers, ...fullData])} + ${htmlTable([headers, ...fullData])}
`); }; +const htmlTable = (data: string[][]): string => { + let table = ``; + + // Generate table header + table += ` + ${data + .shift()! + .map((header) => ``) + .join('')} + `; + + // Generate table rows + for (const row of data) { + table += ` + ${row.map((cell) => ``).join('')} + `; + } + + table += '
${header}
${cell}
'; + return table; +}; + void main().catch((e) => console.error(e));