Merge pull request #5240 from mermaid-js/sidv/runTimeDiffTest

Test

.github/workflows/e2e.yml

@@ -69,15 +69,11 @@ jobs:
  start: pnpm run dev
  wait-on: 'http://localhost:9000'
  browser: chrome
- spec: |
- cypress/integration/rendering/classDiagram.spec.js
- cypress/integration/rendering/flowchart-v2.spec.js
 
  - name: Move runtime data
  if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }}
  run: |
- mkdir -p cypress/snapshots/runtimes
- mv cypress/runtimes cypress/snapshots/runtimes/base
+ mv cypress/snapshots/runtimes/current cypress/snapshots/runtimes/base
 
  e2e:
  runs-on: ubuntu-latest
@@ -149,9 +145,6 @@ jobs:
  # e.g. if this action was run from a fork
  record: ${{ secrets.CYPRESS_RECORD_KEY != '' }}
  parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }}
- spec: |
- cypress/integration/rendering/classDiagram.spec.js
- cypress/integration/rendering/flowchart-v2.spec.js
  env:
  CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
  VITEST_COVERAGE: true
@@ -205,17 +198,12 @@ jobs:
  id: runtime
  if: ${{ needs.e2e.result != 'failure' && github.event_name == 'pull_request' }}
  run: |
- ls -l cypress/snapshots/runtimes
- mv cypress/snapshots/runtimes cypress/snapshots/runtimes/head
- ls -l cypress/snapshots/runtimes
- ls -l cypress/snapshots/runtimes/base
- ls -l cypress/snapshots/runtimes/head
- tree cypress/snapshots/runtimes
+ mv ./snapshots/runtimes/current ./snapshots/runtimes/head
  npm config set ignore-scripts true
  pnpm install --frozen-lockfile
  {
  echo 'runtime_diff<<EOF'
- npx tsx scripts/runTime.ts
+ npx tsx scripts/runTime.ts ./snapshots
  echo EOF
  } >> "$GITHUB_OUTPUT"
 

cypress.config.ts

@@ -21,7 +21,7 @@ export default eyesPlugin(
  });
  on('task', {
  recordRenderTime({ fileName, testName, timeTaken }) {
- const resultsPath = path.join('cypress', 'runtimes');
+ const resultsPath = path.join('cypress', 'snapshots', 'runtimes', 'current');
  if (!fs.existsSync(resultsPath)) {
  fs.mkdirSync(resultsPath, { recursive: true });
  }

scripts/runTime.ts

@@ -1,7 +1,6 @@
 /* eslint-disable no-console */
 import { readFile } from 'fs/promises';
 import { globby } from 'globby';
-import { markdownTable } from 'markdown-table';
 
 interface RunTimes {
  [key: string]: number;
@@ -51,10 +50,14 @@ const percentageDifference = (
 };
 
 const main = async () => {
- const oldStats = await readStats('./cypress/snapshots/runtimes/base/**/*.csv');
- const newStats = await readStats('./cypress/snapshots/runtimes/head/**/*.csv');
+ const base = process.argv[2] || './cypress/snapshots';
+ const oldStats = await readStats(`${base}/runtimes/base/**/*.csv`);
+ const newStats = await readStats(`${base}/runtimes/head/**/*.csv`);
  const fullData: string[][] = [];
  const changed: string[][] = [];
+ let oldRuntimeSum = 0;
+ let newRuntimeSum = 0;
+ let testCount = 0;
  for (const [fileName, runtimes] of Object.entries(newStats)) {
  const oldStat = oldStats[fileName];
  if (!oldStat) {
@@ -65,31 +68,68 @@ const main = async () => {
  if (!oldTimeTaken) {
  continue;
  }
+ oldRuntimeSum += oldTimeTaken;
+ newRuntimeSum += timeTaken;
+ testCount++;
  const delta = timeTaken - oldTimeTaken;
+
  const { change, crossedThreshold } = percentageDifference(oldTimeTaken, timeTaken);
  const out = [
  fileName,
- testName,
- oldTimeTaken.toString(),
- timeTaken.toString(),
- change,
- `${delta.toString()}ms`,
+ testName.replace('#', ''),
+ `${oldTimeTaken}/${timeTaken}`,
+ `${delta.toString()}ms ${change}`,
  ];
- if (crossedThreshold) {
+ if (crossedThreshold && Math.abs(delta) > 25) {
  changed.push(out);
- console.warn(`${testName} (${fileName}): ${timeTaken}ms (${delta}ms, ${change})`);
  }
  fullData.push(out);
  }
  }
- const headers = ['File', 'Test', 'Old Time', 'New Time', '% Change', 'Difference'];
- console.log(markdownTable([headers, ...changed]));
+ const oldAverage = oldRuntimeSum / testCount;
+ const newAverage = newRuntimeSum / testCount;
+ const { change, crossedThreshold } = percentageDifference(oldAverage, newAverage);
+
+ const headers = ['File', 'Test', 'Time Old/New', 'Change (%)'];
+ console.log(`## Runtime Changes
+Old runtime average: ${oldAverage.toFixed(2)}ms
+New runtime average: ${newAverage.toFixed(2)}ms
+Change: ${change} ${crossedThreshold ? '⚠️' : ''}
+ `);
+ console.log(`
+ <details>
+ <summary>Changed tests</summary>
+ ${htmlTable([headers, ...changed])}
+</details>
+`);
  console.log(`
  <details>
  <summary>Full Data</summary>
- ${markdownTable([headers, ...fullData])}
+ ${htmlTable([headers, ...fullData])}
 </details>
 `);
 };
 
+const htmlTable = (data: string[][]): string => {
+ let table = `<table border='1' style="border-collapse: collapse">`;
+
+ // Generate table header
+ table += `<tr>
+ ${data
+ .shift()!
+ .map((header) => `<th>${header}</th>`)
+ .join('')}
+ </tr>`;
+
+ // Generate table rows
+ for (const row of data) {
+ table += `<tr>
+ ${row.map((cell) => `<td>${cell}</td>`).join('')}
+ </tr>`;
+ }
+
+ table += '</table>';
+ return table;
+};
+
 void main().catch((e) => console.error(e));