Integrate performance tests in the build

- update performance scripts to report results for history and regression test
- configure the build workflow for performance tests
- track performance history
- use benchmark action to check for regression
- publish results to GH Pages
- only use the dispatch trigger, for now

Contributed on behalf of STMicroelectronics.

Signed-off-by: Christian W. Damus <cdamus.ext@eclipsesource.com>

.github/workflows/performance-tests.yml

@@ -0,0 +1,59 @@
+name: Performance Tests
+
+on:
+  workflow_dispatch:
+
+jobs:
+  build-and-test-performance:
+    name: ubuntu-latest, Node.js 16.x
+
+    runs-on: ubuntu-latest
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Use Node.js 16.x
+        uses: actions/setup-node@v3
+        with:
+          node-version: "16.x"
+          registry-url: "https://registry.npmjs.org"
+
+      - name: Use Python 3.x
+        uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+
+      - name: Build
+        shell: bash
+        run: |
+          yarn --skip-integrity-check --network-timeout 100000 --ignore-engines
+          yarn build:examples
+        env:
+          NODE_OPTIONS: --max_old_space_size=4096
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
+
+      - name: Performance (browser)
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: yarn performance:startup:browser
+
+      - name: Performance (Electron)
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: yarn performance:startup:electron
+
+      - name: Analyze performance results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Performance Benchmarks
+          tool: "customSmallerIsBetter"
+          output-file-path: performance-result.json
+          alert-threshold: "150%"
+          fail-on-alert: false
+          github-token: ${{ secrets.GITHUB_TOKEN }} # Needed for comments an GH Pages
+          benchmark-data-dir-path: tests/performance
+          auto-push: true # Push to GH Pages
+          comment-on-alert: true # Comment on commit if it causes a performance regression
+          max-items-in-chart: 100 # Don't just collect results forever

.gitignore

@@ -32,3 +32,4 @@ scripts/download
 dependency-check-summary.txt*
 *-trace.json
 .tours
+/performance-result.json

CHANGELOG.md

@@ -6,6 +6,8 @@
 
 ## v1.33.0 - unreleased
 
+- [scripts] integrated start-up performance scripts into nightly master build [#10463](https://github.com/eclipse-theia/theia/pull/10463) - Contributed on behalf of STMicroelectronics
+
 <a name="breaking_changes_1.33.0">[Breaking Changes:](#breaking_changes_1.33.0)</a>
 
 - [core] returns of many methods of `MenuModelRegistry` changed from `CompositeMenuNode` to `MutableCompoundMenuNode`. To mutate a menu, use the `updateOptions` method or add a check for `instanceof CompositeMenuNode`, which will be true in most cases.

scripts/performance/browser-performance.js

@@ -17,7 +17,7 @@
 const puppeteer = require('puppeteer');
 const fsx = require('fs-extra');
 const { resolve } = require('path');
-const { delay, lcp, isLCP, measure } = require('./common-performance');
+const { delay, githubReporting, isLCP, lcp, measure } = require('./common-performance');
 
 const workspacePath = resolve('./workspace');
 const profilesPath = './profiles/';
@@ -73,6 +73,9 @@ let runs = 10;
     if (args.headless !== undefined && args.headless.toString() === 'false') {
         headless = false;
     }
+    if (process.env.GITHUB_ACTIONS) {
+        githubReporting.enabled = true;
+    }
 
     // Verify that the application exists
     const indexHTML = resolve(__dirname, '../../examples/browser/src-gen/frontend/index.html');

scripts/performance/common-performance.js

@@ -40,10 +40,29 @@
  */
 
 const fs = require('fs');
+const { resolve } = require('path');
 
 const performanceTag = braceText('Performance');
 const lcp = 'Largest Contentful Paint (LCP)';
 
+/**
+ * A GitHub performance results record.
+ * 
+ * @typedef PerformanceResult
+ * @property {string} name The performance measurement name
+ * @property {string} unit The performance unit of measure
+ * @property {number} value The performance measurement
+ * @property {number} [range] The standard deviation (the GitHub action calls it a "range") of the measurement
+ */
+
+/**
+ * Configuration of reporting of performance test results in a GitHub build.
+ * 
+ * @property {boolean} enabled whether GitHub result reporting is enabled (`false` by default)
+ * @property {Array<PerformanceResult>} results the performance results, if reporting is enabled
+ */
+var githubReporting = { enabled: false, results: [] };
+
 /**
  * Measure the performance of a `test` function implementing some `scenario` of interest.
  * 
@@ -90,9 +109,35 @@ function logSummary(name, scenario, durations) {
         const stdev = calculateStandardDeviation(mean, durations);
         logDuration(name, 'MEAN', scenario, mean);
         logDuration(name, 'STDEV', scenario, stdev);
+        if (githubReporting.enabled) {
+            githubResult({ name, unit: 'seconds', value: prec(mean), range: prec(stdev) });
+        }
+    } else if (githubReporting.enabled) {
+        // Only one duration
+        githubResult({ name, unit: 'seconds', value: prec(durations[0]) });
     }
 }
 
+function prec(value, precision = 3) {
+    return Number.parseFloat(value.toPrecision(precision));
+}
+
+/**
+ * Report the performance result for GitHub to pick up.
+ * 
+ * @param {PerformanceResult} result the performance result to report
+ */
+function githubResult(result) {
+    const resultsFile = resolve('../..', 'performance-result.json');
+
+    // We append to any previous results that there may have been from another script
+    const previousResults = fs.existsSync(resultsFile) ? JSON.parse(fs.readFileSync(resultsFile, 'utf-8')) : [];
+    githubReporting.results.push(...previousResults);
+
+    githubReporting.results.push(result);
+    fs.writeFileSync(resultsFile, JSON.stringify(githubReporting.results, undefined, 2), 'utf-8');
+}
+
 /**
  * Analyze a performance trace file.
  * 
@@ -227,6 +272,7 @@ function delay(time) {
 }
 
 module.exports = {
+    githubReporting,
     measure, analyzeTrace,
     calculateMean, calculateStandardDeviation,
     duration, logDuration, logSummary,

scripts/performance/electron-performance.js

@@ -17,7 +17,7 @@
 const fsx = require('fs-extra');
 const { resolve } = require('path');
 const { spawn, ChildProcess } = require('child_process');
-const { delay, lcp, isLCP, measure } = require('./common-performance');
+const { delay, githubReporting, isLCP, lcp, measure } = require('./common-performance');
 const traceConfigTemplate = require('./electron-trace-config.json');
 const { exit } = require('process');
 
@@ -80,6 +80,9 @@ let debugging = false;
         runs = parseInt(args.runs.toString());
     }
     debugging = args.debug;
+    if (process.env.GITHUB_ACTIONS) {
+        githubReporting.enabled = true;
+    }
 
     // Verify that the application exists
     const indexHTML = resolve(electronExample, 'src-gen/frontend/index.html');