Integrate performance tests in the build

- update performance scripts to report results for history and regression test
- configure the build workflow for performance tests
- track performance history
- use benchmark action to check for regression
- publish results to GH Pages

Contributed on behalf of STMicroelectronics.

Signed-off-by: Christian W. Damus <cdamus.ext@eclipsesource.com>

.github/workflows/build.yml

@@ -71,6 +71,63 @@ jobs:
         with:
           run: yarn electron test
 
+  performance:
+    name: Performance Tests
+    needs: build # No point in running if build failed
+    if: github.ref == 'refs/heads/master' && github.event_name == 'schedule'
+    runs-on: ubuntu-18.04
+    timeout-minutes: 30
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Use Node.js 14.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: '14.x'
+          registry-url: 'https://registry.npmjs.org'
+
+      - name: Use Python 3.x
+        uses: actions/setup-python@v2
+        with:
+          python-version: '3.x'
+
+      - name: Build
+        shell: bash
+        run: |
+          yarn --skip-integrity-check --network-timeout 100000 --ignore-engines
+          npx electron-replace-ffmpeg
+          npx electron-codecs-test
+          yarn build:examples
+        env:
+          NODE_OPTIONS: --max_old_space_size=4096
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/microsoft/vscode-ripgrep/issues/9
+
+      - name: Performance (browser)
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: yarn performance:startup:browser
+
+      - name: Performance (Electron)
+        uses: GabrielBB/xvfb-action@v1
+        with:
+          run: yarn performance:startup:electron
+
+      - name: Analyze performance results
+        uses: benchmark-action/github-action-benchmark@v1
+        with:
+          name: Performance Benchmarks
+          tool: 'customSmallerIsBetter'
+          output-file-path: performance-result.json
+          alert-threshold: '150%'
+          fail-on-alert: false
+          github-token: ${{ secrets.GITHUB_TOKEN }} # Needed for comments an GH Pages
+          benchmark-data-dir-path: tests/performance
+          auto-push: true # Push to GH Pages
+          comment-on-alert: true # Comment on commit that makes performance regression
+          max-items-in-chart: 100 # Don't just collect results forever
+
   publish:
     needs: build
     if: github.ref == 'refs/heads/master' && github.event_name != 'schedule' # We still publish the manually dispatched workflows: 'workflow_dispatch'.

.gitignore

@@ -31,3 +31,4 @@ dev-packages/electron/compile_commands.json
 scripts/download
 license-check-summary.txt*
 *-trace.json
+/performance-result.json

CHANGELOG.md

@@ -1,5 +1,6 @@
 # Change Log
 
+- [scripts] integrated start-up performance scripts into nightly master build [#](https://github.com/eclipse-theia/theia/pull/) - Contributed on behalf of STMicroelectronics
 - [scripts] added Electron frontend start-up performance measurement script [#10442](https://github.com/eclipse-theia/theia/pull/10442) - Contributed on behalf of STMicroelectronics
 
 ## v1.20.0 - 11/25/2021

scripts/performance/browser-performance.js

@@ -17,7 +17,7 @@
 const puppeteer = require('puppeteer');
 const fsx = require('fs-extra');
 const { resolve } = require('path');
-const { delay, lcp, isLCP, measure } = require('./common-performance');
+const { delay, githubReporting, isLCP, lcp, measure } = require('./common-performance');
 
 const workspacePath = resolve('./workspace');
 const profilesPath = './profiles/';
@@ -73,6 +73,9 @@ let runs = 10;
     if (args.headless !== undefined && args.headless.toString() === 'false') {
         headless = false;
     }
+    if (process.env.GITHUB_ACTIONS) {
+        githubReporting.enabled = true;
+    }
 
     // Verify that the application exists
     const indexHTML = resolve(__dirname, '../../examples/browser/src-gen/frontend/index.html');

scripts/performance/common-performance.js

@@ -40,10 +40,29 @@
  */
 
 const fs = require('fs');
+const { resolve } = require('path');
 
 const performanceTag = braceText('Performance');
 const lcp = 'Largest Contentful Paint (LCP)';
 
+/**
+ * A GitHub performance results record.
+ * 
+ * @typedef PerformanceResult
+ * @property {string} name The performance measurement name
+ * @property {string} unit The performance unit of measure
+ * @property {number} value The performance measurement
+ * @property {number} [range] The standard deviation (the GitHub action calls it a "range") of the measurement
+ */
+
+/**
+ * Configuration of reporting of performance test results in a GitHub build.
+ * 
+ * @property {boolean} enabled whether GitHub result reporting is enabled (`false` by default)
+ * @property {Array<PerformanceResult>} results the performance results, if reporting is enabled
+ */
+var githubReporting = { enabled: false, results: [] };
+
 /**
  * Measure the performance of a `test` function implementing some `scenario` of interest.
  * 
@@ -90,9 +109,35 @@ function logSummary(name, scenario, durations) {
         const stdev = calculateStandardDeviation(mean, durations);
         logDuration(name, 'MEAN', scenario, mean);
         logDuration(name, 'STDEV', scenario, stdev);
+        if (githubReporting.enabled) {
+            githubResult({ name, unit: 'seconds', value: prec(mean), range: prec(stdev) });
+        }
+    } else if (githubReporting.enabled) {
+        // Only one duration
+        githubResult({ name, unit: 'seconds', value: prec(durations[0]) });
     }
 }
 
+function prec(value, precision = 3) {
+    return Number.parseFloat(value.toPrecision(precision));
+}
+
+/**
+ * Report the performance result for GitHub to pick up.
+ * 
+ * @param {PerformanceResult} result the performance result to report
+ */
+function githubResult(result) {
+    const resultsFile = resolve('../..', 'performance-result.json');
+
+    // We append to any previous results that there may have been from another script
+    const previousResults = fs.existsSync(resultsFile) ? JSON.parse(fs.readFileSync(resultsFile, 'utf-8')) : [];
+    githubReporting.results.push(...previousResults);
+
+    githubReporting.results.push(result);
+    fs.writeFileSync(resultsFile, JSON.stringify(githubReporting.results, undefined, 2), 'utf-8');
+}
+
 /**
  * Analyze a performance trace file.
  * 
@@ -227,6 +272,7 @@ function delay(time) {
 }
 
 module.exports = {
+    githubReporting,
     measure, analyzeTrace,
     calculateMean, calculateStandardDeviation,
     duration, logDuration, logSummary,

scripts/performance/electron-performance.js

@@ -17,7 +17,7 @@
 const fsx = require('fs-extra');
 const { resolve } = require('path');
 const { spawn, ChildProcess } = require('child_process');
-const { delay, lcp, isLCP, measure } = require('./common-performance');
+const { delay, githubReporting, isLCP, lcp, measure } = require('./common-performance');
 const traceConfigTemplate = require('./electron-trace-config.json');
 const { exit } = require('process');
 
@@ -80,6 +80,9 @@ let debugging = false;
         runs = parseInt(args.runs.toString());
     }
     debugging = args.debug;
+    if (process.env.GITHUB_ACTIONS) {
+        githubReporting.enabled = true;
+    }
 
     // Verify that the application exists
     const indexHTML = resolve(electronExample, 'src-gen/frontend/index.html');