Better reporting for UI Profiler tests

.github/workflows/benchmark.yml

@@ -25,9 +25,9 @@ on:
           - retrolab
           - testing
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
       reference_project:
         description: "Playwright project to execute on the reference version"
         required: false
@@ -81,9 +81,9 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }}
       # Branch to checkout for scheduled benchmark
-      challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }}
+      challenger_branch: ${{ github.event.inputs.challenger_branch || 'main' }}
       challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }}
-      reference_branch: ${{ github.event.inputs.reference_branch || 'master' }}
+      reference_branch: ${{ github.event.inputs.reference_branch || 'main' }}
       reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }}
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: ${{ github.event.inputs.browser || 'chromium' }}

.github/workflows/memory-leak.yml

@@ -23,5 +23,5 @@ jobs:
     uses: ./.github/workflows/run-memory-leak.yml
     with:
       repository: ${{ github.event.inputs.repository || 'jupyterlab/jupyterlab' }}
-      branch: ${{ github.event.inputs.branch || 'master' }}
+      branch: ${{ github.event.inputs.branch || 'main' }}
       samples: ${{ github.event.inputs.samples || '7' }}

.github/workflows/profiler.yml

@@ -10,7 +10,7 @@ on:
       challenger_branch:
         description: "Git repository reference to the challenger branch"
         required: true
-        default: "master"
+        default: "main"
       challenger_project:
         description: "Playwright project to execute (windowingMode JupyterLab 4; renderCellOnIdle: JupyterLab 2.3 or 3.x)"
         required: false
@@ -27,9 +27,9 @@ on:
           - retrolab
           - testing
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
       reference_project:
         description: "Playwright project to execute on the reference version"
         required: false
@@ -79,9 +79,9 @@ jobs:
     uses: ./.github/workflows/run-profiler.yml
     with:
       challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }}
-      challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }}
+      challenger_branch: ${{ github.event.inputs.challenger_branch || 'main' }}
       challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }}
-      reference_branch: ${{ github.event.inputs.reference_branch || 'master' }}
+      reference_branch: ${{ github.event.inputs.reference_branch || 'main' }}
       reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }}
       browser: ${{ github.event.inputs.browser || 'chromium' }}
       samples: ${{ github.event.inputs.samples || '25' }}

.github/workflows/run-benchmark.yml

@@ -24,9 +24,9 @@ on:
         default: "jupyterlab"
         type: "string"
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
         type: string
       reference_project:
         description: "Playwright project to execute on the reference version"

.github/workflows/run-profiler.yml

@@ -20,9 +20,9 @@ on:
         default: "jupyterlab"
         type: "string"
       reference_branch:
-        description: "Reference branch on the JupyterLab repository (default: master)"
+        description: "Reference branch on the JupyterLab repository (default: main)"
         required: false
-        default: "master"
+        default: "main"
         type: string
       reference_project:
         description: "Playwright project to execute on the reference version"
@@ -44,6 +44,11 @@ on:
         required: false
         default: "profiler-assets"
         type: "string"
+      results_name:
+        description: "Uploaded results name"
+        required: false
+        default: "profiler-results"
+        type: "string"
       grep:
         description: "Tests to include"
         required: false
@@ -220,6 +225,14 @@ jobs:
             benchmarks/tests/report-reference
             benchmarks/tests/report-challenger
 
+      - name: Upload UI Profiler assets
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ inputs.results_name }}
+          path: |
+            benchmarks/tests/results/
+
       - name: Print JupyterLab logs
         if: always()
         run: |

.github/workflows/scheduled-benchmark.yml

@@ -21,8 +21,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -42,8 +42,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -63,8 +63,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -84,8 +84,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution
@@ -105,8 +105,8 @@ jobs:
       # Repository to clone for scheduled benchmark
       challenger: 'jupyterlab/jupyterlab'
       # Branch to checkout for scheduled benchmark
-      challenger_branch: 'master'
-      reference_branch: 'master'
+      challenger_branch: 'main'
+      reference_branch: 'main'
       # Which browser to use (one of 'chromium', 'firefox', 'webkit')
       browser: 'chromium'
       # How many samples to compute the statistical distribution

.github/workflows/scheduled-memory-leak.yml

@@ -11,4 +11,4 @@ jobs:
     uses: ./.github/workflows/run-memory-leak.yml
     with:
       repository: jupyterlab/jupyterlab
-      branch: master
+      branch: main

.github/workflows/test.yml

@@ -57,7 +57,7 @@ jobs:
         uses: actions/checkout@v3
         with:
           repository: jupyterlab/jupyterlab
-          ref: master
+          ref: main
           path: reference
 
       - name: Install dependencies
@@ -153,4 +153,4 @@ jobs:
     uses: ./.github/workflows/run-memory-leak.yml
     with:
       repository: jupyterlab/jupyterlab
-      branch: master
+      branch: main

tests/jupyterlab/ui-profiler.specs.ts

@@ -1,4 +1,4 @@
-import { benchmark, galata } from "@jupyterlab/galata";
+import { benchmark, galata, IJupyterLabPageFixture } from "@jupyterlab/galata";
 import { JSONObject } from "@lumino/coreutils";
 import type {
   IBenchmarkResult,
@@ -157,6 +157,21 @@ test.afterEach(async ({ tmpPath, baseURL }) => {
   await contents.deleteDirectory(tmpPath);
 });
 
+async function acceptKernelDialog(page: IJupyterLabPageFixture) {
+  const dialogLocator = page.locator(".jp-Dialog");
+  try {
+    // Wait up to three seconds for the kernel selection dialog to appear
+    await dialogLocator.waitFor({ timeout: 3000 });
+  } catch {
+    // no-op
+  }
+  // If the kernel dialog shows up, accept default kernel
+  if (await dialogLocator.isVisible()) {
+    await page.click(".jp-Dialog .jp-mod-accept");
+    await dialogLocator.waitFor({ state: "detached" });
+  }
+}
+
 test.describe("Measure execution time", () => {
   for (const [id, scenario] of Object.entries(scenarios)) {
     for (const file of fileNames) {
@@ -180,6 +195,7 @@ test.describe("Measure execution time", () => {
 
         if (openNotebook) {
           await page.notebook.openByPath(notebookPath);
+          await acceptKernelDialog(page);
         }
 
         const result = (await profiler.runBenchmark(
@@ -206,9 +222,18 @@ test.describe("Measure execution time", () => {
               time: time,
               project: testInfo.project.name,
               profiler: true,
+              granular: true,
             })
           );
         }
+        testInfo.attach(`${reference}-${id}:execution-time.json`, {
+          body: JSON.stringify({
+            ...result,
+            reference,
+            backgroundTab: file,
+          }),
+          contentType: "application/json",
+        });
       });
     }
   }
@@ -239,6 +264,7 @@ test.describe("Benchmark style sheets @slow", () => {
 
         if (openNotebook) {
           await page.notebook.openByPath(notebookPath);
+          await acceptKernelDialog(page);
         }
 
         const result = (await profiler.runBenchmark(
@@ -257,7 +283,11 @@ test.describe("Benchmark style sheets @slow", () => {
         >;
 
         testInfo.attach(`${reference}-${id}:style-sheet.json`, {
-          body: JSON.stringify(result),
+          body: JSON.stringify({
+            ...result,
+            reference,
+            backgroundTab: file,
+          }),
           contentType: "application/json",
         });
       });
@@ -287,6 +317,7 @@ test.describe("Benchmark style rules @slow", () => {
 
         if (openNotebook) {
           await page.notebook.openByPath(notebookPath);
+          await acceptKernelDialog(page);
         }
 
         const result = (await profiler.runBenchmark(
@@ -305,7 +336,11 @@ test.describe("Benchmark style rules @slow", () => {
         >;
 
         testInfo.attach(`${reference}-${id}:style-rule.json`, {
-          body: JSON.stringify(result),
+          body: JSON.stringify({
+            ...result,
+            reference,
+            backgroundTab: file,
+          }),
           contentType: "application/json",
         });
       });

tests/package.json

@@ -27,5 +27,8 @@
     "prettier": "^2.8.8",
     "rimraf": "^3.0.2",
     "typescript": "~4.9.0"
+  },
+  "dependencies": {
+    "@actions/core": "^1.10.0"
   }
 }

tests/playwright.config.ts

@@ -133,6 +133,10 @@ export default {
       "@jupyterlab/galata/lib/benchmarkReporter",
       { outputFile: "lab-benchmark.json" },
     ],
+    [
+      "./reporter", {}
+    ],
+    ['json', {  outputFile: 'test-results.json' }]
   ],
   use: {
     // Browser options