Merge pull request #143 from krassowski/feat/integrate-profiler

Integrate ui-profiler, improve dispatch, separate jlpm/yarn

.github/workflows/benchmark.yml

@@ -10,23 +10,48 @@ on:
  description: "Git repository reference to the challenger branch"
  required: true
  challenger_project:
- description: "Playwright project to execute on the challenger version"
+ description: "Playwright project to execute (windowingMode JupyterLab 4; renderCellOnIdle: JupyterLab 2.3 or 3.x)"
  required: false
  default: "jupyterlab"
- type: "string"
+ type: choice
+ options:
+ - jupyterlab
+ - jupyterlab-renderCellOnIdle-on
+ - jupyterlab-renderCellOnIdle-off
+ - jupyterlab-windowingMode-full
+ - jupyterlab-windowingMode-defer
+ - jupyterlab-windowingMode-none
+ - jupyterlab-1-2
+ - retrolab
+ - testing
  reference_branch:
  description: "Reference branch on the JupyterLab repository (default: master)"
  required: false
  default: "master"
  reference_project:
  description: "Playwright project to execute on the reference version"
  required: false
- default: "jupyterlab"
- type: "string"
+ default: ""
+ type: choice
+ options:
+ - jupyterlab
+ - jupyterlab-renderCellOnIdle-on
+ - jupyterlab-renderCellOnIdle-off
+ - jupyterlab-windowingMode-full
+ - jupyterlab-windowingMode-defer
+ - jupyterlab-windowingMode-none
+ - jupyterlab-1-2
+ - retrolab
+ - testing
  browser:
- description: "Which browser to use (one of 'chromium' [default], 'firefox', 'webkit')"
+ description: "Which browser to use (default 'chromium')"
  required: false
  default: "chromium"
+ type: choice
+ options:
+ - chromium
+ - firefox
+ - webkit
  samples:
  description: "Number of samples to compute"
  required: false
@@ -49,7 +74,8 @@ permissions:
 
 jobs:
  test:
- uses: jupyterlab/benchmarks/.github/workflows/run-benchmark.yml@master
+ # uses: jupyterlab/benchmarks/.github/workflows/run-benchmark.yml@master
+ uses: ./.github/workflows/run-benchmark.yml
  with:
  event: ${{ github.event_name }}
  # Repository to clone for scheduled benchmark

.github/workflows/profiler.yml

@@ -0,0 +1,90 @@
+name: JupyterLab UI Profiler
+
+on:
+ workflow_dispatch:
+ inputs:
+ challenger:
+ description: "JupyterLab Git repository with the challenger version (format {owner}/{repo})"
+ required: true
+ default: "jupyterlab/jupyterlab"
+ challenger_branch:
+ description: "Git repository reference to the challenger branch"
+ required: true
+ default: "master"
+ challenger_project:
+ description: "Playwright project to execute (windowingMode JupyterLab 4; renderCellOnIdle: JupyterLab 2.3 or 3.x)"
+ required: false
+ default: "jupyterlab"
+ type: choice
+ options:
+ - jupyterlab
+ - jupyterlab-renderCellOnIdle-on
+ - jupyterlab-renderCellOnIdle-off
+ - jupyterlab-windowingMode-full
+ - jupyterlab-windowingMode-defer
+ - jupyterlab-windowingMode-none
+ - jupyterlab-1-2
+ - retrolab
+ - testing
+ reference_branch:
+ description: "Reference branch on the JupyterLab repository (default: master)"
+ required: false
+ default: "master"
+ reference_project:
+ description: "Playwright project to execute on the reference version"
+ required: false
+ default: ""
+ type: choice
+ options:
+ - jupyterlab
+ - jupyterlab-renderCellOnIdle-on
+ - jupyterlab-renderCellOnIdle-off
+ - jupyterlab-windowingMode-full
+ - jupyterlab-windowingMode-defer
+ - jupyterlab-windowingMode-none
+ - jupyterlab-1-2
+ - retrolab
+ - testing
+ browser:
+ description: "Which browser to use (default 'chromium')"
+ required: false
+ default: "chromium"
+ type: choice
+ options:
+ - chromium
+ - firefox
+ - webkit
+ samples:
+ description: "Number of samples to compute (individual tests multiply this number by their a test-speicific factor)"
+ required: false
+ default: "25"
+ type: string
+ grep:
+ description: "Tests to include (grep expression)"
+ required: false
+ default: ""
+ type: string
+ grep_invert:
+ description: "Tests to skip (grep expression). Benchmarking individual rules requires a lot of time (reduce number of samples!)"
+ required: false
+ default: "rules"
+ type: string
+
+permissions:
+ issues: write
+
+jobs:
+ test:
+ # uses: jupyterlab/benchmarks/.github/workflows/run-profiler.yml@master
+ uses: ./.github/workflows/run-profiler.yml
+ with:
+ challenger: ${{ github.event.inputs.challenger || 'jupyterlab/jupyterlab' }}
+ challenger_branch: ${{ github.event.inputs.challenger_branch || 'master' }}
+ challenger_project: ${{ github.event.inputs.challenger_project || 'jupyterlab' }}
+ reference_branch: ${{ github.event.inputs.reference_branch || 'master' }}
+ reference_project: ${{ github.event.inputs.reference_project || 'jupyterlab' }}
+ browser: ${{ github.event.inputs.browser || 'chromium' }}
+ samples: ${{ github.event.inputs.samples || '25' }}
+ grep: ${{ github.event.inputs.grep || '' }}
+ grep_invert: ${{ github.event.inputs.grep_invert || '' }}
+ artifacts_name: 'profiler-report'

.github/workflows/run-benchmark.yml

@@ -63,6 +63,14 @@ on:
  required: false
  default: "benchmark-assets"
  type: string
+ grep_invert:
+ description: "Tests to skip"
+ required: false
+ default: "@slow"
+ type: string
+
+env:
+ YARN_ENABLE_IMMUTABLE_INSTALLS: 0
 
 permissions:
  issues: write
@@ -154,7 +162,7 @@ jobs:
  git checkout ${OLD_REF_SHA}
  working-directory: reference
 
- - name: Install dependencies
+ - name: Install reference
  run: |
  set -ex
  echo "OLD_REF_SHA=$(git log -n1 --format='%H')" >> $GITHUB_ENV
@@ -169,24 +177,31 @@ jobs:
  set -ex
  pip install ipywidgets plotly
  pip install -v extensions/fixed-data-table
+ pip install jupyterlab-ui-profiler
+ jupyter labextension disable @jupyterlab/ui-profiler:user-interface
 
  jupyter server extension list
  jupyter labextension list
  working-directory: benchmarks
 
- - name: Launch JupyterLab
+ - name: Install testing dependencies
+ shell: bash
+ run: |
+ yarn install
+ working-directory: benchmarks/tests
+
+ - name: Launch JupyterLab (reference)
  shell: bash
  run: |
- jlpm install
  # Mount a volume to overwrite the server configuration
- jlpm start-jlab 2>&1 > /tmp/jupyterlab_server_old.log &
+ yarn start-jlab 2>&1 > /tmp/jupyterlab_server_old.log &
  working-directory: benchmarks/tests
 
  - name: Install browser
  run: |
  set -ex
  # Install only Chromium browser
- jlpm playwright install chromium
+ yarn playwright install chromium
  working-directory: benchmarks/tests
 
  - name: Wait for JupyterLab
@@ -195,16 +210,16 @@ jobs:
  resource: http-get://localhost:9999/lab
  timeout: 360000
 
- - name: Execute benchmark tests
+ - name: Execute benchmark tests (reference)
  continue-on-error: true
  working-directory: benchmarks/tests
  run: |
  set -ex
  # Update test screenshots
- BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 jlpm run test --project ${{ inputs.reference_project }} -u
+ BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 yarn run test --project ${{ inputs.reference_project }} -u --grep-invert '${{ inputs.grep_invert }}'
 
  # -u is needed to generate the benchmark expected file
- jlpm run test --project ${{ inputs.reference_project }} -u
+ yarn run test --project ${{ inputs.reference_project }} -u --grep-invert '${{ inputs.grep_invert }}'
  cp tests-out/lab-benchmark-expected.json /tmp/
 
  - name: Kill the server
@@ -224,7 +239,7 @@ jobs:
  ref: ${{ env.CHALLENGER_REF }}
  path: challenger
 
- - name: Install dependencies
+ - name: Install challenger
  run: |
  set -ex
  # Fix for installing JupyterLab 1
@@ -237,11 +252,10 @@ jobs:
  cp /tmp/lab-benchmark-expected.json ./tests-out
  working-directory: challenger
 
- - name: Launch JupyterLab
+ - name: Launch JupyterLab (challenger)
  shell: bash
  run: |
- # Mount a volume to overwrite the server configuration
- jlpm start-jlab 2>&1 > /tmp/jupyterlab_server_new.log &
+ yarn start-jlab 2>&1 > /tmp/jupyterlab_server_new.log &
  working-directory: benchmarks/tests
 
  - name: Wait for JupyterLab
@@ -250,7 +264,7 @@ jobs:
  resource: http-get://localhost:9999/lab
  timeout: 360000
 
- - name: Execute benchmark tests
+ - name: Execute benchmark tests (challenger)
  continue-on-error: true
  shell: bash
  env:
@@ -259,13 +273,13 @@ jobs:
  run: |
  set -ex
  # Update test screenshots
- BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 jlpm run test --project ${{ inputs.challenger_project }} -u
+ BENCHMARK_NUMBER_SAMPLES=1 PW_VIDEO=1 yarn run test --project ${{ inputs.challenger_project }} -u --grep-invert '${{ inputs.grep_invert }}'
 
  # Copy reference here otherwise it will use the value from the update screenshots
  # command called just before
  cp /tmp/lab-benchmark-expected.json ./tests-out
 
- jlpm run test --project ${{ inputs.challenger_project }}
+ yarn run test --project ${{ inputs.challenger_project }} --grep-invert '${{ inputs.grep_invert }}'
  working-directory: benchmarks/tests
 
  - name: Generate the report
@@ -301,6 +315,12 @@ jobs:
  # Copy the reference data to upload it as artifact
  cp ./tests-out/lab-benchmark-expected.json ./benchmark-results/
 
+ - name: Set job summary
+ if: always()
+ shell: bash
+ run: |
+ cat ./benchmarks/tests/benchmark-results/lab-benchmark.md >> $GITHUB_STEP_SUMMARY
+
  - name: Publish the report
  if: ${{ inputs.event == 'schedule' }}
  uses: actions/github-script@v6

.github/workflows/run-memory-leak.yml

@@ -17,6 +17,9 @@ on:
  default: "7"
  type: string
 
+env:
+ YARN_ENABLE_IMMUTABLE_INSTALLS: 0
+
 jobs:
  memory-leak-test:
  runs-on: ubuntu-22.04
@@ -79,8 +82,7 @@ jobs:
  - name: Launch JupyterLab
  shell: bash
  run: |
- # Mount a volume to overwrite the server configuration
- jlpm start-jlab 2>&1 > /tmp/jupyterlab_server.log &
+ yarn start-jlab 2>&1 > /tmp/jupyterlab_server.log &
  working-directory: benchmarks/memory-leaks
 
  - name: Execute memory leaks test