build-test.yml

# This yml is used for PRs, pre-release, and release build.
# We use the github.event_name to determine what started the workflow to determine which
# situation we are in.

name: Build and Test

permissions:
  deployments: write

on:
  pull_request:
    branches:
      - main
      - 'release'
      - 'release/*'
      - 'release-*'
  check_run:
    types: [rerequested, requested_action]
  push:
    branches:
      - main
      - 'release'
      - 'release/*'
      - 'release-*'
  schedule:
    - cron: '0 9 * * 1-5' # 9am UTC, Monday-Friday (2am PDT, after VS Code pre-release builds which is 11pm PDT)
  workflow_dispatch:

env:
  NODE_VERSION: 18.15.0
  NPM_VERSION: 9.5.0
  PYTHON_VERSION: 3.8
  DENO_VERSION: '~1.37'
  MOCHA_REPORTER_JUNIT: true # Use the mocha-multi-reporters and send output to both console (spec) and JUnit (mocha-junit-reporter). Also enables a reporter which exits the process running the tests if it haven't already.
  CACHE_NPM_DEPS: cache-npm
  CACHE_OUT_DIRECTORY: cache-out-directory
  CACHE_PIP_DEPS: cache-pip
  VSC_JUPYTER_FORCE_LOGGING: 'true'
  VSC_PYTHON_FORCE_LOGGING: 'true'
  VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 'true'
  # Key for the cache created at the end of the the 'Cache ./pythonFiles/lib/python' step.
  CACHE_PYTHONFILES: cache-pvsc-pythonFiles
  COVERAGE_REPORTS: tests-coverage-reports
  TEST_RESULTS_DIRECTORY: .
  TEST_RESULTS_GLOB: '**/test-results*.xml'
  IPYWIDGET_SCREENSHOT_PATH: '*-screenshot.png'
  DISABLE_INSIDERS_EXTENSION: 1 # Disable prompts to install pre-release in tests (else it blocks activation of extension).
  VSC_JUPYTER_INSTRUMENT_CODE_FOR_COVERAGE: true
  VSC_JUPYTER_LOG_KERNEL_OUTPUT: true

jobs:
  # Make sure to cancel previous runs on a push
  cancel_previous_runs:
    runs-on: ubuntu-latest
    steps:
      - name: Cancel Previous Runs
        uses: styfle/cancel-workflow-action@0.12.0
        with:
          access_token: ${{ github.token }}

  build-vsix:
    name: Build VSIX
    runs-on: ubuntu-latest
    if: github.repository == 'microsoft/vscode-jupyter'
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Use Node ${{env.NODE_VERSION}}
        uses: actions/setup-node@v4
        with:
          node-version: ${{env.NODE_VERSION}}

      - name: Use Npm ${{env.NPM_VERSION}}
        run: npm i -g npm@${{env.NPM_VERSION}}

      - name: Use Python ${{env.PYTHON_VERSION}}
        uses: actions/setup-python@v5
        with:
          python-version: ${{env.PYTHON_VERSION}}

      # Caching of npm packages (https://github.com/actions/cache/blob/main/examples.md#node---npm)
      - name: Cache npm on linux/mac
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-

      - run: npm i -g @vscode/vsce

      - name: Build VSIX
        uses: ./.github/actions/build-vsix
        id: build-vsix

      - uses: actions/upload-artifact@v3
        with:
          name: 'ms-toolsai-jupyter-insiders.vsix'
          path: 'ms-toolsai-jupyter-insiders.vsix'

  lint:
    name: Lint
    runs-on: ubuntu-latest
    if: github.repository == 'microsoft/vscode-jupyter'
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Use Node ${{env.NODE_VERSION}}
        uses: actions/setup-node@v4
        with:
          node-version: ${{env.NODE_VERSION}}

      - name: Use Npm ${{env.NPM_VERSION}}
        run: npm i -g npm@${{env.NPM_VERSION}}

      - name: Cache pip files
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: ${{runner.os}}-${{env.CACHE_PIP_DEPS}}-${{env.PYTHON_VERSION}}

      - name: Cache npm files
        uses: actions/cache@v4
        with:
          path: ~/.npm
          key: ${{runner.os}}-${{env.CACHE_NPM_DEPS}}-${{hashFiles('package-lock.json')}}

      - name: Cache the out/ directory
        uses: actions/cache@v4
        with:
          path: ./out
          key: ${{runner.os}}-${{env.CACHE_OUT_DIRECTORY}}-${{hashFiles('src/**')}}

      # This is faster than running `npm ci`, we do not want to build zmq, etc.
      # Let that happen in other jobs, this job needs to be fast
      - name: npm ci
        run: npm ci --ignore-scripts --prefer-offline --no-audit

      - name: npm run postinstall
        run: npm run postinstall

      - name: Verify Translation files
        run: npm run validateTranslationFiles

      - name: Run linting on TypeScript code (eslint)
        run: npm run lint

      - name: Run prettier on JavaScript code
        run: npm run format-check

      - name: Use Python ${{env.PYTHON_VERSION}}
        uses: actions/setup-python@v5
        with:
          python-version: ${{env.PYTHON_VERSION}}

      - name: Run Black on Python code
        run: |
          python -m pip install click==8.0.4
          python -m pip install -U black
          python -m black . --check
        working-directory: pythonFiles

      - name: Run gulp prePublishNonBundle
        run: npm run prePublishNonBundle

      - name: Check dependencies
        run: npm run checkDependencies

      - name: Check changes to package-lock.json
        run: npx gulp validatePackageLockJson

      - name: Validate TELEMETRY files
        run: npm run validateTelemetry

      - name: Verify usage of new Proposed API
        if: github.event_name == 'pull_request'
        run: npx tsx ./build/verifyProposedApiUsage.ts
        env:
          PULL_REQUEST_SHA: ${{ github.sha }}
          BASE_SHA: ${{ github.event.pull_request.base.sha }}

  ts_tests:
    name: Type Script Tests
    runs-on: ${{ matrix.os }}
    if: github.repository == 'microsoft/vscode-jupyter'
    strategy:
      fail-fast: false
      matrix:
        os: ['ubuntu-latest', 'windows-latest']
        test-suite: [ts-unit]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Use Node ${{env.NODE_VERSION}}
        uses: actions/setup-node@v4
        with:
          node-version: ${{env.NODE_VERSION}}

      - name: Use Npm ${{env.NPM_VERSION}}
        run: npm i -g npm@${{env.NPM_VERSION}}

      # Caching of npm packages (https://github.com/actions/cache/blob/main/examples.md#node---npm)
      - name: Cache npm on linux/mac
        uses: actions/cache@v4
        if: matrix.os != 'windows-latest'
        with:
          path: ~/.npm
          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Get npm cache directory
        if: matrix.os == 'windows-latest'
        id: npm-cache
        run: |
          echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
      #   - name: Cache npm on windows
      #     uses: actions/cache@v4
      #     if: matrix.os == 'windows-latest'
      #     with:
      #       path: ${{ steps.npm-cache.outputs.dir }}
      #       key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
      #       restore-keys: |
      #         ${{ runner.os }}-node-

      - name: Cache compiled TS files
        # Use an id for this step so that its cache-hit output can be accessed and checked in the next step.
        id: out-cache
        uses: actions/cache@v4
        with:
          path: ./out
          key: ${{runner.os}}-${{env.CACHE_OUT_DIRECTORY}}-${{hashFiles('src/**')}}

      # This is faster than running `npm ci`, we do not want to build zmq, etc.
      # Let that happen in other jobs, this job needs to be fast
      - name: npm ci
        run: npm ci --ignore-scripts --prefer-offline --no-audit

      - name: npm run postinstall
        run: npm run postinstall

      - name: Compile if not cached
        run: npm run prePublishNonBundle
        if: steps.out-cache.outputs.cache-hit != 'true'

      - name: Run TypeScript unit tests
        id: test_unittests
        run: npm run test:unittests

      - name: Verify there are no unhandled errors
        run: npm run verifyUnhandledErrors

      - name: Create coverage folder (if not created)
        run: npm run createNycFolder

      # Upload unit test coverage reports for later use in the "reports" job.
      - name: Upload unit test coverage reports
        uses: actions/upload-artifact@v3
        if: "(success() || failure()) && !contains(github.ref, 'refs/heads/release') && github.event_name != 'pull_request'"
        with:
          name: ${{env.COVERAGE_REPORTS}}-${{runner.os}}
          path: .nyc_output
          retention-days: 1

  vscodeTests:
    name: Tests # These tests run with Python extension & real Jupyter
    runs-on: ${{ matrix.os }}
    if: github.repository == 'microsoft/vscode-jupyter'
    env:
      VSC_PYTHON_FORCE_LOGGING: 1
      VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 1
    strategy:
      fail-fast: false
      matrix:
        jupyterConnection: [raw] # valid values include raw = uzing zqm, local = using local jupyter, remote = using remote jupyter, web = using remote jupyter in web mode
        python: [python] # valid values include python,conda,noPython
        pythonVersion: ['3.10']
        # Whether we're using stable (<empty>) or pre-release versions of Python packages.
        # When installing pre-release versions, we're only focused on jupyter & related packages.
        # Not pre-release versions of pandas, numpy or other such packages that are not core to Jupyter.
        packageVersion: ['']
        tags: [
            '^[^@]+$|@mandatory|@kernelCore|@python|@jupyter',
            '@widgets',
            '@iw',
            '@webview|@export|@lsp|@variableViewer',
            '@debugger',
            '@notebookPerformance', # Disabled for now, separate PR this will be enabled via cron (want to make PR smaller)
            '@executionPerformance'
          ]
        # We're not running CI on macOS for now because it's one less matrix entry to lower the number of runners used,
        # macOS runners are expensive, and we assume that Ubuntu is enough to cover the UNIX case.
        os: [ubuntu-latest]
        ipywidgetsVersion: ['']
        isScheduled:
          - ${{ github.event_name == 'schedule' && 'true' || '' }} # We need an empty value to preserve the matrix name (as we enforce mandatory tests based on names).
        # More details on includes/excludes can be found here https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#example-including-additional-values-into-combinations
        # Basically with exclude, you can exclude any of the combinations from the result matrix.
        # & with include, you can include additional items to the result matrix.
        exclude:
          - tags: '@notebookPerformance' # Run only as part of scheduled CI runs
            isScheduled: ''
          - tags: '@executionPerformance' # Run only as part of scheduled CI runs
            isScheduled: ''
        include:
          # Mandatory tests
          - jupyterConnection: remote
            python: python
            pythonVersion: '3.10'
            tags: '^[^@]+$|@mandatory'
            os: ubuntu-latest
            ipywidgetsVersion: ''
          - jupyterConnection: remote
            python: python
            pythonVersion: '3.10'
            packageVersion: 'prerelease'
            tags: '^[^@]+$|@mandatory'
            os: ubuntu-latest
            ipywidgetsVersion: ''
          - jupyterConnection: web
            python: python
            pythonVersion: '3.10'
            tags: '^[^@]+$|@mandatory'
            os: ubuntu-latest
            ipywidgetsVersion: ''
          - jupyterConnection: raw
            python: python
            pythonVersion: '3.10'
            tags: '^[^@]+$|@mandatory'
            os: windows-latest
            ipywidgetsVersion: ''
          # IPyWidgets 8
          - jupyterConnection: raw
            python: python
            pythonVersion: '3.10'
            tags: '@widgets'
            os: ubuntu-latest
            ipywidgetsVersion: '8'
          - jupyterConnection: remote
            python: python
            pythonVersion: '3.10'
            tags: '@widgets'
            os: ubuntu-latest
            ipywidgetsVersion: '8'
          # Conda
          - jupyterConnection: raw
            python: conda
            pythonVersion: '3.10'
            tags: '^[^@]+$|@mandatory|@python'
            os: ubuntu-latest
            ipywidgetsVersion: ''
          # Pre-Release Versions
          - jupyterConnection: raw
            python: python
            pythonVersion: '3.10'
            packageVersion: 'prerelease'
            tags: '^[^@]+$|@mandatory|@kernelCore|@python|@jupyter'
            os: ubuntu-latest
            ipywidgetsVersion: ''
          # Without Python
          - jupyterConnection: raw
            python: noPython
            tags: '@nonPython'
            os: ubuntu-latest
            ipywidgetsVersion: ''
          # Misc
          - jupyterConnection: remote
            python: python
            pythonVersion: '3.10'
            tags: '@kernelCore'
            os: ubuntu-latest
            ipywidgetsVersion: ''

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Generate Tag Variable
        run: echo "TAGSVAR=${{ matrix.tags }}}" >> $GITHUB_ENV

      - name: Generate Friendly Variable
        run:
          # Generate a file friendly tag name for the test run (used for the artifacts)
          echo "TAGS_NAME=${TAGSVAR//[^a-zA-Z]/_}" >> $GITHUB_ENV

      - name: Use Python ${{matrix.pythonVersion}}
        uses: actions/setup-python@v5
        id: setupPythonVersion
        if: matrix.python != 'conda' && matrix.python != 'noPython'
        with:
          python-version: ${{matrix.pythonVersion}}

      - name: Install Conda environment from environment.yml
        uses: mamba-org/setup-micromamba@v1
        if: matrix.python == 'conda'
        with:
          cache-downloads: true
          environment-name: functional_test_env
          environment-file: ./build/conda-test-requirements.yml
          create-args: |
            python=${{matrix.pythonVersion}}

      - name: Set CI Path
        shell: bash
        if: matrix.python != 'conda' && matrix.python != 'noPython'
        run: echo "CI_PYTHON_PATH=${{ steps.setupPythonVersion.outputs.python-path }}" >> $GITHUB_ENV

      - name: Set CI Path for Conda
        uses: ./.github/actions/set-python-conda
        id: set-python-conda
        if: matrix.python == 'conda'
        with:
          PYTHON_VERSION: ${{matrix.pythonVersion}}

      - name: Use Node ${{env.NODE_VERSION}}
        uses: actions/setup-node@v4
        with:
          node-version: ${{env.NODE_VERSION}}

      - name: Use Npm ${{env.NPM_VERSION}}
        run: npm i -g npm@${{env.NPM_VERSION}}

      # Start caching

      # Cache Python Dependencies.
      # Caching (https://github.com/actions/cache/blob/main/examples.md#python---pip
      - name: Cache pip on linux
        uses: actions/cache@v4
        if: matrix.os == 'ubuntu-latest' && matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion != '8'
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{hashFiles('build/venv-test-requirements.txt')}}
          restore-keys: |
            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-7

      - name: Cache pip on linux
        uses: actions/cache@v4
        if: matrix.os == 'ubuntu-latest' && matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion == '8'
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{hashFiles('build/venv-test-ipywidgets8-requirements.txt')}}
          restore-keys: |
            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-8

      - name: Cache pip on mac
        uses: actions/cache@v4
        if: matrix.os == 'macos-latest' && matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion != '8'
        with:
          path: ~/Library/Caches/pip
          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{hashFiles('build/venv-test-requirements.txt')}}
          restore-keys: |
            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-7

      - name: Cache pip on mac
        uses: actions/cache@v4
        if: matrix.os == 'macos-latest' && matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion == '8'
        with:
          path: ~/Library/Caches/pip
          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{hashFiles('build/venv-test-ipywidgets8-requirements.txt')}}
          restore-keys: |
            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-8

      - name: Cache pip on windows
        uses: actions/cache@v4
        if: matrix.os == 'windows-latest' && matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion != '8'
        with:
          path: ~\AppData\Local\pip\Cache
          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{hashFiles('build/venv-test-requirements.txt')}}
          restore-keys: |
            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-7

      - name: Cache pip on windows
        uses: actions/cache@v4
        if: matrix.os == 'windows-latest' && matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion == '8'
        with:
          path: ~\AppData\Local\pip\Cache
          key: ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-${{hashFiles('build/venv-test-ipywidgets8-requirements.txt')}}
          restore-keys: |
            ${{ runner.os }}-pip-${{env.PYTHON_VERSION}}-8

      # Caching of npm packages (https://github.com/actions/cache/blob/main/examples.md#node---npm)
      - name: Cache npm on linux/mac
        uses: actions/cache@v4
        if: matrix.os != 'windows-latest'
        with:
          path: ~/.npm
          key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.os }}-node-

      - name: Get npm cache directory
        if: matrix.os == 'windows-latest'
        id: npm-cache
        run: |
          echo "dir=$(npm config get cache)" >> $GITHUB_OUTPUT
      #   - name: Cache npm on windows
      #     uses: actions/cache@v4
      #     if: matrix.os == 'windows-latest'
      #     with:
      #       path: ${{ steps.npm-cache.outputs.dir }}
      #       key: ${{ runner.os }}-node-${{ hashFiles('**/package-lock.json') }}
      #       restore-keys: |
      #         ${{ runner.os }}-node-

      # For faster/better builds of sdists.
      - run: python -m pip install wheel
        shell: bash
        if: matrix.python != 'conda' && matrix.python != 'noPython'

      # Install the pre-release versions of the packages
      - name: Install Pre-Release Python Libs
        if: matrix.python != 'conda' && matrix.python != 'noPython' && matrix.packageVersion == 'prerelease'
        run: |
          python --version
          python -c "import sys;print(sys.executable)"
          python -m pip install jupyter notebook ipykernel ipython nbconvert nbformat jupyterlab traitlets --pre

      # debugpy is not shipped, only installed for local tests.
      # In production, we get debugpy from python extension.
      - name: Install Python Libs
        if: matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion != '8' && matrix.tags != '@notebookPerformance'
        run: |
          python --version
          python -c "import sys;print(sys.executable)"
          python -m pip --disable-pip-version-check install -r build/venv-test-requirements.txt
          python -m pip install packaging # required for installing debugpy
          python ./pythonFiles/install_debugpy.py
          python -m ipykernel install --user

      - name: Install Jupyter Notebook and Lab
        if: matrix.jupyterConnection == 'remote' || matrix.jupyterConnection == 'web'
        run: |
          python -m pip install jupyterlab notebook

      - name: Install Python Libs (IPyWidget 8)
        if: matrix.python != 'conda' && matrix.python != 'noPython' && matrix.ipywidgetsVersion == '8' && matrix.tags != '@notebookPerformance'
        run: |
          python --version
          python -c "import sys;print(sys.executable)"
          python -m pip install ipywidgets -U
          python -m pip --disable-pip-version-check install -r build/venv-test-ipywidgets8-requirements.txt
          python -m pip install packaging # required for installing debugpy
          python ./pythonFiles/install_debugpy.py
          python -m pip install ipywidgets -U
          python -m ipykernel install --user

      - name: Install Python Libs for conda
        shell: bash -l {0}
        if: matrix.python == 'conda'
        run: |
          python -m pip --disable-pip-version-check install -r build/conda-nonconda-test-requirements.txt
          python -m pip install packaging # required for installing debugpy
          python ./pythonFiles/install_debugpy.py

      - name: Install matplotlib widgets into user and system paths
        if:
          matrix.os == 'ubuntu-latest' && matrix.python != 'conda' && matrix.python != 'noPython' && matrix.packageVersion != 'prerelease' && matrix.tags != '^[^@]+$|@mandatory' && matrix.tags != '@notebookPerformance'
          # This test will ensure widgets work when installed in 3 places
          # 1. In python environments site-packages folder (we have other 3rd party widgets in the python env)
          # 2. In user's home folder (ipympl will be installed in there)
          # 3. In system folder (all users) (matplotlib will be installed in there)
        run: |
          # Uninstall ipympl from the sys prefix folder and ensure the widget scripts are installed
          # into the user directory.
          export PYTHON_EXECUTABLE=$(which python)
          echo $PYTHON_EXECUTABLE
          python -m jupyter nbextension uninstall --sys-prefix --py ipympl
          sudo $PYTHON_EXECUTABLE -m jupyter nbextension install --system --py ipympl

      # Run npm install (we need chrome to get downloaded)
      - name: npm ci
        if: matrix.jupyterConnection == 'web'
        uses: ./.github/actions/npm-ci

      # This is faster than running `npm ci`, we do not want to build zmq, etc.
      # Let that happen in other jobs, this job needs to be super fast for linting
      - name: npm ci
        if: matrix.jupyterConnection != 'web'
        run: npm ci --ignore-scripts --prefer-offline --no-audit

      - name: npm run postinstall
        if: matrix.jupyterConnection != 'web'
        run: npm run postinstall

      - name: Install screen capture dependencies
        if: matrix.os == 'ubuntu-latest'
        run: sudo apt-get install imagemagick x11-xserver-utils

        # This step is slow.
      - name: Compile # if not cached
        run: npm run prePublishNonBundle
        # Do not cache for web tests, as the code generated in the compiled code is different for each matrix in web tests
        # Rememeber the compiled code contains injected tests, and the injected tests are different for each matrix in the web
        # if: steps.out-cache.outputs.cache-hit != 'true' && matrix.jupyterConnection != 'web'
        env:
          VSC_JUPYTER_CI_TEST_GREP: ${{ matrix.tags }}
          VSC_JUPYTER_CI_SKIP_WEB_BUNDLE: 1
          VSC_JUPYTER_CI_FAST_COMPILATION: 1

      - name: Compile Performance Test Extension
        if: matrix.tags == '@notebookPerformance'
        run: npm run compile
        working-directory: src/test/vscode-notebook-perf

      # Used by tests for non-python kernels.
      # Test are enabled via env variable `VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST`
      - name: Install Deno
        uses: denoland/setup-deno@v1
        if: matrix.tags != '@notebookPerformance'
        with:
          deno-version: ${{ env.DENO_VERSION}}

      - name: Install Deno Kernel
        if: matrix.tags != '@notebookPerformance'
        run: npx tsx ./build/installDenoKernel.ts

      - name: Create Virtual Env for Tests
        uses: ./.github/actions/create-venv-for-tests
        if: matrix.python != 'conda' && matrix.python != 'noPython' && matrix.os != 'windows-latest' && matrix.jupyterConnection != 'remote' && matrix.tags != '^[^@]+$|@mandatory' && matrix.ipywidgetsVersion != '8' && matrix.jupyterConnection != 'web' && matrix.jupyterConnection != 'remote' && matrix.tags != '@debugger' && matrix.tags != '@webview|@export|@lsp|@variableViewer' && matrix.tags != '@notebookPerformance'

      - name: Create Virtual Env for Tests (ipywidgets 8)
        uses: ./.github/actions/create-venv-for-tests
        if: matrix.python != 'conda' && matrix.python != 'noPython' && matrix.os != 'windows-latest' && matrix.jupyterConnection != 'remote' && matrix.tags != '^[^@]+$|@mandatory' && matrix.ipywidgetsVersion == '8' && matrix.jupyterConnection != 'web' && matrix.jupyterConnection != 'remote' && matrix.tags != '@debugger' && matrix.tags != '@webview|@export|@lsp|@variableViewer' && matrix.tags != '@notebookPerformance'
        with:
          IPyWidgetVersion: '8'

      - name: Create temp folder for user data dir
        run: |
          echo "VSC_JUPYTER_USER_DATA_DIR=$env:USERPROFILE\AppData\Local\Temp" >> $Env:GITHUB_ENV
        if: matrix.os == 'windows-latest'
        shell: pwsh

      - name: Print temp folder for user data dir
        run: |
          echo ${{env.VSC_JUPYTER_USER_DATA_DIR}} is user data dir
        if: matrix.os == 'windows-latest'
        shell: pwsh

      - name: Create temp folder for user data dir
        run: |
          echo "VSC_JUPYTER_USER_DATA_DIR=$(mktemp -d)" >> $GITHUB_ENV
          echo ${{env.VSC_JUPYTER_USER_DATA_DIR}} is user data dir
        if: matrix.os != 'windows-latest'

      # Set the correct xvfb commands to run vscode tests
      # https://code.visualstudio.com/api/working-with-extensions/continuous-integration
      # Note that xvfb github action only runs through xvfb on linux, so only set the commands there
      # as on windows / mac they would get directly passed to run, not xvfb-run
      - name: Set xvfb parameters linux
        if: matrix.os == 'ubuntu-latest'
        run: echo "xvfbCommand=--server-args=\"-screen 0 1024x768x24\"" >> $GITHUB_ENV

      - name: Run Native Notebook with VSCode & Jupyter (ubuntu)
        uses: GabrielBB/xvfb-action@v1.4
        with:
          run: ${{ env.xvfbCommand }} npm run testNativeNotebooksInVSCode
        env:
          VSC_JUPYTER_FORCE_LOGGING: 1
          VSC_PYTHON_FORCE_LOGGING: 1
          VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 1
          VSC_JUPYTER_REMOTE_NATIVE_TEST: ${{ matrix.jupyterConnection == 'remote' }}
          VSC_JUPYTER_NON_RAW_NATIVE_TEST: ${{ matrix.jupyterConnection == 'local' }}
          VSC_JUPYTER_CI_RUN_JAVA_NB_TEST: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_IS_CONDA: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders'
          VSC_JUPYTER_CI_TEST_GREP: ${{ matrix.tags }}
        id: test_notebook_vscode_ubuntu
        if: matrix.python != 'noPython' && matrix.os == 'ubuntu-latest' && matrix.jupyterConnection != 'web' && matrix.tags != '@notebookPerformance'

      - name: Run Notebook Perf Test Without Jupyter
        uses: GabrielBB/xvfb-action@v1.4
        with:
          run: ${{ env.xvfbCommand }} npm run testPerfInVSCode
        env:
          VSC_JUPYTER_FORCE_LOGGING: 1
          VSC_PYTHON_FORCE_LOGGING: 1
          VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 1
          VSC_JUPYTER_PERF_TEST: ${{ matrix.matrix == '@notebookPerformance' }}
          VSC_JUPYTER_NOTEBOOK_PERF_TEST: ${{ matrix.matrix == '@notebookPerformance' }}
          VSC_JUPYTER_CI_TEST_DO_NOT_INSTALL_PYTHON_EXT: ${{ matrix.matrix == '@notebookPerformance' }}
          VSC_JUPYTER_REMOTE_NATIVE_TEST: ${{ matrix.jupyterConnection == 'remote' }}
          VSC_JUPYTER_NON_RAW_NATIVE_TEST: ${{ matrix.jupyterConnection == 'local' }}
          VSC_JUPYTER_CI_RUN_JAVA_NB_TEST: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_IS_CONDA: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders'
          VSC_JUPYTER_CI_TEST_GREP: ${{ matrix.tags }}
        id: test_notebook_perf_vscode_ubuntu
        if: matrix.tags == '@notebookPerformance'

      - name: Run Execution Perf Test With Jupyter
        uses: GabrielBB/xvfb-action@v1.4
        with:
          run: ${{ env.xvfbCommand }} npm run testExecPerfInVSCode
        env:
          VSC_JUPYTER_FORCE_LOGGING: 1
          VSC_PYTHON_FORCE_LOGGING: 1
          VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 1
          VSC_JUPYTER_PERF_TEST: ${{ matrix.matrix == '@executionPerformance' }}
          VSC_JUPYTER_REMOTE_NATIVE_TEST: ${{ matrix.jupyterConnection == 'remote' }}
          VSC_JUPYTER_NON_RAW_NATIVE_TEST: ${{ matrix.jupyterConnection == 'local' }}
          VSC_JUPYTER_CI_RUN_JAVA_NB_TEST: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_IS_CONDA: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders'
          VSC_JUPYTER_CI_TEST_GREP: ${{ matrix.tags }}
        id: test_exec_perf_vscode_ubuntu
        if: matrix.tags == '@executionPerformance'

      - name: Build web bundle for testing
        run: npm run compile-web-test
        if: matrix.python != 'noPython' && matrix.os == 'ubuntu-latest' && matrix.jupyterConnection == 'web'
        env:
          VSC_JUPYTER_CI_TEST_GREP: ${{ matrix.tags }}

      - name: Run Native Notebook with VSCode & Jupyter (web)
        uses: GabrielBB/xvfb-action@v1.4
        with:
          run: npm run testWebExtension
        env:
          VSC_JUPYTER_FORCE_LOGGING: 1
          VSC_PYTHON_FORCE_LOGGING: 1
          VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 1
          VSC_JUPYTER_REMOTE_NATIVE_TEST: ${{ matrix.jupyterConnection == 'web' }}
          VSC_JUPYTER_NON_RAW_NATIVE_TEST: ${{ matrix.jupyterConnection != 'web' }}
          VSC_JUPYTER_CI_RUN_JAVA_NB_TEST: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_IS_CONDA: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders'
          VSC_JUPYTER_CI_TEST_GREP: ${{ matrix.tags }}
        id: test_notebook_vscode_web
        if: matrix.python != 'noPython' && matrix.os == 'ubuntu-latest' && matrix.jupyterConnection == 'web'

      - name: Run Native Notebook with VSCode & Jupyter (windows)
        run: |
          npm run testNativeNotebooksInVSCodeWithoutTestSuffix
        env:
          VSC_JUPYTER_FORCE_LOGGING: 1
          VSC_PYTHON_FORCE_LOGGING: 1
          VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 1
          VSC_JUPYTER_REMOTE_NATIVE_TEST: ${{ matrix.jupyterConnection == 'remote' }}
          VSC_JUPYTER_NON_RAW_NATIVE_TEST: ${{ matrix.jupyterConnection == 'local' }}
          VSC_JUPYTER_CI_RUN_JAVA_NB_TEST: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_IS_CONDA: ${{ matrix.python == 'conda' }}
          VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders'
          VSC_PYTHON_LOG_FILE: ${{env.VSC_JUPYTER_USER_DATA_DIR}}/logs/python.log
          TEST_FILES_SUFFIX: '+(interrupt|execut)*.vscode.test*'
        id: test_notebook_vscode_windows
        if: matrix.python != 'noPython' && matrix.os == 'windows-latest' && matrix.jupyterConnection != 'web'

      - name: Run Native Notebook with VSCode & Jupyter (without Python)
        uses: GabrielBB/xvfb-action@v1.4
        with:
          run: ${{ env.xvfbCommand }} npm run testNativeNotebooksWithoutPythonInVSCode
        env:
          VSC_JUPYTER_FORCE_LOGGING: 1
          VSC_JUPYTER_CI_RUN_NON_PYTHON_NB_TEST: 1
          VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders'
          VSC_JUPYTER_CI_TEST_GREP: ${{ matrix.tags }}
        id: test_notebookWithoutPythonExt_vscode
        if: matrix.python == 'noPython' && matrix.os != 'windows-latest'

      - name: Create coverage folder (if not created)
        run: npm run createNycFolder

      # Upload unit test coverage reports for later use in the "reports" job.
      - name: Upload unit test coverage reports
        uses: actions/upload-artifact@v3
        if: "(success() || failure()) && !contains(github.ref, 'refs/heads/release') && github.event_name != 'pull_request'"
        with:
          name: ${{env.COVERAGE_REPORTS}}-${{runner.os}}
          path: .nyc_output
          retention-days: 1

      - name: Upload VS code logs
        uses: actions/upload-artifact@v3
        if: failure()
        with:
          name: VSCodeLogs-${{matrix.jupyterConnection}}-${{matrix.python}}-${{matrix.pythonVersion}}-${{matrix.packageVersion}}-${{matrix.os}}-${{env.TAGS_NAME}}-${{matrix.ipywidgetsVersion}}
          path: '${{env.VSC_JUPYTER_USER_DATA_DIR}}/logs/**/*'
          retention-days: 1

      - name: Log test results
        if: always()
        run: npm run printTestResults

      - name: Upload test result, screenshots files
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: TestLogs-${{matrix.jupyterConnection}}-${{matrix.python}}-${{matrix.pythonVersion}}-${{matrix.packageVersion}}-${{matrix.os}}-${{env.TAGS_NAME}}-${{matrix.ipywidgetsVersion}}
          path: './logs/*'
          retention-days: 60

      - name: Verify there are no unhandled errors
        run: npm run verifyUnhandledErrors

  smoke-tests:
    timeout-minutes: 30
    name: Smoke tests
    # The value of runs-on is the OS of the current job (specified in the strategy matrix below) instead of being hardcoded.
    runs-on: ${{ matrix.os }}
    if: github.repository == 'microsoft/vscode-jupyter'
    needs: [build-vsix]
    env:
      VSIX_NAME: 'ms-toolsai-jupyter-insiders.vsix'
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python: ['3.10']
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Use Python ${{matrix.python}}
        uses: actions/setup-python@v5
        id: setupPythonVersion
        with:
          python-version: ${{matrix.python}}

      - name: Set CI Path
        shell: bash
        run: echo "CI_PYTHON_PATH=${{ steps.setupPythonVersion.outputs.python-path }}" >> $GITHUB_ENV

      - name: Use Node ${{env.NODE_VERSION}}
        uses: actions/setup-node@v4
        with:
          node-version: ${{env.NODE_VERSION}}

      - name: Use Npm ${{env.NPM_VERSION}}
        run: npm i -g npm@${{env.NPM_VERSION}}

      - name: Download VSIX
        uses: actions/download-artifact@v3
        with:
          name: 'ms-toolsai-jupyter-insiders.vsix'

      # This is faster than running `npm ci`, we do not want to build zmq, etc.
      # Let that happen in other jobs, this job needs to be fast
      - name: npm ci
        run: npm ci --ignore-scripts --prefer-offline --no-audit

      # Run again, as the download of zmq binaries could have failed with 403 errors
      - run: npm run postinstall
        env:
          GITHUB_TOKEN: ${{ github.token }}
        shell: bash

      - name: pip install system test requirements
        run: |
          python -m pip install --upgrade -r build/venv-smoke-test-requirements.txt
        shell: bash

      # Compile the test files.
      - name: Prepare for smoke tests
        run: npx tsc -p ./
        shell: bash

      - name: Run desktop smoke tests
        env:
          DISPLAY: 10
          VSC_JUPYTER_FORCE_LOGGING: 1
          VSC_JUPYTER_CI_TEST_VSC_CHANNEL: 'insiders'
        uses: GabrielBB/xvfb-action@v1.4
        with:
          run: npm run testSmokeLogged

  coverage:
    name: Coverage reports upload
    runs-on: ubuntu-latest
    if: "(success() || failure()) && !contains(github.ref, 'refs/heads/release') && github.event_name != 'pull_request'"
    needs: [ts_tests, vscodeTests]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Node
        uses: actions/setup-node@v4
        with:
          node-version: ${{env.NODE_VERSION}}

      - name: npm ci
        uses: ./.github/actions/npm-ci

      - name: Cache compiled TS files
        id: out-cache
        uses: actions/cache@v4
        with:
          path: ./out
          key: ${{runner.os}}-${{env.CACHE_OUT_DIRECTORY}}-${{hashFiles('src/**')}}

      - name: Compile if not cached
        run: npm run prePublishNonBundle
        if: steps.out-cache.outputs.cache-hit != 'true'

      # It isn't possible to specify a regex for artifact names, so we have to download each artifact manually.
      # The name pattern is ${{env.COVERAGE_REPORTS}}-${{runner.os}}, and possible values for runner.os are `Linux`, `Windows`, or `macOS`.
      # See https://help.github.com/en/actions/reference/contexts-and-expression-syntax-for-github-actions#runner-context
      - name: Download Ubuntu test coverage artifacts
        uses: actions/download-artifact@v3
        with:
          name: ${{env.COVERAGE_REPORTS}}-${{runner.os}}
          path: ${{env.COVERAGE_REPORTS}}-${{runner.os}}

      - name: Extract Ubuntu coverage artifacts to ./nyc_output
        run: |
          pwd
          mkdir .nyc_output
          mv ${{env.COVERAGE_REPORTS}}-${{runner.os}}/* .nyc_output
          rm -r ${{env.COVERAGE_REPORTS}}-${{runner.os}}

      - name: Merge coverage reports
        run: |
          pwd
          npx nyc merge ./.nyc_output
          rm -r .nyc_output
          mkdir .nyc_output
          mv coverage.json ./.nyc_output
        continue-on-error: true

      - name: Generate coverage reports
        run: npm run test:cover:report
        continue-on-error: true

      - name: Upload coverage to codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage/cobertura-coverage.xml
          path_to_write_report: ./coverage/codecov_report.txt
      # Alternative method for uploading
      # - name: Upload coverage to codecov
      #  run: bash <(curl -s https://codecov.io/bash) -f ./coverage/cobertura-coverage.xml -v