wpt.yml

# This workflow will check out wpt and run the WebDriver BiDi tests against our
# implementation sharded without generating report or updating expectations.

name: WPT

# Declare default permissions as read only.
permissions: read-all

env:
  DEBUG: 'bidi:server:*,bidi:mapper:*'
  DEBUG_DEPTH: 10
  FORCE_COLOR: 3
  PIP_DISABLE_PIP_VERSION_CHECK: 1

on:
  push:
    branches: 'main'
  pull_request:
    types:
      # These are the defaults. See
      # https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request
      - opened
      - reopened
      - synchronize

      # Used for `update-expectations`
      - labeled
  workflow_dispatch:
    inputs:
      tests:
        description: Tests to run (e.g. 'network/combined/')
        required: false
        type: string
      verbose:
        description: Verbose logging
        default: false
        required: false
        type: boolean
      auto-commit:
        description: Auto-commit expectations
        default: false
        required: false
        type: boolean

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  wpt-required:
    name: '[Required] WPT sink'
    needs: [wpt]
    runs-on: ubuntu-latest
    if: always()
    steps:
      - if: ${{ needs.wpt.result != 'success' }}
        run: 'exit 1'
      - run: 'exit 0'

  wpt:
    name: ${{ matrix.this_chunk }}/${{ matrix.total_chunks }} ${{ matrix.kind }}-${{ matrix.head }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        # Should be in sync with `update_expectations` job.
        kind: [chromedriver, mapper]
        head: [headless, headful]
        total_chunks: [6]
        this_chunk: [1, 2, 3, 4, 5, 6]
        exclude:
          # Don't run headful mapper, as it takes too long.
          - kind: mapper
            head: headful
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          submodules: true
      - name: Set up Node.js
        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
        with:
          node-version-file: '.nvmrc'
          cache: npm
      - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1
      - name: Install and build npm dependencies
        run: npm ci
      - name: Setup dirs
        run: mkdir -p out
      - name: Set up Python
        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: '3.11'
          cache: 'pip'
      - name: Set up virtualenv
        run: pip install virtualenv
      - name: Set up hosts
        run: ./wpt make-hosts-file | sudo tee -a /etc/hosts
        working-directory: wpt
      # Install chrome, chromedriver and headless shell is required to keep them cached.
      - name: Install all chrome binaries if needed
        uses: ./.github/actions/setup-chrome-binaries
      - name: Run WPT tests
        timeout-minutes: 60
        run: >
          xvfb-run --auto-servernum
          npm run wpt -- "webdriver/tests/bidi/${{ github.event.inputs.tests }}"
        env:
          CHROMEDRIVER: ${{ matrix.kind == 'chromedriver' }}
          HEADLESS: ${{ matrix.head!='headful' }}
          THIS_CHUNK: ${{ matrix.this_chunk }}
          TOTAL_CHUNKS: ${{ matrix.total_chunks }}
          UPDATE_EXPECTATIONS: false
          VERBOSE: ${{ github.event.inputs.verbose }}
          WPT_REPORT: out/wptreport.${{ matrix.kind }}-${{ matrix.head }}-${{ matrix.this_chunk }}.${{ matrix.total_chunks }}.json
      - name: Run WPT interop tests
        if: ${{matrix.kind == 'chromedriver'}}
        run: >
          xvfb-run --auto-servernum
          npm run wpt -- "webdriver/tests/interop/${{ github.event.inputs.tests }}"
        env:
          CHROMEDRIVER: ${{ matrix.kind == 'chromedriver' }}
          CHROMEDRIVER_LOG_NAME: 'chromedriver-interop'
          THIS_CHUNK: ${{ matrix.this_chunk }}
          TOTAL_CHUNKS: ${{ matrix.total_chunks }}
          UPDATE_EXPECTATIONS: false
          VERBOSE: ${{ github.event.inputs.verbose }}
          WPT_REPORT: out/wptreport-interop.${{ matrix.kind }}-${{ matrix.head }}-${{ matrix.this_chunk }}.${{ matrix.total_chunks }}.json
          FAIL_NO_TEST: false
      - name: Upload artifacts
        if: always()
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: ${{ matrix.kind }}-${{ matrix.head }}-${{ matrix.this_chunk }}.${{ matrix.total_chunks }}-artifacts
          path: |
            logs
            out

  update_expectations:
    # The job gets all the sharded reports for a given configuration and updates the
    # expectations for the configuration. It uploads the results to the artifacts.
    name: Update WPT expectations (if required)
    strategy:
      matrix:
        # Should be in sync with `wpt` job.
        kind: [chromedriver, mapper]
        head: [headless, headful]
        exclude:
          # Don't run headful mapper, as it takes too long.
          - kind: mapper
            head: headful
    runs-on: ubuntu-latest
    needs: [wpt-required]
    # Only update expectations if the tests were failed and either `auto-commit`
    # checkbox is set or `update-expectations` label is present.
    if: ${{ failure() && (github.event.inputs.auto-commit == 'true' || contains(github.event.pull_request.labels.*.name, 'update-expectations')) }}
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
        with:
          submodules: true
      - name: Set up Node.js
        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
        with:
          node-version-file: '.nvmrc'
          cache: npm
      - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1
      - name: Install and build npm dependencies
        run: npm ci
      - name: Setup dirs
        run: mkdir -p out
      - name: Set up Python
        uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
        with:
          python-version: '3.11'
          cache: 'pip'
      - name: Set up virtualenv
        run: pip install virtualenv
      - name: Download Artifact
        # Get all the artifacts from the previous WPT run in order to get all the
        # test reports.
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          path: wpt_artifacts
          # Merge the artifacts from all jobs in the same file.
          merge-multiple: true
      - name: Update expectations
        timeout-minutes: 60
        env:
          CHROMEDRIVER: ${{ matrix.kind == 'chromedriver' }}
          HEADLESS: ${{ matrix.head!='headful' }}
          # Do not run tests, only update expectations.
          RUN_TESTS: false
          UPDATE_EXPECTATIONS: true
          VERBOSE: true
        # Find all the reports for the given configuration and update the
        # expectations with each report one-by-one.
        run: >
          find ./wpt_artifacts/
          -name "wptreport.${{ matrix.kind }}-${{ matrix.head }}*.json"
          -exec npm run wpt -- --wpt-report {} \;
      - name: Update interop expectations
        if: ${{ matrix.kind == 'chromedriver' }}
        env:
          CHROMEDRIVER: ${{ matrix.kind == 'chromedriver' }}
          HEADLESS: ${{ matrix.head!='headful' }}
          # Do not run tests, only update expectations.
          RUN_TESTS: false
          UPDATE_EXPECTATIONS: true
          VERBOSE: true
        # Find all the reports for the given configuration and update the
        # expectations with each report one-by-one.
        run: >
          find ./wpt_artifacts/
          -name "wptreport-interop.${{ matrix.kind }}-${{ matrix.head }}*.json"
          -exec npm run wpt -- --wpt-report {} \;
      - name: Move updated expectations
        # Move the expectations from the current config to a separate directory to
        # upload them to artifacts.
        run: |
          mkdir -p ./artifacts/updated-wpt-metadata/${{ matrix.kind }}/${{ matrix.head }}
          mv ./wpt-metadata/${{ matrix.kind }}/${{ matrix.head }}/* ./artifacts/updated-wpt-metadata/${{ matrix.kind }}/${{ matrix.head }}/
      - name: Upload artifacts
        if: success()
        uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3
        with:
          name: updated-wpt-metadata-${{ matrix.kind }}-${{ matrix.head }}
          path: ./artifacts

  commit_updated_expectations:
    # Gets updated wpt expectations for all configurations and commits them in a
    # single commit.
    name: Commit updated expectations
    runs-on: ubuntu-latest
    needs: [update_expectations]
    # Experiment.
    # TODO: remove.
    if: success() || failure()
    steps:
      # Just checkout the repo. No need in setting up Node.js or Python.
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Download Artifact
        # There is no way to download artifacts by wildcard, so we need to download
        # all of them. The `updated-wpt-metadata` directory should contain all the
        # updated expectations from the `update_expectations` matrix.
        uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          path: all-artifacts
          merge-multiple: true
      - name: Replace expectations with the updated ones.
        # Remove the actual expectations and replace them with the updated ones.
        run: |
          rm -rf wpt-metadata/chromedriver wpt-metadata/mapper
          mv all-artifacts/updated-wpt-metadata/* ./wpt-metadata/
      - name: Create Pull Request
        uses: peter-evans/create-pull-request@5e914681df9dc83aa4e4905692ca88beb2f9e91f # v7.0.5
        with:
          token: ${{ secrets.BROWSER_AUTOMATION_BOT_TOKEN }}
          branch: ${{ github.head_ref }}-update-expectations
          delete-branch: true
          base: ${{ github.head_ref }}
          committer: Browser Automation Bot <browser-automation-bot@google.com>
          author: Browser Automation Bot <browser-automation-bot@google.com>
          commit-message: 'test: update the expectations for PR'
          title: 'test: update the expectations for PR'
          body: 'Automatically generated by https://github.com/GoogleChromeLabs/chromium-bidi/blob/main/.github/workflows/wpt.yml'
          push-to-fork: browser-automation-bot/chromium-bidi
          add-paths: |
            wpt-metadata/**/*.ini

  update_report:
    name: WPT report
    needs: wpt
    if: ${{ github.ref == 'refs/heads/main' }}
    runs-on: ubuntu-latest
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    # Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
    permissions:
      contents: read
      pages: write
      id-token: write
    steps:
      - name: Checkout
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
      - name: Set up Node.js
        uses: actions/setup-node@39370e3970a6d050c480ffad4ff0ed4d3fdee5af # v4.1.0
        with:
          node-version-file: '.nvmrc'
          cache: npm
      - uses: google/wireit@83d7f8bed70b7bcfc40f4b9f54f4b7485753991b # setup-github-actions-caching/v2.0.1
      - name: Install and build npm dependencies
        run: npm ci
      - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8
        with:
          pattern: chromedriver-headful-*
          merge-multiple: true
      - name: Generate HTML test report
        run: >
          node tools/wpt-report-builder/builder.mjs \
            --bidi $(ls out/wptreport.*.json) \
            --interop $(ls out/wptreport-interop.*.json) \
            --out out/wptreport.html \
            --out-label-2023 out/wptreport-2023.html
      - name: Prepare Pages
        # TODO: Support merged wpt report as before
        # cp out/wptreport.json out/site/wptreport.json
        run: |
          mkdir -p out/site
          cp out/wptreport.html out/site/index.html
          cp out/wptreport-2023.html out/site/2023.html
      - name: Setup Pages
        uses: actions/configure-pages@983d7736d9b0ae728b81ab479565c72886d7745b # v5.0.0
      - name: Upload Pages artifact
        uses: actions/upload-pages-artifact@56afc609e74202658d3ffba0e8f6dda462b719fa # v3.0.1
        with:
          path: out/site
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@d6db90164ac5ed86f2b6aed7e0febac5b3c0c03e # v4.0.5