e2e.yml

name: End-to-End Tests
permissions:
  contents: read # for checkout
on:
  # Build on pushes branches that have a PR (including drafts)
  pull_request:
  # Build on commits pushed to branches without a PR if it's in the allowlist
  push:
    branches: [next]
jobs:
  install:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    env:
      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
    strategy:
      fail-fast: false
      matrix:
        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
        project: [chromium, firefox]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        name: Install pnpm
        id: pnpm-install
        with:
          run_install: false

      - name: Get pnpm store directory
        id: pnpm-cache
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT

      - name: Cache node modules
        id: cache-node-modules
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ vars.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            v1-${{ runner.os }}-pnpm-store-${{ vars.cache-name }}-
            v1-${{ runner.os }}-pnpm-store-
            v1-${{ runner.os }}-

      - name: Install project dependencies
        run: pnpm install

      - name: Store Playwright's Version
        run: |
          PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
          echo "Playwright's Version: $PLAYWRIGHT_VERSION"
          echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV

      - name: Cache Playwright Browsers for Playwright's Version
        id: cache-playwright-browsers
        uses: actions/cache@v4
        with:
          path: ~/.cache/ms-playwright
          key: playwright-browsers-${{ vars.PLAYWRIGHT_VERSION }}

      - name: Install Playwright Browsers
        if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
        run: npx playwright install --with-deps

      - name: Build CLI
        run: pnpm build:cli --output-logs=full --log-order=grouped

      - name: Build E2E test studio on next
        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }}
        env:
          # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
          # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
          # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
          SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
          SANITY_E2E_PROJECT_ID: ${{ vars.SANITY_E2E_PROJECT_ID }}
          SANITY_E2E_DATASET: ${{ vars.SANITY_E2E_DATASET }}
        run: pnpm e2e:setup && pnpm e2e:build

      - name: Build E2E test studio on PR
        if: ${{ github.event_name == 'pull_request' }}
        env:
          # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
          # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
          # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
          SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
          SANITY_E2E_PROJECT_ID: ${{ vars.SANITY_E2E_PROJECT_ID }}
          SANITY_E2E_DATASET: pr-${{ matrix.project }}-${{ github.event.number }}
        run: pnpm e2e:setup && pnpm e2e:build

      # Caches build from either PR or next
      - name: Cache build
        id: cache-e2e-build
        uses: actions/cache@v4
        env:
          cache-name: cache-e2e-build
        with:
          path: |
            ./*
            !**/node_modules/**
          # Unique key for a workflow run. Should be invalidated in the next run
          key: ${{ runner.os }}-${{ matrix.project }}-${{ vars.cache-name }}-${{ github.run_id }}

  playwright-test:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    needs: [install]
    env:
      TURBO_TOKEN: ${{ secrets.TURBO_TOKEN }}
      TURBO_TEAM: ${{ vars.TURBO_TEAM }}
    strategy:
      fail-fast: false
      matrix:
        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
        project: [chromium, firefox]
        # Add more shards here if needed
        shardIndex: [1, 2]
        shardTotal: [2]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        name: Install pnpm
        id: pnpm-install
        with:
          run_install: false

      - name: Get pnpm store directory
        id: pnpm-cache
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT

      - name: Cache node modules
        id: cache-node-modules
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ vars.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            v1-${{ runner.os }}-pnpm-store-${{ vars.cache-name }}-
            v1-${{ runner.os }}-pnpm-store-
            v1-${{ runner.os }}-

      - name: Install project dependencies
        run: pnpm install

      - name: Store Playwright's Version
        run: |
          PLAYWRIGHT_VERSION=$(npx playwright --version | sed 's/Version //')
          echo "Playwright's Version: $PLAYWRIGHT_VERSION"
          echo "PLAYWRIGHT_VERSION=$PLAYWRIGHT_VERSION" >> $GITHUB_ENV

      - name: Cache Playwright Browsers for Playwright's Version
        id: cache-playwright-browsers
        uses: actions/cache/restore@v4
        with:
          path: ~/.cache/ms-playwright
          key: playwright-browsers-${{ vars.PLAYWRIGHT_VERSION }}

      - name: Install Playwright Browsers
        if: steps.cache-playwright-browsers.outputs.cache-hit != 'true'
        run: npx playwright install --with-deps

      - name: Restore build cache
        uses: actions/cache/restore@v4
        id: restore-build
        env:
          cache-name: cache-e2e-build
        with:
          path: |
            ./*
            !**/node_modules/**
          key: ${{ runner.os }}-${{ matrix.project }}-${{ vars.cache-name }}-${{ github.run_id }}
          # If the cached build from the pervious step is not available. Fail the build
          fail-on-cache-miss: true

      - name: Run E2E tests on next
        if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/next' }}
        env:
          # Missing in docs but in use
          # here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108
          PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }}
          # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
          # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
          # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
          SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
          SANITY_E2E_PROJECT_ID: ${{ vars.SANITY_E2E_PROJECT_ID }}
          SANITY_E2E_DATASET: ${{ vars.SANITY_E2E_DATASET }}
        run: pnpm test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}

      - name: Run E2E tests on PR
        if: ${{ github.event_name == 'pull_request' }}
        env:
          # Missing in docs but in use
          # here https://github.com/microsoft/playwright/blob/main/packages/playwright/src/reporters/blob.ts#L108
          PWTEST_BLOB_REPORT_NAME: ${{ matrix.project }}
          # Update the SANITY_E2E_SESSION_TOKEN on github to the new value once this is merged to next
          # Change the below to `secrets.SANITY_E2E_SESSION_TOKEN`
          # Delete `SANITY_E2E_SESSION_TOKEN_NEW` from github
          SANITY_E2E_SESSION_TOKEN: ${{ secrets.SANITY_E2E_SESSION_TOKEN_NEW }}
          SANITY_E2E_PROJECT_ID: ${{ vars.SANITY_E2E_PROJECT_ID }}
          SANITY_E2E_DATASET: pr-${{ matrix.project }}-${{ github.event.number }}
        run: pnpm test:e2e --project ${{ matrix.project }} --shard ${{ matrix.shardIndex }}/${{ matrix.shardTotal }}

      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: playwright-report
          path: blob-report
          retention-days: 30
  merge-reports:
    if: always()
    needs: [playwright-test]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-node@v4
        with:
          node-version: 18

      - uses: pnpm/action-setup@v4
        name: Install pnpm
        id: pnpm-install
        with:
          run_install: false

      - name: Get pnpm store directory
        id: pnpm-cache
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path)" >> $GITHUB_OUTPUT

      - name: Cache node modules
        id: cache-node-modules
        uses: actions/cache@v4
        env:
          cache-name: cache-node-modules
        with:
          path: ${{ steps.pnpm-cache.outputs.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ vars.cache-name }}-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            v1-${{ runner.os }}-pnpm-store-${{ vars.cache-name }}-
            v1-${{ runner.os }}-pnpm-store-
            v1-${{ runner.os }}-

      - name: Install project dependencies
        run: pnpm install

      - name: Download blob reports from Github Actions Artifacts
        uses: actions/download-artifact@v3
        with:
          name: playwright-report
          path: playwright-report

      - name: Merge into HTML Report
        run: npx playwright merge-reports --reporter html ./playwright-report

      - name: Upload HTML report
        uses: actions/upload-artifact@v3
        with:
          name: html-report--attempt-${{ github.run_attempt }}
          path: playwright-report
          retention-days: 30

  cleanup:
    permissions:
      contents: read
      actions: write # needed to delete the cache
    timeout-minutes: 30
    name: Cleanup (${{ matrix.project }})
    runs-on: ubuntu-latest
    needs: [playwright-test]

    strategy:
      # we want to know if a test fails on a specific node version
      fail-fast: false
      matrix:
        # Be sure to update all instances in this file and `pr-cleanup.yml` if updated
        project: [chromium, firefox]

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup node
        uses: actions/setup-node@v4
        with:
          node-version: 18

      # Delete the cache so it is only used once
      - name: Delete Cache
        run: gh cache delete ${{ runner.os }}-${{ matrix.project }}-${{ vars.cache-name }}-${{ github.run_id }}
        env:
          cache-name: cache-e2e-build
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}