diff --git a/.github/workflows/test.yml b/.github/workflows/lint.yml similarity index 93% rename from .github/workflows/test.yml rename to .github/workflows/lint.yml index 3f69d85..417f831 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/lint.yml @@ -1,4 +1,4 @@ -name: Test +name: Lint on: push: @@ -7,7 +7,7 @@ on: branches: [main] jobs: - build: + lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/preview.yml b/.github/workflows/preview.yml index 998a9ee..9560788 100644 --- a/.github/workflows/preview.yml +++ b/.github/workflows/preview.yml @@ -1,4 +1,4 @@ -name: Screen Reader Test Preview +name: Screen Reader Tests Preview Report on: pull_request: @@ -16,7 +16,7 @@ permissions: # Allow one concurrent deployment concurrency: - group: "aria-at-tests-preview" + group: "aria-at-tests-preview-report" cancel-in-progress: true jobs: @@ -245,10 +245,13 @@ jobs: path: ./playwright-report publish-preview-html-report: - name: Publish Preview HTML Report To Vercel + name: Publish Preview HTML Report if: always() needs: [test-voiceover-preview, test-nvda-preview] runs-on: ubuntu-latest + environment: + name: aria-at-tests-preview-report + url: ${{ steps.deploy-pages.outputs.page_url }} steps: - name: Clear Space run: | @@ -279,24 +282,11 @@ jobs: with: node-version: 20 - - name: Update Deployment Status - Start - uses: bobheadxi/deployments@v1 - id: deployment - with: - step: start - env: aria-at-tests-preview - token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/download-artifact@v4 with: pattern: playwright-report-* path: ./playwright-report - - name: Log Directory Structure - run: | - ls -la ./ - ls -la ./playwright-report - - name: Cache node_modules uses: actions/cache@v3 id: cache-macos-node-modules @@ -311,35 +301,24 @@ jobs: - name: Cleanup Downloads run: rm -rf ./playwright-report || true - - name: Log Directory Structure - run: | - ls -la ./ - ls -la ./html-report + - name: Setup Pages + uses: actions/configure-pages@v4 - - name: Upload HTML Report Artifact - uses: actions/upload-artifact@v4 + - name: Upload HTML Report + uses: actions/upload-pages-artifact@v3 with: - name: html-report path: html-report - - name: Deploy HTML Report to Vercel - uses: amondnet/vercel-action@v25 - id: vercel-action - with: - vercel-token: ${{ secrets.VERCEL_TOKEN }} - vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} - vercel-project-id: ${{ secrets.PREVIEW_VERCEL_PROJECT_ID }} - github-comment: true - github-token: ${{ secrets.GITHUB_TOKEN }} - working-directory: html-report - - - name: Update Deployment Status - Finish - uses: bobheadxi/deployments@v1 - if: always() + - name: Deploy to GitHub Pages + id: deploy-pages + uses: actions/deploy-pages@v4 + + - name: PR Comment + if: ${{ always() && github.event_name == 'pull_request' }} + uses: thollander/actions-comment-pull-request@v2 with: - step: finish - token: ${{ secrets.GITHUB_TOKEN }} - status: ${{ job.status }} - deployment_id: ${{ steps.deployment.outputs.deployment_id }} - env: ${{ steps.deployment.outputs.env }} - env_url: ${{ steps.vercel-action.outputs.preview-url }} + message: | + Playwright Test Report: ${{steps.deploy-pages.outputs.page_url}} + comment_tag: "Playwright Test Report" + pr_number: ${{ inputs.pr_number }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/nvda.yml b/.github/workflows/report.yml similarity index 53% rename from .github/workflows/nvda.yml rename to .github/workflows/report.yml index 52d0e8e..386d6f7 100644 --- a/.github/workflows/nvda.yml +++ b/.github/workflows/report.yml @@ -1,4 +1,4 @@ -name: NVDA Screen Reader Test +name: Screen Reader Tests Report on: push: @@ -16,10 +16,58 @@ permissions: # Allow one concurrent deployment concurrency: - group: "aria-at-tests-nvda" + group: "aria-at-tests-report" cancel-in-progress: true jobs: + macos-install: + name: MacOS Install + runs-on: macos-12 + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: Cache node_modules + uses: actions/cache@v3 + id: cache-macos-node-modules + with: + path: | + node_modules + key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} + + - name: Cache Playwright binaries + uses: actions/cache@v3 + id: cache-macos-playwright + with: + path: | + ~/Library/Caches/ms-playwright + key: macos-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} + + - name: Cache Test Suites + uses: actions/cache@v3 + id: cache-macos-test-suites + with: + path: | + testSuites.json + key: macos-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }} + + - name: Install Dependencies + if: steps.cache-macos-node-modules.outputs.cache-hit != 'true' + run: yarn install --frozen-lockfile + + - name: Install Playwright Browsers + if: steps.cache-macos-playwright.outputs.cache-hit != 'true' + run: yarn test:install + + - name: Generate Test Suites + if: steps.cache-macos-test-suites.outputs.cache-hit != 'true' + run: yarn test:generate + windows-install: name: Windows Install runs-on: windows-2022 @@ -68,6 +116,70 @@ jobs: if: steps.cache-windows-test-suites.outputs.cache-hit != 'true' run: yarn test:generate + test-voiceover: + name: Playwright VoiceOver Tests (${{ matrix.shardIndex }} / 10) + needs: macos-install + runs-on: macos-12 + strategy: + fail-fast: false + matrix: + shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] + + steps: + - uses: actions/checkout@v3 + with: + submodules: recursive + + - uses: actions/setup-node@v3 + with: + node-version: 20 + + - name: Guidepup Setup + uses: guidepup/setup-action@0.13.0 + + - name: Cache node_modules + uses: actions/cache@v3 + with: + path: | + node_modules + key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} + + - name: Cache Playwright + uses: actions/cache@v3 + with: + path: | + ~/Library/Caches/ms-playwright + key: macos-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} + + - name: Cache Test Suites + uses: actions/cache@v3 + with: + path: | + testSuites.json + key: macos-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }} + + - name: Run Tests + run: yarn test ./src/macOsVoiceOver.spec.ts --config macos.config.ts --shard ${{ matrix.shardIndex }}/10 + continue-on-error: true + + - name: Upload Playwright Artifacts + uses: actions/upload-artifact@v4 + if: always() + continue-on-error: true + with: + name: artifacts-macos-voiceover-${{ matrix.shardIndex }} + path: | + ./test-results + ./recordings + + - name: Upload HTML Report Artifacts + uses: actions/upload-artifact@v4 + if: always() + continue-on-error: true + with: + name: playwright-report-macos-voiceover-${{ matrix.shardIndex }} + path: ./playwright-report + test-nvda: name: Playwright NVDA Tests (${{ matrix.shardIndex }} / 20) needs: windows-install @@ -136,7 +248,7 @@ jobs: run: yarn test ./src/windowsNvda.spec.ts --config windows.config.ts --shard ${{ matrix.shardIndex }}/20 continue-on-error: true - - name: Archive Playwright Artifacts + - name: Upload Playwright Artifacts uses: actions/upload-artifact@v4 if: always() continue-on-error: true @@ -146,19 +258,22 @@ jobs: ./test-results ./recordings - - name: Archive Allure Artifacts + - name: Upload HTML Report Artifacts uses: actions/upload-artifact@v4 if: always() continue-on-error: true with: - name: allure-results-windows-nvda-${{ matrix.shardIndex }} - path: ./allure-results + name: playwright-report-windows-nvda-${{ matrix.shardIndex }} + path: ./playwright-report publish-html-report: - name: Publish HTML Report To Vercel + name: Publish HTML Report if: always() - needs: [test-nvda] + needs: [test-voiceover, test-nvda] runs-on: ubuntu-latest + environment: + name: aria-at-tests-report + url: ${{ steps.deploy-pages.outputs.page_url }} steps: - name: Clear Space run: | @@ -189,56 +304,33 @@ jobs: with: node-version: 20 - - name: Update Deployment Status - Start - uses: bobheadxi/deployments@v1 - id: deployment - with: - step: start - env: aria-at-tests-nvda - token: ${{ secrets.GITHUB_TOKEN }} - - uses: actions/download-artifact@v4 with: - pattern: allure-results-windows-nvda-* - path: ./allure-results + pattern: playwright-report-* + path: ./playwright-report - - name: Move Allure results into one directory - run: | - mkdir allure-results-final - for i in allure-results-windows-nvda-{1..30}; do - if [ -d "$i" ]; then - mv $i/* allure-results-final/ - fi - done - working-directory: ./allure-results - - - name: Install Dependencies - run: yarn install --frozen-lockfile + - name: Cache node_modules + uses: actions/cache@v3 + id: cache-macos-node-modules + with: + path: | + node_modules + key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} - - name: Create Allure Test Report - run: npx allure generate ./allure-results/allure-results-final --clean -o ./allure-report + - name: Create Test Report + run: yarn test:report - name: Cleanup Downloads - run: rm -rf ./allure-results || true + run: rm -rf ./playwright-report || true - - name: Deploy to Vercel - uses: amondnet/vercel-action@v25 - id: vercel-action - with: - vercel-token: ${{ secrets.VERCEL_TOKEN }} - vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} - vercel-project-id: ${{ secrets.NVDA_VERCEL_PROJECT_ID }} - github-comment: false - vercel-args: "--prod" - working-directory: allure-report + - name: Setup Pages + uses: actions/configure-pages@v4 - - name: Update Deployment Status - Finish - uses: bobheadxi/deployments@v1 - if: always() + - name: Upload HTML Report + uses: actions/upload-pages-artifact@v3 with: - step: finish - token: ${{ secrets.GITHUB_TOKEN }} - status: ${{ job.status }} - deployment_id: ${{ steps.deployment.outputs.deployment_id }} - env: ${{ steps.deployment.outputs.env }} - env_url: ${{ steps.vercel-action.outputs.preview-url }} + path: html-report + + - name: Deploy to GitHub Pages + id: deploy-pages + uses: actions/deploy-pages@v4 diff --git a/.github/workflows/voiceover.yml b/.github/workflows/voiceover.yml deleted file mode 100644 index 41e2066..0000000 --- a/.github/workflows/voiceover.yml +++ /dev/null @@ -1,227 +0,0 @@ -name: VoiceOver Screen Reader Test - -on: - push: - branches: [main] - # Allows you to run this workflow manually from the Actions tab - workflow_dispatch: - -# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages -permissions: - contents: read - deployments: write - pages: write - id-token: write - pull-requests: write - -# Allow one concurrent deployment -concurrency: - group: "aria-at-tests-voiceover" - cancel-in-progress: true - -jobs: - macos-install: - name: MacOS Install - runs-on: macos-12 - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - uses: actions/setup-node@v3 - with: - node-version: 20 - - - name: Cache node_modules - uses: actions/cache@v3 - id: cache-macos-node-modules - with: - path: | - node_modules - key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} - - - name: Cache Playwright binaries - uses: actions/cache@v3 - id: cache-macos-playwright - with: - path: | - ~/Library/Caches/ms-playwright - key: macos-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} - - - name: Cache Test Suites - uses: actions/cache@v3 - id: cache-macos-test-suites - with: - path: | - testSuites.json - key: macos-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }} - - - name: Install Dependencies - if: steps.cache-macos-node-modules.outputs.cache-hit != 'true' - run: yarn install --frozen-lockfile - - - name: Install Playwright Browsers - if: steps.cache-macos-playwright.outputs.cache-hit != 'true' - run: yarn test:install - - - name: Generate Test Suites - if: steps.cache-macos-test-suites.outputs.cache-hit != 'true' - run: yarn test:generate - - test-voiceover: - name: Playwright VoiceOver Tests (${{ matrix.shardIndex }} / 10) - needs: macos-install - runs-on: macos-12 - strategy: - fail-fast: false - matrix: - shardIndex: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] - - steps: - - uses: actions/checkout@v3 - with: - submodules: recursive - - - uses: actions/setup-node@v3 - with: - node-version: 20 - - - name: Guidepup Setup - uses: guidepup/setup-action@0.13.0 - - - name: Cache node_modules - uses: actions/cache@v3 - with: - path: | - node_modules - key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} - - - name: Cache Playwright - uses: actions/cache@v3 - with: - path: | - ~/Library/Caches/ms-playwright - key: macos-playwright-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} - - - name: Cache Test Suites - uses: actions/cache@v3 - with: - path: | - testSuites.json - key: macos-testSuites-${{ hashFiles('aria-at') }}-${{ hashFiles('ignoredTests.json') }} - - - name: Run Tests - run: yarn test ./src/macOsVoiceOver.spec.ts --config macos.config.ts --shard ${{ matrix.shardIndex }}/10 - continue-on-error: true - - - name: Archive Playwright Artifacts - uses: actions/upload-artifact@v4 - if: always() - continue-on-error: true - with: - name: artifacts-macos-voiceover-${{ matrix.shardIndex }} - path: | - ./test-results - ./recordings - - - name: Archive Allure Artifacts - uses: actions/upload-artifact@v4 - if: always() - continue-on-error: true - with: - name: allure-results-macos-voiceover-${{ matrix.shardIndex }} - path: ./allure-results - - publish-html-report: - name: Publish HTML Report To Vercel - if: always() - needs: [test-voiceover] - runs-on: ubuntu-latest - steps: - - name: Clear Space - run: | - sudo rm -rf /usr/share/dotnet - sudo rm -rf /usr/local/lib/android - sudo rm -rf /opt/ghc - sudo rm -rf /opt/hostedtoolcache/CodeQL - sudo docker image prune --all --force - sudo apt-get purge -y \ - '^llvm-.*' \ - 'php.*' \ - '^mongodb-.*' \ - '^mysql-.*' \ - azure-cli \ - google-chrome-stable \ - firefox \ - powershell \ - microsoft-edge-stable \ - mono-devel - sudo apt-get autoremove -y - sudo apt-get clean - - - uses: actions/checkout@v3 - with: - submodules: recursive - - - uses: actions/setup-node@v3 - with: - node-version: 20 - - - name: Update Deployment Status - Start - uses: bobheadxi/deployments@v1 - id: deployment - with: - step: start - env: production - token: ${{ secrets.GITHUB_TOKEN }} - - - uses: actions/download-artifact@v4 - with: - pattern: allure-results-macos-voiceover-* - path: ./allure-results - - - name: Move Allure results into one directory - run: | - mkdir allure-results-final - for i in allure-results-macos-voiceover-{1..30}; do - if [ -d "$i" ]; then - mv $i/* allure-results-final/ - fi - done - working-directory: ./allure-results - - - name: Install Dependencies From Cache - uses: actions/cache@v3 - id: cache-macos-node-modules - with: - path: | - node_modules - key: macos-modules-${{ hashFiles('yarn.lock') }}-${{ hashFiles('package.json') }} - - - name: Create Allure Test Report - run: npx allure generate ./allure-results/allure-results-final --clean -o ./allure-report - - - name: Cleanup Downloads - run: rm -rf ./allure-results || true - - - name: Deploy to Vercel - uses: amondnet/vercel-action@v25 - id: vercel-action - with: - vercel-token: ${{ secrets.VERCEL_TOKEN }} - vercel-org-id: ${{ secrets.VERCEL_ORG_ID }} - vercel-project-id: ${{ secrets.VOICEOVER_VERCEL_PROJECT_ID }} - github-comment: false - vercel-args: "--prod" - working-directory: allure-report - - - name: Update Deployment Status - Finish - uses: bobheadxi/deployments@v1 - if: always() - with: - step: finish - token: ${{ secrets.GITHUB_TOKEN }} - status: ${{ job.status }} - deployment_id: ${{ steps.deployment.outputs.deployment_id }} - env: ${{ steps.deployment.outputs.env }} - env_url: ${{ steps.vercel-action.outputs.preview-url }} diff --git a/macos.config.ts b/macos.config.ts index 456747b..bde3a2d 100644 --- a/macos.config.ts +++ b/macos.config.ts @@ -13,14 +13,15 @@ const config: PlaywrightTestConfig = { name: "webkit", use: { ...devices["Desktop Safari"], headless: false }, }, - { - name: "firefox", - use: { ...devices["Desktop Firefox"], headless: false }, - }, - { - name: "chromium", - use: { ...devices["Desktop Chrome"], headless: false }, - }, + // TODO: Enable once figure out storage size situation. + // { + // name: "firefox", + // use: { ...devices["Desktop Firefox"], headless: false }, + // }, + // { + // name: "chromium", + // use: { ...devices["Desktop Chrome"], headless: false }, + // }, ], reporter: process.env.CI ? [["github"], ["html", { open: "never" }]] : "list", }; diff --git a/src/attachRecording.ts b/src/attachRecording.ts index e0a95c4..58ff3fd 100644 --- a/src/attachRecording.ts +++ b/src/attachRecording.ts @@ -22,6 +22,11 @@ export const attachRecording = async ({ return; } + // Only attach failed test recordings + if (!test.info().errors?.length) { + return; + } + for (let i = 0; i < EXISTS_RETRIES; i++) { try { accessSync(path, constants.F_OK); diff --git a/windows.config.ts b/windows.config.ts index 0a9f4f1..252432d 100644 --- a/windows.config.ts +++ b/windows.config.ts @@ -9,14 +9,15 @@ const config: PlaywrightTestConfig = { timeout: 5 * 60 * 1000, retries: 1, projects: [ - { - name: "firefox", - use: { ...devices["Desktop Firefox"], headless: false }, - }, { name: "chromium", use: { ...devices["Desktop Chrome"], headless: false }, }, + // TODO: Enable once figure out storage size situation. + // { + // name: "firefox", + // use: { ...devices["Desktop Firefox"], headless: false }, + // }, ], reporter: process.env.CI ? [["github"], ["html", { open: "never" }]] : "list", };