feat(experiments): allow multiple preview images #63655
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Storybook | |
| on: | |
| pull_request: | |
| paths: # Only run if the frontend has changed | |
| - 'frontend/**' | |
| - '.storybook/**' | |
| - 'package.json' | |
| - '.github/workflows/storybook-chromatic.yml' | |
| - 'playwright.config.ts' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| # This is so that the workflow run isn't canceled when a snapshot update is pushed within it by posthog-bot | |
| # We do however cancel from container-images-ci.yml if a commit is pushed by someone OTHER than posthog-bot | |
| cancel-in-progress: false | |
| jobs: | |
| storybook-chromatic: | |
| name: Publish to Chromatic | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 15 | |
| if: github.event.pull_request.head.repo.full_name == github.repository # Don't run on forks | |
| outputs: | |
| storybook-url: ${{ steps.publish.outputs.storybookUrl }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 # 👈 Required to retrieve git history (https://www.chromatic.com/docs/github-actions) | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: 18.12.1 | |
| cache: pnpm | |
| - name: Install dependencies and Chromatic | |
| run: pnpm i -D chromatic | |
| - name: Publish to Chromatic | |
| uses: chromaui/action@v11 | |
| id: publish | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| # 👇 Chromatic projectToken, refer to the manage page to obtain it. | |
| projectToken: ${{ secrets.CHROMATIC_PROJECT_TOKEN }} | |
| visual-regression: | |
| name: Visual regression tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| container: | |
| image: mcr.microsoft.com/playwright:v1.45.0 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| browser: ['chromium', 'webkit'] | |
| shard: [1, 2] | |
| env: | |
| SHARD_COUNT: '2' | |
| CYPRESS_INSTALL_BINARY: '0' | |
| NODE_OPTIONS: --max-old-space-size=6144 | |
| OPT_OUT_CAPTURE: 1 | |
| outputs: | |
| # The below have to be manually listed unfortunately, as GitHub Actions doesn't allow matrix-dependent outputs | |
| chromium-1-added: ${{ steps.diff.outputs.chromium-1-added }} | |
| chromium-1-modified: ${{ steps.diff.outputs.chromium-1-modified }} | |
| chromium-1-deleted: ${{ steps.diff.outputs.chromium-1-deleted }} | |
| chromium-1-total: ${{ steps.diff.outputs.chromium-1-total }} | |
| chromium-1-commitHash: ${{ steps.commit-hash.outputs.chromium-1-commitHash }} | |
| chromium-2-added: ${{ steps.diff.outputs.chromium-2-added }} | |
| chromium-2-modified: ${{ steps.diff.outputs.chromium-2-modified }} | |
| chromium-2-deleted: ${{ steps.diff.outputs.chromium-2-deleted }} | |
| chromium-2-total: ${{ steps.diff.outputs.chromium-2-total }} | |
| chromium-2-commitHash: ${{ steps.commit-hash.outputs.chromium-2-commitHash }} | |
| webkit-1-added: ${{ steps.diff.outputs.webkit-1-added }} | |
| webkit-1-modified: ${{ steps.diff.outputs.webkit-1-modified }} | |
| webkit-1-deleted: ${{ steps.diff.outputs.webkit-1-deleted }} | |
| webkit-1-total: ${{ steps.diff.outputs.webkit-1-total }} | |
| webkit-1-commitHash: ${{ steps.commit-hash.outputs.webkit-1-commitHash }} | |
| webkit-2-added: ${{ steps.diff.outputs.webkit-2-added }} | |
| webkit-2-modified: ${{ steps.diff.outputs.webkit-2-modified }} | |
| webkit-2-deleted: ${{ steps.diff.outputs.webkit-2-deleted }} | |
| webkit-2-total: ${{ steps.diff.outputs.webkit-2-total }} | |
| webkit-2-commitHash: ${{ steps.commit-hash.outputs.webkit-2-commitHash }} | |
| firefox-1-added: ${{ steps.diff.outputs.firefox-1-added }} | |
| firefox-1-modified: ${{ steps.diff.outputs.firefox-1-modified }} | |
| firefox-1-deleted: ${{ steps.diff.outputs.firefox-1-deleted }} | |
| firefox-1-total: ${{ steps.diff.outputs.firefox-1-total }} | |
| firefox-1-commitHash: ${{ steps.commit-hash.outputs.firefox-1-commitHash }} | |
| firefox-2-added: ${{ steps.diff.outputs.firefox-2-added }} | |
| firefox-2-modified: ${{ steps.diff.outputs.firefox-2-modified }} | |
| firefox-2-deleted: ${{ steps.diff.outputs.firefox-2-deleted }} | |
| firefox-2-total: ${{ steps.diff.outputs.firefox-2-total }} | |
| firefox-2-commitHash: ${{ steps.commit-hash.outputs.firefox-2-commitHash }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 1 | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.ref }} | |
| # Use PostHog Bot token when not on forks to enable proper snapshot updating | |
| token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }} | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| - name: Set up Node.js | |
| uses: buildjet/setup-node@v3 | |
| with: | |
| node-version: 18.12.1 | |
| cache: pnpm | |
| - name: Install package.json dependencies with pnpm | |
| run: pnpm install --frozen-lockfile | |
| - name: Install CI utilities with pnpm | |
| run: pnpm install http-server wait-on | |
| - name: Build Storybook | |
| run: pnpm build-storybook --test --quiet # Silence since progress logging results in a massive wall of spam | |
| - name: Serve Storybook in the background | |
| run: | | |
| retries=3 | |
| while [ $retries -gt 0 ]; do | |
| pnpm exec http-server storybook-static --port 6006 --silent & | |
| if pnpm wait-on http://127.0.0.1:6006 --timeout 15; then | |
| break | |
| fi | |
| retries=$((retries-1)) | |
| echo "Failed to serve Storybook, retrying... ($retries retries left)" | |
| done | |
| - name: Run @storybook/test-runner | |
| env: | |
| # Solving this bug by overriding $HOME: https://github.com/microsoft/playwright/issues/6500 | |
| HOME: /root | |
| # Update snapshots for PRs on the main repo, verify on forks, which don't have access to PostHog Bot | |
| VARIANT: ${{ github.event.pull_request.head.repo.full_name == github.repository && 'update' || 'verify' }} | |
| STORYBOOK_SKIP_TAGS: 'test-skip,test-skip-${{ matrix.browser }}' | |
| run: | | |
| pnpm test:visual:ci:$VARIANT --browsers ${{ matrix.browser }} --shard ${{ matrix.shard }}/$SHARD_COUNT | |
| - name: Archive failure screenshots | |
| if: ${{ failure() }} | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: failure-screenshots-${{ matrix.browser }} | |
| path: frontend/__snapshots__/__failures__/ | |
| - name: Count and optimize updated snapshots | |
| id: diff | |
| # Skip on forks | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| run: | | |
| git config --global --add safe.directory '*' # Calm git down about file ownership | |
| git diff --name-status frontend/__snapshots__/ # For debugging | |
| ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l) | |
| MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l) | |
| DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l) | |
| TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l) | |
| # If added or modified, run OptiPNG | |
| if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then | |
| echo "Snapshots updated ($ADDED new, $MODIFIED changed), running OptiPNG" | |
| apt update && apt install -y optipng | |
| optipng -clobber -o4 -strip all | |
| # we don't want to _always_ run OptiPNG | |
| # so, we run it after checking for a diff | |
| # but, the files we diffed might then be changed by OptiPNG | |
| # and as a result they might no longer be different... | |
| # we check again | |
| git diff --name-status frontend/__snapshots__/ # For debugging | |
| ADDED=$(git diff --name-status frontend/__snapshots__/ | grep '^A' | wc -l) | |
| MODIFIED=$(git diff --name-status frontend/__snapshots__/ | grep '^M' | wc -l) | |
| DELETED=$(git diff --name-status frontend/__snapshots__/ | grep '^D' | wc -l) | |
| TOTAL=$(git diff --name-status frontend/__snapshots__/ | wc -l) | |
| if [ $ADDED -gt 0 ] || [ $MODIFIED -gt 0 ]; then | |
| echo "Snapshots updated ($ADDED new, $MODIFIED changed), _even after_ running OptiPNG" | |
| git add frontend/__snapshots__/ playwright/ | |
| fi | |
| fi | |
| echo "${{ matrix.browser }}-${{ matrix.shard }}-added=$ADDED" >> $GITHUB_OUTPUT | |
| echo "${{ matrix.browser }}-${{ matrix.shard }}-modified=$MODIFIED" >> $GITHUB_OUTPUT | |
| echo "${{ matrix.browser }}-${{ matrix.shard }}-deleted=$DELETED" >> $GITHUB_OUTPUT | |
| echo "${{ matrix.browser }}-${{ matrix.shard }}-total=$TOTAL" >> $GITHUB_OUTPUT | |
| - name: Commit updated snapshots | |
| uses: EndBug/add-and-commit@v9 | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| id: commit | |
| with: | |
| add: '["frontend/__snapshots__/", "playwright/"]' | |
| message: 'Update UI snapshots for `${{ matrix.browser }}` (${{ matrix.shard }})' | |
| pull: --rebase --autostash # Make sure we're up to date with other browsers' updates | |
| default_author: github_actions | |
| github_token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }} | |
| - name: Add commit hash to outputs, including browser name | |
| id: commit-hash | |
| if: steps.commit.outputs.pushed == 'true' | |
| run: echo "${{ matrix.browser }}-${{ matrix.shard }}-commitHash=${{ steps.commit.outputs.commit_long_sha }}" >> $GITHUB_OUTPUT | |
| visual-regression-summary: | |
| name: Summarize visual regression tests | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| needs: visual-regression | |
| if: always() # Run even if visual-regression fails for one (or more) of the browsers | |
| steps: | |
| - name: Post comment about updated snapshots | |
| if: github.event.pull_request.head.repo.full_name == github.repository | |
| uses: actions/github-script@v6 | |
| with: | |
| github-token: ${{ secrets.POSTHOG_BOT_GITHUB_TOKEN || github.token }} | |
| script: | | |
| const BROWSERS = ['chromium', 'webkit'] | |
| const diffJobOutputs = ${{ toJson(needs.visual-regression.outputs) }} | |
| const summaryDiff = { total: 0, added: 0, modified: 0, deleted: 0 } | |
| const diffByBrowser = Object.fromEntries(BROWSERS.map(browser => [browser, { | |
| total: 0, added: 0, modified: 0, deleted: 0, commitHashes: [] | |
| }])) | |
| for (const [key, rawValue] of Object.entries(diffJobOutputs)) { | |
| // Split e.g. 'chromium-1-commitHash' into ['chromium', '1' 'commitHash'] | |
| const [browser, shardNumber, diffKey] = key.split('-') | |
| // Sum up the counts - but not the commit hash | |
| if (diffKey === 'commitHash') { | |
| diffByBrowser[browser].commitHashes.push([parseInt(shardNumber), rawValue]) | |
| } else { | |
| const value = parseInt(rawValue) | |
| diffByBrowser[browser][diffKey] += value | |
| summaryDiff[diffKey] += value | |
| } | |
| } | |
| for (const browser of BROWSERS) { | |
| if (diffByBrowser[browser]?.total === undefined) { | |
| diffByBrowser[browser] = null // Null means failure | |
| } | |
| } | |
| if (summaryDiff.total === 0) { | |
| console.log('No changes were made, skipping comment') | |
| return | |
| } | |
| const diffByBrowserDisplay = Object.entries(diffByBrowser).map(([browser, diff]) => { | |
| if (!diff) { | |
| return `- \`${browser}\`: failed` | |
| } | |
| const { added: a, modified: m, deleted: d, commitHashes } = diff | |
| const b = a + m + d > 0 ? '**' : '' // Bold list item if there were changes | |
| let extraInfo = '' | |
| if (b) { | |
| const commitInfo = commitHashes.map( | |
| ([shardNumber, commitHash]) => | |
| `[diff for shard ${shardNumber}](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${commitHash})` | |
| ).join(', ') || "wasn't pushed!" | |
| extraInfo = ` (${commitInfo})` | |
| } | |
| return `- ${b}\`${browser}\`${b}: **${a}** added, **${m}** modified, **${d}** deleted${extraInfo}` | |
| }).join('\n') | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: `## 📸 UI snapshots have been updated | |
| **${summaryDiff.total}** snapshot changes in total. **${summaryDiff.added}** added, **${summaryDiff.modified}** modified, **${summaryDiff.deleted}** deleted: | |
| ${diffByBrowserDisplay} | |
| Triggered by [this commit](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/commits/${{ github.sha }}). | |
| 👉 **[Review this PR's diff of snapshots.](https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request.number }}/files#:~:text=frontend/__snapshots__/)**` | |
| }) |