Skip to content

Regression Tests

Regression Tests #3604

name: Regression Tests
on:
deployment_status:
states: ['success']
env:
PLAYWRIGHT_TEST_BASE_URL: ${{ github.event.deployment_status.target_url }}
VERCEL_PROTECTION_BYPASS: ${{ secrets.VERCEL_PROTECTION_BYPASS_CATALYST_LATEST }}
BIGCOMMERCE_ACCESS_TOKEN: ${{ secrets.BIGCOMMERCE_ACCESS_TOKEN }}
BIGCOMMERCE_STORE_HASH: ${{ secrets.BIGCOMMERCE_STORE_HASH }}
jobs:
generate-lighthouse-audit:
name: Lighthouse Audit
timeout-minutes: 30
runs-on: ubuntu-latest
if: ${{ contains(fromJson('["Production – catalyst-latest", "Preview – catalyst-latest"]'), github.event.deployment_status.environment) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- name: Pull Request Details
id: pr_details
uses: actions/github-script@v7
with:
script: |
const prData = require('./.github/workflows/pull-request-data.js');
await prData({ github, context, core });
- name: Lighthouse house audit on desktop
id: lighthouse_audit_desktop
uses: treosh/lighthouse-ci-action@v11
with:
urls: |
${{ github.event.deployment_status.target_url }}
configPath: '.github/workflows/.lighthouserc-desktop.json'
temporaryPublicStorage: true
runs: 3
- name: Lighthouse audit on mobile
id: lighthouse_audit_mobile
uses: treosh/lighthouse-ci-action@v11
with:
urls: |
${{ github.event.deployment_status.target_url }}
temporaryPublicStorage: true
runs: 3
- uses: pnpm/action-setup@v3
- name: Format lighthouse score on desktop
id: format_lighthouse_score_desktop
uses: actions/github-script@v7
with:
script: |
const lighthouseCommentMaker = require('./.github/workflows/lighthouseCommentMaker.js');
const lighthouseOutputs = {
manifest: ${{ steps.lighthouse_audit_desktop.outputs.manifest }},
links: ${{ steps.lighthouse_audit_desktop.outputs.links }},
preset: "desktop"
};
const comment = lighthouseCommentMaker({ lighthouseOutputs });
core.setOutput("comment", comment);
- name: Format lighthouse score on mobile
id: format_lighthouse_score_mobile
uses: actions/github-script@v7
with:
script: |
const lighthouseCommentMaker = require('./.github/workflows/lighthouseCommentMaker.js');
const lighthouseOutputs = {
manifest: ${{ steps.lighthouse_audit_mobile.outputs.manifest }},
links: ${{ steps.lighthouse_audit_mobile.outputs.links }},
preset: "mobile"
};
const comment = lighthouseCommentMaker({ lighthouseOutputs });
core.setOutput("comment", comment);
- name: Add comment to PR
id: comment_to_pr
uses: marocchino/sticky-pull-request-comment@v2.9.0
if: ${{ steps.pr_details.outputs.pr }}
with:
recreate: true
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
number: ${{ steps.pr_details.outputs.pr }}
header: lighthouse
message: |
# ⚡️🏠 Lighthouse report
*Lighthouse ran against ${{ github.event.deployment_status.target_url }}*
## 🖥️ Desktop
${{ steps.format_lighthouse_score_desktop.outputs.comment }}
## 📱 Mobile
${{ steps.format_lighthouse_score_mobile.outputs.comment }}
ui-tests:
name: Playwright UI Tests
timeout-minutes: 30
runs-on: ubuntu-latest
if: ${{ contains(fromJson('["Production – catalyst-latest", "Preview – catalyst-latest"]'), github.event.deployment_status.environment) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
- name: Use Node.js
uses: actions/setup-node@v4
with:
node-version-file: '.nvmrc'
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Pull Request Details
id: pr_details
uses: actions/github-script@v7
with:
script: |
const prData = require('./.github/workflows/pull-request-data.js');
await prData({ github, context, core });
- name: Install Playwright Browsers
run: |
cd core
npx playwright install --with-deps
- name: Run Playwright tests
run: |
cd core
npx playwright test tests/ui/ --project=tests-chromium
- uses: actions/upload-artifact@v4
if: failure()
with:
name: playwright-report-ui
path: core/playwright-report/
retention-days: 30
- name: Send slack notification
uses: slackapi/slack-github-action@v1.26.0
if: ${{ steps.pr_details.outputs.draft != 'true' && steps.pr_details.outputs.pr && failure() }}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
with:
payload: |
{
"Job": "${{ github.job }}",
"Status": "Failed",
"Environment": "${{ github.event.deployment_status.environment }}",
"Pull_Request": "${{ steps.pr_details.outputs.url }}",
"Commit_Message" : "${{ steps.pr_details.outputs.title }}",
"Job_Run": "https://github.com/bigcommerce/catalyst/actions/runs/${{ github.run_id }}"
}
visual-regression-tests:
name: Playwright Visual Regression Tests
timeout-minutes: 30
runs-on: macos-14
if: ${{ contains(fromJson('["Production – catalyst-latest", "Preview – catalyst-latest"]'), github.event.deployment_status.environment) }}
steps:
- name: Checkout code
uses: actions/checkout@v4
- uses: pnpm/action-setup@v3
- name: Use Node.js
uses: actions/setup-node@v4
with:
node-version-file: '.nvmrc'
cache: 'pnpm'
- name: Install dependencies
run: pnpm install --frozen-lockfile
- name: Pull Request Details
id: pr_details
uses: actions/github-script@v7
with:
script: |
const prData = require('./.github/workflows/pull-request-data.js');
await prData({ github, context, core });
- name: Install Playwright Browsers
run: |
cd core
npx playwright install chromium
- name: Run Playwright tests
run: |
cd core
npx playwright test tests/visual-regression/components/ --project=tests-chromium
- uses: actions/upload-artifact@v4
if: failure()
with:
name: playwright-report-visual-regression
path: core/playwright-report/
retention-days: 30
- name: Send slack notification
uses: slackapi/slack-github-action@v1.26.0
if: ${{ steps.pr_details.outputs.draft != 'true' && steps.pr_details.outputs.pr && failure() }}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
with:
payload: |
{
"Job": "${{ github.job }}",
"Status": "Failed",
"Environment": "${{ github.event.deployment_status.environment }}",
"Pull_Request": "${{ steps.pr_details.outputs.url }}",
"Commit_Message" : "${{ steps.pr_details.outputs.title }}",
"Job_Run": "https://github.com/bigcommerce/catalyst/actions/runs/${{ github.run_id }}"
}