Skip to content

Commit

Permalink
feat: e2e reports in gh-pages (#16179)
Browse files Browse the repository at this point in the history
  • Loading branch information
zomars authored Aug 13, 2024
1 parent 0b03bcb commit f206b3d
Show file tree
Hide file tree
Showing 9 changed files with 220 additions and 17 deletions.
2 changes: 0 additions & 2 deletions .github/actions/yarn-install/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -37,8 +37,6 @@ runs:
with:
path: ${{ steps.yarn-config.outputs.CACHE_FOLDER }}
key: yarn-download-cache-${{ hashFiles('yarn.lock') }}
restore-keys: |
yarn-download-cache-

# Invalidated on yarn.lock changes
- name: Restore node_modules
Expand Down
10 changes: 6 additions & 4 deletions .github/actions/yarn-playwright-install/action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,10 @@ description: "Install playwright, cache and restore if necessary"
runs:
using: "composite"
steps:
- name: Get installed Playwright version
shell: bash
id: playwright-version
run: echo "PLAYWRIGHT_VERSION=$(node -e "console.log(require('./package.json').devDependencies['@playwright/test'])")" >> $GITHUB_ENV
- name: Cache playwright binaries
id: playwright-cache
uses: buildjet/cache@v4
Expand All @@ -11,9 +15,7 @@ runs:
~/Library/Caches/ms-playwright
~/.cache/ms-playwright
${{ github.workspace }}/node_modules/playwright
key: cache-playwright-${{ hashFiles('**/yarn.lock') }}
restore-keys: cache-playwright-
key: ${{ runner.os }}-playwright-${{ env.PLAYWRIGHT_VERSION }}
- name: Yarn playwright install
shell: bash
if: steps.playwright-cache.outputs.cache-hit != 'true'
run: yarn playwright install
run: yarn playwright install --with-deps
58 changes: 58 additions & 0 deletions .github/workflows/cleanup.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,58 @@
name: Delete

on:
delete:
branches-ignore: [main, gh-pages]

# ensures that currently running Playwright workflow of deleted branch gets cancelled
concurrency:
group: ${{ github.event.ref }}
cancel-in-progress: true

jobs:
delete_reports:
name: Delete Reports
runs-on: ubuntu-latest
env:
# Contains all reports for deleted branch
BRANCH_REPORTS_DIR: reports/${{ github.event.ref }}
steps:
- name: Checkout GitHub Pages Branch
uses: actions/checkout@v2
with:
ref: gh-pages
- name: Set Git User
# see: https://github.com/actions/checkout/issues/13#issuecomment-724415212
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
- name: Check for workflow reports
run: |
if [ -z "$(ls -A $BRANCH_REPORTS_DIR)" ]; then
echo "BRANCH_REPORTS_EXIST="false"" >> $GITHUB_ENV
else
echo "BRANCH_REPORTS_EXIST="true"" >> $GITHUB_ENV
fi
- name: Delete reports from repo for branch
if: ${{ env.BRANCH_REPORTS_EXIST == 'true' }}
timeout-minutes: 3
run: |
cd $BRANCH_REPORTS_DIR/..
rm -rf ${{ github.event.ref }}
git add .
git commit -m "workflow: remove all reports for branch ${{ github.event.ref }}"
while true; do
git pull --rebase
if [ $? -ne 0 ]; then
echo "Failed to rebase. Please review manually."
exit 1
fi
git push
if [ $? -eq 0 ]; then
echo "Successfully pushed HTML reports to repo."
exit 0
fi
done
7 changes: 4 additions & 3 deletions .github/workflows/e2e.yml
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,9 @@ jobs:
- name: Run Tests
run: yarn e2e --shard=${{ matrix.shard }}/${{ strategy.job-total }}
- name: Upload Test Results
if: ${{ always() }}
uses: actions/upload-artifact@v4
if: always()
with:
name: test-results-${{ matrix.shard }}_${{ strategy.job-total }}
path: test-results
name: blob-report-${{ matrix.shard }}
path: blob-report
retention-days: 30
25 changes: 25 additions & 0 deletions .github/workflows/merge-reports.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
# https://playwright.dev/docs/test-sharding#merging-reports-from-multiple-shards
on:
workflow_call:
jobs:
merge-reports:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- uses: ./.github/actions/dangerous-git-checkout
- uses: ./.github/actions/yarn-install
- uses: ./.github/actions/yarn-playwright-install
- name: Download blob reports from GitHub Actions Artifacts
uses: actions/download-artifact@v4
with:
path: all-blob-reports
pattern: blob-report-*
merge-multiple: true
- name: Merge into HTML Report
run: yarn playwright merge-reports --reporter html ./all-blob-reports
- name: Upload HTML report
uses: actions/upload-artifact@v4
with:
name: html-report--attempt-${{ github.run_number }}-${{ github.run_attempt }}
path: playwright-report
retention-days: 14
54 changes: 47 additions & 7 deletions .github/workflows/pr.yml
Original file line number Diff line number Diff line change
Expand Up @@ -91,44 +91,50 @@ jobs:
console.log('Found the label?', labelFound);
core.setOutput('run-e2e', labelFound);
deps:
name: Install dependencies
needs: [changes, check-label]
if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
uses: ./.github/workflows/yarn-install.yml

type-check:
name: Type check
needs: [changes, check-label]
needs: [changes, check-label, deps]
if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
uses: ./.github/workflows/check-types.yml
secrets: inherit

lint:
name: Linters
needs: [changes, check-label]
needs: [changes, check-label, deps]
if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
uses: ./.github/workflows/lint.yml
secrets: inherit

unit-test:
name: Tests
needs: [changes, check-label]
needs: [changes, check-label, deps]
if: ${{ needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
uses: ./.github/workflows/unit-tests.yml
secrets: inherit

build-api-v1:
name: Production builds
needs: [changes, check-label]
needs: [changes, check-label, deps]
if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
uses: ./.github/workflows/api-v1-production-build.yml
secrets: inherit

build-api-v2:
name: Production builds
needs: [changes, check-label]
needs: [changes, check-label, deps]
if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
uses: ./.github/workflows/api-v2-production-build.yml
secrets: inherit

build:
name: Production builds
needs: [changes, check-label]
needs: [changes, check-label, deps]
if: ${{ needs.check-label.outputs.run-e2e == 'true' && needs.changes.outputs.has-files-requiring-all-checks == 'true' }}
uses: ./.github/workflows/production-build-without-database.yml
secrets: inherit
Expand Down Expand Up @@ -181,8 +187,42 @@ jobs:
uses: ./.github/workflows/nextjs-bundle-analysis.yml
secrets: inherit

merge-reports:
name: Merge reports
if: ${{ !cancelled() }}
needs: [e2e]
uses: ./.github/workflows/merge-reports.yml
secrets: inherit

publish-report:
name: Publish HTML report
if: ${{ !cancelled() }}
permissions:
contents: write
issues: write
pull-requests: write
needs: [merge-reports]
uses: ./.github/workflows/publish-report.yml
secrets: inherit

required:
needs: [changes, lint, type-check, unit-test, integration-test, check-label, build, build-api-v1, build-api-v2, e2e, e2e-api-v2, e2e-embed, e2e-embed-react, e2e-app-store]
needs:
[
changes,
lint,
type-check,
unit-test,
integration-test,
check-label,
build,
build-api-v1,
build-api-v2,
e2e,
e2e-api-v2,
e2e-embed,
e2e-embed-react,
e2e-app-store,
]
if: always()
runs-on: buildjet-2vcpu-ubuntu-2204
steps:
Expand Down
78 changes: 78 additions & 0 deletions .github/workflows/publish-report.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,78 @@
on:
workflow_call:
permissions:
contents: write
issues: write
pull-requests: write
jobs:
publish-report:
runs-on: ubuntu-latest
continue-on-error: true
env:
# Unique URL path for each workflow run attempt
HTML_REPORT_URL_PATH: reports/${{ github.ref_name }}/${{ github.run_id }}/${{ github.run_attempt }}
steps:
- name: Checkout GitHub Pages Branch
uses: actions/checkout@v4
with:
ref: gh-pages
- name: Set Git User
# see: https://github.com/actions/checkout/issues/13#issuecomment-724415212
run: |
git config --global user.name "github-actions[bot]"
git config --global user.email "41898282+github-actions[bot]@users.noreply.github.com"
- name: Download zipped HTML report
uses: actions/download-artifact@v4
with:
name: html-report--attempt-${{ github.run_number }}-${{ github.run_attempt }}
path: ${{ env.HTML_REPORT_URL_PATH }}
- name: Push HTML Report
timeout-minutes: 3
# commit report, then try push-rebase-loop until it's able to merge the HTML report to the gh-pages branch
# this is necessary when this job running at least twice at the same time (e.g. through two pushes at the same time)
run: |
git add .
git commit -m "workflow: add HTML report for run-id ${{ github.run_id }} (attempt: ${{ github.run_attempt }})"
while true; do
git pull --rebase
if [ $? -ne 0 ]; then
echo "Failed to rebase. Please review manually."
exit 1
fi
git push
if [ $? -eq 0 ]; then
echo "Successfully pushed HTML report to repo."
exit 0
fi
done
- name: Output Report URL as Workflow Annotation
id: url
run: |
FULL_HTML_REPORT_URL=https://calcom.github.io/cal.com/$HTML_REPORT_URL_PATH
echo "::notice title=📋 Published Playwright Test Report::$FULL_HTML_REPORT_URL"
echo "link=$FULL_HTML_REPORT_URL" >> $GITHUB_OUTPUT
- name: Find Comment
uses: peter-evans/find-comment@v2
id: fc
with:
issue-number: ${{ github.event.pull_request.number }}
comment-author: "github-actions[bot]"
body-includes: <!-- GENERATED-E2E-RESULTS -->
- name: Create comment
if: steps.fc.outputs.comment-id == ''
uses: peter-evans/create-or-update-comment@v3
with:
issue-number: ${{ github.event.pull_request.number }}
body: |
<!-- GENERATED-E2E-RESULTS -->
## E2E results are ready!
- [Attempt #${{ github.run_number }}.${{ github.run_attempt }} results](${{ steps.url.outputs.link }})
- name: Update comment
if: steps.fc.outputs.comment-id != ''
uses: peter-evans/create-or-update-comment@v3
with:
comment-id: ${{ steps.fc.outputs.comment-id }}
body: |
- [Attempt #${{ github.run_number }}.${{ github.run_attempt }} results](${{ steps.url.outputs.link }})
1 change: 1 addition & 0 deletions .github/workflows/yarn-install.yml
Original file line number Diff line number Diff line change
Expand Up @@ -15,3 +15,4 @@ jobs:
- uses: actions/checkout@v4
- uses: ./.github/actions/dangerous-git-checkout
- uses: ./.github/actions/yarn-install
- uses: ./.github/actions/yarn-playwright-install
2 changes: 1 addition & 1 deletion playwright.config.ts
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,7 @@ const config: PlaywrightTestConfig = {
maxFailures: headless ? 10 : undefined,
fullyParallel: true,
reporter: [
[process.env.CI ? "github" : "list"],
[process.env.CI ? "blob" : "list"],
["html", { outputFolder: "./test-results/reports/playwright-html-report", open: "never" }],
["junit", { outputFile: "./test-results/reports/results.xml" }],
],
Expand Down

0 comments on commit f206b3d

Please sign in to comment.