From 6caa754021ff231e785f2da44451337138e7fc68 Mon Sep 17 00:00:00 2001 From: Kai Hao Date: Mon, 14 Nov 2022 23:05:57 +0800 Subject: [PATCH 1/3] Post a summary of the flaky tests to the commit --- .github/workflows/end2end-test-playwright.yml | 66 ------ .github/workflows/end2end-test.yml | 93 +++++++- .github/workflows/flaky-tests.yml | 38 --- package-lock.json | 142 +---------- package.json | 2 - packages/report-flaky-tests/action.yml | 6 +- packages/report-flaky-tests/package.json | 3 +- .../__tests__/__snapshots__/run.test.ts.snap | 191 ++++++++++++++- .../src/__tests__/run.test.ts | 222 ++++++++++++++++-- packages/report-flaky-tests/src/github-api.ts | 72 +++--- packages/report-flaky-tests/src/markdown.ts | 28 ++- packages/report-flaky-tests/src/run.ts | 70 ++++-- packages/report-flaky-tests/src/types.ts | 7 + 13 files changed, 602 insertions(+), 338 deletions(-) delete mode 100644 .github/workflows/end2end-test-playwright.yml delete mode 100644 .github/workflows/flaky-tests.yml diff --git a/.github/workflows/end2end-test-playwright.yml b/.github/workflows/end2end-test-playwright.yml deleted file mode 100644 index e0aabb17afa019..00000000000000 --- a/.github/workflows/end2end-test-playwright.yml +++ /dev/null @@ -1,66 +0,0 @@ -name: End-to-End Tests Playwright - -on: - pull_request: - push: - branches: - - trunk - - 'release/**' - - 'wp/**' - -# Cancels all previous workflow runs for pull requests that have not completed. -concurrency: - # The concurrency group contains the workflow name and the branch name for pull requests - # or the commit hash for any other events. - group: ${{ github.workflow }}-${{ github.event_name == 'pull_request' && github.head_ref || github.sha }} - cancel-in-progress: true - -jobs: - e2e: - name: E2E Tests - runs-on: ubuntu-latest - if: ${{ github.repository == 'WordPress/gutenberg' || github.event_name == 'pull_request' }} - strategy: - fail-fast: false - - steps: - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - - - name: Use desired version of NodeJS - uses: actions/setup-node@8c91899e586c5b171469028077307d293428b516 # v3.5.1 - with: - node-version-file: '.nvmrc' - cache: npm - - - name: Npm install and build - run: | - npm ci - npm run build - - - name: Install Playwright dependencies - run: | - npx playwright install chromium firefox webkit --with-deps - - - name: Install WordPress and start the server - run: | - npm run wp-env start - - - name: Run the tests - run: | - xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test:e2e:playwright - - - name: Archive debug artifacts (screenshots, traces) - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 - if: always() - with: - name: failures-artifacts - path: artifacts/test-results - if-no-files-found: ignore - - - name: Archive flaky tests report - uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 - if: always() - with: - name: flaky-tests-report-playwright - path: flaky-tests - if-no-files-found: ignore diff --git a/.github/workflows/end2end-test.yml b/.github/workflows/end2end-test.yml index 172d3b090eaf0b..ad4db13ff6ce8c 100644 --- a/.github/workflows/end2end-test.yml +++ b/.github/workflows/end2end-test.yml @@ -60,6 +60,97 @@ jobs: uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 if: always() with: - name: flaky-tests-report-${{ matrix.part }} + name: flaky-tests-report path: flaky-tests if-no-files-found: ignore + + e2e-playwright: + name: Playwright + runs-on: ubuntu-latest + if: ${{ github.repository == 'WordPress/gutenberg' || github.event_name == 'pull_request' }} + strategy: + fail-fast: false + + steps: + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + + - name: Use desired version of NodeJS + uses: actions/setup-node@8c91899e586c5b171469028077307d293428b516 # v3.5.1 + with: + node-version-file: '.nvmrc' + cache: npm + + - name: Npm install and build + run: | + npm ci + npm run build + + - name: Install Playwright dependencies + run: | + npx playwright install chromium firefox webkit --with-deps + + - name: Install WordPress and start the server + run: | + npm run wp-env start + + - name: Run the tests + run: | + xvfb-run --auto-servernum --server-args="-screen 0 1280x960x24" -- npm run test:e2e:playwright + + - name: Archive debug artifacts (screenshots, traces) + uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 + if: always() + with: + name: failures-artifacts + path: artifacts/test-results + if-no-files-found: ignore + + - name: Archive flaky tests report + uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1 + if: always() + with: + name: flaky-tests-report + path: flaky-tests + if-no-files-found: ignore + + report-to-issues: + name: Report to GitHub + needs: [admin, e2e-playwright] + if: ${{ always() }} + runs-on: ubuntu-latest + steps: + # Checkout defaults to using the branch which triggered the event, which + # isn't necessarily `trunk` (e.g. in the case of a merge). + - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 + with: + ref: trunk + + - uses: actions/download-artifact@v3 + id: download_artifact + # Don't fail the job if there isn't any flaky tests report. + continue-on-error: true + with: + name: flaky-tests-report + path: flaky-tests + + - name: Use desired version of NodeJS + if: ${{ steps.download_artifact.outcome == 'success' }} + uses: actions/setup-node@8c91899e586c5b171469028077307d293428b516 # v3.5.1 + with: + node-version-file: '.nvmrc' + cache: npm + + - name: Npm install and build + if: ${{ steps.download_artifact.outcome == 'success' }} + # TODO: We don't have to build the entire project, just the action itself. + run: | + npm ci + npm run build:packages + + - name: Report flaky tests + if: ${{ steps.download_artifact.outcome == 'success' }} + uses: ./packages/report-flaky-tests + with: + repo-token: '${{ secrets.GITHUB_TOKEN }}' + label: '[Type] Flaky Test' + artifact-path: flaky-tests diff --git a/.github/workflows/flaky-tests.yml b/.github/workflows/flaky-tests.yml deleted file mode 100644 index 6f457f6b2292c5..00000000000000 --- a/.github/workflows/flaky-tests.yml +++ /dev/null @@ -1,38 +0,0 @@ -name: Report Flaky Tests - -on: - workflow_run: - workflows: ['End-to-End Tests', 'End-to-End Tests Playwright'] - types: - - completed - -jobs: - report-to-issues: - name: Report to GitHub issues - runs-on: ubuntu-latest - if: ${{ github.event.workflow_run.conclusion == 'success' }} - steps: - # Checkout defaults to using the branch which triggered the event, which - # isn't necessarily `trunk` (e.g. in the case of a merge). - - uses: actions/checkout@93ea575cb5d8a053eaa0ac8fa3b40d7e05a33cc8 # v3.1.0 - with: - ref: trunk - - - name: Use desired version of NodeJS - uses: actions/setup-node@8c91899e586c5b171469028077307d293428b516 # v3.5.1 - with: - node-version-file: '.nvmrc' - cache: npm - - - name: Npm install and build - # TODO: We don't have to build the entire project, just the action itself. - run: | - npm ci - npm run build:packages - - - name: Report flaky tests - uses: ./packages/report-flaky-tests - with: - repo-token: '${{ secrets.GITHUB_TOKEN }}' - label: '[Type] Flaky Test' - artifact-name-prefix: flaky-tests-report diff --git a/package-lock.json b/package-lock.json index 9edebed07e4c36..7caa04085f536c 100644 --- a/package-lock.json +++ b/package-lock.json @@ -17148,15 +17148,6 @@ "integrity": "sha512-FvUupuM3rlRsRtCN+fDudtmytGO6iHJuuRKS1Ss0pG5z8oX0diNEw94UEL7hgDbpN94rgaK5R7sWm6RrSkZuAQ==", "dev": true }, - "@types/unzipper": { - "version": "0.10.5", - "resolved": "https://registry.npmjs.org/@types/unzipper/-/unzipper-0.10.5.tgz", - "integrity": "sha512-NrLJb29AdnBARpg9S/4ktfPEisbJ0AvaaAr3j7Q1tg8AgcEUsq2HqbNzvgLRoWyRtjzeLEv7vuL39u1mrNIyNA==", - "dev": true, - "requires": { - "@types/node": "*" - } - }, "@types/uuid": { "version": "8.3.1", "resolved": "https://registry.npmjs.org/@types/uuid/-/uuid-8.3.1.tgz", @@ -19167,8 +19158,7 @@ "requires": { "@actions/core": "^1.8.0", "@actions/github": "^5.0.1", - "jest-message-util": "^28.0.2", - "unzipper": "^0.10.11" + "jest-message-util": "^28.0.2" }, "dependencies": { "@actions/core": { @@ -28891,7 +28881,8 @@ "version": "1.6.48", "resolved": "https://registry.npmjs.org/big-integer/-/big-integer-1.6.48.tgz", "integrity": "sha512-j51egjPa7/i+RdiRuJbPdJ2FIUYYPhvYLjzoYbcMMm62ooO6F94fETG4MTs46zPAF9Brs04OajboA/qTGuz78w==", - "dev": true + "dev": true, + "optional": true }, "big.js": { "version": "5.2.2", @@ -28937,16 +28928,6 @@ } } }, - "binary": { - "version": "0.3.0", - "resolved": "https://registry.npmjs.org/binary/-/binary-0.3.0.tgz", - "integrity": "sha1-n2BVO8XOjDOG87VTz/R0Yq3sqnk=", - "dev": true, - "requires": { - "buffers": "~0.1.1", - "chainsaw": "~0.1.0" - } - }, "binary-extensions": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.2.0.tgz", @@ -29437,24 +29418,12 @@ "integrity": "sha512-4/rOEg86jivtPTeOUUT61jJO1Ya1TrR/OkqCSZDyq84WJh3LuuiphBYJN+fm5xufIk4XAFcEwte/8WzC8If/1g==", "dev": true }, - "buffer-indexof-polyfill": { - "version": "1.0.2", - "resolved": "https://registry.npmjs.org/buffer-indexof-polyfill/-/buffer-indexof-polyfill-1.0.2.tgz", - "integrity": "sha512-I7wzHwA3t1/lwXQh+A5PbNvJxgfo5r3xulgpYDB5zckTu/Z9oUK9biouBKQUjEqzaz3HnAT6TYoovmE+GqSf7A==", - "dev": true - }, "buffer-xor": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/buffer-xor/-/buffer-xor-1.0.3.tgz", "integrity": "sha1-JuYe0UIvtw3ULm42cp7VHYVf6Nk=", "dev": true }, - "buffers": { - "version": "0.1.1", - "resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz", - "integrity": "sha1-skV5w77U1tOWru5tmorn9Ugqt7s=", - "dev": true - }, "builtin-modules": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/builtin-modules/-/builtin-modules-1.1.1.tgz", @@ -29853,23 +29822,6 @@ "integrity": "sha512-Jt9tIBkRc9POUof7QA/VwWd+58fKkEEfI+/t1/eOlxKM7ZhrczNzMFefge7Ai+39y1pR/pP6cI19guHy3FSLmw==", "dev": true }, - "chainsaw": { - "version": "0.1.0", - "resolved": "https://registry.npmjs.org/chainsaw/-/chainsaw-0.1.0.tgz", - "integrity": "sha1-XqtQsor+WAdNDVgpE4iCi15fvJg=", - "dev": true, - "requires": { - "traverse": ">=0.3.0 <0.4" - }, - "dependencies": { - "traverse": { - "version": "0.3.9", - "resolved": "https://registry.npmjs.org/traverse/-/traverse-0.3.9.tgz", - "integrity": "sha1-cXuPIgzAu3tE5AUUwisui7xw2Lk=", - "dev": true - } - } - }, "chalk": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/chalk/-/chalk-4.1.1.tgz", @@ -33195,15 +33147,6 @@ "integrity": "sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==", "dev": true }, - "duplexer2": { - "version": "0.1.4", - "resolved": "https://registry.npmjs.org/duplexer2/-/duplexer2-0.1.4.tgz", - "integrity": "sha1-ixLauHjA1p4+eJEFFmKjL8a93ME=", - "dev": true, - "requires": { - "readable-stream": "^2.0.2" - } - }, "duplexify": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/duplexify/-/duplexify-3.6.0.tgz", @@ -35021,7 +34964,7 @@ "eventemitter2": { "version": "1.0.5", "resolved": "https://registry.npmjs.org/eventemitter2/-/eventemitter2-1.0.5.tgz", - "integrity": "sha512-EUFhWUYzqqBZlzBMI+dPU8rnKXfQZEUnitnccQuEIAnvWFHCpt3+4fts2+4dpxLtlsiseVXCMFg37KjYChSxpg==" + "integrity": "sha1-+YNhBRexc3wLncZDvsqTiTwE3xg=" }, "eventemitter3": { "version": "4.0.7", @@ -36767,43 +36710,6 @@ "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", "optional": true }, - "fstream": { - "version": "1.0.12", - "resolved": "https://registry.npmjs.org/fstream/-/fstream-1.0.12.tgz", - "integrity": "sha512-WvJ193OHa0GHPEL+AycEJgxvBEwyfRkN1vhjca23OaPVMCaLCXTd5qAu82AjTcgP1UJmytkOKb63Ypde7raDIg==", - "dev": true, - "requires": { - "graceful-fs": "^4.1.2", - "inherits": "~2.0.0", - "mkdirp": ">=0.5 0", - "rimraf": "2" - }, - "dependencies": { - "glob": { - "version": "7.1.7", - "resolved": "https://registry.npmjs.org/glob/-/glob-7.1.7.tgz", - "integrity": "sha512-OvD9ENzPLbegENnYP5UUfJIirTg4+XwMWGaQfQTY0JenxNvvIKP3U3/tAQSPIu/lHxXYSZmpXlUHeqAIdKzBLQ==", - "dev": true, - "requires": { - "fs.realpath": "^1.0.0", - "inflight": "^1.0.4", - "inherits": "2", - "minimatch": "^3.0.4", - "once": "^1.3.0", - "path-is-absolute": "^1.0.0" - } - }, - "rimraf": { - "version": "2.7.1", - "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-2.7.1.tgz", - "integrity": "sha512-uWjbaKIK3T1OSVptzX7Nl6PvQ3qAGtKEtVRjRuazjfL3Bx5eI409VZSqgND+4UNnmzLVdPj9FqFJNPqBZFve4w==", - "dev": true, - "requires": { - "glob": "^7.1.3" - } - } - } - }, "function-bind": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", @@ -38259,7 +38165,7 @@ "htmlparser2-without-node-native": { "version": "3.9.2", "resolved": "https://registry.npmjs.org/htmlparser2-without-node-native/-/htmlparser2-without-node-native-3.9.2.tgz", - "integrity": "sha512-+FplQXqmY5fRx6vCIp2P5urWaoBCpTNJMXnKP/6mNCcyb+AZWWJzA8D03peXfozlxDL+vpgLK5dJblqEgu8j6A==", + "integrity": "sha1-s+0FDYd9D/NGWWnjOYd7f59mMfY=", "requires": { "domelementtype": "^1.3.0", "domhandler": "^2.3.0", @@ -44503,12 +44409,6 @@ } } }, - "listenercount": { - "version": "1.0.1", - "resolved": "https://registry.npmjs.org/listenercount/-/listenercount-1.0.1.tgz", - "integrity": "sha1-hMinKrWcRyUyFIDJdeZQg0LnCTc=", - "dev": true - }, "listr": { "version": "0.14.3", "resolved": "https://registry.npmjs.org/listr/-/listr-0.14.3.tgz", @@ -60459,38 +60359,6 @@ "os-homedir": "^1.0.0" } }, - "unzipper": { - "version": "0.10.11", - "resolved": "https://registry.npmjs.org/unzipper/-/unzipper-0.10.11.tgz", - "integrity": "sha512-+BrAq2oFqWod5IESRjL3S8baohbevGcVA+teAIOYWM3pDVdseogqbzhhvvmiyQrUNKFUnDMtELW3X8ykbyDCJw==", - "dev": true, - "requires": { - "big-integer": "^1.6.17", - "binary": "~0.3.0", - "bluebird": "~3.4.1", - "buffer-indexof-polyfill": "~1.0.0", - "duplexer2": "~0.1.4", - "fstream": "^1.0.12", - "graceful-fs": "^4.2.2", - "listenercount": "~1.0.1", - "readable-stream": "~2.3.6", - "setimmediate": "~1.0.4" - }, - "dependencies": { - "bluebird": { - "version": "3.4.7", - "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.4.7.tgz", - "integrity": "sha1-9y12C+Cbf3bQjtj66Ysomo0F+rM=", - "dev": true - }, - "graceful-fs": { - "version": "4.2.8", - "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.8.tgz", - "integrity": "sha512-qkIilPUYcNhJpd33n0GBXTB1MMPp14TxEsEs0pTrsSVucApsYzW5V+Q8Qxhik6KU3evy+qkAAowTByymK0avdg==", - "dev": true - } - } - }, "upath": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/upath/-/upath-1.2.0.tgz", diff --git a/package.json b/package.json index b1ebbf430f35fe..93c17bc1d92cdd 100644 --- a/package.json +++ b/package.json @@ -132,7 +132,6 @@ "@types/requestidlecallback": "0.3.4", "@types/semver": "7.3.8", "@types/sprintf-js": "1.1.2", - "@types/unzipper": "0.10.5", "@types/uuid": "8.3.1", "@wordpress/babel-plugin-import-jsx-pragma": "file:packages/babel-plugin-import-jsx-pragma", "@wordpress/babel-plugin-makepot": "file:packages/babel-plugin-makepot", @@ -240,7 +239,6 @@ "terser-webpack-plugin": "5.1.4", "typescript": "4.4.2", "uglify-js": "3.13.7", - "unzipper": "0.10.11", "uuid": "8.3.0", "wd": "1.12.1", "webpack": "5.65.0", diff --git a/packages/report-flaky-tests/action.yml b/packages/report-flaky-tests/action.yml index 7cea6a50c23a3f..5dfcbeb0d9fc60 100644 --- a/packages/report-flaky-tests/action.yml +++ b/packages/report-flaky-tests/action.yml @@ -8,10 +8,10 @@ inputs: description: 'The flaky-test label name' required: true default: 'flaky-test' - artifact-name-prefix: - description: 'The prefix name of the uploaded artifact' + artifact-path: + description: 'The path of the downloaded artifact' required: true - default: 'flaky-tests-report' + default: 'flaky-tests' runs: using: 'node16' main: 'build/index.js' diff --git a/packages/report-flaky-tests/package.json b/packages/report-flaky-tests/package.json index 0bb58464d33238..76528ceb8a3dbb 100644 --- a/packages/report-flaky-tests/package.json +++ b/packages/report-flaky-tests/package.json @@ -28,8 +28,7 @@ "dependencies": { "@actions/core": "^1.8.0", "@actions/github": "^5.0.1", - "jest-message-util": "^28.0.2", - "unzipper": "^0.10.11" + "jest-message-util": "^28.0.2" }, "publishConfig": { "access": "public" diff --git a/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap b/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap index a0a6c8a67011b9..b0ea1a4c054aa4 100644 --- a/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap +++ b/packages/report-flaky-tests/src/__tests__/__snapshots__/run.test.ts.snap @@ -1,6 +1,6 @@ // Jest Snapshot v1, https://goo.gl/fbAQLP -exports[`Report flaky tests should report flaky tests to issue: Created new flaky issue 1`] = ` +exports[`Report flaky tests should report flaky tests to issue on pull request: Created new flaky issue 1`] = ` " **Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.** @@ -14,7 +14,7 @@ Should insert new template part on creation
- Test passed after 2 failed attempts on headBranch. + Test passed after 2 failed attempts on headBranch. \`\`\` @@ -62,7 +62,7 @@ Should insert new template part on creation " `; -exports[`Report flaky tests should report flaky tests to issue: Updated existing flaky issue 1`] = ` +exports[`Report flaky tests should report flaky tests to issue on pull request: Updated existing flaky issue 1`] = ` " **Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.** @@ -148,7 +148,190 @@ should copy only partial selection of text blocks
- Test passed after 1 failed attempt on headBranch. + Test passed after 1 failed attempt on headBranch. + + +\`\`\` +Error: Snapshot comparison failed: + + +

block

+ + + +

B

+ + + +

A block

+ + + +

+ + + +

B block

+ + +Expected: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-expected.txt +Received: /home/runner/work/gutenberg/gutenberg/artifacts/test-results/editor-various-copy-cut-paste-Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-chromium/Copy-cut-paste-should-copy-only-partial-selection-of-text-blocks-2-actual.txt + at /home/runner/work/gutenberg/gutenberg/test/e2e/specs/editor/various/copy-cut-paste.spec.js:253:52 +\`\`\` +
+ +" +`; + +exports[`Report flaky tests should report flaky tests to issue on push: Created new flaky issue 1`] = ` +" +**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.** + +## Test title +Should insert new template part on creation + +## Test path +\`specs/site-editor/template-part.test.js\` + +## Errors + +
+ + Test passed after 2 failed attempts on trunk. + + +\`\`\` + ● Template Part › Template part block › Template part placeholder › Should insert new template part on creation + + expect(jest.fn()).not.toHaveErrored(expected) + + Expected mock function not to be called but it was called with: + [\\"TypeError: Cannot read properties of null (reading 'frameElement') + + at Vo (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/block-editor/index.min.js?ver=dfd3a79ce1dc54c31b6ed591bcb0d55a:3:21925) + at ft (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:43451) + at Wt (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:50270) + at ts (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:112276) + at Fr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:77758) + at Dr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:77686) + at Rr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:77549) + at Nr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:74544) + at ../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:30170 + at unstable_runWithPriority (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react.min.js?ver=6.1-alpha-53362:1:7430)\\"] + at runMicrotasks () + + ● Template Part › Template part block › Template part placeholder › Should insert new template part on creation + + expect(jest.fn()).not.toHaveErrored(expected) + + Expected mock function not to be called but it was called with: + [\\"TypeError: Cannot read properties of null (reading 'frameElement') + + at Vo (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/block-editor/index.min.js?ver=dfd3a79ce1dc54c31b6ed591bcb0d55a:3:21925) + at ft (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:43451) + at Wt (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:50270) + at ts (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:112276) + at Fr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:77758) + at Dr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:77686) + at Rr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:77549) + at Nr (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:74544) + at ../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react-dom.min.js?ver=6.1-alpha-53362:1:30170 + at unstable_runWithPriority (../../http:/localhost:8889/wp-content/plugins/gutenberg/build/vendors/react.min.js?ver=6.1-alpha-53362:1:7430)\\"] + at runMicrotasks () + +\`\`\` +
+ +" +`; + +exports[`Report flaky tests should report flaky tests to issue on push: Updated existing flaky issue 1`] = ` +" +**Flaky test detected. This is an auto-generated issue by GitHub Actions. Please do NOT edit this manually.** + +## Test title +should copy only partial selection of text blocks + +## Test path +\`/test/e2e/specs/editor/various/copy-cut-paste.spec.js\` + +## Errors + + Test passed after 1 failed attempt on trunk. +
+ Test passed after 1 failed attempt on update/post-featured-image-component. +
+ Test passed after 1 failed attempt on add/wp-env/wp-env-core-env. +
+ Test passed after 1 failed attempt on fix/e2e-test. +
+ Test passed after 1 failed attempt on trunk. +
+ Test passed after 1 failed attempt on fix/duotone-site-editor. +
+ Test passed after 1 failed attempt on fix/playwright-snapshots. +
+ Test passed after 1 failed attempt on trunk. +
+ Test passed after 1 failed attempt on trunk. +
+ Test passed after 1 failed attempt on rnmobile/feature/drag-and-drop. +
+ Test passed after 1 failed attempt on rnmobile/feature/drag-and-drop-prevent-text-focus-on-long-press. +
+ Test passed after 1 failed attempt on add/section-concept. +
+ Test passed after 1 failed attempt on trunk. +
+ Test passed after 1 failed attempt on rnmobile/feature/drag-and-drop-refactor-draggable. +
+ Test passed after 1 failed attempt on rnmobile/feature/drag-and-drop-update-chip-animation. +
+ Test passed after 1 failed attempt on fix/comment-reply-link-alignment. +
+ Test passed after 1 failed attempt on rnmobile/feature/drag-and-drop-haptic-feedback. +
+ Test passed after 1 failed attempt on trunk. +
+ Test passed after 1 failed attempt on fix/embed-block-preview-cut-off. +
+ Test passed after 1 failed attempt on trunk. +
+ Test passed after 1 failed attempt on fix/copy-from-non-text-inputs. +
+ Test passed after 1 failed attempt on remove/block-styles-remove-role. +
+ Test passed after 1 failed attempt on docgen/replace-fixtures-with-code. +
+ Test passed after 1 failed attempt on try/use-css-var-for-user-presets. +
+ Test passed after 1 failed attempt on fix/flaky-test-reporter. +
+ Test passed after 1 failed attempt on wp/6.0. +
+ Test passed after 1 failed attempt on wp/6.0. +
+ Test passed after 1 failed attempt on wp/6.0. +
+ Test passed after 1 failed attempt on wp/6.0. +
+ Test passed after 1 failed attempt on wp/6.0. +
+ Test passed after 1 failed attempt on fix/input-field-reset-behavior-moar. +
+ Test passed after 1 failed attempt on refactor/range-control-to-typescript. +
+ Test passed after 1 failed attempt on refactor/range-control-to-typescript. +
+ Test passed after 1 failed attempt on wp/6.0. +
+ Test passed after 1 failed attempt on wp/6.0. +
+ Test passed after 1 failed attempt on wp/6.0. +
+
+ + Test passed after 1 failed attempt on trunk. \`\`\` diff --git a/packages/report-flaky-tests/src/__tests__/run.test.ts b/packages/report-flaky-tests/src/__tests__/run.test.ts index 93b8c4eddf46b0..b064fae579f3e3 100644 --- a/packages/report-flaky-tests/src/__tests__/run.test.ts +++ b/packages/report-flaky-tests/src/__tests__/run.test.ts @@ -2,6 +2,7 @@ * External dependencies */ import * as core from '@actions/core'; +import * as github from '@actions/github'; /** * Internal dependencies @@ -10,22 +11,42 @@ import { run } from '../run'; jest.useFakeTimers( 'modern' ).setSystemTime( new Date( '2020-05-10' ) ); -jest.mock( '@actions/github', () => ( { - context: { - repo: { - owner: 'WordPress', - repo: 'gutenberg', - }, - eventName: 'workflow_run', - payload: { - action: 'completed', - workflow_run: { - html_url: 'runURL', - head_branch: 'headBranch', - id: 100, +const mockPushEventContext = { + runId: 100, + repo: { + owner: 'WordPress', + repo: 'gutenberg', + }, + ref: 'refs/heads/trunk', + sha: 'commitSHA', + eventName: 'push', +}; +const mockPullRequestEventContext = { + runId: 100, + repo: { + owner: 'WordPress', + repo: 'gutenberg', + }, + ref: 'refs/pull/10/merge', + sha: 'mergeSHA', + eventName: 'pull_request', + payload: { + pull_request: { + head: { + ref: 'headBranch', + sha: 'headSHA', }, }, }, +}; +const mockGetContext = jest.fn( + (): typeof mockPushEventContext | typeof mockPullRequestEventContext => + mockPullRequestEventContext +); +jest.mock( '@actions/github', () => ( { + get context() { + return mockGetContext(); + }, } ) ); jest.mock( '@actions/core', () => ( { @@ -35,14 +56,19 @@ jest.mock( '@actions/core', () => ( { } ) ); const mockAPI = { - downloadReportFromArtifact: jest.fn(), fetchAllIssuesLabeledFlaky: jest.fn(), findMergeBaseCommit: jest.fn(), updateIssue: jest.fn(), createIssue: jest.fn(), + createComment: jest.fn(), }; jest.mock( '../github-api', () => ( { - GitHubAPI: jest.fn().mockImplementation( () => mockAPI ), + GitHubAPI: jest.fn( () => mockAPI ), +} ) ); + +jest.mock( 'fs/promises', () => ( { + readdir: jest.fn(), + readFile: jest.fn(), } ) ); describe( 'Report flaky tests', () => { @@ -50,7 +76,7 @@ describe( 'Report flaky tests', () => { jest.clearAllMocks(); } ); - it( 'should report flaky tests to issue', async () => { + it( 'should report flaky tests to issue on pull request', async () => { const existingFlakyTest = await import( '../__fixtures__/should copy only partial selection of text blocks.json' ).then( ( json ) => json.default ); @@ -62,8 +88,11 @@ describe( 'Report flaky tests', () => { ).then( ( json ) => json.default ); ( core.getInput as jest.Mock ) + // token .mockReturnValueOnce( 'repo-token' ) - .mockReturnValueOnce( 'flaky-tests-report' ) + // artifact-path + .mockReturnValueOnce( 'flaky-tests' ) + // label .mockReturnValueOnce( '[Type] Flaky Test' ); // Replacing the cwd for the test for consistent snapshot results. @@ -71,29 +100,146 @@ describe( 'Report flaky tests', () => { '/home/runner/work/gutenberg/gutenberg', process.cwd() ); - mockAPI.downloadReportFromArtifact.mockImplementationOnce( () => [ - existingFlakyTest, - newFlakyTest, - ] ); + + const mockedFs = require( 'fs/promises' ); + mockedFs.readdir.mockImplementationOnce( () => + Promise.resolve( [ + `${ existingFlakyTest.title }.json`, + `${ newFlakyTest.title }.json`, + ] ) + ); + mockedFs.readFile + .mockImplementationOnce( () => + Promise.resolve( JSON.stringify( existingFlakyTest ) ) + ) + .mockImplementationOnce( () => + Promise.resolve( JSON.stringify( newFlakyTest ) ) + ); mockAPI.fetchAllIssuesLabeledFlaky.mockImplementationOnce( () => flakyIssues ); mockAPI.updateIssue.mockImplementationOnce( () => ( { + number: 1, html_url: 'html_url', } ) ); mockAPI.createIssue.mockImplementationOnce( () => ( { + number: 2, html_url: 'html_url', } ) ); + mockAPI.createComment.mockImplementationOnce( () => ( { + html_url: 'comment_html_url', + } ) ); + await run(); const existingFlakyIssue = flakyIssues.find( ( issue ) => issue.title === `[Flaky Test] ${ existingFlakyTest.title }` + )!; + expect( mockAPI.updateIssue ).toHaveBeenCalledWith( + expect.objectContaining( { + issue_number: existingFlakyIssue.number, + state: 'open', + } ) + ); + expect( mockAPI.updateIssue.mock.calls[ 0 ][ 0 ].body ).toMatchSnapshot( + 'Updated existing flaky issue' ); + + expect( mockAPI.createIssue ).toHaveBeenCalledWith( + expect.objectContaining( { + title: `[Flaky Test] ${ newFlakyTest.title }`, + } ) + ); + expect( mockAPI.createIssue.mock.calls[ 0 ][ 0 ].body ).toMatchSnapshot( + 'Created new flaky issue' + ); + + expect( mockAPI.createComment ).toHaveBeenCalledTimes( 1 ); + expect( mockAPI.createComment.mock.calls[ 0 ][ 0 ] ).toBe( 'headSHA' ); + expect( mockAPI.createComment.mock.calls[ 0 ][ 1 ] ) + .toMatchInlineSnapshot( ` + " + **Flaky tests detected.** + Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information. + + 🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/100 + 📝 Reported issues: + - #1 in \`/test/e2e/specs/editor/various/copy-cut-paste.spec.js\` + - #2 in \`specs/site-editor/template-part.test.js\`" + ` ); + } ); + + it( 'should report flaky tests to issue on push', async () => { + mockGetContext.mockImplementation( () => mockPushEventContext ); + + const existingFlakyTest = await import( + '../__fixtures__/should copy only partial selection of text blocks.json' + ).then( ( json ) => json.default ); + const newFlakyTest = await import( + '../__fixtures__/Should insert new template part on creation.json' + ).then( ( json ) => json.default ); + const flakyIssues = await import( + '../__fixtures__/flaky-issues.json' + ).then( ( json ) => json.default ); + + ( core.getInput as jest.Mock ) + // token + .mockReturnValueOnce( 'repo-token' ) + // artifact-path + .mockReturnValueOnce( 'flaky-tests' ) + // label + .mockReturnValueOnce( '[Type] Flaky Test' ); + + // Replacing the cwd for the test for consistent snapshot results. + existingFlakyTest.path = existingFlakyTest.path.replace( + '/home/runner/work/gutenberg/gutenberg', + process.cwd() + ); + + const mockedFs = require( 'fs/promises' ); + mockedFs.readdir.mockImplementationOnce( () => + Promise.resolve( [ + `${ existingFlakyTest.title }.json`, + `${ newFlakyTest.title }.json`, + ] ) + ); + mockedFs.readFile + .mockImplementationOnce( () => + Promise.resolve( JSON.stringify( existingFlakyTest ) ) + ) + .mockImplementationOnce( () => + Promise.resolve( JSON.stringify( newFlakyTest ) ) + ); + + mockAPI.fetchAllIssuesLabeledFlaky.mockImplementationOnce( + () => flakyIssues + ); + + mockAPI.updateIssue.mockImplementationOnce( () => ( { + number: 1, + html_url: 'html_url', + } ) ); + + mockAPI.createIssue.mockImplementationOnce( () => ( { + number: 2, + html_url: 'html_url', + } ) ); + + mockAPI.createComment.mockImplementationOnce( () => ( { + html_url: 'comment_html_url', + } ) ); + + await run(); + + const existingFlakyIssue = flakyIssues.find( + ( issue ) => + issue.title === `[Flaky Test] ${ existingFlakyTest.title }` + )!; expect( mockAPI.updateIssue ).toHaveBeenCalledWith( expect.objectContaining( { issue_number: existingFlakyIssue.number, @@ -112,6 +258,24 @@ describe( 'Report flaky tests', () => { expect( mockAPI.createIssue.mock.calls[ 0 ][ 0 ].body ).toMatchSnapshot( 'Created new flaky issue' ); + + expect( mockAPI.createComment ).toHaveBeenCalledTimes( 1 ); + expect( mockAPI.createComment.mock.calls[ 0 ][ 0 ] ).toBe( + 'commitSHA' + ); + expect( mockAPI.createComment.mock.calls[ 0 ][ 1 ] ) + .toMatchInlineSnapshot( ` + " + **Flaky tests detected.** + Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information. + + 🔍 Workflow run URL: https://github.com/WordPress/gutenberg/actions/runs/100 + 📝 Reported issues: + - #1 in \`/test/e2e/specs/editor/various/copy-cut-paste.spec.js\` + - #2 in \`specs/site-editor/template-part.test.js\`" + ` ); + + mockGetContext.mockImplementation( () => mockPullRequestEventContext ); } ); it( 'should skip for outdated branches', async () => { @@ -132,9 +296,14 @@ describe( 'Report flaky tests', () => { '/home/runner/work/gutenberg/gutenberg', process.cwd() ); - mockAPI.downloadReportFromArtifact.mockImplementationOnce( () => [ - flakyTest, - ] ); + + const mockedFs = require( 'fs/promises' ); + mockedFs.readdir.mockImplementationOnce( () => + Promise.resolve( [ `${ flakyTest.title }.json` ] ) + ); + mockedFs.readFile.mockImplementationOnce( () => + Promise.resolve( JSON.stringify( flakyTest ) ) + ); mockAPI.fetchAllIssuesLabeledFlaky.mockImplementationOnce( () => flakyIssues @@ -151,8 +320,9 @@ describe( 'Report flaky tests', () => { // indicating that the code base at this point is outdated. const flakyIssue = flakyIssues.find( ( issue ) => issue.title === `[Flaky Test] ${ flakyTest.title }` - ); + )!; flakyIssue.state = 'closed'; + // @ts-expect-error: "closed_at" hasn't been typed yet. flakyIssue.closed_at = new Date( '2022-05-15' ).toISOString(); await run(); @@ -160,5 +330,7 @@ describe( 'Report flaky tests', () => { expect( mockAPI.findMergeBaseCommit ).toHaveBeenCalledTimes( 1 ); expect( mockAPI.updateIssue ).not.toHaveBeenCalled(); + + expect( mockAPI.createComment ).not.toHaveBeenCalled(); } ); } ); diff --git a/packages/report-flaky-tests/src/github-api.ts b/packages/report-flaky-tests/src/github-api.ts index b7aa0990085683..8d5d49025f576c 100644 --- a/packages/report-flaky-tests/src/github-api.ts +++ b/packages/report-flaky-tests/src/github-api.ts @@ -2,14 +2,13 @@ * External dependencies */ import { getOctokit } from '@actions/github'; -import * as unzipper from 'unzipper'; import type { GitHub } from '@actions/github/lib/utils'; import type { Endpoints } from '@octokit/types'; /** * Internal dependencies */ -import type { FlakyTestResult } from './types'; +import { isReportComment } from './markdown'; type Octokit = InstanceType< typeof GitHub >; @@ -27,44 +26,6 @@ class GitHubAPI { this.#repo = repo; } - async downloadReportFromArtifact( - runID: number, - artifactNamePrefix: string - ): Promise< FlakyTestResult[] | undefined > { - const { - data: { artifacts }, - } = await this.#octokit.rest.actions.listWorkflowRunArtifacts( { - ...this.#repo, - run_id: runID, - } ); - - const matchArtifact = artifacts.find( ( artifact ) => - artifact.name.startsWith( artifactNamePrefix ) - ); - - if ( ! matchArtifact ) { - return undefined; - } - - const download = await this.#octokit.rest.actions.downloadArtifact( { - ...this.#repo, - artifact_id: matchArtifact.id, - archive_format: 'zip', - } ); - - const { files } = await unzipper.Open.buffer( - Buffer.from( download.data as Buffer ) - ); - const fileBuffers = await Promise.all( - files.map( ( file ) => file.buffer() ) - ); - const parsedFiles = fileBuffers.map( - ( buffer ) => JSON.parse( buffer.toString() ) as FlakyTestResult - ); - - return parsedFiles; - } - async fetchAllIssuesLabeledFlaky( label: string ) { const issues = await this.#octokit.paginate( this.#octokit.rest.issues.listForRepo, @@ -116,6 +77,37 @@ class GitHubAPI { return data; } + + async createComment( sha: string, body: string ) { + const { data: comments } = + await this.#octokit.rest.repos.listCommentsForCommit( { + ...this.#repo, + commit_sha: sha, + } ); + const reportComment = comments.find( ( comment ) => + isReportComment( comment.body ) + ); + + if ( reportComment ) { + const { data } = await this.#octokit.rest.repos.updateCommitComment( + { + ...this.#repo, + comment_id: reportComment.id, + body, + } + ); + + return data; + } + + const { data } = await this.#octokit.rest.repos.createCommitComment( { + ...this.#repo, + commit_sha: sha, + body, + } ); + + return data; + } } export { GitHubAPI }; diff --git a/packages/report-flaky-tests/src/markdown.ts b/packages/report-flaky-tests/src/markdown.ts index 2852623115c674..91e4a7bda22d61 100644 --- a/packages/report-flaky-tests/src/markdown.ts +++ b/packages/report-flaky-tests/src/markdown.ts @@ -9,7 +9,7 @@ import * as core from '@actions/core'; * Internal dependencies */ import { stripAnsi } from './strip-ansi'; -import type { MetaData, FlakyTestResult } from './types'; +import type { MetaData, FlakyTestResult, ReportedIssue } from './types'; type ParsedTestResult = { date: Date; @@ -203,10 +203,36 @@ function parseIssueBody( body: string ) { }; } +const FLAKY_TESTS_REPORT_COMMENT_TOKEN = `flaky-tests-report-comment`; + +function renderCommitComment( { + reportedIssues, + runURL, +}: { + reportedIssues: ReportedIssue[]; + runURL: string; +} ) { + return ` +**Flaky test${ reportedIssues.length > 1 ? 's' : '' } detected.** +Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information. + +🔍 Workflow run URL: ${ runURL } +📝 Reported issues: +${ reportedIssues + .map( ( issue ) => `- #${ issue.issueNumber } in \`${ issue.testPath }\`` ) + .join( '\n' ) }`; +} + +function isReportComment( body: string ) { + return body.startsWith( `` ); +} + export { renderIssueBody, formatTestErrorMessage, formatTestResults, parseFormattedTestResults, parseIssueBody, + renderCommitComment, + isReportComment, }; diff --git a/packages/report-flaky-tests/src/run.ts b/packages/report-flaky-tests/src/run.ts index 307a41cb100787..65323c3d87fa80 100644 --- a/packages/report-flaky-tests/src/run.ts +++ b/packages/report-flaky-tests/src/run.ts @@ -3,7 +3,9 @@ */ import * as github from '@actions/github'; import * as core from '@actions/core'; -import type { WorkflowRunCompletedEvent } from '@octokit/webhooks-types'; +import * as fs from 'fs/promises'; +import * as path from 'path'; +import type { PullRequestEvent } from '@octokit/webhooks-types'; /** * Internal dependencies @@ -14,39 +16,44 @@ import { formatTestErrorMessage, formatTestResults, parseIssueBody, + renderCommitComment, } from './markdown'; +import type { ReportedIssue } from './types'; async function run() { - if ( - github.context.eventName !== 'workflow_run' || - github.context.payload.action !== 'completed' - ) { - return; - } - const token = core.getInput( 'repo-token', { required: true } ); - const artifactNamePrefix = core.getInput( 'artifact-name-prefix', { + const artifactPath = core.getInput( 'artifact-path', { required: true, } ); - const api = new GitHubAPI( token, github.context.repo ); - // Cast the payload type: https://github.com/actions/toolkit/tree/main/packages/github#webhook-payload-typescript-definitions - const { - workflow_run: { head_branch: headBranch, html_url: runURL, id: runID }, - } = github.context.payload as WorkflowRunCompletedEvent; - - const flakyTests = await api.downloadReportFromArtifact( - runID, - artifactNamePrefix + const { runId: runID, repo, ref } = github.context; + const runURL = `https://github.com/${ repo.owner }/${ repo.repo }/actions/runs/${ runID }`; + const api = new GitHubAPI( token, repo ); + + const flakyTestsDir = await fs.readdir( artifactPath ); + const flakyTests = await Promise.all( + flakyTestsDir.map( ( filename ) => + fs + .readFile( path.join( artifactPath, filename ), 'utf-8' ) + .then( ( text ) => JSON.parse( text ) ) + ) ); - if ( ! flakyTests ) { + if ( ! flakyTests || flakyTests.length === 0 ) { // No flaky tests reported in this run. return; } + const headBranch = + github.context.eventName === 'pull_request' + ? // Cast the payload type: https://github.com/actions/toolkit/tree/main/packages/github#webhook-payload-typescript-definitions + ( github.context.payload as PullRequestEvent ).pull_request.head + .ref + : ref.replace( /^refs\/(heads|tag)\//, '' ); + const label = core.getInput( 'label', { required: true } ); const issues = await api.fetchAllIssuesLabeledFlaky( label ); + const reportedIssues: ReportedIssue[] = []; for ( const flakyTest of flakyTests ) { const { title: testTitle } = flakyTest; @@ -143,8 +150,33 @@ async function run() { } ); } + reportedIssues.push( { + testTitle, + testPath, + issueNumber: issue.number, + issueUrl: issue.html_url, + } ); core.info( `Reported flaky test to ${ issue.html_url }` ); } + + if ( reportedIssues.length === 0 ) { + return; + } + + const commitSHA = + github.context.eventName === 'pull_request' + ? // Cast the payload type: https://github.com/actions/toolkit/tree/main/packages/github#webhook-payload-typescript-definitions + ( github.context.payload as PullRequestEvent ).pull_request.head + .sha + : github.context.sha; + const { html_url: commentUrl } = await api.createComment( + commitSHA, + renderCommitComment( { + runURL, + reportedIssues, + } ) + ); + core.info( `Reported the summary of the flaky tests to ${ commentUrl }` ); } function getIssueTitle( testTitle: string ) { diff --git a/packages/report-flaky-tests/src/types.ts b/packages/report-flaky-tests/src/types.ts index e52d6dba471018..5de8770ef75418 100644 --- a/packages/report-flaky-tests/src/types.ts +++ b/packages/report-flaky-tests/src/types.ts @@ -29,3 +29,10 @@ export type MetaData = { totalCommits?: number; baseCommit?: string; }; + +export type ReportedIssue = { + testTitle: string; + testPath: string; + issueNumber: number; + issueUrl: string; +}; From 5779016a4eb5d694a79504104da0add976704caf Mon Sep 17 00:00:00 2001 From: Kai Hao Date: Wed, 16 Nov 2022 16:54:05 +0800 Subject: [PATCH 2/3] Add test for rendering commit comment --- .../src/__tests__/markdown.test.ts | 52 +++++++++++++++++++ packages/report-flaky-tests/src/markdown.ts | 2 +- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/packages/report-flaky-tests/src/__tests__/markdown.test.ts b/packages/report-flaky-tests/src/__tests__/markdown.test.ts index 5d7b7dd62bc873..e8a5b4f25da5d3 100644 --- a/packages/report-flaky-tests/src/__tests__/markdown.test.ts +++ b/packages/report-flaky-tests/src/__tests__/markdown.test.ts @@ -12,7 +12,10 @@ import { renderIssueBody, parseFormattedTestResults, parseIssueBody, + renderCommitComment, + isReportComment, } from '../markdown'; +import { ReportedIssue } from '../types'; jest.useFakeTimers( 'modern' ).setSystemTime( new Date( '2020-05-10' ) ); @@ -226,6 +229,55 @@ describe( 'parseIssueBody', () => { } ); } ); +describe( 'renderCommitComment', () => { + it( 'render the commit comment', () => { + const runURL = 'runURL'; + const reportedIssues: ReportedIssue[] = [ + { + testTitle: 'title1', + testPath: 'path1', + issueNumber: 1, + issueUrl: 'url1', + }, + { + testTitle: 'title2', + testPath: 'path2', + issueNumber: 2, + issueUrl: 'url2', + }, + ]; + + const commentBody = renderCommitComment( { + reportedIssues, + runURL, + } ); + + expect( commentBody ).toMatchInlineSnapshot( ` + " + **Flaky tests detected.** + Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information. + + 🔍 Workflow run URL: runURL + 📝 Reported issues: + - #1 in \`path1\` + - #2 in \`path2\`" + ` ); + } ); +} ); + +describe( 'isReportComment', () => { + it( 'matches the report comment', () => { + const commentBody = renderCommitComment( { + reportedIssues: [], + runURL: '', + } ); + + expect( isReportComment( commentBody ) ).toBe( true ); + + expect( isReportComment( 'random string' ) ).toBe( false ); + } ); +} ); + function renderToDisplayText( html: string ) { const container = document.createElement( 'div' ); container.innerHTML = html; diff --git a/packages/report-flaky-tests/src/markdown.ts b/packages/report-flaky-tests/src/markdown.ts index 91e4a7bda22d61..15d850773ffe65 100644 --- a/packages/report-flaky-tests/src/markdown.ts +++ b/packages/report-flaky-tests/src/markdown.ts @@ -213,7 +213,7 @@ function renderCommitComment( { runURL: string; } ) { return ` -**Flaky test${ reportedIssues.length > 1 ? 's' : '' } detected.** +**Flaky tests detected.** Some tests passed with failed attempts. The failures may not be related to this commit but are still reported for visibility. See [the documentation](https://github.com/WordPress/gutenberg/blob/HEAD/docs/contributors/code/testing-overview.md#flaky-tests) for more information. 🔍 Workflow run URL: ${ runURL } From f9cb3fd092631b8cea53157e78eb969bee9af0dd Mon Sep 17 00:00:00 2001 From: Kai Hao Date: Thu, 15 Dec 2022 16:50:56 +0800 Subject: [PATCH 3/3] Update name from admin to puppeteer --- .github/workflows/end2end-test.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/end2end-test.yml b/.github/workflows/end2end-test.yml index ad4db13ff6ce8c..46278d6384a5b4 100644 --- a/.github/workflows/end2end-test.yml +++ b/.github/workflows/end2end-test.yml @@ -16,8 +16,8 @@ concurrency: cancel-in-progress: true jobs: - admin: - name: Admin - ${{ matrix.part }} + e2e-puppeteer: + name: Puppeteer - ${{ matrix.part }} runs-on: ubuntu-latest if: ${{ github.repository == 'WordPress/gutenberg' || github.event_name == 'pull_request' }} strategy: @@ -115,7 +115,7 @@ jobs: report-to-issues: name: Report to GitHub - needs: [admin, e2e-playwright] + needs: [e2e-puppeteer, e2e-playwright] if: ${{ always() }} runs-on: ubuntu-latest steps: