From 658af081ee2781cc159dbc2826857def3765f281 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 14:42:51 +0530 Subject: [PATCH 01/14] Flatten uploaded images --- .github/workflows/e2e.yml | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 996fc6673e..436624968c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -102,8 +102,30 @@ jobs: token: 6845cc80-77ee-4e17-85a1-026cd95e0766 - name: Upload Artifacts - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: ${{ failure() && steps.cypress.conclusion == 'failure' }} with: - name: error-snapshots + name: error-snapshots-${{ matrix.containers }} + retention-days: 1 path: cypress/snapshots/**/__diff_output__/* + + combineArtifacts: + needs: e2e + runs-on: ubuntu-latest + steps: + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + path: snapshots + pattern: error-snapshots-* + merge-multiple: true + - run: | + mkdir errors + cd snapshots + find . -mindepth 2 -type f -print -exec mv {} ../errors/ \; + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: error-snapshots + retention-days: 10 + path: errors/ \ No newline at end of file From 4ce5d071250ec7b281ad9009e557551bc68ceeca Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 14:46:21 +0530 Subject: [PATCH 02/14] Flatten uploaded images --- .github/workflows/e2e.yml | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 436624968c..29bd6eab56 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -113,19 +113,19 @@ jobs: needs: e2e runs-on: ubuntu-latest steps: - - name: Download All Artifacts - uses: actions/download-artifact@v4 - with: - path: snapshots - pattern: error-snapshots-* - merge-multiple: true - - run: | - mkdir errors - cd snapshots - find . -mindepth 2 -type f -print -exec mv {} ../errors/ \; - - name: Upload Artifacts - uses: actions/upload-artifact@v4 - with: - name: error-snapshots - retention-days: 10 - path: errors/ \ No newline at end of file + - name: Download All Artifacts + uses: actions/download-artifact@v4 + with: + path: snapshots + pattern: error-snapshots-* + merge-multiple: true + - run: | + mkdir errors + cd snapshots + find . -mindepth 2 -type f -print -exec mv {} ../errors/ \; + - name: Upload Artifacts + uses: actions/upload-artifact@v4 + with: + name: error-snapshots + retention-days: 10 + path: errors/ From f58a86d747ab5a773fddf905e23176ebf8581c5e Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 15:30:18 +0530 Subject: [PATCH 03/14] Notify users --- .github/workflows/e2e.yml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 29bd6eab56..cfc8a45d2c 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -111,6 +111,7 @@ jobs: combineArtifacts: needs: e2e + if: ${{ failure() }} runs-on: ubuntu-latest steps: - name: Download All Artifacts @@ -125,7 +126,12 @@ jobs: find . -mindepth 2 -type f -print -exec mv {} ../errors/ \; - name: Upload Artifacts uses: actions/upload-artifact@v4 + id: upload-artifacts with: name: error-snapshots retention-days: 10 path: errors/ + - name: Notify Users + run: | + echo "::error,title=Visual tests failed::You can view images that failed by downloading the error-snapshots artifact: ${{ steps.upload-artifacts.outputs.artifact-url }}" + From a9a8a208a6dc011db8c737cbee4c0eb228af2abb Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 15:50:40 +0530 Subject: [PATCH 04/14] Update cache key --- .github/workflows/e2e.yml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index cfc8a45d2c..3b983589d8 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -8,6 +8,9 @@ on: permissions: contents: read +env: + targetHash: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base.sha || github.event.before }} + jobs: cache: runs-on: ubuntu-latest @@ -25,13 +28,13 @@ jobs: with: save-always: true path: ./cypress/snapshots - key: ${{ runner.os }}-snapshots + key: ${{ runner.os }}-snapshots-${{ env.targetHash }} - name: Switch to base branch if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }} uses: actions/checkout@v4 with: - ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base.sha || 'develop' }} + ref: ${{ env.targetHash }} - name: Cypress run uses: cypress-io/github-action@v4 @@ -66,7 +69,7 @@ jobs: uses: actions/cache/restore@v3 with: path: ./cypress/snapshots - key: ${{ runner.os }}-snapshots + key: ${{ runner.os }}-snapshots-${{ env.targetHash }} # Install NPM dependencies, cache them correctly # and run all Cypress tests @@ -109,6 +112,14 @@ jobs: retention-days: 1 path: cypress/snapshots/**/__diff_output__/* + - name: Save snapshots cache + id: cache-upload + if: ${{ github.event_name == 'push' }} + uses: actions/cache/save@v3 + with: + path: ./cypress/snapshots + key: ${{ runner.os }}-snapshots-${{ github.event.after }} + combineArtifacts: needs: e2e if: ${{ failure() }} From be4721b24db004317f451ac406bb094379ce0f59 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 16:18:07 +0530 Subject: [PATCH 05/14] Fix cache save --- .github/workflows/e2e.yml | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 3b983589d8..9f4fd86b2f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -108,34 +108,38 @@ jobs: uses: actions/upload-artifact@v4 if: ${{ failure() && steps.cypress.conclusion == 'failure' }} with: - name: error-snapshots-${{ matrix.containers }} + name: snapshots-${{ matrix.containers }} retention-days: 1 - path: cypress/snapshots/**/__diff_output__/* - - - name: Save snapshots cache - id: cache-upload - if: ${{ github.event_name == 'push' }} - uses: actions/cache/save@v3 - with: path: ./cypress/snapshots - key: ${{ runner.os }}-snapshots-${{ github.event.after }} combineArtifacts: needs: e2e - if: ${{ failure() }} runs-on: ubuntu-latest steps: - name: Download All Artifacts uses: actions/download-artifact@v4 with: path: snapshots - pattern: error-snapshots-* + pattern: snapshots-* merge-multiple: true - - run: | + + - name: Save snapshots cache + id: cache-upload + if: ${{ github.event_name == 'push' }} + uses: actions/cache/save@v3 + with: + path: | + ./cypress/snapshots + !./**/__diff_output__/* + key: ${{ runner.os }}-snapshots-${{ github.event.after }} + + - if: ${{ failure() }} + run: | mkdir errors cd snapshots - find . -mindepth 2 -type f -print -exec mv {} ../errors/ \; - - name: Upload Artifacts + find . -mindepth 2 -type d -name "*__diff_output__*" -exec sh -c 'mv "$0"/*.png ../errors/' {} \; + - name: Upload Error snapshots + if: ${{ failure() }} uses: actions/upload-artifact@v4 id: upload-artifacts with: @@ -143,6 +147,7 @@ jobs: retention-days: 10 path: errors/ - name: Notify Users + if: ${{ failure() }} run: | echo "::error,title=Visual tests failed::You can view images that failed by downloading the error-snapshots artifact: ${{ steps.upload-artifacts.outputs.artifact-url }}" From 35b3192c2bf69cd3d15e35cb4422ab5c5f7afbd5 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 16:23:26 +0530 Subject: [PATCH 06/14] Fix error message --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 9f4fd86b2f..5326cb4575 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -149,5 +149,5 @@ jobs: - name: Notify Users if: ${{ failure() }} run: | - echo "::error,title=Visual tests failed::You can view images that failed by downloading the error-snapshots artifact: ${{ steps.upload-artifacts.outputs.artifact-url }}" + echo "::error title=Visual tests failed::You can view images that failed by downloading the error-snapshots artifact: ${{ steps.upload-artifacts.outputs.artifact-url }}" From 6f205f89b22d82af5d56fb1613307e2b7b50db69 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 16:24:42 +0530 Subject: [PATCH 07/14] Always run combineArtifacts --- .github/workflows/e2e.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 5326cb4575..754f9ada13 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -115,6 +115,7 @@ jobs: combineArtifacts: needs: e2e runs-on: ubuntu-latest + if: ${{ always() }} steps: - name: Download All Artifacts uses: actions/download-artifact@v4 From a964af67ec5f1954342cd0b2711b97ee26a7c963 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 19:04:25 +0530 Subject: [PATCH 08/14] Handle edge cases in E2E --- .github/workflows/e2e.yml | 38 +++++++++++++++------ cypress/integration/rendering/debug.spec.js | 12 ------- 2 files changed, 27 insertions(+), 23 deletions(-) delete mode 100644 cypress/integration/rendering/debug.spec.js diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 754f9ada13..90769858e0 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -1,3 +1,9 @@ +# We use github cache to save snapshots between runs. +# For PRs and MergeQueues, the target commit is used, and for push events, github.event.previous is used. +# If a snapshot for a given Hash is not found, we checkout that commit, run the tests and cache the snapshots. +# These are then downloaded before running the E2E, providing the reference snapshots. +# If there are any errors, the diff image is uploaded to artifacts, and the user is notified. + name: E2E on: @@ -9,7 +15,8 @@ permissions: contents: read env: - targetHash: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base.sha || github.event.before }} + # For PRs and MergeQueues, the target commit is used, and for push events, github.event.previous is used. + targetHash: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} jobs: cache: @@ -30,6 +37,7 @@ jobs: path: ./cypress/snapshots key: ${{ runner.os }}-snapshots-${{ env.targetHash }} + # If a snapshot for a given Hash is not found, we checkout that commit, run the tests and cache the snapshots. - name: Switch to base branch if: ${{ steps.cache-snapshot.outputs.cache-hit != 'true' }} uses: actions/checkout@v4 @@ -44,6 +52,8 @@ jobs: start: pnpm run dev wait-on: 'http://localhost:9000' browser: chrome + spec: + cypress/integration/rendering/sequencediagram.spec.js e2e: runs-on: ubuntu-latest @@ -64,6 +74,7 @@ jobs: with: node-version: ${{ matrix.node-version }} + # These cached snapshots are downloaded, providing the reference snapshots. - name: Cache snapshots id: cache-snapshot uses: actions/cache/restore@v3 @@ -87,6 +98,8 @@ jobs: # e.g. if this action was run from a fork record: ${{ secrets.CYPRESS_RECORD_KEY != '' }} parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }} + spec: + cypress/integration/rendering/sequencediagram.spec.js env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} VITEST_COVERAGE: true @@ -104,9 +117,10 @@ jobs: verbose: true token: 6845cc80-77ee-4e17-85a1-026cd95e0766 + # We upload the artifacts into numbered archives to prevent overwriting - name: Upload Artifacts uses: actions/upload-artifact@v4 - if: ${{ failure() && steps.cypress.conclusion == 'failure' }} + if: ${{ always() }} with: name: snapshots-${{ matrix.containers }} retention-days: 1 @@ -117,38 +131,40 @@ jobs: runs-on: ubuntu-latest if: ${{ always() }} steps: + # Download all snapshot artifacts and merge them into a single folder - name: Download All Artifacts uses: actions/download-artifact@v4 with: path: snapshots pattern: snapshots-* merge-multiple: true - + + # For successful push events, we save the snapshots cache - name: Save snapshots cache id: cache-upload - if: ${{ github.event_name == 'push' }} + if: ${{ github.event_name == 'push' && needs.e2e.result != 'failure' }} uses: actions/cache/save@v3 with: - path: | - ./cypress/snapshots - !./**/__diff_output__/* + path: ./snapshots key: ${{ runner.os }}-snapshots-${{ github.event.after }} - - if: ${{ failure() }} + - name: Flatten images to a folder + if: ${{ needs.e2e.result == 'failure' }} run: | mkdir errors cd snapshots find . -mindepth 2 -type d -name "*__diff_output__*" -exec sh -c 'mv "$0"/*.png ../errors/' {} \; + - name: Upload Error snapshots - if: ${{ failure() }} + if: ${{ needs.e2e.result == 'failure' }} uses: actions/upload-artifact@v4 id: upload-artifacts with: name: error-snapshots retention-days: 10 path: errors/ + - name: Notify Users - if: ${{ failure() }} + if: ${{ needs.e2e.result == 'failure' }} run: | echo "::error title=Visual tests failed::You can view images that failed by downloading the error-snapshots artifact: ${{ steps.upload-artifacts.outputs.artifact-url }}" - diff --git a/cypress/integration/rendering/debug.spec.js b/cypress/integration/rendering/debug.spec.js deleted file mode 100644 index 56ad0f15f8..0000000000 --- a/cypress/integration/rendering/debug.spec.js +++ /dev/null @@ -1,12 +0,0 @@ -import { imgSnapshotTest } from '../../helpers/util.ts'; - -describe('Flowchart', () => { - it('34: testing the label width in percy', () => { - imgSnapshotTest( - `graph TD - A[Christmas] - `, - { theme: 'forest', fontFamily: '"Noto Sans SC", sans-serif' } - ); - }); -}); From a871a68f3c8bce5419bcd6f04deaeddd0defa473 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 19:05:06 +0530 Subject: [PATCH 09/14] Remove spec selector --- .github/workflows/e2e.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 90769858e0..817635cbf6 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -52,8 +52,6 @@ jobs: start: pnpm run dev wait-on: 'http://localhost:9000' browser: chrome - spec: - cypress/integration/rendering/sequencediagram.spec.js e2e: runs-on: ubuntu-latest @@ -98,8 +96,6 @@ jobs: # e.g. if this action was run from a fork record: ${{ secrets.CYPRESS_RECORD_KEY != '' }} parallel: ${{ secrets.CYPRESS_RECORD_KEY != '' }} - spec: - cypress/integration/rendering/sequencediagram.spec.js env: CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }} VITEST_COVERAGE: true From f1fc874da8d4bdef88add516c7e6c046fdb9b755 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 19:20:39 +0530 Subject: [PATCH 10/14] Debug --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 817635cbf6..56bdd30a01 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -16,12 +16,13 @@ permissions: env: # For PRs and MergeQueues, the target commit is used, and for push events, github.event.previous is used. - targetHash: ${{ github.event.pull_request.base.sha || github.event.merge_group.base_sha || github.event.before }} + targetHash: ${{ github.event_name == 'pull_request' && github.event.pull_request.base.sha || github.event_name == 'merge_group' && github.event.merge_group.base_sha || github.event.before }} jobs: cache: runs-on: ubuntu-latest steps: + - run: echo "${{ github.event }}" - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 - name: Setup Node.js From 279e31bc559caade5d63ce0712273f02a77f244d Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 19:23:13 +0530 Subject: [PATCH 11/14] Debug --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 56bdd30a01..674f276727 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -22,7 +22,7 @@ jobs: cache: runs-on: ubuntu-latest steps: - - run: echo "${{ github.event }}" + - run: echo "${{ toJSON(github) }}" - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 - name: Setup Node.js From f5dd24bce4488cb636f384b949c0ea5583eda860 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 19:29:45 +0530 Subject: [PATCH 12/14] Remove :: --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 674f276727..30c25cb914 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -22,7 +22,7 @@ jobs: cache: runs-on: ubuntu-latest steps: - - run: echo "${{ toJSON(github) }}" + - run: echo "${{ toJSON(github) }}" | sed 's/::/:/g' - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 - name: Setup Node.js From 3935f6b389e715382c0701c499ca6825fe34b299 Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 19:31:25 +0530 Subject: [PATCH 13/14] Remove :: --- .github/workflows/e2e.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 30c25cb914..7d3bd3628f 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -22,7 +22,8 @@ jobs: cache: runs-on: ubuntu-latest steps: - - run: echo "${{ toJSON(github) }}" | sed 's/::/:/g' + - run: | + echo "${{ toJSON(github) }}" | sed 's/::/:/g' - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 - name: Setup Node.js From 8bfb269b373611d3b4884177c47f1c9ed71289ce Mon Sep 17 00:00:00 2001 From: Sidharth Vinod Date: Fri, 19 Jan 2024 19:33:59 +0530 Subject: [PATCH 14/14] Remove :: --- .github/workflows/e2e.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/e2e.yml b/.github/workflows/e2e.yml index 7d3bd3628f..aab0e1facd 100644 --- a/.github/workflows/e2e.yml +++ b/.github/workflows/e2e.yml @@ -23,7 +23,7 @@ jobs: runs-on: ubuntu-latest steps: - run: | - echo "${{ toJSON(github) }}" | sed 's/::/:/g' + echo '${{ toJSON(github) }}' | sed 's/::/:/g' - uses: actions/checkout@v4 - uses: pnpm/action-setup@v2 - name: Setup Node.js