E2E Test Stability Review #702
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Test Stability Review | |
| on: | |
| schedule: | |
| - cron: '0 02 * * 1-5' # 10pm EST/2am UTC, weekdays | |
| env: | |
| BUILDTYPE: vagovprod | |
| jobs: | |
| build: | |
| name: Build | |
| runs-on: ubuntu-16-cores-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install dependencies | |
| uses: ./.github/workflows/install | |
| timeout-minutes: 30 | |
| with: | |
| key: ${{ hashFiles('yarn.lock') }} | |
| yarn_cache_folder: .cache/yarn | |
| path: | | |
| .cache/yarn | |
| node_modules | |
| - name: Configure AWS Credentials | |
| uses: ./.github/workflows/configure-aws-credentials | |
| with: | |
| aws_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws_region: us-gov-west-1 | |
| - name: Get Mapbox Token | |
| uses: ./.github/workflows/inject-secrets | |
| with: | |
| ssm_parameter: /dsva-vagov/vets-website/dev/mapbox_token | |
| env_variable_name: MAPBOX_TOKEN | |
| - name: Build | |
| run: yarn build --verbose --buildtype=${{ env.BUILDTYPE }} | |
| timeout-minutes: 30 | |
| - name: Generate build details | |
| run: | | |
| cat > build/${{ env.BUILDTYPE }}/BUILD.txt << EOF | |
| BUILDTYPE=${{ env.BUILDTYPE }} | |
| NODE_ENV=production | |
| BRANCH_NAME=$(echo "${GITHUB_REF#refs/heads/}") | |
| CHANGE_TARGET=null | |
| RUN_ID=${{ github.run_id }} | |
| RUN_NUMBER=${{ github.run_number }} | |
| REF=${{ github.sha }} | |
| BUILDTIME=$(date +%s) | |
| EOF | |
| - name: Compress and archive build | |
| run: tar -C build/${{ env.BUILDTYPE }} -cjf ${{ env.BUILDTYPE }}.tar.bz2 . | |
| - name: Upload build artifact | |
| uses: ./.github/workflows/upload-artifact | |
| with: | |
| name: ${{ env.BUILDTYPE }}.tar.bz2 | |
| path: ${{ env.BUILDTYPE }}.tar.bz2 | |
| retention-days: 1 | |
| testing-reports-prep: | |
| name: Testing Reports Prep | |
| runs-on: ubuntu-latest | |
| continue-on-error: true | |
| outputs: | |
| app_list: ${{ env.APPLICATION_LIST }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 | |
| - name: Install dependencies | |
| uses: ./.github/workflows/install | |
| timeout-minutes: 30 | |
| with: | |
| key: ${{ hashFiles('yarn.lock') }} | |
| yarn_cache_folder: .cache/yarn | |
| path: | | |
| .cache/yarn | |
| node_modules | |
| - name: Generate new application list | |
| run: yarn generate-app-list | |
| # exports app list and assigns to environmental variable at this step | |
| - name: Configure AWS credentials | |
| uses: ./.github/workflows/configure-aws-credentials | |
| with: | |
| aws_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws_region: us-gov-west-1 | |
| - name: Get va-vsp-bot token | |
| uses: ./.github/workflows/inject-secrets | |
| with: | |
| ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN | |
| env_variable_name: VA_VSP_BOT_GITHUB_TOKEN | |
| - name: Init Dashboard Data Repo | |
| uses: ./.github/workflows/init-data-repo | |
| - name: Set Up BigQuery Creds | |
| uses: ./.github/workflows/configure-bigquery | |
| - name: Upload app list to BigQuery | |
| run: yarn generate-app-list | |
| working-directory: qa-standards-dashboard-data | |
| fetch-e2e-allow-list: | |
| name: Fetch E2E Test Stability Allow List | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 | |
| - name: Configure AWS credentials | |
| uses: ./.github/workflows/configure-aws-credentials | |
| with: | |
| aws_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws_region: us-gov-west-1 | |
| - name: Get va-vsp-bot token | |
| uses: ./.github/workflows/inject-secrets | |
| with: | |
| ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN | |
| env_variable_name: VA_VSP_BOT_GITHUB_TOKEN | |
| - name: Init Dashboard Data Repo | |
| uses: ./.github/workflows/init-data-repo | |
| - name: Set Up BigQuery Creds | |
| uses: ./.github/workflows/configure-bigquery | |
| - name: Fetch E2E Test Stability Allow List | |
| run: yarn get-allow-list | |
| working-directory: qa-standards-dashboard-data | |
| env: | |
| TEST_TYPE: e2e | |
| - name: Archive E2E Test Stability Allow List | |
| if: ${{ always() }} | |
| uses: ./.github/workflows/upload-artifact | |
| with: | |
| name: e2e-allow-list | |
| path: qa-standards-dashboard-data/e2e_allow_list.json | |
| cypress-tests-prep: | |
| name: Prep for Cypress Tests | |
| runs-on: ubuntu-latest | |
| needs: [fetch-e2e-allow-list] | |
| outputs: | |
| cypress-tests-to-stress-test: ${{ steps.cypress-tests-to-stress-test.outputs.tests }} | |
| num_containers: ${{ steps.containers.outputs.num }} | |
| ci_node_index: ${{ steps.matrix.outputs.ci_node_index }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 | |
| with: | |
| fetch-depth: 0 | |
| - name: Install dependencies | |
| uses: ./.github/workflows/install | |
| timeout-minutes: 30 | |
| with: | |
| key: ${{ hashFiles('yarn.lock') }} | |
| yarn_cache_folder: .cache/yarn | |
| path: | | |
| .cache/yarn | |
| node_modules | |
| - name: Download E2E Test Stability Allow List | |
| uses: ./.github/workflows/download-artifact | |
| with: | |
| name: e2e-allow-list | |
| path: . | |
| - name: Set NUM_CONTAINERS, CI_NODE_INDEX, TESTS variables | |
| run: node script/github-actions/select-cypress-tests.js | |
| env: | |
| RUN_FULL_SUITE: true | |
| CHANGED_FILE_PATHS: ${{ steps.get-changed-apps.outputs.changed_files }} | |
| APP_URLS: ${{ steps.get-changed-apps.outputs.urls }} | |
| IS_STRESS_TEST: true | |
| TEST_TYPE: e2e | |
| - name: Set output of CYPRESS_TESTS_TO_STRESS_TEST | |
| id: cypress-tests-to-stress-test | |
| run: echo tests=$CYPRESS_TESTS_TO_STRESS_TEST >> $GITHUB_OUTPUT | |
| - name: Upload artifact of Cypress Tests to Stress Test | |
| if: ${{ steps.cypress-tests-to-stress-test.outputs.tests == 'true' }} | |
| uses: ./.github/workflows/upload-artifact | |
| with: | |
| name: e2e-tests-to-stress-test | |
| path: e2e_tests_to_stress_test.json | |
| - name: Set output of NUM_CONTAINERS | |
| id: containers | |
| run: echo num=$NUM_CONTAINERS >> $GITHUB_OUTPUT | |
| - name: Set output of CI_NODE_INDEX | |
| id: matrix | |
| run: echo ci_node_index=$CI_NODE_INDEX >> $GITHUB_OUTPUT | |
| stress-test-cypress-tests: | |
| name: E2E Test Stability Review | |
| runs-on: ubuntu-16-cores-latest | |
| timeout-minutes: 1200 | |
| needs: [build, cypress-tests-prep] | |
| if: | | |
| needs.build.result == 'success' && | |
| needs.cypress-tests-prep.result == 'success' | |
| container: | |
| image: public.ecr.aws/cypress-io/cypress/browsers:node16.13.2-chrome100-ff98 | |
| strategy: | |
| fail-fast: false | |
| max-parallel: 20 | |
| matrix: | |
| ci_node_index: | |
| [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19] | |
| env: | |
| CI_NODE_INDEX: ${{ needs.cypress-tests-prep.outputs.ci_node_index }} | |
| steps: | |
| # The following statement is included in each step because of a bug in | |
| # GitHub's branch protection: | |
| # | |
| # if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| # | |
| # Previously, if no tests were selected, the branch protection check that | |
| # requires the cypress-tests job to run was not satisfied. This update forces | |
| # the job to always run, and skips each step if no tests are selected. | |
| # Previously, the above conditional was included in the job's if statement. | |
| - name: Checkout | |
| uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 | |
| - name: Configure AWS credentials | |
| if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| uses: ./.github/workflows/configure-aws-credentials | |
| with: | |
| aws_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws_region: us-gov-west-1 | |
| - name: Checkout vets-website | |
| if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 | |
| - name: Download production build artifact | |
| if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| uses: ./.github/workflows/download-artifact | |
| with: | |
| name: vagovprod.tar.bz2 | |
| - name: Unpack build | |
| if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| run: | | |
| mkdir -p build/vagovprod | |
| tar -C build/vagovprod -xjf vagovprod.tar.bz2 | |
| - name: Install dependencies | |
| if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| uses: ./.github/workflows/install | |
| timeout-minutes: 20 | |
| with: | |
| key: ${{ hashFiles('yarn.lock') }} | |
| yarn_cache_folder: .cache/yarn | |
| path: | | |
| .cache/yarn | |
| /github/home/.cache/Cypress | |
| node_modules | |
| - name: Start server | |
| if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| run: node src/platform/testing/e2e/test-server.js --buildtype=vagovprod --port=3001 & | |
| - name: Download Tests to verify | |
| uses: ./.github/workflows/download-artifact | |
| with: | |
| name: e2e-tests-to-stress-test | |
| path: . | |
| - name: Run Cypress tests | |
| if: needs.cypress-tests-prep.outputs.tests != '[]' | |
| run: node script/github-actions/run-cypress-stress-tests.js | |
| timeout-minutes: 1200 | |
| env: | |
| CYPRESS_CI: true | |
| STEP: ${{ matrix.ci_node_index }} | |
| TESTS: ${{ needs.cypress-tests-prep.outputs.cypress-tests-to-stress-test }} | |
| APP_URLS: '' | |
| NUM_CONTAINERS: ${{ needs.cypress-tests-prep.outputs.num_containers }} | |
| IS_STRESS_TEST: true | |
| - name: Archive test videos | |
| if: ${{ needs.cypress-tests-prep.outputs.tests != '[]' && failure() }} | |
| uses: ./.github/workflows/upload-artifact | |
| with: | |
| name: cypress-stress-test-videos-${{ matrix.ci_node_index }} | |
| path: cypress/videos | |
| - name: Archive test screenshots | |
| if: ${{ needs.cypress-tests-prep.outputs.tests != '[]' && failure() }} | |
| uses: ./.github/workflows/upload-artifact | |
| with: | |
| name: cypress-screenshot-artifacts-${{ matrix.ci_node_index }} | |
| path: cypress/screenshots | |
| - name: Archive Mochawesome test results | |
| if: ${{ needs.cypress-tests-prep.outputs.tests != '[]' && always() }} | |
| uses: ./.github/workflows/upload-artifact | |
| with: | |
| name: cypress-stress-test-results-${{ matrix.ci_node_index }} | |
| path: cypress/results | |
| retention-days: 1 | |
| update-e2e-allow-list: | |
| name: Update E2E Test Stability Allow List | |
| runs-on: ubuntu-latest | |
| needs: | |
| [ | |
| testing-reports-prep, | |
| cypress-tests-prep, | |
| stress-test-cypress-tests, | |
| fetch-e2e-allow-list, | |
| ] | |
| if: ${{ always() && needs.cypress-tests-prep.outputs.tests != '[]' && (needs.stress-test-cypress-tests.result == 'success' || needs.stress-test-cypress-tests.result == 'failure') }} | |
| continue-on-error: true | |
| env: | |
| APPLICATION_LIST: ${{ needs.testing-reports-prep.outputs.app_list }} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@cd7d8d697e10461458bc61a30d094dc601a8b017 | |
| - name: Configure AWS credentials | |
| uses: ./.github/workflows/configure-aws-credentials | |
| with: | |
| aws_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws_region: us-gov-west-1 | |
| - name: Get va-vsp-bot token | |
| uses: ./.github/workflows/inject-secrets | |
| with: | |
| ssm_parameter: /devops/VA_VSP_BOT_GITHUB_TOKEN | |
| env_variable_name: VA_VSP_BOT_GITHUB_TOKEN | |
| - name: Init Dashboard Data Repo | |
| uses: ./.github/workflows/init-data-repo | |
| - name: Set Up BigQuery Creds | |
| uses: ./.github/workflows/configure-bigquery | |
| - name: Get AWS IAM role | |
| uses: ./.github/workflows/inject-secrets | |
| with: | |
| ssm_parameter: /frontend-team/github-actions/parameters/AWS_FRONTEND_NONPROD_ROLE | |
| env_variable_name: AWS_FRONTEND_NONPROD_ROLE | |
| - name: Set UUID for Mochawesome reports | |
| run: echo "UUID=$(uuidgen)" >> $GITHUB_ENV | |
| - name: Configure AWS Credentials (2) | |
| uses: ./.github/workflows/configure-aws-credentials | |
| with: | |
| aws_id: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| aws_key: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| aws_region: us-gov-west-1 | |
| role: ${{ env.AWS_FRONTEND_NONPROD_ROLE }} | |
| role_duration: 900 | |
| session_name: vsp-frontendteam-githubaction | |
| - name: Download Cypress E2E Mochawesome test results | |
| uses: ./.github/workflows/download-artifact | |
| with: | |
| pattern: cypress-stress-test-results-* | |
| path: qa-standards-dashboard-data/src/allow-list/data | |
| merge-multiple: true | |
| - name: Download E2E Test Stability Allow List | |
| uses: ./.github/workflows/download-artifact | |
| with: | |
| name: e2e-allow-list | |
| path: qa-standards-dashboard-data | |
| - name: Copy test results to mochawesome directory | |
| run: cp -R qa-standards-dashboard-data/src/allow-list/data qa-standards-dashboard-data/src/testing-reports/data | |
| - name: Download Cypress E2E video artifacts | |
| if: ${{ needs.stress-test-cypress-tests.result == 'failure' }} | |
| uses: ./.github/workflows/download-artifact | |
| with: | |
| pattern: cypress-stress-test-videos-* | |
| path: qa-standards-dashboard-data/videos/${{ env.UUID }} | |
| merge-multiple: true | |
| - name: Update E2E Test Stability Allow List | |
| run: yarn update-allow-list | |
| working-directory: qa-standards-dashboard-data | |
| env: | |
| TEST_TYPE: e2e | |
| IS_CI: false | |
| GITHUB_WORKFLOW_URL: ${{ github.server_url}}/${{ github.repository }}/actions/runs/${{ github.run_id }} | |
| - name: Create Cypress E2E report and post results to BigQuery | |
| run: yarn cypress-mochawesome-to-bigquery | |
| working-directory: qa-standards-dashboard-data | |
| env: | |
| IS_MASTER_BUILD: false | |
| TEST_EXECUTIONS_TABLE_NAME: cypress_test_suite_executions | |
| TEST_RESULTS_TABLE_NAME: cypress_test_results | |
| TEST_REPORTS_FOLDER_NAME: vets-website-cypress-stress-test-reports | |
| TEST_RETRIES_TABLE_NAME: cypress_retry_records | |
| NUM_CONTAINERS: ${{ needs.cypress-tests-prep.outputs.num_containers }} | |
| IS_STRESS_TEST: true | |
| - name: Upload Cypress E2E test videos to s3 | |
| if: ${{ needs.stress-test-cypress-tests.result == 'failure' }} | |
| run: aws s3 cp qa-standards-dashboard-data/videos/${{ env.UUID }} s3://testing-tools-testing-reports/vets-website-cypress-stress-test-reports/videos/${{ env.UUID }} --acl public-read --region us-gov-west-1 --recursive | |
| - name: Upload Cypress E2E test report to s3 | |
| run: aws s3 cp qa-standards-dashboard-data/mochawesome-report s3://testing-tools-testing-reports/vets-website-cypress-stress-test-reports --acl public-read --region us-gov-west-1 --recursive | |
| - name: Publish E2E Test Stability Review Results | |
| if: ${{ always() }} | |
| uses: ./.github/workflows/checks-action | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| name: E2E Test Stability Review Summary | |
| conclusion: ${{ needs.stress-test-cypress-tests.result }} | |
| output: | | |
| {"summary":${{ env.MOCHAWESOME_REPORT_RESULTS }}} |