[e2e] iOS Detox Test Run #4201
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # | |
| # Detox e2e tests in CI | |
| # | |
| name: '[e2e] iOS Detox Test Run' | |
| on: | |
| push: | |
| branches: | |
| - 'dependabot/**' | |
| pull_request: | |
| types: | |
| - synchronize | |
| - opened | |
| paths: | |
| - 'VAMobile/src/**' | |
| - 'VAMobile/package.json' | |
| workflow_dispatch: | |
| inputs: | |
| run_full_test: | |
| description: 'Run full e2e test?' | |
| type: boolean | |
| required: false | |
| tests_to_run: | |
| description: 'List tests to test in ["test name", "test name"] format. Leave blank if running full e2e test.' | |
| type: string | |
| required: false | |
| testRail_name: | |
| description: 'Name testRail run (will default to date if blank)' | |
| type: string | |
| required: false | |
| run_testRail: | |
| description: 'Add results to testRail?' | |
| type: boolean | |
| required: false | |
| schedule: | |
| - cron: '0 4 * * 1,2,3,4,5' | |
| workflow_run: | |
| workflows: ['[Release] New Release Issue'] | |
| types: | |
| - in_progress | |
| workflow_call: | |
| secrets: | |
| TEST_RAIL_USER: | |
| description: "TestRail robot userid" | |
| required: true | |
| TEST_RAIL_KEY: | |
| description: "TestRail api key" | |
| required: true | |
| concurrency: | |
| group: ios-detox-${{ github.ref }} | |
| cancel-in-progress: true | |
| defaults: | |
| run: | |
| working-directory: VAMobile | |
| env: | |
| # IAM staging app client secret | |
| APP_CLIENT_SECRET: ${{ secrets.APP_CLIENT_SECRET }} | |
| # IAM production app client secret | |
| APP_CLIENT_SECRET_PROD: ${{ secrets.APP_CLIENT_SECRET_PROD }} | |
| # Slack API token | |
| SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }} | |
| jobs: | |
| start_slack_thread: | |
| if: github.event_name == 'schedule' | |
| name: Start Slack thread | |
| uses: ./.github/workflows/start_slack_thread.yml | |
| secrets: inherit | |
| with: | |
| channel_name: va-mobile-build-alerts | |
| message: 'Starting E2E iOS tests. Please see :thread: for results. This process may take a while.' | |
| find_detox_tests_to_run: | |
| uses: ./.github/workflows/e2e_detox_mapping.yml | |
| output_detox_tests_to_run: | |
| runs-on: macos-latest-xl | |
| needs: find_detox_tests_to_run | |
| outputs: | |
| output1: ${{ steps.matrix_value.outputs.matrix }} | |
| output2: ${{ steps.matrix_value.outputs.individual_matrix}} | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: 'Get Matrix Value' | |
| id: matrix_value | |
| run: | | |
| if [[ "${{ inputs.run_full_test }}" == "true" ]] || [[ ${{ github.event_name }} == 'schedule' ]] || [[ "${{ inputs.tests_to_run}}" != "" ]] || [[ "${{ github.event.pull_request.user.login }}" == "dependabot[bot]" ]]; then | |
| if [[ "${{ inputs.run_full_test }}" == "true" ]] || [[ ${{ github.event_name }} == 'schedule' ]] || [[ "${{ github.event.pull_request.user.login }}" == "dependabot[bot]" ]]; then | |
| e2eNames=$(gh api repos/department-of-veterans-affairs/va-mobile-app/contents/VAMobile/e2e/tests | jq --compact-output 'del(.[] | select(.name == "utils.ts")) | [.[].name]') | |
| echo "matrix=$e2eNames" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=" >> "$GITHUB_OUTPUT" | |
| else | |
| if [[ "${{ inputs.tests_to_run}}" != "" ]]; then | |
| echo "matrix=${{ toJson(inputs.tests_to_run)}}" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "matrix=" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| else | |
| if [[ "${{ needs.find_detox_tests_to_run.outputs.test_run }}" != "" ]]; then | |
| echo "${{needs.find_detox_tests_to_run.outputs.test_matrix}}" | |
| if [[ "${{needs.find_detox_tests_to_run.outputs.test_matrix}}" == "[]" ]]; then | |
| if [[ "${{ inputs.run_full_test }}" == "true" ]] || [[ ${{ github.event_name }} == 'schedule' ]] || [[ "${{ github.event.pull_request.user.login }}" == "dependabot[bot]" ]]; then | |
| e2eNames=$(gh api repos/department-of-veterans-affairs/va-mobile-app/contents/VAMobile/e2e/tests | jq --compact-output 'del(.[] | select(.name == "utils.ts")) | [.[].name]') | |
| echo "matrix=$e2eNames" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=" >> "$GITHUB_OUTPUT" | |
| else | |
| if [[ "${{ inputs.tests_to_run}}" != "" ]]; then | |
| echo "matrix=${{ toJson(inputs.tests_to_run)}}" >> "$GITHUB_OUTPUT" | |
| echo "individual_matrix=true" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "matrix=" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| else | |
| resp=${{toJson(needs.find_detox_tests_to_run.outputs.test_matrix)}} | |
| echo "matrix=$resp" >> "$GITHUB_OUTPUT" | |
| fi | |
| fi | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| matrix-e2e-ios: | |
| if: (!cancelled()) && needs.output_detox_tests_to_run.outputs.output1 != '' | |
| runs-on: macos-14-xlarge | |
| needs: [start_slack_thread, output_detox_tests_to_run] | |
| env: | |
| # Path to write secret key to. Also used by fastlane | |
| APPSTORE_CONNECT_FILEPATH: './AppStoreConnectKey.p8' | |
| # Xcode project file name | |
| IOS_PROJ_FILE: 'VAMobile.xcodeproj' | |
| # Xcode scheme to build | |
| IOS_SCHEME: 'VAMobileRelease' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| testsuite: ${{fromJson(needs.output_detox_tests_to_run.outputs.output1)}} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Decode base64 encoded App Store Connect API key and GoogleService-Info.plist | |
| working-directory: VAMobile/ios | |
| run: | | |
| echo "${{ secrets.APPSTORE_CONNECT_BASE64 }}" | base64 --decode > "$APPSTORE_CONNECT_FILEPATH" | |
| echo "${{ secrets.IOS_GS_PLIST_BASE64 }}" | base64 --decode > GoogleService-Info.plist | |
| - name: Setup Xcode | |
| uses: maxim-lobanov/setup-xcode@v1 | |
| with: | |
| xcode-version: latest-stable | |
| - name: Setup node and restore yarn cache | |
| uses: actions/setup-node@v3 | |
| with: | |
| node-version-file: 'VAMobile/.nvmrc' | |
| cache: 'yarn' | |
| cache-dependency-path: './VAMobile/yarn.lock' | |
| - name: Install macOS dependencies | |
| run: | | |
| brew tap wix/brew | |
| brew install applesimutils | |
| env: | |
| HOMEBREW_NO_AUTO_UPDATE: 1 | |
| HOMEBREW_NO_INSTALL_CLEANUP: 1 | |
| - name: Install npm dependencies | |
| run: yarn install --frozen-lockfile --non-interactive | |
| - name: Set app environment variables | |
| run: yarn env:staging | |
| - name: Install pods | |
| working-directory: VAMobile/ios | |
| run: pod install | |
| - name: Install Ruby gems | |
| working-directory: VAMobile/ios | |
| run: bundle install | |
| - name: Bundle iOS app | |
| run: yarn e2e:ios-build | |
| - name: Run e2e tests for iOS | |
| id: run_e2e_tests | |
| run: | | |
| if [[ "${{needs.output_detox_tests_to_run.outputs.output2}}" == "" ]]; then | |
| yarn e2e:ios-test /e2e/tests/${{matrix.testsuite}} --updateSnapshot --debug-synchronization 15000 | |
| echo "NAV_AF_TEST_RUN=" >> "$GITHUB_OUTPUT" | |
| else | |
| echo "NAV_AF_TEST_RUN=true" >> "$GITHUB_OUTPUT" | |
| yarn e2e:ios-test /e2e/tests/Navigation.e2e AvailabilityFramework.e2e ${{matrix.testsuite}}.e2e --updateSnapshot --debug-synchronization 15000 | |
| fi | |
| continue-on-error: true | |
| - name: Rerun e2e tests for iOS if failed | |
| if: steps.run_e2e_tests.outcome == 'failure' | |
| id: rerun_e2e_tests | |
| run: | | |
| yarn jest:clear; | |
| if [[ "${{ steps.run_e2e_tests.outputs.NAV_AF_TEST_RUN}}" != "" ]]; then | |
| yarn e2e:ios-test /e2e/tests/Navigation.e2e AvailabilityFramework.e2e ${{matrix.testsuite}}.e2e --updateSnapshot --debug-synchronization 15000 | |
| else | |
| yarn e2e:ios-test /e2e/tests/${{matrix.testsuite}} --updateSnapshot --debug-synchronization 15000 | |
| fi | |
| - name: Upload e2e-junit | |
| if: failure() || success() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: e2e-junit-${{matrix.testsuite}} | |
| path: VAMobile/e2e/test_reports/e2e-junit.xml | |
| - name: Upload artifacts on failure | |
| if: failure() || steps.run_e2e_tests.outcome == 'failure' | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: detox-artifacts-${{ runner.os }}-${{ github.run_id }}-${{matrix.testsuite}} | |
| path: VAMobile/artifacts | |
| retention-days: 1 | |
| - name: Fail workflow if needed(View e2e step for details) | |
| if: steps.run_e2e_tests.outcome == 'failure' && steps.rerun_e2e_tests.outcome == 'failure' | |
| run: exit 1 | |
| output-slack-results-and-update-detox-failure-ticket: | |
| if: (!cancelled()) && github.event_name == 'schedule' | |
| needs: [matrix-e2e-ios, start_slack_thread] | |
| runs-on: macos-latest-xl | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| ref: ${{ github.ref }} | |
| - name: Download results | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: VAMobile/e2e/test_reports | |
| - name: Inform Slack | |
| id: inform_slack | |
| shell: bash | |
| run: | | |
| if grep -rq "<failure>" .; then | |
| echo SLACK_THREAD_TS=${ts} >> $GITHUB_OUTPUT | |
| ts=$(curl -X POST -H 'Authorization: Bearer '"$SLACK_API_TOKEN"' ' \ | |
| -H 'Content-type: application/json' \ | |
| --data '{"channel":"'${{needs.start_slack_thread.outputs.channel_id}}'","thread_ts":"'${{needs.start_slack_thread.outputs.thread_ts}}'","text":"Failed! Please check workflow for results: https://github.com/department-of-veterans-affairs/va-mobile-app/actions"}' \ | |
| https://slack.com/api/chat.postMessage| | |
| jq -r '.ts') | |
| echo SLACK_THREAD_TS=${ts} >> $GITHUB_OUTPUT | |
| echo "DATE_OF_ISSUE=$(date -d '+%A %b %d, %Y')" >> $GITHUB_ENV | |
| echo "TEST_FAILURE=true" >> $GITHUB_OUTPUT | |
| else | |
| ts=$(curl -X POST -H 'Authorization: Bearer '"$SLACK_API_TOKEN"' ' \ | |
| -H 'Content-type: application/json' \ | |
| --data '{"channel":"'${{needs.start_slack_thread.outputs.channel_id}}'","thread_ts":"'${{needs.start_slack_thread.outputs.thread_ts}}'","text":"Success! All E2E tests have passed"}' \ | |
| https://slack.com/api/chat.postMessage| | |
| jq -r '.ts') | |
| fi | |
| - name: Find failing tests | |
| id: failing_tests | |
| shell: bash | |
| run: | | |
| if grep -rq "<failure>" .; then | |
| failing_tests_list=$(grep -rl "<failure>") | |
| failing_tests="" | |
| for i in ${failing_tests_list[@]}; do | |
| remove_before=${i%.ts*} | |
| remove_after=${remove_before#*test_reports/} | |
| echo "$remove_after" | |
| if [ -z "${failing_tests}" ]; then | |
| failing_tests=$(echo $failing_tests"$remove_after") | |
| else | |
| failing_tests=$(echo $failing_tests", ""$remove_after") | |
| fi | |
| done | |
| echo "FAILING_TEST=$failing_tests" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Github CLI Authentication | |
| run: | | |
| echo "${{ secrets.GITHUB_TOKEN }}" >> token.txt | |
| gh auth login --with-token < token.txt | |
| - name: Update a ticket on failure (if needed) | |
| if: steps.inform_slack.outputs.TEST_FAILURE != '' | |
| id: find_if_ticket_already_exists | |
| run: | | |
| old_body=$(gh issue list --json body --jq '.[] | .body' --state open --search "Bug - Detox - Fix Overnight Failures in:title") | |
| if [[ "$old_body" != "" ]]; then | |
| ticket_number=$(gh issue list --json number --jq '.[] | .number' --state open --search "Bug - Detox - Fix Overnight Failures in:title") | |
| new_body=$(echo "$old_body"$'\n'"- "${{ env.DATE_OF_ISSUE}}" on iOS: "${{steps.failing_tests.outputs.FAILING_TEST}}) | |
| echo "$new_body" | |
| echo "TICKET_EXISTS=true" >> "$GITHUB_OUTPUT" | |
| gh issue edit $ticket_number -b "$new_body" | |
| else | |
| echo "TICKET_EXISTS=" >> "$GITHUB_OUTPUT" | |
| fi | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Make a ticket on failure (if needed) | |
| if: steps.inform_slack.outputs.TEST_FAILURE != '' && steps.find_if_ticket_already_exists.outputs.TICKET_EXISTS == '' | |
| uses: JasonEtco/create-an-issue@v2 | |
| with: | |
| filename: .github/ISSUE_TEMPLATE/detox_nightly_build_failure.md | |
| search_existing: open | |
| env: | |
| dateOfIssue: ${{ env.DATE_OF_ISSUE }} | |
| OS: "iOS" | |
| issues: ${{steps.failing_tests.outputs.FAILING_TEST}} | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| id: create-issue | |
| matrix_send_test_results_to_testrail: | |
| if: (!cancelled()) && github.event.inputs.run_testRail == 'true' | |
| needs: [matrix-e2e-ios, output_detox_tests_to_run] | |
| name: Update testRail Results | |
| uses: ./.github/workflows/update_testrail_results.yml | |
| with: | |
| testRail_name: ${{ inputs.testRail_name }} | |
| test_OS_name: "iOS" | |
| secrets: inherit | |
| update-test-names: | |
| if: (!cancelled()) && github.event.inputs.run_testRail == 'true' | |
| needs: [matrix-e2e-ios, output_detox_tests_to_run, matrix_send_test_results_to_testrail] | |
| runs-on: ubuntu-latest | |
| continue-on-error: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v3 | |
| - name: update-test-names | |
| run: | | |
| for ((i=0; ; i+=250)); do | |
| echo "i = $i" | |
| getTestCaseNewName=$(curl -X GET -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/get_tests/${{ needs.matrix_send_test_results_to_testrail.outputs.test_run_id_number }}&offset=$i") | |
| echo $(jq --compact-output '[.tests[] | .title |= sub("(]).*$"; "]")]' <<< "$getTestCaseNewName") >> input.json | |
| if [[ $(jq '._links.next == null' <<< "$getTestCaseNewName") == 'true' ]]; | |
| then | |
| break | |
| fi | |
| done | |
| getTestCaseNewName=$(jq --compact-output -s 'add' input.json) | |
| groupTestCasesByTitle=$(echo $getTestCaseNewName | jq --compact-output '[group_by(.title)[] | [.[0].title, .[0].status_id, map(.id)[]]]') | |
| getFirstTestNewName=$(echo $groupTestCasesByTitle | jq --compact-output '[.[] | .[0]]') | |
| getCasesID=$(curl -X GET -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/get_cases/29") | |
| updateRunWithNewCases=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"suite_id": 92, "include_all": false, "case_ids": '$(echo $getCasesID | jq --argjson testRailNames "$getFirstTestNewName" --compact-output '.cases | map(select(.title == $testRailNames[])) | map(.id)')'}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/update_run/${{needs.matrix_send_test_results_to_testrail.outputs.test_run_id_number}}") | |
| getTestCases=$(curl -X GET -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/get_tests/${{needs.matrix_send_test_results_to_testrail.outputs.test_run_id_number}}") | |
| for i in $(echo $getTestCases | jq -r '.tests[].id'); do | |
| getTestName=$(echo $getTestCases | jq --argjson testIDs "$i" -r '.tests | map(select(.id == $testIDs)) | .[0].title') | |
| getFailedTests=$(echo $getTestCaseNewName | jq --compact-output 'map(select(.status_id == 5)) | map(.title)') | |
| isTestFailed=$(echo $getFailedTests | jq 'any(. == "'"$getTestName"'")') | |
| getRetestTests=$(echo $getTestCaseNewName | jq --compact-output 'map(select(.status_id == 4)) | map(.title)') | |
| isTestRetest=$(echo $getRetestTests | jq 'any(. == "'"$getTestName"'")') | |
| getPassedTests=$(echo $getTestCaseNewName | jq --compact-output 'map(select(.status_id == 1)) | map(.title)') | |
| isTestPassed=$(echo $getPassedTests | jq 'any(. == "'"$getTestName"'")') | |
| if [[ "$isTestFailed" == "true" ]]; then | |
| resp5=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"status_id": 5}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/add_result/"$i"") | |
| elif [[ "$isTestRetest" == "true" ]]; then | |
| resp5=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"status_id": 4}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/add_result/"$i"") | |
| elif [[ "$isTestPassed" == "true" ]]; then | |
| resp5=$(curl -X POST -H 'Content-Type: application/json' \ | |
| -u "${{secrets.TEST_RAIL_USER}}:${{secrets.TEST_RAIL_KEY}}" \ | |
| -d '{"status_id": 1}' \ | |
| "https://dsvavsp.testrail.io//index.php?/api/v2/add_result/"$i"") | |
| fi | |
| done |