E2E Performance Tests #61
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: E2E Performance Tests | |
| on: | |
| workflow_call: | |
| inputs: | |
| PR_NUMBER: | |
| description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests. | |
| type: string | |
| required: true | |
| workflow_dispatch: | |
| inputs: | |
| PR_NUMBER: | |
| description: A PR number to run performance tests against. If the PR is already merged, the merge commit will be used. If not, the PR will be merged locally before running the performance tests. | |
| type: string | |
| required: true | |
| jobs: | |
| buildBaseline: | |
| runs-on: ubuntu-latest-xl | |
| name: Build apk from latest release as a baseline | |
| outputs: | |
| VERSION: ${{ steps.getMostRecentRelease.outputs.VERSION }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Get most recent release version | |
| id: getMostRecentRelease | |
| run: echo "VERSION=$(gh release list --limit 1 | awk '{ print $1 }')" >> "$GITHUB_OUTPUT" | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: Check if there's an existing artifact for this baseline | |
| id: checkForExistingArtifact | |
| uses: xSAVIKx/artifact-exists-action@3c5206b1411c0d2fc0840f56b7140646933d9d6a | |
| with: | |
| name: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }} | |
| - name: Skip build if there's already an existing artifact for the baseline | |
| if: ${{ fromJSON(steps.checkForExistingArtifact.outputs.exists) }} | |
| run: echo 'APK for baseline ${{ steps.getMostRecentRelease.outputs.VERSION }} already exists, reusing existing build' | |
| - name: Checkout "Baseline" commit (last release) | |
| if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.exists) }} | |
| run: | | |
| git fetch origin tag ${{ steps.getMostRecentRelease.outputs.VERSION }} --no-tags --depth=1 | |
| git switch --detach ${{ steps.getMostRecentRelease.outputs.VERSION }} | |
| - name: Configure MapBox SDK | |
| run: ./scripts/setup-mapbox-sdk.sh ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }} | |
| - name: Build APK | |
| if: ${{ !fromJSON(steps.checkForExistingArtifact.outputs.exists) }} | |
| uses: Expensify/App/.github/actions/composite/buildAndroidAPK@main | |
| with: | |
| ARTIFACT_NAME: baseline-apk-${{ steps.getMostRecentRelease.outputs.VERSION }} | |
| buildDelta: | |
| runs-on: ubuntu-latest-xl | |
| name: Build apk from delta ref | |
| outputs: | |
| DELTA_REF: ${{ steps.getDeltaRef.outputs.DELTA_REF }} | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Get pull request details | |
| id: getPullRequestDetails | |
| uses: Expensify/App/.github/actions/javascript/getPullRequestDetails@main | |
| with: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| PULL_REQUEST_NUMBER: ${{ inputs.PR_NUMBER }} | |
| USER: ${{ github.actor }} | |
| - name: Merged PR - Get merge commit sha for the pull request | |
| if: ${{ fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} | |
| id: getMergeCommitShaIfMergedPR | |
| run: | | |
| MERGE_COMMIT_SHA=${{ steps.getPullRequestDetails.outputs.MERGE_COMMIT_SHA }} | |
| git fetch origin "$MERGE_COMMIT_SHA" --no-tags --depth=1 | |
| echo "MERGE_COMMIT_SHA=$MERGE_COMMIT_SHA" >> "$GITHUB_OUTPUT" | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: Unmerged PR - Fetch head ref of unmerged PR | |
| if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} | |
| run: | | |
| git fetch origin ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} --no-tags --depth=1 | |
| - name: Unmerged PR - Set dummy git credentials before merging | |
| if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} | |
| run: | | |
| git config --global user.email "test@test.com" | |
| git config --global user.name "Test" | |
| - name: Unmerged PR - Merge pull request locally and get merge commit sha | |
| if: ${{ !fromJSON(steps.getPullRequestDetails.outputs.IS_MERGED) }} | |
| id: getMergeCommitShaIfUnmergedPR | |
| run: | | |
| git merge --allow-unrelated-histories --no-commit ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} | |
| git checkout ${{ steps.getPullRequestDetails.outputs.HEAD_COMMIT_SHA }} | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: Determine "delta ref" | |
| id: getDeltaRef | |
| run: echo "DELTA_REF=${{ steps.getMergeCommitShaIfMergedPR.outputs.MERGE_COMMIT_SHA || steps.getMergeCommitShaIfUnmergedPR.outputs.MERGE_COMMIT_SHA }}" >> "$GITHUB_OUTPUT" | |
| env: | |
| GITHUB_TOKEN: ${{ github.token }} | |
| - name: Checkout "delta ref" | |
| run: git checkout ${{ steps.getDeltaRef.outputs.DELTA_REF }} | |
| - name: Configure MapBox SDK | |
| run: ./scripts/setup-mapbox-sdk.sh ${{ secrets.MAPBOX_SDK_DOWNLOAD_TOKEN }} | |
| - name: Build APK | |
| uses: Expensify/App/.github/actions/composite/buildAndroidAPK@main | |
| with: | |
| ARTIFACT_NAME: delta-apk-${{ steps.getDeltaRef.outputs.DELTA_REF }} | |
| runTestsInAWS: | |
| runs-on: ubuntu-latest | |
| needs: [buildBaseline, buildDelta] | |
| name: Run E2E tests in AWS device farm | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Setup Node | |
| uses: Expensify/App/.github/actions/composite/setupNode@main | |
| - name: Make zip directory for everything to send to AWS Device Farm | |
| run: mkdir zip | |
| - name: Download baseline APK | |
| uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b | |
| id: downloadBaselineAPK | |
| with: | |
| name: baseline-apk-${{ needs.buildBaseline.outputs.VERSION }} | |
| path: zip | |
| # The downloaded artifact will be a file named "app-e2e-release.apk" so we have to rename it | |
| - name: Rename baseline APK | |
| run: mv "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadBaselineAPK.outputs.download-path}}/app-e2eRelease-main.apk" | |
| - name: Download delta APK | |
| uses: actions/download-artifact@e9ef242655d12993efdcda9058dee2db83a2cb9b | |
| id: downloadDeltaAPK | |
| with: | |
| name: delta-apk-${{ needs.buildDelta.outputs.DELTA_REF }} | |
| path: zip | |
| - name: Rename delta APK | |
| run: mv "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2e-release.apk" "${{steps.downloadDeltaAPK.outputs.download-path}}/app-e2eRelease-delta.apk" | |
| - name: Copy e2e code into zip folder | |
| run: cp -r tests/e2e zip | |
| - name: Zip everything in the zip directory up | |
| run: zip -qr App.zip ./zip | |
| - name: Configure AWS Credentials | |
| uses: Expensify/App/.github/actions/composite/configureAwsCredentials@main | |
| with: | |
| AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} | |
| AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} | |
| AWS_REGION: us-west-2 | |
| - name: Schedule AWS Device Farm test run on main branch | |
| uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b | |
| id: schedule-awsdf-main | |
| with: | |
| name: App E2E Performance Regression Tests | |
| project_arn: ${{ secrets.AWS_PROJECT_ARN }} | |
| device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }} | |
| app_file: zip/app-e2eRelease-main.apk | |
| app_type: ANDROID_APP | |
| test_type: APPIUM_NODE | |
| test_package_file: App.zip | |
| test_package_type: APPIUM_NODE_TEST_PACKAGE | |
| test_spec_file: tests/e2e/TestSpecMain.yml | |
| test_spec_type: APPIUM_NODE_TEST_SPEC | |
| remote_src: false | |
| file_artifacts: Customer Artifacts.zip | |
| log_artifacts: debug.log | |
| cleanup: true | |
| - name: Print logs if run failed | |
| if: failure() | |
| run: | | |
| echo ${{ steps.schedule-awsdf-main.outputs.data }} | |
| unzip "Customer Artifacts.zip" -d mainResults | |
| cat ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/debug.log | |
| - name: Unzip AWS Device Farm main results | |
| run: unzip "Customer Artifacts.zip" -d mainResults | |
| - name: Delete Customer Artifacts.zip | |
| run: rm "Customer Artifacts.zip" | |
| - name: Schedule AWS Device Farm test run on delta branch | |
| uses: realm/aws-devicefarm/test-application@7b9a91236c456c97e28d384c9e476035d5ea686b | |
| with: | |
| name: App E2E Performance Regression Tests | |
| project_arn: ${{ secrets.AWS_PROJECT_ARN }} | |
| device_pool_arn: ${{ secrets.AWS_DEVICE_POOL_ARN }} | |
| app_file: zip/app-e2eRelease-delta.apk | |
| app_type: ANDROID_APP | |
| test_type: APPIUM_NODE | |
| test_package_file: App.zip | |
| test_package_type: APPIUM_NODE_TEST_PACKAGE | |
| test_spec_file: tests/e2e/TestSpecDelta.yml | |
| test_spec_type: APPIUM_NODE_TEST_SPEC | |
| remote_src: false | |
| file_artifacts: Customer Artifacts.zip | |
| cleanup: true | |
| - name: Unzip AWS Device Farm delta results | |
| run: unzip "Customer Artifacts.zip" -d deltaResults | |
| - name: Compare results | |
| run: node tests/e2e/merge.js --mainPath ./mainResults/Host_Machine_Files/\$WORKING_DIRECTORY/main.json --deltaPath ./deltaResults//Host_Machine_Files/\$WORKING_DIRECTORY/delta.json --outputPath ./output.md | |
| - name: Print results | |
| run: cat "./output.md" | |
| # - name: Print AWS Device Farm verbose run results | |
| # if: ${{ always() && runner.debug != null && fromJSON(runner.debug) }} | |
| # run: cat "./Host_Machine_Files/debug.log" | |
| # TODO: Once tests are more reliable we should uncomment this | |
| # - name: Check if test failed, if so post the results and add the DeployBlocker label | |
| # run: | | |
| # if grep -q '🔴' ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md; then | |
| # gh pr edit ${{ inputs.PR_NUMBER }} --add-label DeployBlockerCash | |
| # gh pr comment ${{ inputs.PR_NUMBER }} -F ./Host_Machine_Files/\$WORKING_DIRECTORY/output.md | |
| # gh pr comment ${{ inputs.PR_NUMBER }} -b "@Expensify/mobile-deployers 📣 Please look into this performance regression as it's a deploy blocker." | |
| # else | |
| # echo '✅ no performance regression detected' | |
| # fi | |
| # env: | |
| # GITHUB_TOKEN: ${{ github.token }} |