diff --git a/.github/workflows/end-to-end.yml b/.github/workflows/end-to-end.yml index 61ae9d72c3..a2534a64ad 100644 --- a/.github/workflows/end-to-end.yml +++ b/.github/workflows/end-to-end.yml @@ -69,7 +69,7 @@ jobs: name: End to end Tests needs: build-end-to-end-tests runs-on: macos-15 - timeout-minutes: 90 + timeout-minutes: 240 strategy: matrix: test-tag: [release, privacy, securityTest, adClick] @@ -87,13 +87,57 @@ jobs: name: duckduckgo-ios-app path: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app - - name: Install Maestro - run: | - export MAESTRO_VERSION=1.39.1; curl -Ls "https://get.maestro.mobile.dev" | bash - - name: End to End tests + id: upload + uses: mobile-dev-inc/action-maestro-cloud@v1.9.7 + with: + api-key: ${{ secrets.ROBIN_API_KEY }} + project-id: ${{ secrets.ROBIN_PROJECT_KEY }} + name: ${{ matrix.test-tag }}_${{ github.sha }} + timeout: 5 + app-file: DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app + workspace: .maestro + include-tags: ${{ matrix.test-tag }} + env: ONBOARDING_COMPLETED=true + ios-version: 17 + + - name: Process Robin Results + if: always() run: | - export PATH="$PATH":"$HOME/.maestro/bin"; maestro cloud --apiKey ${{ secrets.ROBIN_API_KEY }} --project-id ${{ secrets.ROBIN_PROJECT_KEY }} -e ONBOARDING_COMPLETED=true --fail-on-timeout=true --fail-on-cancellation=true --timeout=150 --ios-version=17 --include-tags=${{ matrix.test-tag }} DerivedData/Build/Products/Debug-iphonesimulator/DuckDuckGo.app .maestro/ + echo "Debug: MAESTRO_CLOUD_UPLOAD_STATUS = " + echo "${{ steps.upload.outputs.MAESTRO_CLOUD_UPLOAD_STATUS }}" + + echo "Debug: MAESTRO_CLOUD_FLOW_RESULTS = " + echo "${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}" + + # Parse the JSON string + RESULTS='${{ steps.upload.outputs.MAESTRO_CLOUD_FLOW_RESULTS }}' + + # Check if RESULTS is empty or null + if [ -z "$RESULTS" ]; then + echo "Error: No results found" + exit 1 + fi + + # Check if any flow was not successful + NON_SUCCESS_FLOWS=$(echo "$RESULTS" | jq -r '[.[] | select(.status != "SUCCESS")]') + NON_SUCCESS_COUNT=$(echo "$NON_SUCCESS_FLOWS" | jq -r 'length') + + # Check if NON_SUCCESS_COUNT is a number + if ! [[ "$NON_SUCCESS_COUNT" =~ ^[0-9]+$ ]]; then + echo "Error: Invalid non-success count" + exit 1 + fi + + if [ "$NON_SUCCESS_COUNT" -gt 0 ]; then + echo "Error: Some flows were not successful" + echo "Details of non-successful flows:" + echo "$NON_SUCCESS_FLOWS" | jq '.' + exit 1 + else + echo "All flows were successful" + fi + shell: bash # notify-failure: # name: Notify on failure