test(ci): enable emulator/simulator tests in CI, but accept errors

This gets us started with automated app testing in CI, logs should show 1 test pass, 1 test fail
invertase · Apr 27, 2022 · 80a5692 · 80a5692
1 parent 2db1b92
commit 80a5692
Show file tree

Hide file tree

Showing 2 changed files with 102 additions and 79 deletions.
diff --git a/.github/workflows/tests_e2e_android.yml b/.github/workflows/tests_e2e_android.yml
@@ -32,6 +32,17 @@ jobs:
       CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
       EMULATOR_COMMAND: "-avd TestingAVD -noaudio -gpu swiftshader_indirect -camera-back none -no-snapshot -no-window -no-boot-anim -nojni -memory 2048 -timezone 'Europe/London' -cores 2"
       EMULATOR_EXECUTABLE: qemu-system-x86_64-headless
+    strategy:
+      fail-fast: false
+      matrix:
+        # Refactor to make these dynamic with a low/high bracket only on schedule, not push
+        # For now this is just the fastest combo (api/arch/target/snapshot-warm-time) based on testing
+        api-level: [29]
+        arch: [x86_64]
+        target: [google_apis]
+        first-boot-delay: [600]
+        # This is useful for benchmarking, do 0, 1, 2, etc (up to 256 max job-per-matrix limit) for averages
+        iteration: [0]
     steps:
       - uses: actions/checkout@v2
         with:
@@ -72,21 +83,77 @@ jobs:
           max_attempts: 3
           command: yarn --no-audit --prefer-offline && cd RNGoogleMobileAdsExample && DETOX_DISABLE_POSTINSTALL=1 yarn --no-audit --prefer-offline
 
-      - uses: actions/cache@v2
-        name: Gradle Cache
+      - name: Setup Gradle
+        uses: gradle/gradle-build-action@v2
         with:
-          path: ~/.gradle/caches
-          key: ${{ runner.os }}-gradle-v1-${{ hashFiles('**/*.gradle*') }}
-          restore-keys: ${{ runner.os }}-gradle-v1
+          # Only write to the cache for builds on the 'main' branches, stops branches evicting main cache
+          # Builds on other branches will only read from main branch cache writes
+          # Comment this and the with: above out for performance testing on a branch
+          cache-read-only: ${{ github.ref != 'refs/heads/main' }}
 
-      - name: Build Android App
+      - name: Warm Gradle Cache
+        # This makes sure we fetch gradle network resources with a retry
         uses: nick-invision/retry@v2
         with:
           timeout_minutes: 15
           retry_wait_seconds: 60
           max_attempts: 3
           command: yarn tests:android:build
 
+        # This appears to be 'Cache Size: ~1230 MB (1290026823 B)' based on watching action logs
+        # Repo limit is 10GB; branch caches are independent; branches may read default branch cache.
+        # We don't want branches to evict main branch snapshot, so save on main, read-only all else
+      - name: AVD cache
+        uses: actions/cache@v3
+        id: avd-cache
+        with:
+          path: |
+            ~/.android/avd/*
+            ~/.android/adb*
+          key: avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1-${{ hashFiles('~/.android/avd/**/snapshots/**') }}
+          restore-keys: |
+            avd-${{ matrix.api-level }}-${{ matrix.arch }}-${{matrix.target}}-v1
+
+      - name: AVD Boot and Snapshot Creation
+        # Only generate a snapshot for saving if we are on main branch with a cache miss
+        # Comment the if out to generate snapshots on branch for performance testing
+        if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: ${{ matrix.api-level }}
+          avd-name: TestingAVD
+          force-avd-creation: false
+          target: ${{ matrix.target }}
+          arch: ${{ matrix.arch }}
+          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          sdcard-path-or-size: 100M
+          disable-animations: true
+          # Give the emulator a little time to run and do first boot stuff before taking snapshot
+          script: echo "Generated AVD snapshot for caching."
+
+        # This step is separate so pure install time may be calculated as a step
+      - name: Emulator Snapshot After Firstboot Warmup
+        # Only generate a snapshot for saving if we are on main branch with a cache miss
+        # Switch the if statements via comment if generating snapshots for performance testing
+        # if: matrix.first-boot-delay != '0'
+        if: steps.avd-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main'
+        env:
+          FIRST_BOOT_DELAY: ${{ matrix.first-boot-delay }}
+        uses: reactivecircus/android-emulator-runner@v2
+        with:
+          api-level: ${{ matrix.api-level }}
+          avd-name: TestingAVD
+          force-avd-creation: false
+          target: ${{ matrix.target }}
+          arch: ${{ matrix.arch }}
+          emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          sdcard-path-or-size: 100M
+          disable-animations: true
+          # Give the emulator a little time to run and do first boot stuff before taking snapshot
+          script: |
+            sleep $FIRST_BOOT_DELAY
+            echo "First boot warmup completed."
+
       - name: Metro Bundler Cache
         uses: actions/cache@v2
         with:
@@ -107,75 +174,22 @@ jobs:
           curl --output /dev/null --silent --head --fail "http://localhost:8081/index.bundle?platform=android&dev=true&minify=false&inlineSourceMap=true"
           echo "...javascript bundle ready."
 
-      # - name: Download Emulator Image
-      #   # This can fail on network request, wrap with retry
-      #   uses: nick-invision/retry@v2
-      #   with:
-      #     timeout_minutes: 10
-      #     retry_wait_seconds: 60
-      #     max_attempts: 3
-      #     command: echo "y" | $ANDROID_HOME/cmdline-tools/latest/bin/sdkmanager --install "system-images;android-30;google_apis;x86_64"
-
-      # - name: Create Emulator
-      #   run: echo "no" | $ANDROID_HOME/cmdline-tools/latest/bin/avdmanager create avd --force --name TestingAVD --device "Nexus 5X" -k 'system-images;android-30;google_apis;x86_64' -g google_apis
-
-      # # These Emulator start steps are the current best practice to do retries on multi-line commands with persistent (nohup) processes
-      # - name: Start Android Emulator
-      #   id: emu1
-      #   timeout-minutes: 5
-      #   continue-on-error: true
-      #   run: |
-      #     echo "Starting emulator"
-      #     nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
-      #     $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
-
-      # - name: Start Android Emulator Retry 1
-      #   id: emu2
-      #   if: steps.emu1.outcome=='failure'
-      #   timeout-minutes: 5
-      #   continue-on-error: true
-      #   run: |
-      #     echo "Starting emulator, second attempt"
-      #     $ANDROID_HOME/platform-tools/adb devices
-      #     sudo killall -9 $EMULATOR_EXECUTABLE || true
-      #     sleep 2
-      #     nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
-      #     $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
-
-      # - name: Start Android Emulator Retry 2
-      #   id: emu3
-      #   if: steps.emu2.outcome=='failure'
-      #   timeout-minutes: 5
-      #   continue-on-error: true
-      #   run: |
-      #     echo "Starting emulator, third attempt"
-      #     $ANDROID_HOME/platform-tools/adb devices
-      #     sudo killall -9 $EMULATOR_EXECUTABLE || true
-      #     sleep 2
-      #     nohup $ANDROID_HOME/emulator/emulator $EMULATOR_COMMAND &
-      #     $ANDROID_HOME/platform-tools/adb wait-for-device shell 'while [[ -z $(getprop sys.boot_completed | tr -d '\r') ]]; do sleep 1; done'
-
-      # - name: Emulator Status
-      #   if: always()
-      #   run: |
-      #     if ${{ steps.emu1.outcome=='success' || steps.emu2.outcome=='success' || steps.emu3.outcome=='success' }}; then
-      #        echo "Emulator Started"
-      #     else
-      #        exit 1
-      #     fi
-
-      # - name: Detox Test
-      #   # Detox uses Espresso to choreograph steps in reaction to UI events, so we need to send a stream of taps.
-      #   timeout-minutes: 40
-      #   run: |
-      #     $ANDROID_HOME/platform-tools/adb devices
-      #     $ANDROID_HOME/platform-tools/adb shell settings put global window_animation_scale 0.0
-      #     $ANDROID_HOME/platform-tools/adb shell settings put global transition_animation_scale 0.0
-      #     $ANDROID_HOME/platform-tools/adb shell settings put global animator_duration_scale 0.0
-      #     nohup sh -c "until false; do $ANDROID_HOME/platform-tools/adb shell input tap 100 800; sleep 0.2; done" &
-      #     nohup sh -c "$ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt" &
-      #     yarn tests:android:test
-      #   shell: bash
+      - name: Run Emulator Tests
+        uses: reactivecircus/android-emulator-runner@v2
+        timeout-minutes: 30
+        continue-on-error: true
+        with:
+          api-level: ${{ matrix.api-level }}
+          avd-name: TestingAVD
+          force-avd-creation: false
+          target: ${{ matrix.target }}
+          arch: ${{ matrix.arch }}
+          emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
+          sdcard-path-or-size: 100M
+          disable-animations: true
+          script: |
+            $ANDROID_HOME/platform-tools/adb logcat '*:D' > adb-log.txt &
+            yarn tests:android:test
 
       - name: Submit Coverage
         # This can fail on timeouts etc, wrap with retry

diff --git a/.github/workflows/tests_e2e_ios.yml b/.github/workflows/tests_e2e_ios.yml
@@ -66,7 +66,7 @@ jobs:
         id: detox-cache
         with:
           path: ~/Library/Detox/ios
-          key: ${{ runner.os }}-detox-framework-cache-${{ steps.workflow-variables.outputs.xcode-version }}
+          key: ${{ runner.os }}-detox-framework-cache-${{ steps.workflow-variables.outputs.xcode-version }}-v1
 
       # Detox is compiled during yarn install, using Xcode, set up cache first
       - uses: hendrikmuhs/ccache-action@v1
@@ -112,6 +112,14 @@ jobs:
           max_attempts: 3
           command: yarn tests:ios:pod:install
 
+      - name: Build Detox Framework Cache
+        uses: nick-invision/retry@v2
+        with:
+          timeout_minutes: 10
+          retry_wait_seconds: 60
+          max_attempts: 3
+          command: cd RNGoogleMobileAdsExample && yarn detox build-framework-cache
+
       - name: Build iOS App
         run: |
           export PATH="/usr/lib/ccache:/usr/local/opt/ccache/libexec:$PATH"
@@ -160,9 +168,10 @@ jobs:
         continue-on-error: true
         run: nohup sh -c "sleep 30 && xcrun simctl spawn booted log stream --level debug --style compact > simulator.log 2>&1 &"
 
-      # - name: Detox Test
-      #   timeout-minutes: 10
-      #   run: yarn tests:ios:test
+      - name: Detox Test
+        timeout-minutes: 10
+        continue-on-error: true
+        run: yarn tests:ios:test
 
       - name: Compress Simulator Log
         continue-on-error: true