From 0a5c58b3da9345cd13874a55f20b07e413fa138c Mon Sep 17 00:00:00 2001 From: LucasZF Date: Wed, 1 Dec 2021 19:09:24 -0300 Subject: [PATCH] Nuke Android/Unity apk flaky tests (#437) --- .github/workflows/ci.yml | 109 ++++++++++--- .../Assets/Scripts/SmokeTester.cs | 87 ++++++----- scripts/smoke-test-droid.ps1 | 143 +++++++++++------- 3 files changed, 222 insertions(+), 117 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aa8e43fa6..caec346af 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -270,9 +270,11 @@ jobs: name: Smoke Test - Android ${{ matrix.api-level }} Unity ${{ matrix.unity-version }} runs-on: macos-latest strategy: + max-parallel: 5 fail-fast: false matrix: - api-level: [21, 22, 23, 24, 25, 26, 27, 28, 29] + api-level: [21, 27, 29] + avd-target: [default] unity-version: [2019.4.31f1, 2020.3.21f1, 2021.1.26f1] steps: - name: Checkout @@ -284,24 +286,52 @@ jobs: name: droid-testapp-${{ matrix.unity-version }} path: samples/artifacts/builds/Android - - name: AVD cache - uses: actions/cache@v2 - id: avd-cache - with: - path: | - ~/.android/avd/* - ~/.android/adb* - key: avd-${{ matrix.api-level }}-${{ matrix.unity-version }} - - - name: Smoke test + - name: Android emulator setup + Smoke test + id: smoke-test + continue-on-error: true + timeout-minutes: 10 uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330 with: api-level: ${{ matrix.api-level }} + target: ${{ matrix.avd-target }} force-avd-creation: false ram-size: 2048M - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + arch: x86 + cores: 2 + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + disable-animations: false + script: sudo pwsh ./scripts/smoke-test-droid.ps1 + + - name: Kill emulator if AVD failed. + continue-on-error: true + if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }} + run: | + adb emu kill + sleep 7 + + - name: Android emulator setup + Smoke test (Retry) + id: smoke-test-retry + continue-on-error: true + timeout-minutes: 10 + # We only want to retry the tests if the previous fail happened on the emulator startup. + if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }} + uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330 + with: + api-level: ${{ matrix.api-level }} + target: ${{ matrix.avd-target }} + ram-size: 2048M + cores: 2 + arch: x86 + force-avd-creation: false + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: false - script: pwsh ./scripts/smoke-test-droid.ps1 + script: sudo pwsh ./scripts/smoke-test-droid.ps1 + + - name: Throw error if Smoke test failed + # We want to throw an error if the smoke test failed. + # We will ignore flaky errors from the emulator setup. + if: ${{ (steps.smoke-test.outcome != 'success' && steps.smoke-test.outputs.smoke-status == 'Completed') || (steps.smoke-test-retry.outcome != 'success' && steps.smoke-test-retry.outputs.smoke-status == 'Completed') }} + run: exit -1 - name: Upload screenshot if smoke test failed if: ${{ failure() }} @@ -315,9 +345,11 @@ jobs: name: Smoke Test - Android ${{ matrix.api-level }} Unity ${{ matrix.unity-version }} runs-on: macos-latest strategy: + max-parallel: 2 fail-fast: false matrix: api-level: [30] + avd-target: [google_apis] #api-level 30 image is only available with google services. unity-version: [2019.4.31f1, 2020.3.21f1, 2021.1.26f1] steps: @@ -330,25 +362,52 @@ jobs: name: droid-testapp-${{ matrix.unity-version }} path: samples/artifacts/builds/Android - - name: AVD cache - uses: actions/cache@v2 - id: avd-cache + - name: Android emulator setup + Smoke test + id: smoke-test + continue-on-error: true + timeout-minutes: 10 + uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330 with: - path: | - ~/.android/avd/* - ~/.android/adb* - key: avd-${{ matrix.api-level }}-${{ matrix.unity-version }}-gservices - - - name: Smoke test + api-level: ${{ matrix.api-level }} + target: ${{ matrix.avd-target }} + force-avd-creation: false + ram-size: 2048M + arch: x86 + cores: 2 + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + disable-animations: false + script: sudo pwsh ./scripts/smoke-test-droid.ps1 + + - name: Kill emulator if AVD failed. + continue-on-error: true + if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }} + run: | + adb emu kill + sleep 7 + + - name: Android emulator setup + Smoke test (Retry) + id: smoke-test-retry + continue-on-error: true + timeout-minutes: 10 + # We only want to retry the tests if the previous fail happened on the emulator startup. + if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }} uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330 with: api-level: ${{ matrix.api-level }} - target: google_apis + target: ${{ matrix.avd-target }} ram-size: 2048M + cores: 2 + arch: x86 force-avd-creation: false - emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none + emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none disable-animations: false - script: pwsh ./scripts/smoke-test-droid.ps1 + script: sudo pwsh ./scripts/smoke-test-droid.ps1 + + - name: Throw error if Smoke test failed + # We want to throw an error if the smoke test failed. + # We will ignore flaky errors from the emulator setup. + if: ${{ (steps.smoke-test.outcome != 'success' && steps.smoke-test.outputs.smoke-status == 'Completed') || (steps.smoke-test-retry.outcome != 'success' && steps.smoke-test-retry.outputs.smoke-status == 'Completed') }} + run: exit -1 - name: Upload screenshot if smoke test failed if: ${{ failure() }} diff --git a/samples/unity-of-bugs/Assets/Scripts/SmokeTester.cs b/samples/unity-of-bugs/Assets/Scripts/SmokeTester.cs index b67cb1e1c..cac26810c 100644 --- a/samples/unity-of-bugs/Assets/Scripts/SmokeTester.cs +++ b/samples/unity-of-bugs/Assets/Scripts/SmokeTester.cs @@ -1,4 +1,4 @@ -#if !UNITY_EDITOR +#if !UNITY_EDITOR #if UNITY_IOS #define SENTRY_NATIVE_IOS #elif UNITY_ANDROID @@ -53,52 +53,69 @@ public void Start() public static void SmokeTest() { - var evt = new ManualResetEventSlim(); - - var requests = new List(); - void Verify(HttpRequestMessage message) + try { - requests.Add(message.Content.ReadAsStringAsync().Result); - evt.Set(); - } + Debug.Log("SMOKE TEST: Start"); + var evt = new ManualResetEventSlim(); + + var requests = new List(); + void Verify(HttpRequestMessage message) + { + Debug.Log("SMOKE TEST: Verify invoked."); + requests.Add(message.Content.ReadAsStringAsync().Result); + evt.Set(); + } - var options = new SentryUnityOptions(); - options.Dsn = "https://key@sentry/project"; - options.Debug = true; - // TODO: Must be set explicitly for the time being. - options.RequestBodyCompressionLevel = CompressionLevelWithAuto.Auto; - options.DiagnosticLogger = new ConsoleDiagnosticLogger(SentryLevel.Debug); - options.CreateHttpClientHandler = () => new TestHandler(Verify); + var options = new SentryUnityOptions(); + options.Dsn = "https://key@sentry/project"; + options.Debug = true; + // TODO: Must be set explicitly for the time being. + options.RequestBodyCompressionLevel = CompressionLevelWithAuto.Auto; + options.DiagnosticLogger = new ConsoleDiagnosticLogger(SentryLevel.Debug); + options.CreateHttpClientHandler = () => new TestHandler(Verify); #if SENTRY_NATIVE_IOS - SentryNativeIos.Configure(options); + Debug.Log("SMOKE TEST: Configure Native iOS."); + SentryNativeIos.Configure(options); #elif SENTRY_NATIVE_ANDROID - SentryNativeAndroid.Configure(options); + Debug.Log("SMOKE TEST: Configure Native Android."); + SentryNativeAndroid.Configure(options); #endif - SentryUnity.Init(options); + Debug.Log("SMOKE TEST: SentryUnity Init."); + SentryUnity.Init(options); - var guid = Guid.NewGuid().ToString(); - Debug.LogError(guid); - SentrySdk.CaptureMessage(guid); + Debug.Log("SMOKE TEST: SentryUnity Init OK."); - if (!evt.Wait(TimeSpan.FromSeconds(3))) - { - // 1 = timeout - Application.Quit(1); - } + var guid = Guid.NewGuid().ToString(); + Debug.LogError(guid); + SentrySdk.CaptureMessage(guid); - if (!requests.Any(r => r.Contains(guid))) - { - // 2 event captured but guid not there. - Application.Quit(2); - } + if (!evt.Wait(TimeSpan.FromSeconds(3))) + { + // 1 = timeout + Application.Quit(1); + } + + if (!requests.Any(r => r.Contains(guid))) + { + // 2 event captured but guid not there. + Application.Quit(2); + } - // On Android we'll grep logcat for this string instead of relying on exit code: - Debug.Log("SMOKE TEST: PASS"); + // On Android we'll grep logcat for this string instead of relying on exit code: + Debug.Log("SMOKE TEST: PASS"); - // Test passed: Exit Code 200 to avoid false positive from a graceful exit unrelated to this test run - Application.Quit(200); + // Test passed: Exit Code 200 to avoid false positive from a graceful exit unrelated to this test run + Application.Quit(200); + + } + catch (Exception ex) + { + Debug.Log("SMOKE TEST: FAILED"); + Debug.LogError(ex); + Application.Quit(-1); + } } private class TestHandler : HttpClientHandler diff --git a/scripts/smoke-test-droid.ps1 b/scripts/smoke-test-droid.ps1 index e8a690972..40dcddca1 100644 --- a/scripts/smoke-test-droid.ps1 +++ b/scripts/smoke-test-droid.ps1 @@ -9,27 +9,48 @@ Set-Variable -Name "ApkPath" -Value "samples/artifacts/builds/Android" Set-Variable -Name "ApkFileName" -Value "IL2CPP_Player.apk" Set-Variable -Name "ProcessName" -Value "io.sentry.samples.unityofbugs" Set-Variable -Name "TestActivityName" -Value "io.sentry.samples.unityofbugs/com.unity3d.player.UnityPlayerActivity" +$LogcatCache = $null function TakeScreenshot { param ( $deviceId ) adb -s $deviceId shell "screencap -p /storage/emulated/0/screen.png" adb pull "/storage/emulated/0/screen.png" "$ApkPath" adb shell "rm /storage/emulated/0/screen.png" - } function WriteDeviceLog { param ( $deviceId ) - adb -s $device logcat -d - # | select-string "Unity|unity|sentry|Sentry|SMOKE" + Write-Output $LogcatCache +} + +function WriteDeviceUiLog { + param ( $deviceId ) + Write-Output "`n`nUI XML Log" + adb -s $deviceId exec-out uiautomator dump /dev/tty } function DateTimeNow { return Get-Date -UFormat "%T %Z" } +function CheckAndCloseActiveSystemAlerts { + param ($deviceId) + $uiInfoXml = adb -s $deviceId exec-out uiautomator dump /dev/tty + if (($uiInfoXml | select-string "android:id/alertTitle|has stopped|Close app") -ne $null) + { + Write-Warning "Active system alert found on $deviceId. Closing it." + adb shell input keyevent 4 + } +} + +function SignalActionSmokeStatus { + param ($smokeStatus) + echo "::set-output name=smoke-status::$smokeStatus" +} + # Filter device List $RawAdbDeviceList = adb devices + $DeviceList = @() foreach ($device in $RawAdbDeviceList) { @@ -42,6 +63,7 @@ $DeviceCount = $DeviceList.Count If ($DeviceCount -eq 0) { + SignalActionSmokeStatus("Completed") Throw "It seems like no devices were found $RawAdbDeviceList" } Else @@ -56,101 +78,108 @@ If (Test-Path -Path "$ApkPath/$ApkFileName" ) } Else { + SignalActionSmokeStatus("Completed") Throw "Expected APK on $ApkPath/$ApkFileName but it was not found." } # Test foreach ($device in $DeviceList) { - $deviceSdk = adb shell getprop ro.build.version.sdk - $deviceApi = adb shell getprop ro.build.version.release - Write-Output "" - Write-Output "Checking device $device with SDK $deviceSdk and API $deviceApi" - Write-Output "" - - Write-Output "Removing previous APP if found." - $stdout = adb -s $device uninstall $ProcessName + $deviceSdk = adb -s $device shell getprop ro.build.version.sdk + $deviceApi = adb -s $device shell getprop ro.build.version.release + Write-Output "`nChecking device $device with SDK $deviceSdk and API $deviceApi`n" + + $stdout = adb -s $device shell "pm list packages -f" + if (($stdout | select-string $ProcessName) -ne $null) + { + Write-Output "Removing previous APP." + $stdout = adb -s $device uninstall $ProcessName + } + + # Move device to home screen + $stdout = adb -s $device shell input keyevent KEYCODE_HOME + Write-Output "Installing test app..." $stdout = (adb -s $device install -r $ApkPath/$ApkFileName) If($stdout -notcontains "Success") { + SignalActionSmokeStatus("Completed") Throw "Failed to Install APK: $stdout." } - $FlakyRetry = 3 - While ($FlakyRetry -gt 0) - { - $AppStarted = 'False' + $AppStarted = 'False' - Write-Output "Clearing logcat from $device." - adb -s $device logcat -c + Write-Output "Clearing logcat from $device." + adb -s $device logcat -c - Write-Output "Starting Test..." + Write-Output "Starting Test..." - adb -s $device shell am start -n $TestActivityName -e test smoke - #despite calling start, the app might not be started yet. + adb -s $device shell am start -n $TestActivityName -e test smoke + #despite calling start, the app might not be started yet. - Write-Output (DateTimeNow) - $Timeout = 45 - While ($Timeout -gt 0) - { - #Get a list of active processes - $processIsRunning = (adb -s $device shell ps) - #And filter by ProcessName - $processIsRunning = $processIsRunning | select-string $ProcessName - - If ($processIsRunning -eq $null -And $AppStarted -eq 'True') - { - $Timeout = -2 - break - } - ElseIf ($processIsRunning -ne $null -And $AppStarted -eq 'False') - { - # Some devices might take a while to start the test, so we wait for the activity to start before checking if it was closed. - $AppStarted = 'True' - } - Write-Output "Waiting Process on $device to complete, waiting $Timeout seconds" - Start-Sleep -Seconds 1 - $Timeout-- - } - Write-Output (DateTimeNow) - - $stdout = adb -s $device logcat -d | select-string 'Unity : Timeout while trying detaching' - If ($stdout -eq $null) + Write-Output (DateTimeNow) + $Timeout = 45 + While ($Timeout -gt 0) + { + #Get a list of active processes + $processIsRunning = (adb -s $device shell ps) + #And filter by ProcessName + $processIsRunning = $processIsRunning | select-string $ProcessName + + If ($processIsRunning -eq $null -And $AppStarted -eq 'True') { + $Timeout = -2 break } - Else + ElseIf ($processIsRunning -ne $null -And $AppStarted -eq 'False') { - Write-Warning "Test was flaky, retrying." - $FlakyRetry-- - Start-Sleep -Seconds 3 + # Some devices might take a while to start the test, so we wait for the activity to start before checking if it was closed. + $AppStarted = 'True' } + Write-Output "Waiting Process on $device to complete, waiting $Timeout seconds" + Start-Sleep -Seconds 1 + $Timeout-- + CheckAndCloseActiveSystemAlerts($device) } - If ($Timeout -eq 0) + Write-Output (DateTimeNow) + SignalActionSmokeStatus("Completed") + $LogcatCache = adb -s $device logcat -d + $stdout = $LogcatCache | select-string 'SMOKE TEST: PASS' + If ($stdout -ne $null) + { + Write-Output "`n`n`n Final logcat" + WriteDeviceLog($device) + Write-Output "`n`n`n $stdout" + } + ElseIf ($Timeout -eq 0) { Write-Warning "Test Timeout, see Logcat info for more information below." WriteDeviceLog($device) Write-Output "PS info." adb -s $device shell ps + WriteDeviceUiLog($device) TakeScreenshot($device) Throw "Test Timeout" } - - $stdout = adb -s $device logcat -d | select-string 'SMOKE TEST: PASS' - If ($stdout -ne $null) + ElseIf (($LogcatCache | select-string 'Unity : Timeout while trying detaching primary window.')) { - Write-Output "$stdout" + SignalActionSmokeStatus("Flaky") + Write-Warning "Test was flaky, unity failed to initialize." + WriteDeviceLog($device) + WriteDeviceUiLog($device) + TakeScreenshot($device) + Throw "Test was flaky, unity failed to initialize." } Else { Write-Warning "Process completed but Smoke test was not signaled." WriteDeviceLog($device) + WriteDeviceUiLog($device) TakeScreenshot($device) Throw "Smoke Test Failed." } } -Write-Output "Test completed." +Write-Output "`nTest completed."