Skip to content

Commit

Permalink
Nuke Android/Unity apk flaky tests (#437)
Browse files Browse the repository at this point in the history
  • Loading branch information
@lucas-zimerman
lucas-zimerman authored Dec 1, 2021
1 parent 940a85e commit 0a5c58b
Show file tree
Hide file tree
Showing 3 changed files with 222 additions and 117 deletions.
109 changes: 84 additions & 25 deletions .github/workflows/ci.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -270,9 +270,11 @@ jobs:
name: Smoke Test - Android ${{ matrix.api-level }} Unity ${{ matrix.unity-version }}
runs-on: macos-latest
strategy:
max-parallel: 5
fail-fast: false
matrix:
api-level: [21, 22, 23, 24, 25, 26, 27, 28, 29]
api-level: [21, 27, 29]
avd-target: [default]
unity-version: [2019.4.31f1, 2020.3.21f1, 2021.1.26f1]
steps:
- name: Checkout
Expand All @@ -284,24 +286,52 @@ jobs:
name: droid-testapp-${{ matrix.unity-version }}
path: samples/artifacts/builds/Android

- name: AVD cache
uses: actions/cache@v2
id: avd-cache
with:
path: |
~/.android/avd/*
~/.android/adb*
key: avd-${{ matrix.api-level }}-${{ matrix.unity-version }}

- name: Smoke test
- name: Android emulator setup + Smoke test
id: smoke-test
continue-on-error: true
timeout-minutes: 10
uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330
with:
api-level: ${{ matrix.api-level }}
target: ${{ matrix.avd-target }}
force-avd-creation: false
ram-size: 2048M
emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
arch: x86
cores: 2
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
disable-animations: false
script: sudo pwsh ./scripts/smoke-test-droid.ps1

- name: Kill emulator if AVD failed.
continue-on-error: true
if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }}
run: |
adb emu kill
sleep 7
- name: Android emulator setup + Smoke test (Retry)
id: smoke-test-retry
continue-on-error: true
timeout-minutes: 10
# We only want to retry the tests if the previous fail happened on the emulator startup.
if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }}
uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330
with:
api-level: ${{ matrix.api-level }}
target: ${{ matrix.avd-target }}
ram-size: 2048M
cores: 2
arch: x86
force-avd-creation: false
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
disable-animations: false
script: pwsh ./scripts/smoke-test-droid.ps1
script: sudo pwsh ./scripts/smoke-test-droid.ps1

- name: Throw error if Smoke test failed
# We want to throw an error if the smoke test failed.
# We will ignore flaky errors from the emulator setup.
if: ${{ (steps.smoke-test.outcome != 'success' && steps.smoke-test.outputs.smoke-status == 'Completed') || (steps.smoke-test-retry.outcome != 'success' && steps.smoke-test-retry.outputs.smoke-status == 'Completed') }}
run: exit -1

- name: Upload screenshot if smoke test failed
if: ${{ failure() }}
Expand All @@ -315,9 +345,11 @@ jobs:
name: Smoke Test - Android ${{ matrix.api-level }} Unity ${{ matrix.unity-version }}
runs-on: macos-latest
strategy:
max-parallel: 2
fail-fast: false
matrix:
api-level: [30]
avd-target: [google_apis]
#api-level 30 image is only available with google services.
unity-version: [2019.4.31f1, 2020.3.21f1, 2021.1.26f1]
steps:
Expand All @@ -330,25 +362,52 @@ jobs:
name: droid-testapp-${{ matrix.unity-version }}
path: samples/artifacts/builds/Android

- name: AVD cache
uses: actions/cache@v2
id: avd-cache
- name: Android emulator setup + Smoke test
id: smoke-test
continue-on-error: true
timeout-minutes: 10
uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330
with:
path: |
~/.android/avd/*
~/.android/adb*
key: avd-${{ matrix.api-level }}-${{ matrix.unity-version }}-gservices

- name: Smoke test
api-level: ${{ matrix.api-level }}
target: ${{ matrix.avd-target }}
force-avd-creation: false
ram-size: 2048M
arch: x86
cores: 2
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
disable-animations: false
script: sudo pwsh ./scripts/smoke-test-droid.ps1

- name: Kill emulator if AVD failed.
continue-on-error: true
if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }}
run: |
adb emu kill
sleep 7
- name: Android emulator setup + Smoke test (Retry)
id: smoke-test-retry
continue-on-error: true
timeout-minutes: 10
# We only want to retry the tests if the previous fail happened on the emulator startup.
if: ${{ steps.smoke-test.outputs.smoke-status != 'Completed' }}
uses: reactivecircus/android-emulator-runner@2b2ebf2e518e38a17180117fc2b677006db27330
with:
api-level: ${{ matrix.api-level }}
target: google_apis
target: ${{ matrix.avd-target }}
ram-size: 2048M
cores: 2
arch: x86
force-avd-creation: false
emulator-options: -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
emulator-options: -no-snapshot-save -no-window -gpu swiftshader_indirect -noaudio -no-boot-anim -camera-back none
disable-animations: false
script: pwsh ./scripts/smoke-test-droid.ps1
script: sudo pwsh ./scripts/smoke-test-droid.ps1

- name: Throw error if Smoke test failed
# We want to throw an error if the smoke test failed.
# We will ignore flaky errors from the emulator setup.
if: ${{ (steps.smoke-test.outcome != 'success' && steps.smoke-test.outputs.smoke-status == 'Completed') || (steps.smoke-test-retry.outcome != 'success' && steps.smoke-test-retry.outputs.smoke-status == 'Completed') }}
run: exit -1

- name: Upload screenshot if smoke test failed
if: ${{ failure() }}
Expand Down
87 changes: 52 additions & 35 deletions samples/unity-of-bugs/Assets/Scripts/SmokeTester.cs
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
#if !UNITY_EDITOR
#if !UNITY_EDITOR
#if UNITY_IOS
#define SENTRY_NATIVE_IOS
#elif UNITY_ANDROID
Expand Down Expand Up @@ -53,52 +53,69 @@ public void Start()

public static void SmokeTest()
{
var evt = new ManualResetEventSlim();

var requests = new List<string>();
void Verify(HttpRequestMessage message)
try
{
requests.Add(message.Content.ReadAsStringAsync().Result);
evt.Set();
}
Debug.Log("SMOKE TEST: Start");
var evt = new ManualResetEventSlim();

var requests = new List<string>();
void Verify(HttpRequestMessage message)
{
Debug.Log("SMOKE TEST: Verify invoked.");
requests.Add(message.Content.ReadAsStringAsync().Result);
evt.Set();
}

var options = new SentryUnityOptions();
options.Dsn = "https://key@sentry/project";
options.Debug = true;
// TODO: Must be set explicitly for the time being.
options.RequestBodyCompressionLevel = CompressionLevelWithAuto.Auto;
options.DiagnosticLogger = new ConsoleDiagnosticLogger(SentryLevel.Debug);
options.CreateHttpClientHandler = () => new TestHandler(Verify);
var options = new SentryUnityOptions();
options.Dsn = "https://key@sentry/project";
options.Debug = true;
// TODO: Must be set explicitly for the time being.
options.RequestBodyCompressionLevel = CompressionLevelWithAuto.Auto;
options.DiagnosticLogger = new ConsoleDiagnosticLogger(SentryLevel.Debug);
options.CreateHttpClientHandler = () => new TestHandler(Verify);

#if SENTRY_NATIVE_IOS
SentryNativeIos.Configure(options);
Debug.Log("SMOKE TEST: Configure Native iOS.");
SentryNativeIos.Configure(options);
#elif SENTRY_NATIVE_ANDROID
SentryNativeAndroid.Configure(options);
Debug.Log("SMOKE TEST: Configure Native Android.");
SentryNativeAndroid.Configure(options);
#endif

SentryUnity.Init(options);
Debug.Log("SMOKE TEST: SentryUnity Init.");
SentryUnity.Init(options);

var guid = Guid.NewGuid().ToString();
Debug.LogError(guid);
SentrySdk.CaptureMessage(guid);
Debug.Log("SMOKE TEST: SentryUnity Init OK.");

if (!evt.Wait(TimeSpan.FromSeconds(3)))
{
// 1 = timeout
Application.Quit(1);
}
var guid = Guid.NewGuid().ToString();
Debug.LogError(guid);
SentrySdk.CaptureMessage(guid);

if (!requests.Any(r => r.Contains(guid)))
{
// 2 event captured but guid not there.
Application.Quit(2);
}
if (!evt.Wait(TimeSpan.FromSeconds(3)))
{
// 1 = timeout
Application.Quit(1);
}

if (!requests.Any(r => r.Contains(guid)))
{
// 2 event captured but guid not there.
Application.Quit(2);
}

// On Android we'll grep logcat for this string instead of relying on exit code:
Debug.Log("SMOKE TEST: PASS");
// On Android we'll grep logcat for this string instead of relying on exit code:
Debug.Log("SMOKE TEST: PASS");

// Test passed: Exit Code 200 to avoid false positive from a graceful exit unrelated to this test run
Application.Quit(200);
// Test passed: Exit Code 200 to avoid false positive from a graceful exit unrelated to this test run
Application.Quit(200);

}
catch (Exception ex)
{
Debug.Log("SMOKE TEST: FAILED");
Debug.LogError(ex);
Application.Quit(-1);
}
}

private class TestHandler : HttpClientHandler
Expand Down
Loading

0 comments on commit 0a5c58b

Please sign in to comment.