Skip to content

Commit e58a8f1

Browse files
jpnurmiFlash0verjamescrosswell
authored
iOS & Android integration tests (#4559)
Co-authored-by: Stefan Pölz <38893694+Flash0ver@users.noreply.github.com> Co-authored-by: James Crosswell <jamescrosswell@users.noreply.github.com>
1 parent 6c2ad0e commit e58a8f1

36 files changed

+1350
-74
lines changed

.github/workflows/build.yml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,7 @@ jobs:
253253
# integration tests, e.g. Directory.Build.props, nuget.config, ...
254254
sparse-checkout: |
255255
integration-test
256+
scripts
256257
.github
257258
258259
- name: Fetch NuGet Packages

.github/workflows/device-tests-android.yml

Lines changed: 38 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -81,6 +81,8 @@ jobs:
8181
8282
- name: Checkout
8383
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
84+
with:
85+
submodules: recursive
8486

8587
- name: Download test app artifact
8688
uses: actions/download-artifact@634f93cb2916e3fdff6788551b99b062d0335ce0 # v5.0.0
@@ -94,11 +96,11 @@ jobs:
9496
# Cached AVD setup per https://github.com/ReactiveCircus/android-emulator-runner/blob/main/README.md
9597

9698
- name: Run Tests
97-
id: first-run
99+
id: first-test-run
98100
continue-on-error: true
99101
timeout-minutes: 40
100102
uses: reactivecircus/android-emulator-runner@1dcd0090116d15e7c562f8db72807de5e036a4ed # v2.34.0
101-
with:
103+
with: &android-emulator-test
102104
api-level: ${{ matrix.api-level }}
103105
target: ${{ env.ANDROID_EMULATOR_TARGET }}
104106
force-avd-creation: false
@@ -110,10 +112,33 @@ jobs:
110112
script: pwsh scripts/device-test.ps1 android -Run -Tfm ${{ matrix.tfm }}
111113

112114
- name: Retry Tests (if previous failed to run)
113-
if: steps.first-run.outcome == 'failure'
115+
if: steps.first-test-run.outcome == 'failure'
114116
timeout-minutes: 40
115117
uses: reactivecircus/android-emulator-runner@1dcd0090116d15e7c562f8db72807de5e036a4ed # v2.34.0
118+
with: *android-emulator-test
119+
120+
- name: Setup Environment
121+
uses: ./.github/actions/environment
122+
123+
- name: Select Java 17
124+
uses: actions/setup-java@dded0888837ed1f317902acf8a20df0ad188d165 # v5.0.0
125+
with:
126+
distribution: ${{ runner.os == 'Windows' && runner.arch == 'ARM64' && 'microsoft' || 'temurin' }}
127+
java-version: '17'
128+
129+
- name: Checkout github-workflows
130+
uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
116131
with:
132+
repository: getsentry/github-workflows
133+
ref: a5e409bd5bad4c295201cdcfe862b17c50b29ab7 # v2.14.1
134+
path: modules/github-workflows
135+
136+
- name: Run Integration Tests
137+
id: first-integration-test-run
138+
continue-on-error: true
139+
timeout-minutes: 40
140+
uses: reactivecircus/android-emulator-runner@1dcd0090116d15e7c562f8db72807de5e036a4ed # v2.34.0
141+
with: &android-emulator-integration-test
117142
api-level: ${{ matrix.api-level }}
118143
target: ${{ env.ANDROID_EMULATOR_TARGET }}
119144
force-avd-creation: false
@@ -122,11 +147,19 @@ jobs:
122147
disk-size: ${{ env.ANDROID_EMULATOR_DISK_SIZE }}
123148
emulator-options: ${{ env.ANDROID_EMULATOR_OPTIONS }}
124149
disable-animations: false
125-
script: pwsh scripts/device-test.ps1 android -Run -Tfm ${{ matrix.tfm }}
150+
script: pwsh integration-test/android.Tests.ps1
151+
152+
- name: Retry Integration Tests (if previous failed to run)
153+
if: steps.first-integration-test-run.outcome == 'failure'
154+
timeout-minutes: 40
155+
uses: reactivecircus/android-emulator-runner@1dcd0090116d15e7c562f8db72807de5e036a4ed # v2.34.0
156+
with: *android-emulator-integration-test
126157

127158
- name: Upload results
128159
if: success() || failure()
129160
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
130161
with:
131162
name: device-test-android-${{ matrix.api-level }}-${{ matrix.tfm }}-results
132-
path: test_output
163+
path: |
164+
test_output
165+
integration-test/mobile-app/test_output

.github/workflows/device-tests-ios.yml

Lines changed: 18 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,17 +35,32 @@ jobs:
3535
run: pwsh ./scripts/device-test.ps1 ios -Build
3636

3737
- name: Run Tests
38-
id: first-run
38+
id: first-test-run
3939
continue-on-error: true
4040
run: pwsh scripts/device-test.ps1 ios -Run
4141

4242
- name: Retry Tests (if previous failed to run)
43-
if: steps.first-run.outcome == 'failure'
43+
if: steps.first-test-run.outcome == 'failure'
4444
run: pwsh scripts/device-test.ps1 ios -Run
4545

46+
- name: Run Integration Tests
47+
id: first-integration-test-run
48+
continue-on-error: true
49+
uses: getsentry/github-workflows/sentry-cli/integration-test/@a5e409bd5bad4c295201cdcfe862b17c50b29ab7 # v2.14.1
50+
with:
51+
path: integration-test/ios.Tests.ps1
52+
53+
- name: Retry Integration Tests (if previous failed to run)
54+
if: steps.first-integration-test-run.outcome == 'failure'
55+
uses: getsentry/github-workflows/sentry-cli/integration-test/@a5e409bd5bad4c295201cdcfe862b17c50b29ab7 # v2.14.1
56+
with:
57+
path: integration-test/ios.Tests.ps1
58+
4659
- name: Upload results
4760
if: success() || failure()
4861
uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2
4962
with:
5063
name: device-test-ios-results
51-
path: test_output
64+
path: |
65+
test_output
66+
integration-test/mobile-app/test_output

integration-test/android.Tests.ps1

Lines changed: 166 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,166 @@
1+
# This file contains test cases for https://pester.dev/
2+
Set-StrictMode -Version Latest
3+
$ErrorActionPreference = 'Stop'
4+
. $PSScriptRoot/pester.ps1
5+
. $PSScriptRoot/../scripts/device-test-utils.ps1
6+
7+
BeforeDiscovery {
8+
# Skip Android integration tests unless an emulator has been already started
9+
# by Android Device Tests, or manually when testing locally. This avoids
10+
# slowing down non-Device Test CI builds further.
11+
Install-XHarness
12+
$script:emulator = Get-AndroidEmulatorId
13+
}
14+
15+
Describe 'MAUI app' -ForEach @(
16+
@{ tfm = "net9.0-android35.0" }
17+
) -Skip:(-not $script:emulator) {
18+
BeforeAll {
19+
Remove-Item -Path "$PSScriptRoot/mobile-app" -Recurse -Force -ErrorAction SilentlyContinue
20+
Copy-Item -Path "$PSScriptRoot/net9-maui" -Destination "$PSScriptRoot/mobile-app" -Recurse -Force
21+
Push-Location $PSScriptRoot/mobile-app
22+
23+
function InstallAndroidApp
24+
{
25+
param([string] $Dsn)
26+
$dsn = $Dsn.Replace('http://', 'http://key@') + '/0'
27+
28+
# replace {{SENTRY_DSN}} in MauiProgram.cs
29+
(Get-Content MauiProgram.cs) `
30+
-replace '\{\{SENTRY_DSN\}\}', $dsn `
31+
| Set-Content MauiProgram.cs
32+
33+
$arch = [System.Runtime.InteropServices.RuntimeInformation]::OSArchitecture.ToString().ToLower()
34+
$rid = "android-$arch"
35+
36+
Write-Host "::group::Build Sentry.Maui.Device.IntegrationTestApp.csproj"
37+
dotnet build Sentry.Maui.Device.IntegrationTestApp.csproj `
38+
--configuration Release `
39+
--framework $tfm `
40+
--runtime $rid
41+
| ForEach-Object { Write-Host $_ }
42+
Write-Host '::endgroup::'
43+
$LASTEXITCODE | Should -Be 0
44+
45+
Write-Host "::group::Install bin/Release/$tfm/$rid/io.sentry.dotnet.maui.device.integrationtestapp-Signed.apk"
46+
xharness android install -v `
47+
--app bin/Release/$tfm/$rid/io.sentry.dotnet.maui.device.integrationtestapp-Signed.apk `
48+
--package-name io.sentry.dotnet.maui.device.integrationtestapp `
49+
--output-directory=test_output
50+
| ForEach-Object { Write-Host $_ }
51+
Write-Host '::endgroup::'
52+
$LASTEXITCODE | Should -Be 0
53+
}
54+
55+
function RunAndroidApp
56+
{
57+
param(
58+
[string] $Dsn,
59+
[string] $TestArg = 'None'
60+
)
61+
62+
try
63+
{
64+
# Setup port forwarding for accessing sentry-server at 127.0.0.1:8000 from the emulator
65+
$port = $Dsn.Split(':')[2].Split('/')[0]
66+
xharness android adb -v -- reverse tcp:$port tcp:$port
67+
68+
Write-Host "::group::Run Android app (TestArg=$TestArg)"
69+
xharness android adb -v `
70+
-- shell am start -S -n io.sentry.dotnet.maui.device.integrationtestapp/.MainActivity `
71+
-e SENTRY_TEST_ARG $TestArg
72+
| ForEach-Object { Write-Host $_ }
73+
Write-Host '::endgroup::'
74+
$LASTEXITCODE | Should -Be 0
75+
76+
do
77+
{
78+
Write-Host "Waiting for app..."
79+
Start-Sleep -Seconds 1
80+
81+
$procid = (& xharness android adb -- shell pidof "io.sentry.dotnet.maui.device.integrationtestapp") -replace '\s', ''
82+
$activity = (& xharness android adb -- shell dumpsys activity activities) -match "io\.sentry\.dotnet\.maui\.device\.integrationtestapp"
83+
84+
} while ($procid -and $activity)
85+
}
86+
finally
87+
{
88+
xharness android adb -v -- reverse --remove-all
89+
}
90+
}
91+
92+
function UninstallAndroidApp
93+
{
94+
Write-Host "::group::Uninstall io.sentry.dotnet.maui.device.integrationtestapp"
95+
xharness android uninstall -v `
96+
--package-name io.sentry.dotnet.maui.device.integrationtestapp
97+
| ForEach-Object { Write-Host $_ }
98+
$LASTEXITCODE | Should -Be 0
99+
Write-Host '::endgroup::'
100+
}
101+
102+
# Helper to dump server stderr if the test server reported errors
103+
function Dump-ServerErrors {
104+
param(
105+
[Parameter(Mandatory)]
106+
$Result
107+
)
108+
if ($Result.HasErrors()) {
109+
Write-Host '::group::sentry-server stderr'
110+
$Result.ServerStdErr | ForEach-Object { Write-Host $_ }
111+
Write-Host '::endgroup::'
112+
}
113+
}
114+
}
115+
116+
AfterAll {
117+
Pop-Location
118+
}
119+
120+
AfterEach {
121+
UninstallAndroidApp
122+
}
123+
124+
It 'Managed crash' {
125+
$result = Invoke-SentryServer {
126+
param([string]$url)
127+
InstallAndroidApp -Dsn $url
128+
RunAndroidApp -Dsn $url -TestArg "Managed"
129+
RunAndroidApp -Dsn $url
130+
}
131+
132+
Dump-ServerErrors -Result $result
133+
$result.HasErrors() | Should -BeFalse
134+
$result.Envelopes() | Should -AnyElementMatch "`"type`":`"System.ApplicationException`""
135+
$result.Envelopes() | Should -Not -AnyElementMatch "`"type`":`"SIGABRT`""
136+
}
137+
138+
It 'Java crash' {
139+
$result = Invoke-SentryServer {
140+
param([string]$url)
141+
InstallAndroidApp -Dsn $url
142+
RunAndroidApp -Dsn $url -TestArg "Java"
143+
RunAndroidApp -Dsn $url
144+
}
145+
146+
Dump-ServerErrors -Result $result
147+
$result.HasErrors() | Should -BeFalse
148+
$result.Envelopes() | Should -AnyElementMatch "`"type`":`"RuntimeException`""
149+
$result.Envelopes() | Should -Not -AnyElementMatch "`"type`":`"System.\w+Exception`""
150+
}
151+
152+
It 'Null reference exception' {
153+
$result = Invoke-SentryServer {
154+
param([string]$url)
155+
InstallAndroidApp -Dsn $url
156+
RunAndroidApp -Dsn $url -TestArg "NullReferenceException"
157+
RunAndroidApp -Dsn $url
158+
}
159+
160+
Dump-ServerErrors -Result $result
161+
$result.HasErrors() | Should -BeFalse
162+
$result.Envelopes() | Should -AnyElementMatch "`"type`":`"System.NullReferenceException`""
163+
# TODO: fix redundant RuntimeException (#3954)
164+
{ $result.Envelopes() | Should -Not -AnyElementMatch "`"type`":`"SIGSEGV`"" } | Should -Throw
165+
}
166+
}

integration-test/common.ps1

Lines changed: 1 addition & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -1,54 +1,4 @@
1-
# So that this works in VS Code testing integration. Otherwise the script is run within its directory.
2-
3-
# In CI, the module is loaded automatically
4-
if (!(Test-Path env:CI ))
5-
{
6-
Import-Module $PSScriptRoot/../../github-workflows/sentry-cli/integration-test/action.psm1 -Force
7-
}
8-
9-
function ShouldAnyElementMatch ($ActualValue, [string]$ExpectedValue, [switch] $Negate, [string] $Because)
10-
{
11-
<#
12-
.SYNOPSIS
13-
Asserts whether any item in the collection matches the expected value
14-
.EXAMPLE
15-
'foo','bar','foobar' | Should -AnyElementMatch 'oob'
16-
17-
This should pass because 'oob' is a substring of 'foobar'.
18-
#>
19-
20-
$filtered = $ActualValue | Where-Object { $_ -match $ExpectedValue }
21-
[bool] $succeeded = @($filtered).Count -gt 0
22-
if ($Negate) { $succeeded = -not $succeeded }
23-
24-
if (-not $succeeded)
25-
{
26-
if ($Negate)
27-
{
28-
$failureMessage = "Expected string '$ExpectedValue' to match no elements in collection @($($ActualValue -join ', '))$(if($Because) { " because $Because"})."
29-
}
30-
else
31-
{
32-
$failureMessage = "Expected string '$ExpectedValue' to match any element in collection @($($ActualValue -join ', '))$(if($Because) { " because $Because"})."
33-
}
34-
}
35-
else
36-
{
37-
$failureMessage = $null
38-
}
39-
40-
return [pscustomobject]@{
41-
Succeeded = $succeeded
42-
FailureMessage = $failureMessage
43-
}
44-
}
45-
46-
BeforeDiscovery {
47-
Add-ShouldOperator -Name AnyElementMatch `
48-
-InternalName 'ShouldAnyElementMatch' `
49-
-Test ${function:ShouldAnyElementMatch} `
50-
-SupportsArrayInput
51-
}
1+
. $PSScriptRoot/pester.ps1
522

533
AfterAll {
544
Pop-Location

0 commit comments

Comments
 (0)