ci: Use xcrun for crash UI test (#5512)

philipphofmann · web-flow · commit fc6557e1428b · 2025-06-30T16:29:04.000+02:00
XCTest isn't built for crashing during tests. Instead of using XCTest to press a button and let a test app crash, we now use UserDefaults to tell the test app to crash during launch. We then simply launch the app again and wait to see if it keeps running. This is basically the same as the testCrash of the SwiftUITestSample without using XCTests.
diff --git a/.github/workflows/ui-tests-critical.yml b/.github/workflows/ui-tests-critical.yml
@@ -48,4 +48,28 @@ jobs:
             xcode: "16.2"
         command:
           - fastlane_command: ui_critical_tests_ios_swiftui_envelope
-          - fastlane_command: ui_critical_tests_ios_swiftui_crash
+
+  run-swiftui-crash-test:
+    name: Run SwiftUI Crash Test
+    runs-on: macos-15
+    steps:
+      - uses: actions/checkout@v4
+
+      - run: ./scripts/ci-select-xcode.sh 16.2
+
+      - run: make init-ci-build
+      - run: make xcode-ci
+
+      - name: Boot simulator
+        run: ./scripts/ci-boot-simulator.sh
+
+      - name: Run SwiftUI Crash Test
+        run: |
+          ./TestSamples/SwiftUICrashTest/test-crash-and-relaunch.sh --screenshots-dir "swiftui-crash-test-screenshots"
+
+      - name: Upload SwiftUI Crash Test Screenshots
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: swiftui-crash-test-screenshots
+          path: swiftui-crash-test-screenshots
diff --git a/.gitignore b/.gitignore
@@ -96,6 +96,7 @@ Samples/visionOS-Swift/visionOS-Swift.xcodeproj
 Samples/watchOS-Swift/watchOS-Swift.xcodeproj
 Samples/SentrySampleShared/SentrySampleShared.xcodeproj
 TestSamples/SwiftUITestSample/SwiftUITestSample.xcodeproj
+TestSamples/SwiftUICrashTest/SwiftUICrashTest.xcodeproj
 
 Sentry.xcframework*
 Sentry-Dynamic.xcframework*
diff --git a/Makefile b/Makefile
@@ -188,3 +188,4 @@ xcode-ci:
 	xcodegen --spec Samples/visionOS-Swift/visionOS-Swift.yml
 	xcodegen --spec Samples/watchOS-Swift/watchOS-Swift.yml
 	xcodegen --spec TestSamples/SwiftUITestSample/SwiftUITestSample.yml
+	xcodegen --spec TestSamples/SwiftUICrashTest/SwiftUICrashTest.yml
diff --git a/Sentry.xcworkspace/contents.xcworkspacedata b/Sentry.xcworkspace/contents.xcworkspacedata
diff --git a/TestSamples/SwiftUICrashTest/SwiftUICrashTest.xcconfig b/TestSamples/SwiftUICrashTest/SwiftUICrashTest.xcconfig
@@ -0,0 +1,29 @@
+#include "../SwiftUITestSample/Shared/Config/Architectures.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/BuildOptions.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Deployment.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Linking.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Localization.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Packaging.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/SearchPaths.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Signing.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Versioning.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/CodeGeneration.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangLanguage.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangCppLanguage.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangModules.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangObjCLanguage.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangPreprocessing.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangWarnings.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangWarningsCpp.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangWarningsObjC.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/AssetCatalog.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/ClangAnalyzer.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Swift.xcconfig"
+#include "../SwiftUITestSample/Shared/Config/Metal.xcconfig"
+
+PRODUCT_NAME = SwiftUICrashTest
+PRODUCT_BUNDLE_IDENTIFIER = io.sentry.tests.SwiftUICrashTest
+GENERATE_INFOPLIST_FILE = YES
+
+SUPPORTED_PLATFORMS = iphoneos iphonesimulator
+MARKETING_VERSION = 1
diff --git a/TestSamples/SwiftUICrashTest/SwiftUICrashTest.yml b/TestSamples/SwiftUICrashTest/SwiftUICrashTest.yml
@@ -0,0 +1,30 @@
+name: SwiftUICrashTest
+createIntermediateGroups: true
+generateEmptyDirectories: true
+configs:
+  Debug: debug
+  Release: release
+projectReferences:
+  Sentry:
+    path: ../../Sentry.xcodeproj
+fileGroups:
+  - SwiftUICrashTest.yml
+options:
+  bundleIdPrefix: io.sentry
+targets:
+  SwiftUICrashTest:
+    type: application
+    platform: auto
+    dependencies:
+      - target: Sentry/Sentry
+    sources:
+      - SwiftUICrashTest
+    configFiles:
+      Debug: SwiftUICrashTest.xcconfig
+      Release: SwiftUICrashTest.xcconfig
+
+schemes:
+  SwiftUICrashTest:
+    build:
+      targets:
+        SwiftUICrashTest: all
diff --git a/TestSamples/SwiftUICrashTest/SwiftUICrashTest/Assets.xcassets/AccentColor.colorset/Contents.json b/TestSamples/SwiftUICrashTest/SwiftUICrashTest/Assets.xcassets/AccentColor.colorset/Contents.json
@@ -0,0 +1,11 @@
+{
+  "colors": [
+    {
+      "idiom": "universal"
+    }
+  ],
+  "info": {
+    "author": "xcode",
+    "version": 1
+  }
+}
diff --git a/TestSamples/SwiftUICrashTest/SwiftUICrashTest/Assets.xcassets/AppIcon.appiconset/Contents.json b/TestSamples/SwiftUICrashTest/SwiftUICrashTest/Assets.xcassets/AppIcon.appiconset/Contents.json
@@ -0,0 +1,35 @@
+{
+  "images": [
+    {
+      "idiom": "universal",
+      "platform": "ios",
+      "size": "1024x1024"
+    },
+    {
+      "appearances": [
+        {
+          "appearance": "luminosity",
+          "value": "dark"
+        }
+      ],
+      "idiom": "universal",
+      "platform": "ios",
+      "size": "1024x1024"
+    },
+    {
+      "appearances": [
+        {
+          "appearance": "luminosity",
+          "value": "tinted"
+        }
+      ],
+      "idiom": "universal",
+      "platform": "ios",
+      "size": "1024x1024"
+    }
+  ],
+  "info": {
+    "author": "xcode",
+    "version": 1
+  }
+}
diff --git a/TestSamples/SwiftUICrashTest/SwiftUICrashTest/Assets.xcassets/Contents.json b/TestSamples/SwiftUICrashTest/SwiftUICrashTest/Assets.xcassets/Contents.json
@@ -0,0 +1,6 @@
+{
+  "info": {
+    "author": "xcode",
+    "version": 1
+  }
+}
diff --git a/TestSamples/SwiftUICrashTest/SwiftUICrashTest/ContentView.swift b/TestSamples/SwiftUICrashTest/SwiftUICrashTest/ContentView.swift
@@ -0,0 +1,18 @@
+import Sentry
+import SwiftUI
+
+struct ContentView: View {
+    var body: some View {
+        VStack {
+            Image(systemName: "globe")
+                .imageScale(.large)
+                .foregroundStyle(.tint)
+            Text("Hello, world!")
+        }
+        .padding()
+    }
+}
+
+#Preview {
+    ContentView()
+}
diff --git a/TestSamples/SwiftUICrashTest/SwiftUICrashTest/SwiftUICrashTestApp.swift b/TestSamples/SwiftUICrashTest/SwiftUICrashTest/SwiftUICrashTestApp.swift
@@ -0,0 +1,25 @@
+import Sentry
+import SwiftUI
+
+@main
+struct SwiftUICrashTestApp: App {
+
+    init() {
+        SentrySDK.start { options in
+            options.dsn = "https://6cc9bae94def43cab8444a99e0031c28@o447951.ingest.sentry.io/5428557"
+        }
+
+        let userDefaultsKey = "crash-on-launch"
+        if UserDefaults.standard.bool(forKey: userDefaultsKey) {
+
+            UserDefaults.standard.removeObject(forKey: userDefaultsKey)
+            SentrySDK.crash()
+        }
+    }
+
+    var body: some Scene {
+        WindowGroup {
+            ContentView()
+        }
+    }
+}
diff --git a/TestSamples/SwiftUICrashTest/test-crash-and-relaunch.sh b/TestSamples/SwiftUICrashTest/test-crash-and-relaunch.sh
@@ -0,0 +1,138 @@
+#!/bin/bash
+
+set -euo pipefail
+
+# Launches the SwiftUI Crash Test app and validates that it crashes and relaunches correctly.
+# This test run requires one booted simulator to work. So make sure to boot one simulator before
+# running this script.
+
+# Background:
+# XCTest isn't built for crashing during tests. Instead of using XCTest to press a button and
+# let a test app crash, we now use UserDefaults to tell the test app to crash during launch.
+# We then simply launch the app again via `xcrun simctl launch` and wait to see if it keeps
+# running. This is basically the same as the testCrash of the SwiftUITestSample without using
+# XCTests.
+
+
+BUNDLE_ID="io.sentry.tests.SwiftUICrashTest"
+USER_DEFAULT_KEY="crash-on-launch"
+DEVICE_ID="booted"
+SCREENSHOTS_DIR="test-crash-and-relaunch-simulator-screenshots"
+
+usage() {
+    echo "Usage: $0"
+    echo "  -s|--screenshots-dir <dir>      Screenshots directory (default: test-crash-and-relaunch-simulator-screenshots)"
+    echo "  -h|--help                       Show this help message"
+    exit 1
+}
+
+# Parse arguments
+while [[ $# -gt 0 ]]; do
+    case $1 in
+        -s|--screenshots-dir)
+            SCREENSHOTS_DIR="$2"
+            shift 2
+            ;;
+        -h|--help)
+            usage
+            ;;
+        *)
+            echo "Unknown option: $1"
+            usage
+            ;;
+    esac
+done
+
+# Echo with timestamp
+log() {
+    echo "[$(date '+%H:%M:%S')] $1"
+}
+
+# Take screenshot with timestamp and custom name
+take_simulator_screenshot() {
+    local name="$1"
+    
+    # Create screenshots directory if it doesn't exist
+    mkdir -p "$SCREENSHOTS_DIR"
+    
+    # Generate timestamp-based filename with custom name
+    timestamp=$(date '+%H%M%S')
+    screenshot_name="$SCREENSHOTS_DIR/${timestamp}_${name}.png"
+    
+    # Take screenshot
+    xcrun simctl io booted screenshot "$screenshot_name" 2>/dev/null || true
+}
+
+log "Removing previous screenshots directory."
+rm -rf "$SCREENSHOTS_DIR"
+
+log "Starting crash test and relaunch test."
+log "This test crashes the app and validates that it can relaunch after a crash without crashing again."
+
+log "🔨 Building SwiftUI Crash Test app for simulator 🔨"
+
+xcodebuild -workspace Sentry.xcworkspace \
+    -scheme SwiftUICrashTest \
+    -destination "platform=iOS Simulator,name=iPhone 16" \
+    -derivedDataPath DerivedData \
+    -configuration Debug \
+    CODE_SIGNING_REQUIRED=NO \
+    build 2>&1 | tee raw-build.log | xcbeautify
+
+log "Installing app on simulator."
+xcrun simctl install $DEVICE_ID DerivedData/Build/Products/Debug-iphonesimulator/SwiftUICrashTest.app
+
+take_simulator_screenshot "after-install"
+
+log "Terminating app if running."
+xcrun simctl terminate $DEVICE_ID $BUNDLE_ID 2>/dev/null || true
+
+# Phase 1: Let the app crash
+
+log "Setting crash flag."
+xcrun simctl spawn $DEVICE_ID defaults write $BUNDLE_ID $USER_DEFAULT_KEY -bool true
+
+log "Launching app with expected crash."
+xcrun simctl launch $DEVICE_ID $BUNDLE_ID
+
+# Check every 100ms for 5 seconds if the app is still running.
+for i in {1..50}; do
+    if xcrun simctl listapps $DEVICE_ID | grep "$BUNDLE_ID" | grep -q "Running"; then
+        sleep 0.1
+    else
+        log "✅ App crashed as expected after $(echo "scale=1; $i * 0.1" | bc) seconds."
+        break
+    fi
+    
+    if [ "$i" -eq 50 ]; then
+        log "❌ App is still running after 5 seconds but it should have crashed instead."
+        exit 1
+    fi
+done
+
+take_simulator_screenshot "after-crash"
+
+# Phase 2: Test normal operation
+
+log "Removing crash flag..."
+xcrun simctl spawn $DEVICE_ID defaults delete $BUNDLE_ID $USER_DEFAULT_KEY
+
+log "Relaunching app after crash."
+xcrun simctl launch $DEVICE_ID $BUNDLE_ID
+
+take_simulator_screenshot "after-crash-check"
+
+log "Waiting for 5 seconds to check if the app is still running."
+sleep 5
+
+take_simulator_screenshot "after-crash-check-after-sleep"
+
+if xcrun simctl spawn booted launchctl list | grep "$BUNDLE_ID"; then
+    log "✅ App is still running"
+else
+    log "❌ App is not running"    
+    exit 1
+fi
+
+log "✅ Test completed successfully." 
+exit 0
diff --git a/fastlane/Fastfile b/fastlane/Fastfile
@@ -200,14 +200,6 @@ platform :ios do
     )
   end
 
-  lane :ui_critical_tests_ios_swiftui_crash do
-    run_ui_tests(
-      scheme: "SwiftUITestSampleCrash",
-      result_bundle_name: "ui_critical_tests_ios_swiftui_crash",
-      device: "iPhone 16 (18.5)"
-    )
-  end
-
   lane :ui_critical_tests_ios_swiftui_envelope do
     run_ui_tests(
       scheme: "SwiftUITestSampleEnvelope",
