diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml new file mode 100644 index 00000000000..7792e15660c --- /dev/null +++ b/.github/workflows/eval-maestro-tests.yml @@ -0,0 +1,156 @@ +name: Maestro UI Tests +on: + push: + branches: + - main + + pull_request: + paths: + - 'Sources/**' + - 'Tests/**' + - '.github/workflows/ui-tests.yml' + - 'fastlane/**' + - '.sauce/config.yml' + - 'scripts/ci-select-xcode.sh' + - 'Samples/iOS-Swift/**' + + +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-sample-for-maestro: + strategy: + fail-fast: false + matrix: + include: + - sample-name: iOS-ObjectiveC + lane_name: build_ios_objc_for_sim_tests + artifact-name: maestro-ios-objc + app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-ObjectiveC.app + - sample-name: iOS-Swift + lane_name: build_ios_swift_for_sim_tests + artifact-name: maestro-ios-swift + app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-Swift.app + - sample-name: iOS-SwiftUI + lane_name: build_ios_swift_ui_for_sim_tests + artifact-name: maestro-ios-swift-ui + app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-SwiftUI.app + name: Build ${{matrix.sample-name}} Sample + runs-on: macos-14 + + steps: + - uses: actions/checkout@v4 + - run: ./scripts/ci-select-xcode.sh 15.4 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - run: bundle exec fastlane ${{matrix.lane_name}} + - uses: actions/upload-artifact@v4 + with: + name: ${{matrix.artifact-name}} + path: ${{matrix.app_path}} + + eval-maestro-swift-ui-tests: + name: Test ${{matrix.sample-name}} on iOS ${{matrix.os-version}} on ${{matrix.device}} Simulator + needs: [build-sample-for-maestro] + runs-on: ${{matrix.runs-on}} + strategy: + fail-fast: false + matrix: + include: + - sample-name: iOS-ObjectiveC + artifact-name: maestro-ios-objc + app: "Sample.app" + test_suit: "Samples/iOS-ObjectiveC/.maestro" + runs-on: macos-14 + xcode: "14.3.1" + device: "iPhone 14" + os-version: "16.4" + + - sample-name: iOS-SwiftUI + artifact-name: maestro-ios-swift-ui + app: "Sample.app" + test_suit: "Samples/iOS-SwiftUI/.maestro" + runs-on: macos-12 + xcode: "13.4.1" + device: "iPhone 8" + os-version: "15.5" + - sample-name: iOS-SwiftUI + artifact-name: maestro-ios-swift-ui + app: "Sample.app" + test_suit: "Samples/iOS-SwiftUI/.maestro" + runs-on: macos-14 + xcode: "14.3.1" + device: "iPhone 14" + os-version: "16.4" + + - sample-name: iOS-Swift + artifact-name: maestro-ios-swift + app: "Sample.app" + test_suit: "Samples/iOS-Swift/.maestro" + runs-on: macos-12 + xcode: "13.4.1" + device: "iPhone 8" + os-version: "15.5" + - sample-name: iOS-Swift + artifact-name: maestro-ios-swift + app: "Sample.app" + test_suit: "Samples/iOS-Swift/.maestro" + runs-on: macos-13 + xcode: "14.3.1" + device: "iPhone 14" + os-version: "16.4" + - sample-name: iOS-Swift + artifact-name: maestro-ios-swift + app: "Sample.app" + test_suit: "Samples/iOS-Swift/.maestro" + runs-on: macos-14 + xcode: "15.4" + device: "iPhone 15" + os-version: "17.5" + + steps: + - uses: actions/checkout@v4 + - name: Create ${{matrix.device}} (${{matrix.os-version}}) Simulator using Xcode ${{matrix.xcode}} + run: ./scripts/create-simulator.sh "${{matrix.xcode}}" "${{matrix.os-version}}" "${{matrix.device}}" + - name: Install Maestro + run: brew tap mobile-dev-inc/tap && brew install mobile-dev-inc/tap/maestro@1.38 + - name: Install iDB Companion + run: brew tap facebook/fb && brew install facebook/fb/idb-companion + + - uses: actions/download-artifact@v4 + with: + name: ${{matrix.artifact-name}} + path: ${{ matrix.app }} + - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3 + with: + model: ${{matrix.device}} + os_version: ${{matrix.os-version}} + - name: Run Maestro Flows + env: + # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57 + MAESTRO_DRIVER_STARTUP_TIMEOUT: 1500000 # 25 min, CI can be slow at times + MAX_ATTEMPTS: 3 # Maestro test suits re runs in case of flakyness + run: | + xcrun simctl install booted ${{ matrix.app}} + + # Avoid exit after command fails + set +e + # Add retries for flakyness + CURR_ATTEMPT=0 + RESULT=1 + while [[ $CURR_ATTEMPT -lt $MAX_ATTEMPTS ]] && [[ $RESULT -ne 0 ]]; do + maestro test ${{matrix.test_suit}} --format junit --debug-output .logs + RESULT=$? + done + exit $RESULT + + - name: Store Maestro Logs + uses: actions/upload-artifact@v4 + if: failure() + with: + name: maestro-logs-${{matrix.sample-name}}-${{matrix.device}}-${{matrix.os-version}} + path: .logs diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index b51d55ac9fd..a9468125d0c 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -79,9 +79,9 @@ jobs: path: | ~/Library/Logs/scan/*.log ./fastlane/test_output/** - - ios-swift-ui-tests: - name: iOS-Swift UI Tests ${{matrix.device}} + + ui-tests-swift: + name: UI Tests for iOS-Swift ${{matrix.device}} Simulator runs-on: ${{matrix.runs-on}} strategy: fail-fast: false @@ -105,10 +105,10 @@ jobs: - name: Create iOS 16.4 simulator if: ${{ matrix.device == 'iPhone 14 (16.4)' }} - run: ./scripts/create-simulator.sh 14.3.1 16.4 16-4 + run: ./scripts/create-simulator.sh 14.3.1 16.4 "iPhone 14" - name: Run Fastlane - run: fastlane ui_tests_ios_swift device:"${{matrix.device}}" + run: fastlane ui_tests_ios_swift device:"${{matrix.device}}" - name: Archiving Raw Test Logs uses: actions/upload-artifact@v4 diff --git a/.gitignore b/.gitignore index 4773a516c44..4412ff87a76 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ Package.resolved Cartfile.resolved *.log + +# Maestro Logs +.logs/.maestro/ diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a5a5fdb7d3..8561c2ed7e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: fail_fast: true args: - "check-versions" - + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: @@ -20,6 +20,7 @@ repos: - id: check-symlinks - id: check-xml - id: check-yaml + args: [--allow-multiple-documents] - id: detect-private-key - id: end-of-file-fixer - id: no-commit-to-branch @@ -30,7 +31,7 @@ repos: - id: check-github-actions - id: check-github-workflows args: [--verbose] - + - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.10.0.1 hooks: @@ -45,7 +46,7 @@ repos: types_or: ["objective-c", "objective-c++", "c", "c++"] args: - "format-clang" - + - id: format-swift name: Format Swift entry: make diff --git a/Samples/iOS-ObjectiveC/.maestro/crash.yaml b/Samples/iOS-ObjectiveC/.maestro/crash.yaml new file mode 100644 index 00000000000..0c1f25a24e1 --- /dev/null +++ b/Samples/iOS-ObjectiveC/.maestro/crash.yaml @@ -0,0 +1,5 @@ +appId: io.sentry.iOS-ObjectiveC +--- +- launchApp +- tapOn: "crash" +- launchApp diff --git a/Samples/iOS-Swift/.maestro/breadcrumbData.yaml b/Samples/iOS-Swift/.maestro/breadcrumbData.yaml new file mode 100644 index 00000000000..9714333408b --- /dev/null +++ b/Samples/iOS-Swift/.maestro/breadcrumbData.yaml @@ -0,0 +1,5 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Extra" +- assertVisible: "{ category: ui.lifecycle, parentViewController: UITabBarController, beingPresented: false, window_isKeyWindow: true, is_window_rootViewController: false }" diff --git a/Samples/iOS-Swift/.maestro/captureError.yaml b/Samples/iOS-Swift/.maestro/captureError.yaml new file mode 100644 index 00000000000..d089c9331e2 --- /dev/null +++ b/Samples/iOS-Swift/.maestro/captureError.yaml @@ -0,0 +1,4 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Capture Error" diff --git a/Samples/iOS-Swift/.maestro/captureException.yaml b/Samples/iOS-Swift/.maestro/captureException.yaml new file mode 100644 index 00000000000..7c2f05f304f --- /dev/null +++ b/Samples/iOS-Swift/.maestro/captureException.yaml @@ -0,0 +1,4 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Capture NSException" diff --git a/Samples/iOS-Swift/.maestro/checkFramesCount.yaml b/Samples/iOS-Swift/.maestro/checkFramesCount.yaml new file mode 100644 index 00000000000..e41fc81f5da --- /dev/null +++ b/Samples/iOS-Swift/.maestro/checkFramesCount.yaml @@ -0,0 +1,8 @@ +appId: io.sentry.sample.iOS-Swift +env: + MAESTRO_FLOW_NAME: "Check Frames Proportions" +--- +- launchApp +- tapOn: "Extra" +- tapOn: "Frames Count" +- assertVisible: "Frames Count OK" diff --git a/Samples/iOS-Swift/.maestro/checkFramesProportions.yaml b/Samples/iOS-Swift/.maestro/checkFramesProportions.yaml new file mode 100644 index 00000000000..bc738cbdb46 --- /dev/null +++ b/Samples/iOS-Swift/.maestro/checkFramesProportions.yaml @@ -0,0 +1,8 @@ +appId: io.sentry.sample.iOS-Swift +env: + MAESTRO_FLOW_NAME: "Check Frames Proportions" +--- +- launchApp +- tapOn: "Extra" +- tapOn: "Frames Proportions" +- assertVisible: "Frames Proportions OK" diff --git a/Samples/iOS-Swift/.maestro/corruptEnvelope.yaml b/Samples/iOS-Swift/.maestro/corruptEnvelope.yaml new file mode 100644 index 00000000000..fe8f53b171d --- /dev/null +++ b/Samples/iOS-Swift/.maestro/corruptEnvelope.yaml @@ -0,0 +1,9 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Extra" +- tapOn: "Corrupt Envelope" +- tapOn: "Close SDK" +- tapOn: "Corrupt Envelope" +- stopApp +- launchApp diff --git a/Samples/iOS-Swift/.maestro/crash.yml b/Samples/iOS-Swift/.maestro/crash.yml new file mode 100644 index 00000000000..14628f22deb --- /dev/null +++ b/Samples/iOS-Swift/.maestro/crash.yml @@ -0,0 +1,5 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Crash the app" +- launchApp diff --git a/Samples/iOS-Swift/.maestro/showImage.yaml b/Samples/iOS-Swift/.maestro/showImage.yaml new file mode 100644 index 00000000000..475f051c640 --- /dev/null +++ b/Samples/iOS-Swift/.maestro/showImage.yaml @@ -0,0 +1,6 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Transactions" +- tapOn: "Load Image Controller" +- assertVisible: "ASSERT: SUCCESS" diff --git a/Samples/iOS-Swift/.maestro/showNib.yaml b/Samples/iOS-Swift/.maestro/showNib.yaml new file mode 100644 index 00000000000..f4e5822bd43 --- /dev/null +++ b/Samples/iOS-Swift/.maestro/showNib.yaml @@ -0,0 +1,7 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Transactions" +- tapOn: "Nib Controller" +- assertVisible: "a lonely button" +- assertVisible: "ASSERT: SUCCESS" diff --git a/Samples/iOS-Swift/.maestro/showSplitView.yaml b/Samples/iOS-Swift/.maestro/showSplitView.yaml new file mode 100644 index 00000000000..e2f1f4bb27b --- /dev/null +++ b/Samples/iOS-Swift/.maestro/showSplitView.yaml @@ -0,0 +1,6 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Transactions" +- tapOn: "Split Controller" +- assertVisible: "ASSERT: SUCCESS" diff --git a/Samples/iOS-Swift/.maestro/showTableView.yaml b/Samples/iOS-Swift/.maestro/showTableView.yaml new file mode 100644 index 00000000000..1fd55840bfa --- /dev/null +++ b/Samples/iOS-Swift/.maestro/showTableView.yaml @@ -0,0 +1,7 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Transactions" +- tapOn: "Table Controller" +- assertVisible: "Gradient Table View" +- assertVisible: "ASSERT: SUCCESS" diff --git a/Samples/iOS-Swift/.maestro/showTextView.yaml b/Samples/iOS-Swift/.maestro/showTextView.yaml new file mode 100644 index 00000000000..ddab6175b86 --- /dev/null +++ b/Samples/iOS-Swift/.maestro/showTextView.yaml @@ -0,0 +1,8 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Transactions" +- tapOn: "Lorem Ipsum Controller" +- copyTextFrom: + id: "LoremIpsumTextView" +- assertTrue: "maestro.copiedText.startsWith('Lorem ipsum dolor sit amet')" diff --git a/Samples/iOS-Swift/.maestro/uiClickTransaction.yaml b/Samples/iOS-Swift/.maestro/uiClickTransaction.yaml new file mode 100644 index 00000000000..a561339e617 --- /dev/null +++ b/Samples/iOS-Swift/.maestro/uiClickTransaction.yaml @@ -0,0 +1,5 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Transactions" +- tapOn: "UIClick Transaction" diff --git a/Samples/iOS-Swift/.maestro/useAfterFreeAfterUIImageNamedEmptyString.yaml b/Samples/iOS-Swift/.maestro/useAfterFreeAfterUIImageNamedEmptyString.yaml new file mode 100644 index 00000000000..d56d26562d6 --- /dev/null +++ b/Samples/iOS-Swift/.maestro/useAfterFreeAfterUIImageNamedEmptyString.yaml @@ -0,0 +1,10 @@ +appId: io.sentry.sample.iOS-Swift +--- +# We received a customer report that ASAN reports a use-after-free error after +# calling UIImage(named:) with an empty string argument. Recording another +# transaction leads to the ASAN error. +- launchApp +# this primes the state required according to the customer report, by setting a UIImageView.image property to a UIImage(named: "") +- tapOn: "Use-after-free" +# this causes another transaction to be recorded which hits the codepath necessary for the ASAN to trip +- tapOn: "Extra" diff --git a/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard b/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard index 971b70b2a4a..ee4385a3a81 100644 --- a/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard +++ b/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard @@ -698,6 +698,7 @@ + @@ -903,10 +904,10 @@ - + - + + - + + + - + +