Merge branch 'kw-full-e2e-matrix' into kw-explore-new-e2e

.github/workflows/eval-maestro-tests.yml

@@ -21,59 +21,136 @@ concurrency:
  cancel-in-progress: true
 
 jobs:
- build-swift-ui:
- name: Maestro Build SwiftUI for UI Tests Simulator
- runs-on: macos-13
+ build-sample-for-maestro:
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - sample-name: iOS-ObjectiveC
+ lane_name: build_ios_objc_for_sim_tests
+ artifact-name: maestro-ios-objc
+ app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-ObjectiveC.app
+ - sample-name: iOS-Swift
+ lane_name: build_ios_swift_for_sim_tests
+ artifact-name: maestro-ios-swift
+ app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-Swift.app
+ - sample-name: iOS-SwiftUI
+ lane_name: build_ios_swift_ui_for_sim_tests
+ artifact-name: maestro-ios-swift-ui
+ app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-SwiftUI.app
+ name: Build ${{matrix.sample-name}} Sample
+ runs-on: macos-14
 
  steps:
  - uses: actions/checkout@v4
- - run: ./scripts/ci-select-xcode.sh 14.3
+ - run: ./scripts/ci-select-xcode.sh 15.4
  - uses: ruby/setup-ruby@v1
  with:
  bundler-cache: true
- - run: bundle exec fastlane build_ios_swift_ui_for_tests
+ - run: bundle exec fastlane ${{matrix.lane_name}}
  - uses: actions/upload-artifact@v4
  with:
- name: maestro-ios-swift-ui-for-tests
- path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-SwiftUI.app
+ name: ${{matrix.artifact-name}}
+ path: ${{matrix.app_path}}
 
  eval-maestro-swift-ui-tests:
- name: Maestro UI Tests for SwiftUI on ${{matrix.device}} Simulator
- needs: [build-swift-ui]
+ name: Test ${{matrix.sample-name}} on iOS ${{matrix.os-version}} on ${{matrix.device}} Simulator
+ needs: [build-sample-for-maestro]
  runs-on: ${{matrix.runs-on}}
  strategy:
  fail-fast: false
  matrix:
  include:
- # - runs-on: macos-12
- # xcode: "13.4.1"
+ - sample-name: iOS-ObjectiveC
+ artifact-name: maestro-ios-objc
+ app: "Sample.app"
+ test_suit: "Samples/iOS-ObjectiveC/.maestro"
+ runs-on: macos-14
+ xcode: "14.3.1"
+ device: "iPhone 14"
+ os-version: "16.4"
+
+ - sample-name: iOS-SwiftUI
+ artifact-name: maestro-ios-swift-ui
+ app: "Sample.app"
+ test_suit: "Samples/iOS-SwiftUI/.maestro"
+ runs-on: macos-12
+ xcode: "13.4.1"
+ device: "iPhone 8"
+ os-version: "15.5"
+ - sample-name: iOS-SwiftUI
+ artifact-name: maestro-ios-swift-ui
+ app: "Sample.app"
+ test_suit: "Samples/iOS-SwiftUI/.maestro"
+ runs-on: macos-14
+ xcode: "14.3.1"
+ device: "iPhone 14"
+ os-version: "16.4"
 
- - runs-on: macos-14
+ - sample-name: iOS-Swift
+ artifact-name: maestro-ios-swift
+ app: "Sample.app"
+ test_suit: "Samples/iOS-Swift/.maestro"
+ runs-on: macos-12
+ xcode: "13.4.1"
+ device: "iPhone 8"
+ os-version: "15.5"
+ - sample-name: iOS-Swift
+ artifact-name: maestro-ios-swift
+ app: "Sample.app"
+ test_suit: "Samples/iOS-Swift/.maestro"
+ runs-on: macos-13
+ xcode: "14.3.1"
+ device: "iPhone 14"
+ os-version: "16.4"
+ - sample-name: iOS-Swift
+ artifact-name: maestro-ios-swift
+ app: "Sample.app"
+ test_suit: "Samples/iOS-Swift/.maestro"
+ runs-on: macos-14
  xcode: "15.4"
  device: "iPhone 15"
- os_version: "17.2"
- app: "iOS-SwiftUI.app"
+ os-version: "17.5"
 
  steps:
  - uses: actions/checkout@v4
- - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}}
+ - name: Create ${{matrix.device}} (${{matrix.os-version}}) Simulator using Xcode ${{matrix.xcode}}
+ run: ./scripts/create-simulator.sh "${{matrix.xcode}}" "${{matrix.os-version}}" "${{matrix.device}}"
  - name: Install Maestro
- run: brew tap mobile-dev-inc/tap && mobile-dev-inc/tap/maestro@1.38
+ run: brew tap mobile-dev-inc/tap && brew install mobile-dev-inc/tap/maestro@1.38
  - name: Install iDB Companion
  run: brew tap facebook/fb && brew install facebook/fb/idb-companion
+
  - uses: actions/download-artifact@v4
  with:
- name: maestro-ios-swift-ui-for-tests
+ name: ${{matrix.artifact-name}}
  path: ${{ matrix.app }}
  - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3
  with:
  model: ${{matrix.device}}
- os_version: ${{matrix.os_version}}
+ os_version: ${{matrix.os-version}}
  - name: Run Maestro Flows
- run: |
+ env:
  # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57
- export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow
-
+ MAESTRO_DRIVER_STARTUP_TIMEOUT: 1500000 # 25 min, CI can be slow at times
+ MAX_ATTEMPTS: 3 # Maestro test suits re runs in case of flakyness
+ run: |
  xcrun simctl install booted ${{ matrix.app}}
 
- maestro test .maestro --format junit
+ # Avoid exit after command fails
+ set +e
+ # Add retries for flakyness
+ CURR_ATTEMPT=0
+ RESULT=1
+ while [[ $CURR_ATTEMPT -lt $MAX_ATTEMPTS ]] && [[ $RESULT -ne 0 ]]; do
+ maestro test ${{matrix.test_suit}} --format junit --debug-output .logs
+ RESULT=$?
+ done
+ exit $RESULT
+
+ - name: Store Maestro Logs
+ uses: actions/upload-artifact@v4
+ if: failure()
+ with:
+ name: maestro-logs-${{matrix.sample-name}}-${{matrix.device}}-${{matrix.os-version}}
+ path: .logs

.github/workflows/ui-tests.yml

@@ -79,9 +79,9 @@ jobs:
  path: |
  ~/Library/Logs/scan/*.log
  ./fastlane/test_output/**
- 
- ios-swift-ui-tests:
- name: iOS-Swift UI Tests ${{matrix.device}}
+
+ ui-tests-swift:
+ name: UI Tests for iOS-Swift ${{matrix.device}} Simulator
  runs-on: ${{matrix.runs-on}}
  strategy:
  fail-fast: false
@@ -105,10 +105,10 @@ jobs:
 
  - name: Create iOS 16.4 simulator
  if: ${{ matrix.device == 'iPhone 14 (16.4)' }}
- run: ./scripts/create-simulator.sh 14.3.1 16.4 16-4
+ run: ./scripts/create-simulator.sh 14.3.1 16.4 "iPhone 14"
 
  - name: Run Fastlane
- run: fastlane ui_tests_ios_swift device:"${{matrix.device}}" 
+ run: fastlane ui_tests_ios_swift device:"${{matrix.device}}"
 
  - name: Archiving Raw Test Logs
  uses: actions/upload-artifact@v4

.gitignore

@@ -81,3 +81,6 @@ Package.resolved
 
 Cartfile.resolved
 *.log
+
+# Maestro Logs
+.logs/.maestro/

Samples/iOS-ObjectiveC/.maestro/crash.yaml

@@ -0,0 +1,5 @@
+appId: io.sentry.iOS-ObjectiveC
+---
+- launchApp
+- tapOn: "crash"
+- launchApp

Samples/iOS-Swift/.maestro/breadcrumbData.yaml

@@ -0,0 +1,5 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Extra"
+- assertVisible: "{ category: ui.lifecycle, parentViewController: UITabBarController, beingPresented: false, window_isKeyWindow: true, is_window_rootViewController: false }"

Samples/iOS-Swift/.maestro/captureError.yaml

@@ -0,0 +1,4 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Capture Error"

Samples/iOS-Swift/.maestro/captureException.yaml

@@ -0,0 +1,4 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Capture NSException"

Samples/iOS-Swift/.maestro/checkFramesCount.yaml

@@ -0,0 +1,8 @@
+appId: io.sentry.sample.iOS-Swift
+env:
+ MAESTRO_FLOW_NAME: "Check Frames Proportions"
+---
+- launchApp
+- tapOn: "Extra"
+- tapOn: "Frames Count"
+- assertVisible: "Frames Count OK"

Samples/iOS-Swift/.maestro/checkFramesProportions.yaml

@@ -0,0 +1,8 @@
+appId: io.sentry.sample.iOS-Swift
+env:
+ MAESTRO_FLOW_NAME: "Check Frames Proportions"
+---
+- launchApp
+- tapOn: "Extra"
+- tapOn: "Frames Proportions"
+- assertVisible: "Frames Proportions OK"

Samples/iOS-Swift/.maestro/corruptEnvelope.yaml

@@ -0,0 +1,9 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Extra"
+- tapOn: "Corrupt Envelope"
+- tapOn: "Close SDK"
+- tapOn: "Corrupt Envelope"
+- stopApp
+- launchApp

Samples/iOS-Swift/.maestro/crash.yml

@@ -0,0 +1,5 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Crash the app"
+- launchApp

Samples/iOS-Swift/.maestro/showImage.yaml

@@ -0,0 +1,6 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Load Image Controller"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showNib.yaml

@@ -0,0 +1,7 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Nib Controller"
+- assertVisible: "a lonely button"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showSplitView.yaml

@@ -0,0 +1,6 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Split Controller"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showTableView.yaml

@@ -0,0 +1,7 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Table Controller"
+- assertVisible: "Gradient Table View"
+- assertVisible: "ASSERT: SUCCESS"

Samples/iOS-Swift/.maestro/showTextView.yaml

@@ -0,0 +1,8 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "Lorem Ipsum Controller"
+- copyTextFrom:
+ id: "LoremIpsumTextView"
+- assertTrue: "maestro.copiedText.startsWith('Lorem ipsum dolor sit amet')"

Samples/iOS-Swift/.maestro/uiClickTransaction.yaml

@@ -0,0 +1,5 @@
+appId: io.sentry.sample.iOS-Swift
+---
+- launchApp
+- tapOn: "Transactions"
+- tapOn: "UIClick Transaction"

Samples/iOS-Swift/.maestro/useAfterFreeAfterUIImageNamedEmptyString.yaml

@@ -0,0 +1,10 @@
+appId: io.sentry.sample.iOS-Swift
+---
+# We received a customer report that ASAN reports a use-after-free error after
+# calling UIImage(named:) with an empty string argument. Recording another
+# transaction leads to the ASAN error.
+- launchApp
+# this primes the state required according to the customer report, by setting a UIImageView.image property to a UIImage(named: "")
+- tapOn: "Use-after-free"
+# this causes another transaction to be recorded which hits the codepath necessary for the ASAN to trip
+- tapOn: "Extra"