From 94da44ecf8f72bb215854a00a608dcac76c43b97 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Thu, 3 Oct 2024 13:33:52 +0200 Subject: [PATCH 01/23] Add CI step to build and test swift ui example using maestro --- .github/workflows/eval-maestro-tests.yml | 64 ++++++++++++++++++++++++ .pre-commit-config.yaml | 7 +-- fastlane/Fastfile | 19 ++++++- maestro/crash.yaml | 5 ++ 4 files changed, 91 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/eval-maestro-tests.yml create mode 100644 maestro/crash.yaml diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml new file mode 100644 index 00000000000..0af547c3f8e --- /dev/null +++ b/.github/workflows/eval-maestro-tests.yml @@ -0,0 +1,64 @@ +name: UI Tests +on: + push: + branches: + - main + + pull_request: + paths: + - 'Sources/**' + - 'Tests/**' + - '.github/workflows/ui-tests.yml' + - 'fastlane/**' + - '.sauce/config.yml' + - 'scripts/ci-select-xcode.sh' + - 'Samples/iOS-Swift/**' + + +# https://docs.github.com/en/actions/using-jobs/using-concurrency#example-using-a-fallback-value +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +jobs: + build-swift-ui: + name: Maestro Build SwiftUI for UI Tests Simulator + runs-on: macos-13 + + steps: + - uses: actions/checkout@v4 + - run: ./scripts/ci-select-xcode.sh 14.3 + - uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - run: bundle exec fastlane build_ios_swift_ui_for_tests + - uses: actions/upload-artifact@v4 + with: + name: maestro-ios-swift-ui-for-tests + path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-SwiftUI.app + + eval-maestro-swift-ui-tests: + name: Maestro UI Tests for SwiftUI on ${{matrix.device}} Simulator + needs: [build-swift-ui] + runs-on: ${{matrix.runs-on}} + strategy: + fail-fast: false + matrix: + include: + # - runs-on: macos-12 + # xcode: "13.4.1" + + - runs-on: macos-13 + xcode: "14.3" + + steps: + - uses: actions/checkout@v4 + - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} + - run: which maestro || brew tap mobile-dev-inc/tap && brew install maestro + - run: brew tap facebook/fb && brew install idb-companion + - uses: actions/download-artifact@v4 + with: + name: maestro-ios-swift-ui-for-tests + path: iOS-SwiftUI.app + - run: maestro start-device --platform ios --os-version 16 + - run: maestro test crash.yaml diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7a5a5fdb7d3..8561c2ed7e4 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -11,7 +11,7 @@ repos: fail_fast: true args: - "check-versions" - + - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.3.0 hooks: @@ -20,6 +20,7 @@ repos: - id: check-symlinks - id: check-xml - id: check-yaml + args: [--allow-multiple-documents] - id: detect-private-key - id: end-of-file-fixer - id: no-commit-to-branch @@ -30,7 +31,7 @@ repos: - id: check-github-actions - id: check-github-workflows args: [--verbose] - + - repo: https://github.com/shellcheck-py/shellcheck-py rev: v0.10.0.1 hooks: @@ -45,7 +46,7 @@ repos: types_or: ["objective-c", "objective-c++", "c", "c++"] args: - "format-clang" - + - id: format-swift name: Format Swift entry: make diff --git a/fastlane/Fastfile b/fastlane/Fastfile index c8e396ba27f..3a3bb33ce3a 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -21,7 +21,7 @@ platform :ios do ) new_version = version.split("-", -1)[0] - # We also need to replace the MARKETING_VERSION otherwise the build will fail with + # We also need to replace the MARKETING_VERSION otherwise the build will fail with # error: The CFBundleShortVersionString of an App Clip ('8.9.0-beta.1') must match that of its containing parent app ('8.9.0'). sh "sed -i '' 's/MARKETING_VERSION = #{version}/MARKETING_VERSION = #{new_version}/g' ../Samples/iOS-Swift/iOS-Swift.xcodeproj/project.pbxproj" @@ -120,6 +120,23 @@ platform :ios do delete_keychain(name: "fastlane_tmp_keychain") unless is_ci end + lane :build_ios_swift_ui_for_tests do + + setup_ci( + force: true + ) + + build_app( + workspace: "Sentry.xcworkspace", + scheme: "iOS-SwiftUI", + configuration: "Debug", + derived_data_path: "DerivedData", + destination: "generic/platform=iOS Simulator", + xcargs: "-sdk 'iphonesimulator'", + skip_archive: true + ) + end + lane :build_ios_swift_ui_test do setup_ci( diff --git a/maestro/crash.yaml b/maestro/crash.yaml new file mode 100644 index 00000000000..3e35679e6ac --- /dev/null +++ b/maestro/crash.yaml @@ -0,0 +1,5 @@ +appId: io.sentry.iOS-SwiftUI +--- +- launchApp +- tapOn: "Crash" +- launchApp From b347de77cbb7a3cf080baff3d949281299470994 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Thu, 3 Oct 2024 13:37:33 +0200 Subject: [PATCH 02/23] Fix the workflow name --- .github/workflows/eval-maestro-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 0af547c3f8e..5837594e528 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -1,4 +1,4 @@ -name: UI Tests +name: Maestro UI Tests on: push: branches: From d1589ed7d0c6597be8370f10bdb2c330ba80aea2 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Thu, 3 Oct 2024 14:05:16 +0200 Subject: [PATCH 03/23] Fix booting simulator --- .github/workflows/eval-maestro-tests.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 5837594e528..9e2b0332c4b 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -60,5 +60,11 @@ jobs: with: name: maestro-ios-swift-ui-for-tests path: iOS-SwiftUI.app - - run: maestro start-device --platform ios --os-version 16 - - run: maestro test crash.yaml + - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3 + with: + model: 'iPhone 8' + os_version: '16.1' + - run: | + # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57 + export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow + maestro test maestro --format junit From 3ea3e8c248ccf818e1b9ab86c152b92097dc7ca5 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Thu, 3 Oct 2024 14:16:22 +0200 Subject: [PATCH 04/23] Use available ios simulator --- .github/workflows/eval-maestro-tests.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 9e2b0332c4b..178046af6ee 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -49,7 +49,9 @@ jobs: # xcode: "13.4.1" - runs-on: macos-13 - xcode: "14.3" + xcode: "15.4" + device: "iPhone 15" + os_version: "17.2" steps: - uses: actions/checkout@v4 From e5c20d92948c2a3b5f22992f679e04a543d6ed0a Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Thu, 3 Oct 2024 14:22:23 +0200 Subject: [PATCH 05/23] fix ios sim and xcode select --- .github/workflows/eval-maestro-tests.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 178046af6ee..10e13994d5b 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -48,7 +48,7 @@ jobs: # - runs-on: macos-12 # xcode: "13.4.1" - - runs-on: macos-13 + - runs-on: macos-14 xcode: "15.4" device: "iPhone 15" os_version: "17.2" @@ -64,8 +64,8 @@ jobs: path: iOS-SwiftUI.app - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3 with: - model: 'iPhone 8' - os_version: '16.1' + model: ${{matrix.device}} + os_version: ${{matrix.os_version}} - run: | # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57 export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow From 5bbccd082191d3d1ee86e9fa020cfce37db7e31d Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Thu, 3 Oct 2024 14:43:26 +0200 Subject: [PATCH 06/23] Fix install app before running tests --- .github/workflows/eval-maestro-tests.yml | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 10e13994d5b..edec6c630ee 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -52,6 +52,7 @@ jobs: xcode: "15.4" device: "iPhone 15" os_version: "17.2" + app: "iOS-SwiftUI.app" steps: - uses: actions/checkout@v4 @@ -61,12 +62,16 @@ jobs: - uses: actions/download-artifact@v4 with: name: maestro-ios-swift-ui-for-tests - path: iOS-SwiftUI.app + path: ${{ matrix.app }} - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3 with: model: ${{matrix.device}} os_version: ${{matrix.os_version}} - - run: | + - name: Maestro Test + run: | # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57 export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow + + xcrun simctl install booted ${{ matrix.app}} + maestro test maestro --format junit From c6c9d9610ac98d08299133e7909b6515dc72ca58 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Thu, 3 Oct 2024 17:12:08 +0200 Subject: [PATCH 07/23] move maestro to .maestro --- .github/workflows/eval-maestro-tests.yml | 10 ++++++---- {maestro => .maestro}/crash.yaml | 0 2 files changed, 6 insertions(+), 4 deletions(-) rename {maestro => .maestro}/crash.yaml (100%) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index edec6c630ee..e3f54a46dcf 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -57,8 +57,10 @@ jobs: steps: - uses: actions/checkout@v4 - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} - - run: which maestro || brew tap mobile-dev-inc/tap && brew install maestro - - run: brew tap facebook/fb && brew install idb-companion + - name: Install Maestro + run: brew tap mobile-dev-inc/tap && mobile-dev-inc/tap/maestro@1.38 + - name: Install iDB Companion + run: brew tap facebook/fb && brew install facebook/fb/idb-companion - uses: actions/download-artifact@v4 with: name: maestro-ios-swift-ui-for-tests @@ -67,11 +69,11 @@ jobs: with: model: ${{matrix.device}} os_version: ${{matrix.os_version}} - - name: Maestro Test + - name: Run Maestro Flows run: | # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57 export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow xcrun simctl install booted ${{ matrix.app}} - maestro test maestro --format junit + maestro test .maestro --format junit diff --git a/maestro/crash.yaml b/.maestro/crash.yaml similarity index 100% rename from maestro/crash.yaml rename to .maestro/crash.yaml From f946eedc5c87e91b37f06a785527daef910cc887 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Fri, 4 Oct 2024 20:03:25 +0200 Subject: [PATCH 08/23] Build all samples for maestro --- .github/workflows/eval-maestro-tests.yml | 78 +++++++++++++++++------- .github/workflows/ui-tests.yml | 8 +-- .maestro/crash.yaml | 5 -- Samples/iOS-Swift/.maestro/crash.yaml | 5 ++ fastlane/Fastfile | 36 ++++++++++- 5 files changed, 99 insertions(+), 33 deletions(-) delete mode 100644 .maestro/crash.yaml create mode 100644 Samples/iOS-Swift/.maestro/crash.yaml diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index e3f54a46dcf..5e6da7ff288 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -21,38 +21,70 @@ concurrency: cancel-in-progress: true jobs: - build-swift-ui: - name: Maestro Build SwiftUI for UI Tests Simulator - runs-on: macos-13 + build-sample-for-maestro: + strategy: + fail-fast: false + matrix: + include: + - sample-name: ObjectiveC + lane_name: build_ios_objc_for_sim_tests + artifact-name: maestro-ios-objc + app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-ObjectiveC.app + - sample-name: Swift + lane_name: build_ios_swift_for_sim_tests + artifact-name: maestro-ios-swift + app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-Swift.app + - sample-name: SwiftUI + lane_name: build_ios_swift_ui_for_sim_tests + artifact-name: maestro-ios-swift-ui + app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-SwiftUI.app + name: Build ${{matrix.sample-name}} Sample for Maestro + runs-on: macos-14 steps: - uses: actions/checkout@v4 - - run: ./scripts/ci-select-xcode.sh 14.3 + - run: ./scripts/ci-select-xcode.sh 15.4 - uses: ruby/setup-ruby@v1 with: bundler-cache: true - - run: bundle exec fastlane build_ios_swift_ui_for_tests + - run: bundle exec fastlane ${{matrix.lane_name}} - uses: actions/upload-artifact@v4 with: - name: maestro-ios-swift-ui-for-tests - path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-SwiftUI.app + name: ${{matrix.artifact-name}} + path: ${{matrix.app_path}} eval-maestro-swift-ui-tests: - name: Maestro UI Tests for SwiftUI on ${{matrix.device}} Simulator - needs: [build-swift-ui] + name: Maestro Test of ${{matrix.sample-name}} on iOS ${{matrix.os-version}} on ${{matrix.device}} Simulator + needs: [build-sample-for-maestro] runs-on: ${{matrix.runs-on}} strategy: fail-fast: false matrix: include: - # - runs-on: macos-12 - # xcode: "13.4.1" - - - runs-on: macos-14 - xcode: "15.4" - device: "iPhone 15" - os_version: "17.2" - app: "iOS-SwiftUI.app" + - sample-name: iOS-ObjectiveC + runs-on: macos-13 + xcode: "14.3" + device: "iPhone 8" + os-version: "16.1" + artifact-name: maestro-ios-objc + app: "Sample.app" + test_suit: "Samples/iOS-ObjectiveC/.maestro" + - sample-name: iOS-SwiftUI + runs-on: macos-12 + xcode: "13.4.1" + device: "iPhone 8" + os-version: "15.2" + artifact-name: maestro-ios-swift-ui + app: "Sample.app" + test_suit: "Samples/iOS-SwiftUI/.maestro" + - sample-name: iOS-SwiftUI + runs-on: macos-14 + xcode: "14.3.1" + device: "iPhone 14" + os-version: "16.4" + artifact-name: maestro-ios-swift-ui + app: "Sample.app" + test_suit: "Samples/iOS-SwiftUI/.maestro" steps: - uses: actions/checkout@v4 @@ -63,17 +95,17 @@ jobs: run: brew tap facebook/fb && brew install facebook/fb/idb-companion - uses: actions/download-artifact@v4 with: - name: maestro-ios-swift-ui-for-tests + name: ${{matrix.artifact-name}} path: ${{ matrix.app }} - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3 with: model: ${{matrix.device}} - os_version: ${{matrix.os_version}} + os-version: ${{matrix.os-version}} - name: Run Maestro Flows - run: | + env: # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57 - export MAESTRO_DRIVER_STARTUP_TIMEOUT=1500000 # 25 min. CI is extremely slow - + MAESTRO_DRIVER_STARTUP_TIMEOUT: 1500000 # 25 min, CI can be slow at times + run: | xcrun simctl install booted ${{ matrix.app}} - maestro test .maestro --format junit + maestro test ${{matrix.test_suit}} --format junit diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index b51d55ac9fd..49d74d41636 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -79,9 +79,9 @@ jobs: path: | ~/Library/Logs/scan/*.log ./fastlane/test_output/** - - ios-swift-ui-tests: - name: iOS-Swift UI Tests ${{matrix.device}} + + ui-tests-swift: + name: UI Tests for iOS-Swift ${{matrix.device}} Simulator runs-on: ${{matrix.runs-on}} strategy: fail-fast: false @@ -108,7 +108,7 @@ jobs: run: ./scripts/create-simulator.sh 14.3.1 16.4 16-4 - name: Run Fastlane - run: fastlane ui_tests_ios_swift device:"${{matrix.device}}" + run: fastlane ui_tests_ios_swift device:"${{matrix.device}}" - name: Archiving Raw Test Logs uses: actions/upload-artifact@v4 diff --git a/.maestro/crash.yaml b/.maestro/crash.yaml deleted file mode 100644 index 3e35679e6ac..00000000000 --- a/.maestro/crash.yaml +++ /dev/null @@ -1,5 +0,0 @@ -appId: io.sentry.iOS-SwiftUI ---- -- launchApp -- tapOn: "Crash" -- launchApp diff --git a/Samples/iOS-Swift/.maestro/crash.yaml b/Samples/iOS-Swift/.maestro/crash.yaml new file mode 100644 index 00000000000..14628f22deb --- /dev/null +++ b/Samples/iOS-Swift/.maestro/crash.yaml @@ -0,0 +1,5 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Crash the app" +- launchApp diff --git a/fastlane/Fastfile b/fastlane/Fastfile index 3a3bb33ce3a..64e33df9a32 100644 --- a/fastlane/Fastfile +++ b/fastlane/Fastfile @@ -120,7 +120,41 @@ platform :ios do delete_keychain(name: "fastlane_tmp_keychain") unless is_ci end - lane :build_ios_swift_ui_for_tests do + lane :build_ios_objc_for_sim_tests do + + setup_ci( + force: true + ) + + build_app( + workspace: "Sentry.xcworkspace", + scheme: "iOS-ObjectiveC", + configuration: "Debug", + derived_data_path: "DerivedData", + destination: "generic/platform=iOS Simulator", + xcargs: "-sdk 'iphonesimulator'", + skip_archive: true + ) + end + + lane :build_ios_swift_for_sim_tests do + + setup_ci( + force: true + ) + + build_app( + workspace: "Sentry.xcworkspace", + scheme: "iOS-Swift", + configuration: "Debug", + derived_data_path: "DerivedData", + destination: "generic/platform=iOS Simulator", + xcargs: "-sdk 'iphonesimulator'", + skip_archive: true + ) + end + + lane :build_ios_swift_ui_for_sim_tests do setup_ci( force: true From 98f20602993ffd228f8d4bdd47479fd3d0820a42 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Mon, 7 Oct 2024 12:00:04 +0200 Subject: [PATCH 09/23] add crash flow for objc and swift ui --- Samples/iOS-ObjectiveC/.maestro/crash.yaml | 5 +++++ Samples/iOS-SwiftUI/.maestro/crash.yaml | 5 +++++ 2 files changed, 10 insertions(+) create mode 100644 Samples/iOS-ObjectiveC/.maestro/crash.yaml create mode 100644 Samples/iOS-SwiftUI/.maestro/crash.yaml diff --git a/Samples/iOS-ObjectiveC/.maestro/crash.yaml b/Samples/iOS-ObjectiveC/.maestro/crash.yaml new file mode 100644 index 00000000000..0c1f25a24e1 --- /dev/null +++ b/Samples/iOS-ObjectiveC/.maestro/crash.yaml @@ -0,0 +1,5 @@ +appId: io.sentry.iOS-ObjectiveC +--- +- launchApp +- tapOn: "crash" +- launchApp diff --git a/Samples/iOS-SwiftUI/.maestro/crash.yaml b/Samples/iOS-SwiftUI/.maestro/crash.yaml new file mode 100644 index 00000000000..3e35679e6ac --- /dev/null +++ b/Samples/iOS-SwiftUI/.maestro/crash.yaml @@ -0,0 +1,5 @@ +appId: io.sentry.iOS-SwiftUI +--- +- launchApp +- tapOn: "Crash" +- launchApp From 441f6497135cfc4976ed4e73a833448c2697a219 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Mon, 7 Oct 2024 12:15:44 +0200 Subject: [PATCH 10/23] Execute ios swift flows in ci --- .github/workflows/eval-maestro-tests.yml | 38 ++++++++++++++++++++---- 1 file changed, 32 insertions(+), 6 deletions(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 5e6da7ff288..717e336df4e 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -62,29 +62,55 @@ jobs: matrix: include: - sample-name: iOS-ObjectiveC + artifact-name: maestro-ios-objc + app: "Sample.app" + test_suit: "Samples/iOS-ObjectiveC/.maestro" runs-on: macos-13 xcode: "14.3" device: "iPhone 8" os-version: "16.1" - artifact-name: maestro-ios-objc - app: "Sample.app" - test_suit: "Samples/iOS-ObjectiveC/.maestro" + - sample-name: iOS-SwiftUI + artifact-name: maestro-ios-swift-ui + app: "Sample.app" + test_suit: "Samples/iOS-SwiftUI/.maestro" runs-on: macos-12 xcode: "13.4.1" device: "iPhone 8" os-version: "15.2" + - sample-name: iOS-SwiftUI artifact-name: maestro-ios-swift-ui app: "Sample.app" test_suit: "Samples/iOS-SwiftUI/.maestro" - - sample-name: iOS-SwiftUI runs-on: macos-14 xcode: "14.3.1" device: "iPhone 14" os-version: "16.4" - artifact-name: maestro-ios-swift-ui + + - sample-name: iOS-Swift + artifact-name: maestro-ios-swift app: "Sample.app" - test_suit: "Samples/iOS-SwiftUI/.maestro" + test_suit: "Samples/iOS-Swift/.maestro" + runs-on: macos-12 + xcode: "13.4.1" + device: "iPhone 8" + os-version: "15.5" + - sample-name: iOS-Swift + artifact-name: maestro-ios-swift + app: "Sample.app" + test_suit: "Samples/iOS-Swift/.maestro" + runs-on: macos-13 + xcode: "15.2" + device: "iPhone 14" + os-version: "16.4" + - sample-name: iOS-Swift + artifact-name: maestro-ios-swift + app: "Sample.app" + test_suit: "Samples/iOS-Swift/.maestro" + runs-on: macos-14 + xcode: "15.4" + device: "iPhone 15" + os-version: "17.2" steps: - uses: actions/checkout@v4 From efb7d6e6e1a7f1332c4eb4347c2c6204d3fa6047 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Mon, 7 Oct 2024 13:01:17 +0200 Subject: [PATCH 11/23] Fix missing brew install for maestro --- .github/workflows/eval-maestro-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 717e336df4e..348b7431f14 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -116,7 +116,7 @@ jobs: - uses: actions/checkout@v4 - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} - name: Install Maestro - run: brew tap mobile-dev-inc/tap && mobile-dev-inc/tap/maestro@1.38 + run: brew tap mobile-dev-inc/tap && brew install mobile-dev-inc/tap/maestro@1.38 - name: Install iDB Companion run: brew tap facebook/fb && brew install facebook/fb/idb-companion - uses: actions/download-artifact@v4 From 1f38675f61eb2e4741e4818757af18434262522d Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Mon, 7 Oct 2024 14:19:48 +0200 Subject: [PATCH 12/23] use default xcode version for maestro --- .github/workflows/eval-maestro-tests.yml | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 348b7431f14..743d3240225 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -66,8 +66,7 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-ObjectiveC/.maestro" runs-on: macos-13 - xcode: "14.3" - device: "iPhone 8" + device: "iPhone 14" os-version: "16.1" - sample-name: iOS-SwiftUI @@ -75,7 +74,6 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-SwiftUI/.maestro" runs-on: macos-12 - xcode: "13.4.1" device: "iPhone 8" os-version: "15.2" - sample-name: iOS-SwiftUI @@ -83,7 +81,6 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-SwiftUI/.maestro" runs-on: macos-14 - xcode: "14.3.1" device: "iPhone 14" os-version: "16.4" @@ -92,7 +89,6 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-Swift/.maestro" runs-on: macos-12 - xcode: "13.4.1" device: "iPhone 8" os-version: "15.5" - sample-name: iOS-Swift @@ -100,7 +96,6 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-Swift/.maestro" runs-on: macos-13 - xcode: "15.2" device: "iPhone 14" os-version: "16.4" - sample-name: iOS-Swift @@ -108,13 +103,11 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-Swift/.maestro" runs-on: macos-14 - xcode: "15.4" device: "iPhone 15" os-version: "17.2" steps: - uses: actions/checkout@v4 - - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} - name: Install Maestro run: brew tap mobile-dev-inc/tap && brew install mobile-dev-inc/tap/maestro@1.38 - name: Install iDB Companion From a7ce5024aa0404790330ca596cf417910919c7b8 Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Mon, 7 Oct 2024 14:27:19 +0200 Subject: [PATCH 13/23] Update jobs names --- .github/workflows/eval-maestro-tests.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 743d3240225..71889975f49 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -26,19 +26,19 @@ jobs: fail-fast: false matrix: include: - - sample-name: ObjectiveC + - sample-name: iOS-ObjectiveC lane_name: build_ios_objc_for_sim_tests artifact-name: maestro-ios-objc app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-ObjectiveC.app - - sample-name: Swift + - sample-name: iOS-Swift lane_name: build_ios_swift_for_sim_tests artifact-name: maestro-ios-swift app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-Swift.app - - sample-name: SwiftUI + - sample-name: iOS-SwiftUI lane_name: build_ios_swift_ui_for_sim_tests artifact-name: maestro-ios-swift-ui app_path: DerivedData/Build/Products/Debug-iphonesimulator/iOS-SwiftUI.app - name: Build ${{matrix.sample-name}} Sample for Maestro + name: Build ${{matrix.sample-name}} Sample runs-on: macos-14 steps: @@ -54,7 +54,7 @@ jobs: path: ${{matrix.app_path}} eval-maestro-swift-ui-tests: - name: Maestro Test of ${{matrix.sample-name}} on iOS ${{matrix.os-version}} on ${{matrix.device}} Simulator + name: Test ${{matrix.sample-name}} on iOS ${{matrix.os-version}} on ${{matrix.device}} Simulator needs: [build-sample-for-maestro] runs-on: ${{matrix.runs-on}} strategy: From 693349596d66497e8ebf261d594cd76217fcae8c Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Mon, 7 Oct 2024 14:28:20 +0200 Subject: [PATCH 14/23] fix os_version not os-version --- .github/workflows/eval-maestro-tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 71889975f49..7c59873c5af 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -119,7 +119,7 @@ jobs: - uses: futureware-tech/simulator-action@bfa03d93ec9de6dacb0c5553bbf8da8afc6c2ee9 # pin@v3 with: model: ${{matrix.device}} - os-version: ${{matrix.os-version}} + os_version: ${{matrix.os-version}} - name: Run Maestro Flows env: # https://github.com/facebook/react-native/blob/24e7f7d25629a7af6d877a0b79fed2faaab96437/.github/actions/maestro-ios/action.yml#L57 From 7033e0ab9a58bc483811aaa896843f22d0ac47bb Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Mon, 7 Oct 2024 15:42:23 +0200 Subject: [PATCH 15/23] Manually create requested simulators --- .github/workflows/eval-maestro-tests.yml | 17 +++++++++++++---- .github/workflows/ui-tests.yml | 2 +- scripts/create-simulator.sh | 6 ++++-- 3 files changed, 18 insertions(+), 7 deletions(-) diff --git a/.github/workflows/eval-maestro-tests.yml b/.github/workflows/eval-maestro-tests.yml index 7c59873c5af..55d26e71d88 100644 --- a/.github/workflows/eval-maestro-tests.yml +++ b/.github/workflows/eval-maestro-tests.yml @@ -65,22 +65,25 @@ jobs: artifact-name: maestro-ios-objc app: "Sample.app" test_suit: "Samples/iOS-ObjectiveC/.maestro" - runs-on: macos-13 + runs-on: macos-14 + xcode: "14.3.1" device: "iPhone 14" - os-version: "16.1" + os-version: "16.4" - sample-name: iOS-SwiftUI artifact-name: maestro-ios-swift-ui app: "Sample.app" test_suit: "Samples/iOS-SwiftUI/.maestro" runs-on: macos-12 + xcode: "13.4.1" device: "iPhone 8" - os-version: "15.2" + os-version: "15.5" - sample-name: iOS-SwiftUI artifact-name: maestro-ios-swift-ui app: "Sample.app" test_suit: "Samples/iOS-SwiftUI/.maestro" runs-on: macos-14 + xcode: "14.3.1" device: "iPhone 14" os-version: "16.4" @@ -89,6 +92,7 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-Swift/.maestro" runs-on: macos-12 + xcode: "13.4.1" device: "iPhone 8" os-version: "15.5" - sample-name: iOS-Swift @@ -96,6 +100,7 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-Swift/.maestro" runs-on: macos-13 + xcode: "14.3.1" device: "iPhone 14" os-version: "16.4" - sample-name: iOS-Swift @@ -103,15 +108,19 @@ jobs: app: "Sample.app" test_suit: "Samples/iOS-Swift/.maestro" runs-on: macos-14 + xcode: "15.4" device: "iPhone 15" - os-version: "17.2" + os-version: "17.5" steps: - uses: actions/checkout@v4 + - name: Create ${{matrix.device}} (${{matrix.os-version}}) Simulator using Xcode ${{matrix.xcode}} + run: ./scripts/create-simulator.sh "${{matrix.xcode}}" "${{matrix.os-version}}" "${{matrix.device}}" - name: Install Maestro run: brew tap mobile-dev-inc/tap && brew install mobile-dev-inc/tap/maestro@1.38 - name: Install iDB Companion run: brew tap facebook/fb && brew install facebook/fb/idb-companion + - uses: actions/download-artifact@v4 with: name: ${{matrix.artifact-name}} diff --git a/.github/workflows/ui-tests.yml b/.github/workflows/ui-tests.yml index 49d74d41636..a9468125d0c 100644 --- a/.github/workflows/ui-tests.yml +++ b/.github/workflows/ui-tests.yml @@ -105,7 +105,7 @@ jobs: - name: Create iOS 16.4 simulator if: ${{ matrix.device == 'iPhone 14 (16.4)' }} - run: ./scripts/create-simulator.sh 14.3.1 16.4 16-4 + run: ./scripts/create-simulator.sh 14.3.1 16.4 "iPhone 14" - name: Run Fastlane run: fastlane ui_tests_ios_swift device:"${{matrix.device}}" diff --git a/scripts/create-simulator.sh b/scripts/create-simulator.sh index c612068f1ba..ec9a1fb3069 100755 --- a/scripts/create-simulator.sh +++ b/scripts/create-simulator.sh @@ -5,8 +5,10 @@ set -euo pipefail XCODE_VERSION="${1}" SIM_RUNTIME="${2}" -SIM_RUNTIME_WITH_DASH="${3}" +DEVICE_TYPE="${3}" # A valid available device type. Find these by running "xcrun simctl list devicetypes". + +SIM_RUNTIME_WITH_DASH="${SIM_RUNTIME//./-}" sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes sudo ln -s "/Applications/Xcode_${XCODE_VERSION}.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime" "/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS ${SIM_RUNTIME}.simruntime" -xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-${SIM_RUNTIME_WITH_DASH}" +xcrun simctl create custom-test-device "${DEVICE_TYPE}" "com.apple.CoreSimulator.SimRuntime.iOS-${SIM_RUNTIME_WITH_DASH}" From 8ee95e256bacd1abe434ba24dbe93c1c817e00cb Mon Sep 17 00:00:00 2001 From: Krystof Woldrich Date: Tue, 8 Oct 2024 13:30:03 +0200 Subject: [PATCH 16/23] Add corrupted envelope ui test --- .../iOS-Swift/.maestro/corrupt-envelope.yaml | 9 +++++ .../iOS-Swift/Base.lproj/Main.storyboard | 40 +++++++++++-------- .../iOS-Swift/ExtraViewController.swift | 24 +++++++++++ 3 files changed, 57 insertions(+), 16 deletions(-) create mode 100644 Samples/iOS-Swift/.maestro/corrupt-envelope.yaml diff --git a/Samples/iOS-Swift/.maestro/corrupt-envelope.yaml b/Samples/iOS-Swift/.maestro/corrupt-envelope.yaml new file mode 100644 index 00000000000..fe8f53b171d --- /dev/null +++ b/Samples/iOS-Swift/.maestro/corrupt-envelope.yaml @@ -0,0 +1,9 @@ +appId: io.sentry.sample.iOS-Swift +--- +- launchApp +- tapOn: "Extra" +- tapOn: "Corrupt Envelope" +- tapOn: "Close SDK" +- tapOn: "Corrupt Envelope" +- stopApp +- launchApp diff --git a/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard b/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard index 971b70b2a4a..ddb0c1e0b7d 100644 --- a/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard +++ b/Samples/iOS-Swift/iOS-Swift/Base.lproj/Main.storyboard @@ -903,10 +903,10 @@ - + - + + - + + + - + +