ref: compile out more code from platforms without profiling (#3159) #6102
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| - release/** | |
| pull_request: | |
| paths: | |
| - 'Sources/**' | |
| - 'Tests/**' | |
| - 'test-server/**' | |
| - 'Samples/**' | |
| - '.github/workflows/test.yml' | |
| - 'fastlane/**' | |
| - 'scripts/tests-with-thread-sanitizer.sh' | |
| - 'scripts/ci-select-xcode.sh' | |
| - 'scripts/xcode-test.sh' | |
| - '.codecov.yml' | |
| - Sentry.xcodeproj | |
| # run the workflow any time an Xcode scheme changes for a sample app | |
| - 'Samples/tvOS-Swift/tvOS-Swift.xcodeproj/xcshareddata/xcschemes/tvOS-Swift.xcscheme' | |
| - 'Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS13-Swift.xcscheme' | |
| - 'Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-SwiftUITests.xcscheme' | |
| - 'Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-Swift.xcscheme' | |
| - 'Samples/macOS-Swift/macOS-Swift.xcodeproj/xcshareddata/xcschemes/macOS-Swift.xcscheme' | |
| - 'Samples/iOS-ObjectiveC/iOS-ObjectiveC.xcodeproj/xcshareddata/xcschemes/iOS-ObjectiveC.xcscheme' | |
| jobs: | |
| build-test-server: | |
| name: Build test server | |
| runs-on: macos-13 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Cache for Test Server | |
| id: cache_test_server | |
| uses: actions/cache@v3 | |
| with: | |
| path: ./test-server/.build | |
| key: test-server-${{ hashFiles('./test-server') }} | |
| restore-keys: | | |
| test-server-${{ hashFiles('./test-server') }} | |
| test-server- | |
| - name: Build Test Server | |
| if: steps.cache_test_server.outputs.cache-hit != 'true' | |
| working-directory: test-server | |
| run: >- | |
| swift build -c release 2>&1 | tee test-server-build.log | |
| - name: Copy exec | |
| working-directory: test-server | |
| run: cp $(swift build --show-bin-path -c release)/Run test-server-exec | |
| - name: Archiving DerivedData | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| name: test-server | |
| path: | | |
| ./test-server/test-server-exec | |
| - name: Archiving Raw Test Logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || cancelled() }} | |
| with: | |
| name: test-server-build-log-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} | |
| path: | | |
| raw-test-output.log | |
| unit-tests: | |
| name: Unit ${{matrix.platform}} - Xcode ${{matrix.xcode}} - OS ${{matrix.test-destination-os}} | |
| runs-on: ${{matrix.runs-on}} | |
| timeout-minutes: 20 | |
| needs: build-test-server | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| # Can't run tests on watchOS because XCTest is not available | |
| include: | |
| # iOS 13.7 | |
| - runs-on: macos-11 | |
| platform: 'iOS' | |
| xcode: '13.2.1' | |
| test-destination-os: '13.7' | |
| # iOS 14 | |
| - runs-on: macos-11 | |
| platform: 'iOS' | |
| xcode: '13.2.1' | |
| test-destination-os: '14.5' | |
| # iOS 15 | |
| - runs-on: macos-12 | |
| platform: 'iOS' | |
| xcode: '13.4.1' | |
| test-destination-os: 'latest' | |
| # iOS 16 | |
| - runs-on: macos-13 | |
| platform: 'iOS' | |
| xcode: '14.3' | |
| test-destination-os: 'latest' | |
| # macOS 11 | |
| - runs-on: macos-11 | |
| platform: 'macOS' | |
| xcode: '13.2.1' | |
| test-destination-os: 'latest' | |
| # macOS 12 | |
| - runs-on: macos-12 | |
| platform: 'macOS' | |
| xcode: '13.4.1' | |
| test-destination-os: 'latest' | |
| # macOS 13 | |
| - runs-on: macos-13 | |
| platform: 'macOS' | |
| xcode: '14.3' | |
| test-destination-os: 'latest' | |
| # Catalyst. We only test the latest version, as | |
| # the risk something breaking on Catalyst and not | |
| # on an older iOS or macOS version is low. | |
| - runs-on: macos-13 | |
| platform: 'Catalyst' | |
| xcode: '14.3' | |
| test-destination-os: 'latest' | |
| # tvOS 15 | |
| - runs-on: macos-12 | |
| platform: 'tvOS' | |
| xcode: '13.4.1' | |
| test-destination-os: 'latest' | |
| # tvOS 16 | |
| - runs-on: macos-13 | |
| platform: 'tvOS' | |
| xcode: '14.3' | |
| test-destination-os: 'latest' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/download-artifact@v3 | |
| with: | |
| name: test-server | |
| - name: Print hardware info | |
| run: system_profiler SPHardwareDataType | |
| - name: Allow test-server to run | |
| run: chmod +x ./test-server-exec | |
| - run: ./test-server-exec & | |
| - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} | |
| # Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435 | |
| - name: Prepare iOS 13.7 simulator | |
| if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '13.7'}} | |
| run: | | |
| sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes | |
| sudo ln -s /Applications/Xcode_11.7.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 13.7.simruntime | |
| xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-13-7" | |
| # Workaround with a symlink pointed out in: https://github.com/actions/virtual-environments/issues/551#issuecomment-637344435 | |
| - name: Prepare iOS 14.5 simulator | |
| if: ${{ matrix.platform == 'iOS' && matrix.test-destination-os == '14.5'}} | |
| run: | | |
| sudo mkdir -p /Library/Developer/CoreSimulator/Profiles/Runtimes | |
| sudo ln -s /Applications/Xcode_12.5.1.app/Contents/Developer/Platforms/iPhoneOS.platform/Library/Developer/CoreSimulator/Profiles/Runtimes/iOS.simruntime /Library/Developer/CoreSimulator/Profiles/Runtimes/iOS\ 14.5.simruntime | |
| xcrun simctl create custom-test-device "iPhone 8" "com.apple.CoreSimulator.SimRuntime.iOS-14-5" | |
| - name: Install Slather | |
| run: gem install slather | |
| # We split building and running tests in two steps so we know how long running the tests takes. | |
| - name: Build tests | |
| id: build_tests | |
| run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci build-for-testing | |
| - name: Run tests | |
| # We call a script with the platform so the destination | |
| # passed to xcodebuild doesn't end up in the job name, | |
| # because GitHub Actions don't provide an easy way of | |
| # manipulating string in expressions. | |
| run: ./scripts/xcode-test.sh ${{matrix.platform}} ${{matrix.test-destination-os}} $GITHUB_REF_NAME ci test-without-building | |
| - name: Slowest Tests | |
| if: ${{ always() }} | |
| run: ./scripts/xcode-slowest-tests.sh | |
| - name: Archiving DerivedData Logs | |
| uses: actions/upload-artifact@v3 | |
| if: steps.build_tests.outcome == 'failure' | |
| with: | |
| name: derived-data-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} | |
| path: | | |
| /Users/runner/Library/Developer/Xcode/DerivedData/**/Logs/** | |
| - name: Archiving Raw Test Logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || cancelled() }} | |
| with: | |
| name: raw-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} | |
| path: | | |
| raw-test-output.log | |
| - name: Archiving Crash Logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || cancelled() }} | |
| with: | |
| name: crash-logs-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} | |
| path: | | |
| ~/Library/Logs/DiagnosticReports/** | |
| # We can upload all coverage reports, because codecov merges them. | |
| # See https://docs.codecov.io/docs/merging-reports | |
| # Checkout .codecov.yml to see the config of Codecov | |
| # We don't upload codecov for release branches, as we don't want a failing coverage check to block a release. | |
| # We don't upload codecov for scheduled runs as CodeCov only accepts a limited amount of uploads per commit. | |
| - name: Push code coverage to codecov | |
| id: codecov_1 | |
| uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4 | |
| if: ${{ contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }} | |
| with: | |
| # Although public repos should not have to specify a token there seems to be a bug with the Codecov GH action, which can | |
| # be solved by specifying the token, see https://github.com/codecov/codecov-action/issues/557#issuecomment-1224970469 | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: false | |
| verbose: true | |
| # Sometimes codecov uploads etc can fail. Retry one time to rule out e.g. intermittent network failures. | |
| - name: Push code coverage to codecov | |
| id: codecov_2 | |
| uses: codecov/codecov-action@eaaf4bedf32dbdc6b720b63067d99c4d77d6047d # pin@v3.1.4 | |
| if: ${{ steps.codecov_1.outcome == 'failure' && contains(matrix.platform, 'iOS') && !contains(github.ref, 'release') && github.event.schedule == '' }} | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| fail_ci_if_error: true | |
| verbose: true | |
| # We don't run all unit tests with Thread Sanitizer enabled because | |
| # that adds a significant overhead. | |
| thread-sanitizer: | |
| name: Unit iOS - Thread Sanitizer | |
| runs-on: macos-13 | |
| # When there are threading issues the tests sometimes keep hanging | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Cache for Test Server | |
| uses: actions/cache@v3 | |
| id: cache_test_server | |
| with: | |
| path: ./test-server/.build | |
| key: ${{ runner.os }}-spm-${{ hashFiles('./test-server/Package.resolved') }} | |
| - run: swift build | |
| if: steps.cache_test_server.outputs.cache-hit != 'true' | |
| working-directory: test-server | |
| - name: Run Test Server in Background | |
| run: swift run & | |
| working-directory: test-server | |
| - run: ./scripts/ci-select-xcode.sh | |
| - name: Running tests with ThreadSanitizer | |
| run: ./scripts/tests-with-thread-sanitizer.sh | |
| - name: Archiving Test Logs | |
| uses: actions/upload-artifact@v3 | |
| with: | |
| path: thread-sanitizer.log | |
| ui-tests: | |
| name: UI Tests for ${{matrix.target}} on Simulators | |
| runs-on: macos-13 | |
| strategy: | |
| matrix: | |
| target: ['ios_swift', 'ios_objc', 'tvos_swift'] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: ./scripts/ci-select-xcode.sh | |
| # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry | |
| - name: Run Fastlane | |
| run: for i in {1..2}; do fastlane ui_tests_${{matrix.target}} && break ; done | |
| shell: sh | |
| - name: Archiving Raw Test Logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || cancelled() }} | |
| with: | |
| name: raw-uitest-output-${{matrix.target}} | |
| path: | | |
| ~/Library/Logs/scan/*.log | |
| ./fastlane/test_output/** | |
| # SwiftUI only supports iOS 14+ so we run it in a separate matrix here | |
| ui-tests-swift-ui: | |
| name: UI Tests for SwiftUI on ${{matrix.device}} Simulator | |
| runs-on: ${{matrix.runs-on}} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - runs-on: macos-12 | |
| xcode: '13.4.1' | |
| device: 'iPhone 8 (15.2)' | |
| - runs-on: macos-13 | |
| xcode: '14.3' | |
| device: 'iPhone 8 (16.1)' | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: ./scripts/ci-select-xcode.sh ${{matrix.xcode}} | |
| # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry | |
| - name: Run Fastlane | |
| run: for i in {1..2}; do fastlane ui_tests_ios_swiftui device:"${{matrix.device}}" && break ; done | |
| shell: sh | |
| - name: Archiving Raw Test Logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || cancelled() }} | |
| with: | |
| name: raw-swiftui-test-output-${{matrix.platform}}-xcode-${{matrix.xcode}}-os-${{matrix.test-destination-os}} | |
| path: | | |
| ~/Library/Logs/scan/*.log | |
| ./fastlane/test_output/** | |
| ui-tests-address-sanitizer: | |
| name: UI Tests with Address Sanitizer | |
| runs-on: macos-13 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - run: ./scripts/ci-select-xcode.sh | |
| # GitHub Actions sometimes fail to launch the UI tests. Therefore we retry | |
| - name: Run Fastlane | |
| run: for i in {1..2}; do fastlane ui_tests_ios_swift device:"$iPhone 8 (15.5)" address_sanitizer:true && break ; done | |
| shell: sh | |
| - name: Archiving Raw Test Logs | |
| uses: actions/upload-artifact@v3 | |
| if: ${{ failure() || cancelled() }} | |
| with: | |
| name: raw-uitest-output-asan | |
| path: | | |
| ~/Library/Logs/scan/*.log | |
| ./fastlane/test_output/** |