ci: Remove test retries (#2242)

Instead of retrying the failing tests multiple times, we should fix them or disable them. Retry can hide flaky tests. Furthermore, retries slow down CI.
getsentry · Oct 4, 2022 · 347a8e9 · 347a8e9
1 parent 4a257fb
commit 347a8e9
Show file tree

Hide file tree

Showing 5 changed files with 12 additions and 48 deletions.
diff --git a/Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-Swift.xcscheme b/Samples/iOS-Swift/iOS-Swift.xcodeproj/xcshareddata/xcschemes/iOS-Swift.xcscheme
@@ -37,6 +37,11 @@
  BlueprintName = "iOS-SwiftUITests"
  ReferencedContainer = "container:iOS-Swift.xcodeproj">
  </BuildableReference>
+ <SkippedTests>
+ <Test
+ Identifier = "SentryFileIOTrackingIntegrationTests/test_DataConsistency_readPath_disabled()">
+ </Test>
+ </SkippedTests>
  </TestableReference>
  </Testables>
  </TestAction>

diff --git a/Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme b/Sentry.xcodeproj/xcshareddata/xcschemes/Sentry.xcscheme
@@ -61,6 +61,9 @@
  <Test
  Identifier = "SentrySessionGeneratorTests/testSendSessions()">
  </Test>
+ <Test
+ Identifier = "SentrySystemEventBreadcrumbsTest/testTimezoneChangeNotificationBreadcrumb_disabled()">
+ </Test>
  </SkippedTests>
  </TestableReference>
  </Testables>

diff --git a/Tests/SentryTests/Integrations/Breadcrumbs/SentrySystemEventBreadcrumbsTest.swift b/Tests/SentryTests/Integrations/Breadcrumbs/SentrySystemEventBreadcrumbsTest.swift
@@ -252,7 +252,7 @@ class SentrySystemEventBreadcrumbsTest: XCTestCase {
  }
  }
 
- func testTimezoneChangeNotificationBreadcrumb() {
+ func testTimezoneChangeNotificationBreadcrumb_disabled() {
  let scope = Scope()
  sut = fixture.getSut(scope: scope, currentDevice: nil)
 

diff --git a/Tests/SentryTests/Integrations/Performance/IO/SentryFileIOTrackingIntegrationTests.swift b/Tests/SentryTests/Integrations/Performance/IO/SentryFileIOTrackingIntegrationTests.swift
@@ -205,7 +205,7 @@ class SentryFileIOTrackingIntegrationTests: XCTestCase {
  XCTAssertEqual(randomValue, readValue)
  }
 
- func test_DataConsistency_readPath() {
+ func test_DataConsistency_readPath_disabled() {
  SentrySDK.start(options: fixture.getOptions())
 
  let randomValue = UUID().uuidString

diff --git a/scripts/xcode-test.sh b/scripts/xcode-test.sh
@@ -53,8 +53,6 @@ esac
 
 echo "CONFIGURATION: $CONFIGURATION"
 
-XCODE_MAJOR_VERSION=$(echo $XCODE | sed -E 's/([0-9]*)\.[0-9]*\.?[0-9]+/\1/g')
-
 if [ $PLATFORM == "iOS" -a $OS == "12.4" ]; then
  # Skip some tests that fail on iOS 12.4.
  env NSUnbufferedIO=YES xcodebuild -workspace Sentry.xcworkspace \
@@ -63,51 +61,9 @@ if [ $PLATFORM == "iOS" -a $OS == "12.4" ]; then
  -skip-testing:"SentryTests/SentrySDKTests/testMemoryFootprintOfAddingBreadcrumbs" \
  -skip-testing:"SentryTests/SentrySDKTests/testMemoryFootprintOfTransactions" \
  test | tee raw-test-output.log | xcpretty -t && exit ${PIPESTATUS[0]}
-elif [ $XCODE_MAJOR_VERSION == "13" ] || [ $XCODE_MAJOR_VERSION == "14" ]; then
- # We can retry flaky tests that fail with the -retry-tests-on-failure option introduced in Xcode 13.
- env NSUnbufferedIO=YES xcodebuild -retry-tests-on-failure -test-iterations 3 -workspace Sentry.xcworkspace \
- -scheme Sentry -configuration $CONFIGURATION \
- GCC_GENERATE_TEST_COVERAGE_FILES=YES GCC_INSTRUMENT_PROGRAM_FLOW_ARCS=YES -destination "$DESTINATION" \
- test | tee raw-test-output.log | xcpretty -t && exit ${PIPESTATUS[0]}
-elif [ $XCODE_MAJOR_VERSION == "12" ]; then
- # To retry flaky tests in Xcode <13, Run the suite normally without them, then run the suite with just the known flaky tests up to 3 times in a bash loop because xcodebuild didn't get the -retry-tests-on-failure option until version 13.
- env NSUnbufferedIO=YES xcodebuild -workspace Sentry.xcworkspace \
- -scheme Sentry -configuration $CONFIGURATION \
- GCC_GENERATE_TEST_COVERAGE_FILES=YES GCC_INSTRUMENT_PROGRAM_FLOW_ARCS=YES -destination "$DESTINATION" \
- -skip-testing:"SentryTests/SentrySessionTrackerTests" \
- test | tee raw-test-output.log | xcpretty -t
- nonflaky_test_status=${PIPESTATUS[0]}
-
- # try the flaky tests once. if they pass, exit with the combined status of flaky and nonflaky tests.
- # if they fail, retry them twice more.
-
+else
  env NSUnbufferedIO=YES xcodebuild -workspace Sentry.xcworkspace \
  -scheme Sentry -configuration $CONFIGURATION \
  GCC_GENERATE_TEST_COVERAGE_FILES=YES GCC_INSTRUMENT_PROGRAM_FLOW_ARCS=YES -destination "$DESTINATION" \
- -only-testing:"SentryTests/SentrySessionTrackerTests" \
- test | tee raw-test-output.log | xcpretty -t
- flaky_test_status=${PIPESTATUS[0]}
- if [ $flaky_test_status -eq 0 ]; then
- exit $nonflaky_test_status
- else
- for i in {1..2}; do
- bash -c "env NSUnbufferedIO=YES xcodebuild -workspace Sentry.xcworkspace \
- -scheme Sentry -configuration $CONFIGURATION \
- GCC_GENERATE_TEST_COVERAGE_FILES=YES GCC_INSTRUMENT_PROGRAM_FLOW_ARCS=YES -destination '$DESTINATION' \
- -only-testing:'SentryTests/SentrySessionTrackerTests' \
- test-without-building | tee raw-test-output.log | xcpretty -t && exit ${PIPESTATUS[0]}"
- flaky_test_status=$?
- if [ $flaky_test_status -eq 0 ]; then
- exit $nonflaky_test_status
- fi
- done
-
- # combine flaky/nonflaky statuses for the exit status of this script so that if either have failed, the script will fail
- all_test_status=$nonflaky_test_status
- let 'all_test_status|=flaky_test_status'
- exit $all_test_status
- fi
-else
- # The branches above are exhaustive at the time they were written. This will help us catch unexpected deviations with future changes.
- exit 1
+ test | tee raw-test-output.log | xcpretty -t && exit ${PIPESTATUS[0]}
 fi