introduce retry total tests limit for Cucumber 9.0+

lib/datadog/ci/configuration/components.rb

@@ -113,7 +113,8 @@ def activate_ci!(settings)
             library_settings_client: build_library_settings_client(settings, test_visibility_api)
           )
           @test_retries = TestRetries::Component.new(
-            retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts
+            retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts,
+            retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit
           )
           # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component
           @test_optimisation = build_test_optimisation(settings, test_visibility_api)

lib/datadog/ci/configuration/settings.rb

@@ -94,6 +94,12 @@ def self.add_settings!(base)
                 o.default 5
               end
 
+              option :retry_failed_tests_total_limit do |o|
+                o.type :int
+                o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT
+                o.default 100
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

lib/datadog/ci/contrib/cucumber/configuration_override.rb

@@ -21,7 +21,9 @@ def retry_attempts
             end
 
             def retry_total_tests
-              super
+              super if !datadog_test_retries_component&.retry_failed_tests_enabled
+
+              datadog_test_retries_component&.retry_failed_tests_total_limit
             end
 
             def datadog_test_retries_component

lib/datadog/ci/ext/settings.rb

@@ -16,6 +16,7 @@ module Settings
         ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE"
         ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING = "DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING"
         ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_MAX_ATTEMPTS"
+        ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_TOTAL_LIMIT"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

lib/datadog/ci/test_retries/component.rb

@@ -7,12 +7,16 @@ module TestRetries
       # - retrying failed tests - improve success rate of CI pipelines
       # - retrying new tests - detect flaky tests as early as possible to prevent them from being merged
       class Component
-        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts
+        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit
 
-        def initialize(retry_failed_tests_max_attempts:)
+        def initialize(
+          retry_failed_tests_max_attempts:,
+          retry_failed_tests_total_limit:
+        )
           # enabled only by remote settings
           @retry_failed_tests_enabled = false
           @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts
+          @retry_failed_tests_total_limit = retry_failed_tests_total_limit
         end
 
         def configure(library_settings)

sig/datadog/ci/ext/settings.rbs

@@ -13,6 +13,7 @@ module Datadog
         ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE: String
         ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING: String
         ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS: String
+        ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT: String
 
         DD_SITE_ALLOWLIST: Array[String]
       end

sig/datadog/ci/test_retries/component.rbs

@@ -2,15 +2,13 @@ module Datadog
   module CI
     module TestRetries
       class Component
-        @retry_failed_tests_enabled: bool
-
-        @retry_failed_tests_max_attempts: Integer
-
         attr_reader retry_failed_tests_enabled: bool
 
         attr_reader retry_failed_tests_max_attempts: Integer
 
-        def initialize: (retry_failed_tests_max_attempts: Integer) -> void
+        attr_reader retry_failed_tests_total_limit: Integer
+
+        def initialize: (retry_failed_tests_max_attempts: Integer, retry_failed_tests_total_limit: Integer) -> void
 
         def configure: (Datadog::CI::Remote::LibrarySettings library_settings) -> void
       end

spec/datadog/ci/contrib/cucumber/instrumentation_spec.rb

@@ -6,6 +6,8 @@
 RSpec.describe "Cucumber formatter" do
   let(:cucumber_features_root) { File.join(__dir__, "features") }
   let(:enable_retries) { false }
+  let(:single_test_retries_count) { 5 }
+  let(:total_test_retries_limit) { 100 }
 
   before do
     allow(Datadog::CI::Git::LocalRepository).to receive(:root).and_return(cucumber_features_root)
@@ -18,10 +20,15 @@
     let(:itr_enabled) { true }
     let(:code_coverage_enabled) { true }
     let(:tests_skipping_enabled) { true }
+
     let(:flaky_test_retries_enabled) { enable_retries }
+    let(:retry_failed_tests_max_attempts) { single_test_retries_count }
+    let(:retry_failed_tests_total_limit) { total_test_retries_limit }
+
     let(:bundle_path) { "step_definitions/helpers" }
   end
 
+  let(:cucumber_9_or_above) { Gem::Version.new("9.0.0") <= Datadog::CI::Contrib::Cucumber::Integration.version }
   let(:cucumber_8_or_above) { Gem::Version.new("8.0.0") <= Datadog::CI::Contrib::Cucumber::Integration.version }
   let(:cucumber_4_or_above) { Gem::Version.new("4.0.0") <= Datadog::CI::Contrib::Cucumber::Integration.version }
 
@@ -519,5 +526,51 @@
 
       expect(test_session_span).to have_pass_status
     end
+
+    context "when max retries attempts configuration value is too low" do
+      let(:single_test_retries_count) { 1 }
+      let(:expected_test_run_code) { 2 }
+
+      it "retries the test once" do
+        # 1 initial run of flaky test + 1 retry + 1 passing = 3 spans
+        expect(test_spans).to have(3).items
+        retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
+        expect(retries_count).to eq(1)
+
+        failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
+        expect(failed_spans).to have(2).items
+        expect(passed_spans).to have(1).items
+
+        expect(test_suite_spans).to have(1).item
+        expect(test_suite_spans.first).to have_fail_status
+
+        expect(test_session_span).to have_fail_status
+      end
+    end
+
+    context "when total limit of failed tests to retry is zero" do
+      before do
+        skip("cucumber-ruby earlier than 9.0.0 does not support total test retries limit") unless cucumber_9_or_above
+      end
+
+      let(:total_test_retries_limit) { 0 }
+      let(:expected_test_run_code) { 2 }
+
+      it "does not retry the test" do
+        # 1 initial run of flaky test + 1 passing = 2 spans
+        expect(test_spans).to have(2).items
+        retries_count = test_spans.count { |span| span.get_tag("test.is_retry") == "true" }
+        expect(retries_count).to eq(0)
+
+        failed_spans, passed_spans = test_spans.partition { |span| span.get_tag("test.status") == "fail" }
+        expect(failed_spans).to have(1).items
+        expect(passed_spans).to have(1).items
+
+        expect(test_suite_spans).to have(1).item
+        expect(test_suite_spans.first).to have_fail_status
+
+        expect(test_session_span).to have_fail_status
+      end
+    end
   end
 end

spec/datadog/ci/test_retries/component_spec.rb

@@ -2,9 +2,15 @@
 
 RSpec.describe Datadog::CI::TestRetries::Component do
   let(:library_settings) { instance_double(Datadog::CI::Remote::LibrarySettings) }
-
   let(:retry_failed_tests_max_attempts) { 1 }
-  subject(:component) { described_class.new(retry_failed_tests_max_attempts: retry_failed_tests_max_attempts) }
+  let(:retry_failed_tests_total_limit) { 12 }
+
+  subject(:component) do
+    described_class.new(
+      retry_failed_tests_max_attempts: retry_failed_tests_max_attempts,
+      retry_failed_tests_total_limit: retry_failed_tests_total_limit
+    )
+  end
 
   describe "#configure" do
     subject { component.configure(library_settings) }
@@ -39,4 +45,10 @@
 
     it { is_expected.to eq(retry_failed_tests_max_attempts) }
   end
+
+  describe "#retry_failed_tests_total_limit" do
+    subject { component.retry_failed_tests_total_limit }
+
+    it { is_expected.to eq(retry_failed_tests_total_limit) }
+  end
 end

spec/support/contexts/ci_mode.rb

@@ -28,6 +28,9 @@
   let(:use_single_threaded_coverage) { false }
   let(:flaky_test_retries_enabled) { false }
 
+  let(:retry_failed_tests_max_attempts) { 5 }
+  let(:retry_failed_tests_total_limit) { 100 }
+
   let(:itr_correlation_id) { "itr_correlation_id" }
   let(:itr_skippable_tests) { [] }
 
@@ -84,12 +87,23 @@
     allow_any_instance_of(Datadog::CI::TestOptimisation::Coverage::Transport).to receive(:send_events).and_return([])
 
     Datadog.configure do |c|
+      # library switch
       c.ci.enabled = ci_enabled
+
+      # test visibility
       c.ci.force_test_level_visibility = force_test_level_visibility
+
+      # test optimisation
       c.ci.itr_enabled = itr_enabled
       c.ci.git_metadata_upload_enabled = git_metadata_upload_enabled
       c.ci.itr_code_coverage_excluded_bundle_path = bundle_path
       c.ci.itr_code_coverage_use_single_threaded_mode = use_single_threaded_coverage
+
+      # test retries
+      c.ci.retry_failed_tests_max_attempts = retry_failed_tests_max_attempts
+      c.ci.retry_failed_tests_total_limit = retry_failed_tests_total_limit
+
+      # instrumentation
       unless integration_name == :no_instrument
         c.ci.instrument integration_name, integration_options
       end