Merge pull request #212 from DataDog/anmarchenko/flaky_test_retries_c…

…ucumber

[SDTEST-437] Retry failed tests for cucumber

lib/datadog/ci/configuration/components.rb

@@ -9,6 +9,7 @@
 require_relative "../test_optimisation/component"
 require_relative "../test_optimisation/coverage/transport"
 require_relative "../test_optimisation/coverage/writer"
+require_relative "../test_retries/component"
 require_relative "../test_visibility/component"
 require_relative "../test_visibility/flush"
 require_relative "../test_visibility/null_component"
@@ -27,13 +28,14 @@ module CI
     module Configuration
       # Adds CI behavior to Datadog trace components
       module Components
-        attr_reader :test_visibility, :test_optimisation, :git_tree_upload_worker, :ci_remote
+        attr_reader :test_visibility, :test_optimisation, :git_tree_upload_worker, :ci_remote, :test_retries
 
         def initialize(settings)
           @test_optimisation = nil
           @test_visibility = TestVisibility::NullComponent.new
           @git_tree_upload_worker = DummyWorker.new
           @ci_remote = nil
+          @test_retries = nil
 
           # Activate CI mode if enabled
           if settings.ci.enabled
@@ -110,6 +112,10 @@ def activate_ci!(settings)
           @ci_remote = Remote::Component.new(
             library_settings_client: build_library_settings_client(settings, test_visibility_api)
           )
+          @test_retries = TestRetries::Component.new(
+            retry_failed_tests_max_attempts: settings.ci.retry_failed_tests_max_attempts,
+            retry_failed_tests_total_limit: settings.ci.retry_failed_tests_total_limit
+          )
           # @type ivar @test_optimisation: Datadog::CI::TestOptimisation::Component
           @test_optimisation = build_test_optimisation(settings, test_visibility_api)
           @test_visibility = TestVisibility::Component.new(

lib/datadog/ci/configuration/settings.rb

@@ -88,6 +88,18 @@ def self.add_settings!(base)
                 o.default true
               end
 
+              option :retry_failed_tests_max_attempts do |o|
+                o.type :int
+                o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS
+                o.default 5
+              end
+
+              option :retry_failed_tests_total_limit do |o|
+                o.type :int
+                o.env CI::Ext::Settings::ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT
+                o.default 100
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

lib/datadog/ci/contrib/cucumber/configuration_override.rb

@@ -0,0 +1,37 @@
+# frozen_string_literal: true
+
+require_relative "formatter"
+
+module Datadog
+  module CI
+    module Contrib
+      module Cucumber
+        # Changes behaviour of Cucumber::Configuration class
+        module ConfigurationOverride
+          def self.included(base)
+            base.prepend(InstanceMethods)
+          end
+
+          # Instance methods for configuration
+          module InstanceMethods
+            def retry_attempts
+              super if !datadog_test_retries_component&.retry_failed_tests_enabled
+
+              datadog_test_retries_component&.retry_failed_tests_max_attempts
+            end
+
+            def retry_total_tests
+              super if !datadog_test_retries_component&.retry_failed_tests_enabled
+
+              datadog_test_retries_component&.retry_failed_tests_total_limit
+            end
+
+            def datadog_test_retries_component
+              Datadog.send(:components).test_retries
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -20,7 +20,7 @@ def initialize(config)
 
             @current_test_suite = nil
 
-            @failed_tests_count = 0
+            @failed_test_suites_count = 0
 
             bind_events(config)
           end
@@ -46,10 +46,12 @@ def on_test_run_started(event)
           end
 
           def on_test_run_finished(event)
+            finish_current_test_suite
+
             if event.respond_to?(:success)
               finish_session(event.success)
             else
-              finish_session(@failed_tests_count.zero?)
+              finish_session(@failed_test_suites_count.zero?)
             end
           end
 
@@ -86,7 +88,6 @@ def on_test_case_finished(event)
             return if test_span.nil?
 
             finish_span(test_span, event.result)
-            @failed_tests_count += 1 if test_span.failed?
           end
 
           def on_test_step_started(event)
@@ -128,8 +129,6 @@ def finish_span(span, result)
           end
 
           def finish_session(result)
-            finish_current_test_suite
-
             test_session = test_visibility_component.active_test_session
             test_module = test_visibility_component.active_test_module
 
@@ -155,6 +154,7 @@ def start_test_suite(test_suite_name)
 
           def finish_current_test_suite
             @current_test_suite&.finish
+            @failed_test_suites_count += 1 if @current_test_suite&.failed?
 
             @current_test_suite = nil
           end

lib/datadog/ci/contrib/cucumber/instrumentation.rb

@@ -6,7 +6,7 @@ module Datadog
   module CI
     module Contrib
       module Cucumber
-        # Instrumentation for Cucumber
+        # Instrumentation for Cucumber::Runtime class
         module Instrumentation
           def self.included(base)
             base.prepend(InstanceMethods)

lib/datadog/ci/contrib/cucumber/patcher.rb

@@ -3,6 +3,7 @@
 require "datadog/tracing/contrib/patcher"
 
 require_relative "instrumentation"
+require_relative "configuration_override"
 
 module Datadog
   module CI
@@ -20,6 +21,7 @@ def target_version
 
           def patch
             ::Cucumber::Runtime.include(Instrumentation)
+            ::Cucumber::Configuration.include(ConfigurationOverride)
           end
         end
       end

lib/datadog/ci/contrib/rspec/example_group.rb

@@ -26,7 +26,7 @@ def run(reporter = ::RSpec::Core::NullReporter)
               success = super
               return success unless test_suite
 
-              if success && test_suite.passed_tests_count > 0
+              if success && test_suite.any_passed?
                 test_suite.passed!
               elsif success
                 test_suite.skipped!

lib/datadog/ci/ext/settings.rb

@@ -15,6 +15,8 @@ module Settings
         ENV_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_EXCLUDED_BUNDLE_PATH"
         ENV_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE = "DD_CIVISIBILITY_ITR_CODE_COVERAGE_USE_SINGLE_THREADED_MODE"
         ENV_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING = "DD_CIVISIBILITY_ITR_TEST_IMPACT_ANALYSIS_USE_ALLOCATION_TRACING"
+        ENV_RETRY_FAILED_TESTS_MAX_ATTEMPTS = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_MAX_ATTEMPTS"
+        ENV_RETRY_FAILED_TESTS_TOTAL_LIMIT = "DD_CIVISIBILITY_RETRY_FAILED_TESTS_TOTAL_LIMIT"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = %w[

lib/datadog/ci/ext/test.rb

@@ -8,7 +8,7 @@ module Ext
       module Test
         CONTEXT_ORIGIN = "ciapp-test"
 
-        # Base test visibility tags
+        # Test visibility tags
         TAG_FRAMEWORK = "test.framework"
         TAG_FRAMEWORK_VERSION = "test.framework_version"
         TAG_NAME = "test.name"
@@ -23,7 +23,8 @@ module Test
         TAG_CODEOWNERS = "test.codeowners"
         TAG_PARAMETERS = "test.parameters"
 
-        # ITR tags
+        # Test optimisation tags
+        TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
         TAG_ITR_TEST_SKIPPING_ENABLED = "test.itr.tests_skipping.enabled"
         TAG_ITR_TEST_SKIPPING_TYPE = "test.itr.tests_skipping.type"
         TAG_ITR_TEST_SKIPPING_COUNT = "test.itr.tests_skipping.count"
@@ -32,11 +33,9 @@ module Test
         TAG_ITR_UNSKIPPABLE = "test.itr.unskippable"
         TAG_ITR_FORCED_RUN = "test.itr.forced_run"
 
-        # Code coverage tags
-        TAG_CODE_COVERAGE_ENABLED = "test.code_coverage.enabled"
-
-        # Special tags, not sent to the backend.
-        # these tags are special and used to correlate tests with the test sessions, suites, and modules
+        # Internal tags, they are not sent to the backend.
+        # These tags are internal to this library and used to correlate tests with
+        # the test sessions, suites, and modules.
         TAG_TEST_SESSION_ID = "_test.session_id"
         TAG_TEST_MODULE_ID = "_test.module_id"
         TAG_TEST_SUITE_ID = "_test.suite_id"
@@ -50,8 +49,7 @@ module Test
         TAG_RUNTIME_VERSION = "runtime.version"
 
         # Tags for browser tests
-        # true if Datadog RUM was detected in the page(s) loaded by Selenium
-        TAG_IS_RUM_ACTIVE = "test.is_rum_active"
+        TAG_IS_RUM_ACTIVE = "test.is_rum_active" # true if Datadog RUM was detected in the page(s) loaded by Selenium
         TAG_BROWSER_DRIVER = "test.browser.driver"
         # version of selenium driver used
         TAG_BROWSER_DRIVER_VERSION = "test.browser.driver_version"
@@ -60,6 +58,9 @@ module Test
         # version of the browser, if multiple browsers or multiple versions then this tag is empty
         TAG_BROWSER_VERSION = "test.browser.version"
 
+        # Tags for test retries
+        TAG_IS_RETRY = "test.is_retry" # true if test was retried by datadog-ci library
+
         # internal APM tag to mark a span as a test span
         TAG_SPAN_KIND = "span.kind"
         SPAN_KIND_TEST = "test"
@@ -68,8 +69,7 @@ module Test
         INHERITABLE_TAGS = [TAG_FRAMEWORK, TAG_FRAMEWORK_VERSION].freeze
 
         # could be either "test" or "suite" depending on whether we skip individual tests or whole suites
-        # we use test skipping for Ruby
-        ITR_TEST_SKIPPING_MODE = "test"
+        ITR_TEST_SKIPPING_MODE = "test" # we always skip tests (not suites) in Ruby
         ITR_TEST_SKIP_REASON = "Skipped by Datadog's intelligent test runner"
         ITR_UNSKIPPABLE_OPTION = :datadog_itr_unskippable
 

lib/datadog/ci/ext/transport.rb

@@ -36,6 +36,7 @@ module Transport
         DD_API_SETTINGS_RESPONSE_CODE_COVERAGE_KEY = "code_coverage"
         DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY = "tests_skipping"
         DD_API_SETTINGS_RESPONSE_REQUIRE_GIT_KEY = "require_git"
+        DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY = "flaky_test_retries_enabled"
         DD_API_SETTINGS_RESPONSE_DEFAULT = {DD_API_SETTINGS_RESPONSE_ITR_ENABLED_KEY => false}.freeze
 
         DD_API_GIT_SEARCH_COMMITS_PATH = "/api/v2/git/repository/search_commits"

lib/datadog/ci/remote/component.rb

@@ -28,6 +28,7 @@ def configure(test_session)
           end
 
           test_optimisation.configure(library_configuration, test_session)
+          test_retries.configure(library_configuration)
         end
 
         private
@@ -36,6 +37,10 @@ def test_optimisation
           Datadog.send(:components).test_optimisation
         end
 
+        def test_retries
+          Datadog.send(:components).test_retries
+        end
+
         def git_tree_upload_worker
           Datadog.send(:components).git_tree_upload_worker
         end

lib/datadog/ci/remote/library_settings.rb

@@ -70,6 +70,12 @@ def tests_skipping_enabled?
           @tests_skipping_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_TESTS_SKIPPING_KEY)
         end
 
+        def flaky_test_retries_enabled?
+          return @flaky_test_retries_enabled if defined?(@flaky_test_retries_enabled)
+
+          @flaky_test_retries_enabled = bool(Ext::Transport::DD_API_SETTINGS_RESPONSE_FLAKY_TEST_RETRIES_KEY)
+        end
+
         private
 
         def bool(key)

lib/datadog/ci/test.rb

@@ -142,7 +142,14 @@ def parameters
       private
 
       def record_test_result(datadog_status)
-        test_suite&.record_test_result(datadog_status)
+        test_id = Utils::TestRun.skippable_test_id(name, test_suite_name, parameters)
+
+        # if this test was already executed in this test suite, mark it as retried
+        if test_suite&.test_executed?(test_id)
+          set_tag(Ext::Test::TAG_IS_RETRY, "true")
+        end
+
+        test_suite&.record_test_result(test_id, datadog_status)
       end
     end
   end

lib/datadog/ci/test_retries/component.rb

@@ -0,0 +1,28 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module TestRetries
+      # Encapsulates the logic to enable test retries, including:
+      # - retrying failed tests - improve success rate of CI pipelines
+      # - retrying new tests - detect flaky tests as early as possible to prevent them from being merged
+      class Component
+        attr_reader :retry_failed_tests_enabled, :retry_failed_tests_max_attempts, :retry_failed_tests_total_limit
+
+        def initialize(
+          retry_failed_tests_max_attempts:,
+          retry_failed_tests_total_limit:
+        )
+          # enabled only by remote settings
+          @retry_failed_tests_enabled = false
+          @retry_failed_tests_max_attempts = retry_failed_tests_max_attempts
+          @retry_failed_tests_total_limit = retry_failed_tests_total_limit
+        end
+
+        def configure(library_settings)
+          @retry_failed_tests_enabled = library_settings.flaky_test_retries_enabled?
+        end
+      end
+    end
+  end
+end