Merge pull request #253 from DataDog/anmarchenko/knapsack_standalone_…

…integration

[SDTEST-1129] Extract Knapsack instrumentation as standalone integration

Rakefile

@@ -76,9 +76,6 @@ TEST_METADATA = {
   "knapsack_rspec" => {
     "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby"
   },
-  "knapsack_rspec_go" => {
-    "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby"
-  },
   "selenium" => {
     "selenium-4-capybara-3" => "❌ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ❌ 3.4 / ✅ jruby"
   },
@@ -156,7 +153,6 @@ namespace :spec do
     ci_queue_minitest
     ci_queue_rspec
     knapsack_rspec
-    knapsack_rspec_go
     selenium timecop
   ].each do |contrib|
     desc "" # "Explicitly hiding from `rake -T`"

lib/datadog/ci.rb

@@ -406,9 +406,16 @@ def test_optimisation
 end
 
 # Integrations
+
+# Test frameworks (manual instrumentation)
 require_relative "ci/contrib/cucumber/integration"
-require_relative "ci/contrib/rspec/integration"
 require_relative "ci/contrib/minitest/integration"
+require_relative "ci/contrib/rspec/integration"
+
+# Test runners (instrumented automatically when corresponding frameworks are instrumented)
+require_relative "ci/contrib/knapsack/integration"
+
+# Additional test libraries (auto instrumented later on test session start)
 require_relative "ci/contrib/selenium/integration"
 require_relative "ci/contrib/simplecov/integration"
 

lib/datadog/ci/configuration/settings.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../contrib/instrumentation"
 require_relative "../ext/settings"
 require_relative "../utils/bundle"
 
@@ -8,8 +9,6 @@ module CI
     module Configuration
       # Adds CI behavior to ddtrace settings
       module Settings
-        InvalidIntegrationError = Class.new(StandardError)
-
         def self.extended(base)
           base = base.singleton_class unless base.is_a?(Class)
           add_settings!(base)
@@ -126,23 +125,11 @@ def self.add_settings!(base)
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 
-                integration = fetch_integration(integration_name)
-                integration.configure(options, &block)
-
-                return unless integration.enabled
-
-                patch_results = integration.patch
-                next if patch_results == true
-
-                error_message = <<-ERROR
-                  Available?: #{patch_results[:available]}, Loaded?: #{patch_results[:loaded]},
-                  Compatible?: #{patch_results[:compatible]}, Patchable?: #{patch_results[:patchable]}"
-                ERROR
-                Datadog.logger.warn("Unable to patch #{integration_name} (#{error_message})")
+                Contrib::Instrumentation.instrument(integration_name, options, &block)
               end
 
               define_method(:[]) do |integration_name|
-                fetch_integration(integration_name).configuration
+                Contrib::Instrumentation.fetch_integration(integration_name).configuration
               end
 
               option :trace_flush
@@ -151,11 +138,6 @@ def self.add_settings!(base)
                 o.type :hash
                 o.default({})
               end
-
-              define_method(:fetch_integration) do |name|
-                Datadog::CI::Contrib::Integration.registry[name] ||
-                  raise(InvalidIntegrationError, "'#{name}' is not a valid integration.")
-              end
             end
           end
         end

lib/datadog/ci/contrib/cucumber/formatter.rb

@@ -3,6 +3,7 @@
 require_relative "../../ext/test"
 require_relative "../../git/local_repository"
 require_relative "../../utils/test_run"
+require_relative "../instrumentation"
 require_relative "ext"
 
 module Datadog
@@ -38,9 +39,9 @@ def on_test_run_started(event)
             test_visibility_component.start_test_session(
               tags: {
                 CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-                CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s
+                CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s
               },
-              service: configuration[:service_name]
+              service: datadog_configuration[:service_name]
             )
             test_visibility_component.start_test_module(Ext::FRAMEWORK)
           end
@@ -61,7 +62,7 @@ def on_test_case_started(event)
             # @type var tags: Hash[String, String]
             tags = {
               CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK,
-              CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Cucumber::Integration.version.to_s,
+              CI::Ext::Test::TAG_FRAMEWORK_VERSION => datadog_integration.version.to_s,
               CI::Ext::Test::TAG_SOURCE_FILE => Git::LocalRepository.relative_to_root(event.test_case.location.file),
               CI::Ext::Test::TAG_SOURCE_START => event.test_case.location.line.to_s
             }
@@ -81,7 +82,7 @@ def on_test_case_started(event)
               event.test_case.name,
               test_suite_name,
               tags: tags,
-              service: configuration[:service_name]
+              service: datadog_configuration[:service_name]
             )
             if event.test_case.match_tags?("@#{CI::Ext::Test::ITR_UNSKIPPABLE_OPTION}")
               test_span&.itr_unskippable!
@@ -199,7 +200,11 @@ def ok?(result, strict)
             end
           end
 
-          def configuration
+          def datadog_integration
+            CI::Contrib::Instrumentation.fetch_integration(:cucumber)
+          end
+
+          def datadog_configuration
             Datadog.configuration.ci[:cucumber]
           end
 

lib/datadog/ci/contrib/cucumber/integration.rb

@@ -9,30 +9,21 @@ module CI
     module Contrib
       module Cucumber
         # Description of Cucumber integration
-        class Integration
-          include Datadog::CI::Contrib::Integration
-
+        class Integration < Contrib::Integration
           MINIMUM_VERSION = Gem::Version.new("3.0.0")
 
-          register_as :cucumber
-
-          def self.version
+          def version
             Gem.loaded_specs["cucumber"]&.version
           end
 
-          def self.loaded?
+          def loaded?
             !defined?(::Cucumber).nil? && !defined?(::Cucumber::Runtime).nil?
           end
 
-          def self.compatible?
+          def compatible?
             super && version >= MINIMUM_VERSION
           end
 
-          # test environments should not auto instrument test libraries
-          def auto_instrument?
-            false
-          end
-
           def new_configuration
             Configuration::Settings.new
           end

lib/datadog/ci/contrib/cucumber/patcher.rb

@@ -14,10 +14,6 @@ module Patcher
 
           module_function
 
-          def target_version
-            Integration.version
-          end
-
           def patch
             ::Cucumber::Runtime.include(Instrumentation)
           end

lib/datadog/ci/contrib/instrumentation.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+module Datadog
+  module CI
+    module Contrib
+      module Instrumentation
+        class InvalidIntegrationError < StandardError; end
+
+        @registry = {}
+
+        def self.registry
+          @registry
+        end
+
+        def self.register_integration(integration_class)
+          @registry[integration_name(integration_class)] = integration_class.new
+        end
+
+        # Manual instrumentation of a specific integration.
+        #
+        # This method is called when user has `c.ci.instrument :integration_name` in their code.
+        def self.instrument(integration_name, options = {}, &block)
+          integration = fetch_integration(integration_name)
+          integration.configure(options, &block)
+
+          return unless integration.enabled
+
+          patch_results = integration.patch
+          if patch_results[:ok]
+            # try to patch dependant integrations (for example knapsack that depends on rspec)
+            dependants = integration.dependants
+              .map { |name| fetch_integration(name) }
+              .filter { |integration| integration.patchable? }
+
+            Datadog.logger.debug("Found dependent integrations for #{integration_name}: #{dependants}")
+
+            dependants.each do |dependent_integration|
+              dependent_integration.patch
+            end
+          else
+            error_message = <<-ERROR
+                  Available?: #{patch_results[:available]}, Loaded?: #{patch_results[:loaded]},
+                  Compatible?: #{patch_results[:compatible]}, Patchable?: #{patch_results[:patchable]}"
+            ERROR
+            Datadog.logger.warn("Unable to patch #{integration_name} (#{error_message})")
+          end
+        end
+
+        # This method instruments all additional test libraries (ex: selenium-webdriver) that need to be instrumented
+        # later in the test suite run.
+        #
+        # It is intended to be called when test session starts to add additional capabilities to test visibility.
+        #
+        # This method does not automatically instrument test frameworks (ex: RSpec, Cucumber, etc), it requires
+        # test framework to be already instrumented.
+        def self.instrument_on_session_start
+          Datadog.logger.debug("Instrumenting all late instrumented integrations...")
+
+          @registry.each do |name, integration|
+            next unless integration.late_instrument?
+
+            Datadog.logger.debug "#{name} is allowed to be late instrumented"
+
+            patch_results = integration.patch
+            if patch_results[:ok]
+              Datadog.logger.debug("#{name} is patched")
+            else
+              Datadog.logger.debug("#{name} is not patched (#{patch_results})")
+            end
+          end
+        end
+
+        def self.fetch_integration(name)
+          @registry[name] ||
+            raise(InvalidIntegrationError, "'#{name}' is not a valid integration.")
+        end
+
+        # take the parent module name and downcase it
+        # for example for Datadog::CI::Contrib::RSpec::Integration it will be :rspec
+        def self.integration_name(subclass)
+          result = subclass.name&.split("::")&.[](-2)&.downcase&.to_sym
+          raise "Integration name could not be derived for #{subclass}" if result.nil?
+          result
+        end
+      end
+    end
+  end
+end