Merge pull request #259 from DataDog/anmarchenko/auto_instrumentation

[SDTEST-228] Auto instrumentation

.standard_todo.yml

@@ -10,5 +10,7 @@ ignore:
       - Style/Alias
   - spec/datadog/ci/contrib/minitest/instrumentation_spec.rb:
       - Lint/ConstantDefinitionInBlock
+  - spec/datadog/ci/contrib/minitest_auto_instrument/instrumentation_spec.rb:
+      - Lint/ConstantDefinitionInBlock
   - spec/datadog/ci/contrib/timecop/instrumentation_spec.rb:
       - Lint/ConstantDefinitionInBlock

README.md

@@ -1,22 +1,20 @@
-# Datadog Test Visibility for Ruby
+# Datadog Test Optimization for Ruby
 
 [![Gem Version](https://badge.fury.io/rb/datadog-ci.svg)](https://badge.fury.io/rb/datadog-ci)
 [![YARD documentation](https://img.shields.io/badge/YARD-documentation-blue)](https://datadoghq.dev/datadog-ci-rb/)
-[![codecov](https://codecov.io/gh/DataDog/datadog-ci-rb/branch/main/graph/badge.svg)](https://app.codecov.io/gh/DataDog/datadog-ci-rb/branch/main)
-[![CircleCI](https://dl.circleci.com/status-badge/img/gh/DataDog/datadog-ci-rb/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DataDog/datadog-ci-rb/tree/main)
 
 Datadog's Ruby Library for instrumenting your tests.
 Learn more on our [official website](https://docs.datadoghq.com/tests/) and check out our [documentation for this library](https://docs.datadoghq.com/tests/setup/ruby/?tab=cloudciprovideragentless).
 
 ## Features
 
 - [Test Visibility](https://docs.datadoghq.com/tests/) - collect metrics and results for your tests
-- [Intelligent test runner](https://docs.datadoghq.com/intelligent_test_runner/) - save time by selectively running only tests affected by code changes
-- [Auto test retries](https://docs.datadoghq.com/tests/auto_test_retries/?tab=ruby) - retrying failing tests up to N times to avoid failing your build due to flaky tests
-- [Early flake detection](https://docs.datadoghq.com/tests/early_flake_detection?tab=ruby) - Datadog’s test flakiness solution that identifies flakes early by running newly added tests multiple times
+- [Test impact analysis](https://docs.datadoghq.com/tests/test_impact_analysis/) - save time by selectively running only tests affected by code changes
+- [Flaky test management](https://docs.datadoghq.com/tests/flaky_test_management/) - track, alert, search your flaky tests in Datadog UI
+- [Auto test retries](https://docs.datadoghq.com/tests/flaky_test_management/auto_test_retries/?tab=ruby) - retrying failing tests up to N times to avoid failing your build due to flaky tests
+- [Early flake detection](https://docs.datadoghq.com/tests/flaky_test_management/early_flake_detection/?tab=ruby) - Datadog’s test flakiness solution that identifies flakes early by running newly added tests multiple times
 - [Search and manage CI tests](https://docs.datadoghq.com/tests/search/)
 - [Enhance developer workflows](https://docs.datadoghq.com/tests/developer_workflows)
-- [Flaky test management](https://docs.datadoghq.com/tests/guides/flaky_test_management/)
 - [Add custom measures to your tests](https://docs.datadoghq.com/tests/guides/add_custom_measures/?tab=ruby)
 - [Browser tests integration with Datadog RUM](https://docs.datadoghq.com/tests/browser_tests)
 
@@ -37,7 +35,7 @@ If you used [test visibility for Ruby](https://docs.datadoghq.com/tests/setup/ru
 ## Setup
 
 - [Test visibility setup](https://docs.datadoghq.com/tests/setup/ruby/?tab=cloudciprovideragentless)
-- [Intelligent test runner setup](https://docs.datadoghq.com/intelligent_test_runner/setup/ruby) (test visibility setup is required before setting up intelligent test runner)
+- [Test impact analysis setup](https://docs.datadoghq.com/tests/test_impact_analysis/setup/ruby/?tab=cloudciprovideragentless) (test visibility setup is required before setting up test impact analysis)
 
 ## Contributing
 

Rakefile

@@ -58,6 +58,9 @@ TEST_METADATA = {
   "minitest" => {
     "minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby"
   },
+  "minitest_auto_instrument" => {
+    "minitest-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby"
+  },
   "activesupport" => {
     "activesupport-4" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby",
     "activesupport-5" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ✅ jruby",
@@ -76,6 +79,9 @@ TEST_METADATA = {
   "knapsack_rspec" => {
     "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby"
   },
+  "knapsack_auto_instrument" => {
+    "knapsack_pro-7-rspec-3" => "✅ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ✅ 3.4 / ❌ jruby"
+  },
   "selenium" => {
     "selenium-4-capybara-3" => "❌ 2.7 / ✅ 3.0 / ✅ 3.1 / ✅ 3.2 / ✅ 3.3 / ❌ 3.4 / ✅ jruby"
   },
@@ -149,10 +155,12 @@ namespace :spec do
     rspec
     minitest
     minitest_shoulda_context
+    minitest_auto_instrument
     activesupport
     ci_queue_minitest
     ci_queue_rspec
     knapsack_rspec
+    knapsack_auto_instrument
     selenium timecop
   ].each do |contrib|
     desc "" # "Explicitly hiding from `rake -T`"

exe/ddcirb

@@ -2,4 +2,6 @@
 
 require "datadog/ci/cli/cli"
 
-Datadog::CI::CLI.exec(ARGV.first)
+command = ARGV.shift
+
+Datadog::CI::CLI.exec(command, ARGV)

lib/datadog/ci/auto_instrument.rb

@@ -0,0 +1,3 @@
+require "datadog/ci"
+
+Datadog::CI::Contrib::Instrumentation.auto_instrument

lib/datadog/ci/cli/cli.rb

@@ -1,14 +1,17 @@
 require "datadog"
 require "datadog/ci"
 
+require_relative "command/exec"
 require_relative "command/skippable_tests_percentage"
 require_relative "command/skippable_tests_percentage_estimate"
 
 module Datadog
   module CI
     module CLI
-      def self.exec(action)
+      def self.exec(action, args = [])
         case action
+        when "exec"
+          Command::Exec.new(args).exec
         when "skipped-tests", "skippable-tests"
           Command::SkippableTestsPercentage.new.exec
         when "skipped-tests-estimate", "skippable-tests-estimate"
@@ -17,6 +20,7 @@ def self.exec(action)
           puts("Usage: bundle exec ddcirb [command] [options]. Available commands:")
           puts("  skippable-tests - calculates the exact percentage of skipped tests and prints it to stdout or file")
           puts("  skippable-tests-estimate - estimates the percentage of skipped tests and prints it to stdout or file")
+          puts("  exec YOUR_TEST_COMMAND - automatically instruments your test command with Datadog and executes it")
         end
       end
     end

lib/datadog/ci/cli/command/exec.rb

@@ -0,0 +1,29 @@
+require_relative "base"
+require_relative "../../test_optimisation/skippable_percentage/estimator"
+
+module Datadog
+  module CI
+    module CLI
+      module Command
+        class Exec < Base
+          def initialize(args)
+            super()
+
+            @args = args
+          end
+
+          def exec
+            rubyopts = [
+              "-rdatadog/ci/auto_instrument"
+            ]
+
+            existing_rubyopt = ENV["RUBYOPT"]
+            ENV["RUBYOPT"] = existing_rubyopt ? "#{existing_rubyopt} #{rubyopts.join(" ")}" : rubyopts.join(" ")
+
+            Kernel.exec(*@args)
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/contrib/cucumber/integration.rb

@@ -17,7 +17,7 @@ def version
           end
 
           def loaded?
-            !defined?(::Cucumber).nil? && !defined?(::Cucumber::Runtime).nil?
+            !defined?(::Cucumber).nil? && !defined?(::Cucumber::Runtime).nil? && !defined?(::Cucumber::Configuration).nil?
           end
 
           def compatible?

lib/datadog/ci/contrib/instrumentation.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require "datadog/core/utils/only_once"
+
 module Datadog
   module CI
     module Contrib
@@ -16,34 +18,63 @@ def self.register_integration(integration_class)
           @registry[integration_name(integration_class)] = integration_class.new
         end
 
+        # Auto instrumentation of all integrations.
+        #
+        # Registers a :script_compiled tracepoint to watch for new Ruby files being loaded.
+        # On every file load it checks if any of the integrations are patchable now.
+        # Only the integrations that are available in the environment are checked.
+        def self.auto_instrument
+          Datadog.logger.debug("Auto instrumenting all integrations...")
+
+          auto_instrumented_integrations = fetch_auto_instrumented_integrations
+          if auto_instrumented_integrations.empty?
+            Datadog.logger.warn(
+              "Auto instrumentation was requested, but no available integrations were found. " \
+              "Tests will be run without Datadog instrumentation."
+            )
+            return
+          end
+
+          # note that `Kernel.require` might be called from a different thread, so
+          # there is a possibility of concurrent execution of this tracepoint
+          mutex = Mutex.new
+          script_compiled_tracepoint = TracePoint.new(:script_compiled) do |tp|
+            all_patched = true
+
+            mutex.synchronize do
+              auto_instrumented_integrations.each do |integration|
+                next if integration.patched?
+
+                all_patched = false
+                next unless integration.loaded?
+
+                auto_configure_datadog
+
+                Datadog.logger.debug("#{integration.class} is loaded")
+                patch_integration(integration)
+              end
+
+              if all_patched
+                Datadog.logger.debug("All expected integrations are patched, disabling the script_compiled tracepoint")
+
+                tp.disable
+              end
+            end
+          end
+          script_compiled_tracepoint.enable
+        end
+
         # Manual instrumentation of a specific integration.
         #
         # This method is called when user has `c.ci.instrument :integration_name` in their code.
         def self.instrument(integration_name, options = {}, &block)
           integration = fetch_integration(integration_name)
+          # when manually instrumented, it might be configured via code
           integration.configure(options, &block)
 
           return unless integration.enabled
 
-          patch_results = integration.patch
-          if patch_results[:ok]
-            # try to patch dependant integrations (for example knapsack that depends on rspec)
-            dependants = integration.dependants
-              .map { |name| fetch_integration(name) }
-              .filter { |integration| integration.patchable? }
-
-            Datadog.logger.debug("Found dependent integrations for #{integration_name}: #{dependants}")
-
-            dependants.each do |dependent_integration|
-              dependent_integration.patch
-            end
-          else
-            error_message = <<-ERROR
-                  Available?: #{patch_results[:available]}, Loaded?: #{patch_results[:loaded]},
-                  Compatible?: #{patch_results[:compatible]}, Patchable?: #{patch_results[:patchable]}"
-            ERROR
-            Datadog.logger.warn("Unable to patch #{integration_name} (#{error_message})")
-          end
+          patch_integration(integration, with_dependencies: true)
         end
 
         # This method instruments all additional test libraries (ex: selenium-webdriver) that need to be instrumented
@@ -58,15 +89,11 @@ def self.instrument_on_session_start
 
           @registry.each do |name, integration|
             next unless integration.late_instrument?
+            next unless integration.enabled
 
             Datadog.logger.debug "#{name} is allowed to be late instrumented"
 
-            patch_results = integration.patch
-            if patch_results[:ok]
-              Datadog.logger.debug("#{name} is patched")
-            else
-              Datadog.logger.debug("#{name} is not patched (#{patch_results})")
-            end
+            patch_integration(integration)
           end
         end
 
@@ -82,6 +109,58 @@ def self.integration_name(subclass)
           raise "Integration name could not be derived for #{subclass}" if result.nil?
           result
         end
+
+        def self.patch_integration(integration, with_dependencies: false)
+          patch_results = integration.patch
+
+          if patch_results[:ok]
+            Datadog.logger.debug("#{integration.class} is patched")
+
+            return unless with_dependencies
+
+            # try to patch dependant integrations (for example knapsack that depends on rspec)
+            dependants = integration.dependants
+              .map { |name| fetch_integration(name) }
+              .filter { |integration| integration.patchable? }
+
+            Datadog.logger.debug("Found dependent integrations for #{integration.class}: #{dependants}")
+
+            dependants.each do |dependent_integration|
+              patch_integration(dependent_integration, with_dependencies: true)
+            end
+
+          else
+            Datadog.logger.debug("Attention: #{integration.class} is not patched (#{patch_results})")
+          end
+        end
+
+        def self.fetch_auto_instrumented_integrations
+          @registry.filter_map do |name, integration|
+            # ignore integrations that are not in the Gemfile or have incompatible versions
+            next unless integration.compatible?
+
+            # late instrumented integrations will be patched when the test session starts
+            next if integration.late_instrument?
+
+            Datadog.logger.debug("#{name} should be auto instrumented")
+            integration
+          end
+        end
+
+        def self.auto_configure_datadog
+          configure_once.run do
+            Datadog.logger.debug("Applying Datadog configuration in CI mode...")
+            Datadog.configure do |c|
+              c.ci.enabled = true
+              c.tracing.enabled = true
+            end
+          end
+        end
+
+        # This is not thread safe, it is synchronized by the caller in the tracepoint
+        def self.configure_once
+          @configure_once ||= Datadog::Core::Utils::OnlyOnce.new
+        end
       end
     end
   end

lib/datadog/ci/contrib/integration.rb

@@ -93,9 +93,7 @@ def patcher
 
         # @!visibility private
         def patch
-          # @type var patcher_klass: untyped
-          patcher_klass = patcher
-          if !patchable? || patcher_klass.nil?
+          if !patchable? || patcher.nil?
             return {
               ok: false,
               available: available?,
@@ -105,10 +103,14 @@ def patch
             }
           end
 
-          patcher_klass.patch
+          patcher.patch
           {ok: true}
         end
 
+        def patched?
+          patcher&.patched?
+        end
+
         # Can the patch for this integration be applied automatically?
         # @return [Boolean] can the tracer activate this instrumentation without explicit user input?
         def late_instrument?

lib/datadog/ci/contrib/minitest/integration.rb

@@ -17,7 +17,8 @@ def version
           end
 
           def loaded?
-            !defined?(::Minitest).nil?
+            !defined?(::Minitest).nil? && !defined?(::Minitest::Runnable).nil? && !defined?(::Minitest::Test).nil? &&
+              !defined?(::Minitest::CompositeReporter).nil?
           end
 
           def compatible?

lib/datadog/ci/contrib/patcher.rb

@@ -1,7 +1,6 @@
 # frozen_string_literal: true
 
 require "datadog/core/utils/only_once"
-require "datadog/core/telemetry/logger"
 
 module Datadog
   module CI
@@ -42,7 +41,6 @@ def patch
           # @param e [Exception]
           def on_patch_error(e)
             Datadog.logger.error("Failed to apply #{patch_name} patch. Cause: #{e} Location: #{Array(e.backtrace).first}")
-            Datadog::Core::Telemetry::Logger.report(e, description: "Failed to apply #{patch_name} patch")
 
             @patch_error_result = {
               type: e.class.name,

lib/datadog/ci/contrib/rspec/integration.rb

@@ -22,7 +22,9 @@ def version
 
           def loaded?
             !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? &&
-              !defined?(::RSpec::Core::Example).nil?
+              !defined?(::RSpec::Core::Example).nil? &&
+              !defined?(::RSpec::Core::Runner).nil? &&
+              !defined?(::RSpec::Core::ExampleGroup).nil?
           end
 
           def compatible?

lib/datadog/ci/test_retries/strategy/retry_new.rb

@@ -101,7 +101,7 @@ def calculate_total_retries_limit(library_settings, test_session)
             end
             @total_limit = (tests_count * percentage_limit / 100.0).ceil
             Datadog.logger.debug do
-              "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}%) of #{tests_count}"
+              "Retry new tests total limit is [#{@total_limit}] (#{percentage_limit}% of #{tests_count})"
             end
           end