Merge pull request #194 from DataDog/anmarchenko/rake_skippable_tests…

…_percentage

[SDTEST-477] add command line tool to compute a percentage of skippable tests for RSpec

Appraisals

@@ -14,7 +14,7 @@ end
 alias original_appraise appraise
 
 REMOVED_GEMS = {
-  check: %w[rbs steep],
+  check: %w[rbs steep ruby_memcheck],
   development: %w[ruby-lsp ruby-lsp-rspec debug irb]
 }
 RUBY_VERSION = Gem::Version.new(RUBY_ENGINE_VERSION)

Steepfile

@@ -16,6 +16,7 @@ target :lib do
   library "tmpdir"
   library "fileutils"
   library "socket"
+  library "optparse"
 
   repo_path "vendor/rbs"
   library "ddtrace"

datadog-ci.gemspec

@@ -23,6 +23,9 @@ Gem::Specification.new do |spec|
   spec.homepage = "https://github.com/DataDog/datadog-ci-rb"
   spec.license = "BSD-3-Clause"
 
+  spec.bindir = "exe"
+  spec.executables = ["ddcirb"]
+
   spec.metadata["allowed_push_host"] = "https://rubygems.org"
   spec.metadata["changelog_uri"] = "https://github.com/DataDog/datadog-ci-rb/blob/main/CHANGELOG.md"
   spec.metadata["homepage_uri"] = spec.homepage
@@ -36,6 +39,7 @@ Gem::Specification.new do |spec|
       README.md
       ext/**/*
       lib/**/*
+      exe/**/*
     ]].select { |fn| File.file?(fn) } # We don't want directories, only files
       .reject { |fn| fn.end_with?(".so", ".bundle") } # Exclude local native binary artifacts
 

docs/CommandLineInterface.md

@@ -0,0 +1,64 @@
+# Command line interface
+
+This library provides experimental command line interface `ddcirb` to get the percentage of the tests
+that will be skipped for the current test run (only when using RSpec).
+
+## Usage
+
+This tool must be used on the same runner as your tests are running on with the same ENV variables.
+Run the command in the same folder where you usually run your tests. Gem datadog-ci must be installed.
+
+Available commands:
+
+- `bundle exec ddcirb skipped-tests` - outputs the percentage of skipped tests to stdout. Note that it runs your specs
+in dry run mode, don't forget to set RAILS_ENV=test environment variable.
+- `bundle exec ddcirb skipped-tests-estimate` - estimates the percentage of skipped tests and outputs to stdout without loading
+your test suite and running it in dry run mode. ATTENTION: this is considerably faster but could be very inaccurate.
+
+Example usage:
+
+```bash
+$ RAILS_ENV=test bundle exec ddcirb skipped-tests
+0.45
+```
+
+Available arguments:
+
+- `-f, --file` - output to a file (example: `bundle exec ddcirb skipped-tests -f out`)
+- `--verbose` - enable verbose output for debugging purposes (example: `bundle exec ddcirb skipped-tests --verbose`)
+- `--spec-path` - path to the folder with RSpec tests (default: `spec`, example: `bundle exec ddcirb skipped-tests --spec-path="myapp/spec"`)
+- `--rspec-opts` - additional options to pass to the RSpec when running it in dry run mode (example: `bundle exec ddcirb skipped-tests --rspec-opts="--require rails_helper"`)
+
+## Example usage in Circle CI
+
+This tool could be used to determine [Circle CI parallelism](https://support.circleci.com/hc/en-us/articles/14928385117851-How-to-dynamically-set-job-parallelism) dynamically:
+
+```yaml
+version: 2.1
+
+setup: true
+
+orbs:
+  continuation: circleci/continuation@0.2.0
+
+jobs:
+  determine-parallelism:
+    docker:
+      - image: cimg/base:edge
+    resource_class: medium
+    steps:
+      - checkout
+      - run:
+          name: Determine parallelism
+          command: |
+            PARALLELISM=$(RAILS_ENV=test bundle exec ddcirb skipped-tests)
+            echo "{\"parallelism\": $PARALLELISM}" > pipeline-parameters.json
+      - continuation/continue:
+          configuration_path: .circleci/continue_config.yml
+          parameters: pipeline-parameters.json
+
+workflows:
+  build-setup:
+    jobs:
+      - determine-parallelism
+```

exe/ddcirb

@@ -0,0 +1,5 @@
+#!/usr/bin/env ruby
+
+require "datadog/ci/cli/cli"
+
+Datadog::CI::CLI.exec(ARGV.first)

lib/datadog/ci/cli/cli.rb

@@ -0,0 +1,24 @@
+require "datadog"
+require "datadog/ci"
+
+require_relative "command/skippable_tests_percentage"
+require_relative "command/skippable_tests_percentage_estimate"
+
+module Datadog
+  module CI
+    module CLI
+      def self.exec(action)
+        case action
+        when "skipped-tests", "skippable-tests"
+          Command::SkippableTestsPercentage.new.exec
+        when "skipped-tests-estimate", "skippable-tests-estimate"
+          Command::SkippableTestsPercentageEstimate.new.exec
+        else
+          puts("Usage: bundle exec ddcirb [command] [options]. Available commands:")
+          puts("  skippable-tests - calculates the exact percentage of skipped tests and prints it to stdout or file")
+          puts("  skippable-tests-estimate - estimates the percentage of skipped tests and prints it to stdout or file")
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/cli/command/base.rb

@@ -0,0 +1,58 @@
+require "optparse"
+
+module Datadog
+  module CI
+    module CLI
+      module Command
+        class Base
+          def exec
+            action = build_action
+            result = action&.call
+
+            validate!(action)
+            output(result)
+          end
+
+          private
+
+          def build_action
+          end
+
+          def options
+            return @options if defined?(@options)
+
+            ddcirb_options = {}
+            OptionParser.new do |opts|
+              opts.banner = "Usage: bundle exec ddcirb [command] [options]\n Available commands: skippable-tests, skippable-tests-estimate"
+
+              opts.on("-f", "--file FILENAME", "Output result to file FILENAME")
+              opts.on("--verbose", "Verbose output to stdout")
+
+              command_options(opts)
+            end.parse!(into: ddcirb_options)
+
+            @options = ddcirb_options
+          end
+
+          def command_options(opts)
+          end
+
+          def validate!(action)
+            if action.nil? || action.failed
+              Datadog.logger.error("ddcirb failed, exiting")
+              Kernel.exit(1)
+            end
+          end
+
+          def output(result)
+            if options[:file]
+              File.write(options[:file], result)
+            else
+              print(result)
+            end
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/cli/command/skippable_tests_percentage.rb

@@ -0,0 +1,27 @@
+require_relative "base"
+require_relative "../../test_optimisation/skippable_percentage/calculator"
+
+module Datadog
+  module CI
+    module CLI
+      module Command
+        class SkippableTestsPercentage < Base
+          private
+
+          def build_action
+            ::Datadog::CI::TestOptimisation::SkippablePercentage::Calculator.new(
+              rspec_cli_options: (options[:"rspec-opts"] || "").split,
+              verbose: !options[:verbose].nil?,
+              spec_path: options[:"spec-path"] || "spec"
+            )
+          end
+
+          def command_options(opts)
+            opts.on("--rspec-opts=[OPTIONS]", "Command line options to pass to RSpec")
+            opts.on("--spec-path=[SPEC_PATH]", "Relative path to the spec directory, example: spec")
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/cli/command/skippable_tests_percentage_estimate.rb

@@ -0,0 +1,21 @@
+require_relative "base"
+require_relative "../../test_optimisation/skippable_percentage/estimator"
+
+module Datadog
+  module CI
+    module CLI
+      module Command
+        class SkippableTestsPercentageEstimate < Base
+          private
+
+          def build_action
+            ::Datadog::CI::TestOptimisation::SkippablePercentage::Estimator.new(
+              verbose: !options[:verbose].nil?,
+              spec_path: options[:"spec-path"] || "spec"
+            )
+          end
+        end
+      end
+    end
+  end
+end

lib/datadog/ci/configuration/components.rb

@@ -19,6 +19,7 @@
 require_relative "../test_visibility/serializers/factories/test_suite_level"
 require_relative "../test_visibility/transport"
 require_relative "../transport/adapters/telemetry_webmock_safe_adapter"
+require_relative "../test_visibility/null_transport"
 require_relative "../transport/api/builder"
 require_relative "../utils/parsing"
 require_relative "../utils/test_run"
@@ -203,6 +204,9 @@ def build_test_visibility_api(settings)
         end
 
         def build_tracing_transport(settings, api)
+          # NullTransport ignores traces
+          return TestVisibility::NullTransport.new if settings.ci.discard_traces
+          # nil means that default legacy APM transport will be used (only for very old Datadog Agent versions)
           return nil if api.nil?
 
           TestVisibility::Transport.new(
@@ -213,7 +217,8 @@ def build_tracing_transport(settings, api)
         end
 
         def build_coverage_writer(settings, api)
-          return nil if api.nil?
+          # nil means that coverage event will be ignored
+          return nil if api.nil? || settings.ci.discard_traces
 
           TestOptimisation::Coverage::Writer.new(
             transport: TestOptimisation::Coverage::Transport.new(api: api)

lib/datadog/ci/configuration/settings.rb

@@ -117,6 +117,12 @@ def self.add_settings!(base)
                 o.default true
               end
 
+              # internal only
+              option :discard_traces do |o|
+                o.type :bool
+                o.default false
+              end
+
               define_method(:instrument) do |integration_name, options = {}, &block|
                 return unless enabled
 

lib/datadog/ci/contrib/rspec/configuration/settings.rb

@@ -24,6 +24,12 @@ class Settings < Datadog::CI::Contrib::Settings
                 Utils::Configuration.fetch_service_name(Ext::DEFAULT_SERVICE_NAME)
               end
             end
+
+            # internal only
+            option :dry_run_enabled do |o|
+              o.type :bool
+              o.default false
+            end
           end
         end
       end

lib/datadog/ci/contrib/rspec/example.rb

@@ -17,7 +17,7 @@ def self.included(base)
 
           module InstanceMethods
             def run(*args)
-              return super if ::RSpec.configuration.dry_run?
+              return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
 
               test_name = full_description.strip

lib/datadog/ci/contrib/rspec/example_group.rb

@@ -16,7 +16,7 @@ def self.included(base)
           # Instance methods for configuration
           module ClassMethods
             def run(reporter = ::RSpec::Core::NullReporter)
-              return super if ::RSpec.configuration.dry_run?
+              return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
               return super unless top_level?
 

lib/datadog/ci/contrib/rspec/knapsack_pro/runner.rb

@@ -15,7 +15,7 @@ def self.included(base)
 
             module InstanceMethods
               def knapsack__run_specs(*args)
-                return super if ::RSpec.configuration.dry_run?
+                return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
                 return super unless datadog_configuration[:enabled]
 
                 test_session = test_visibility_component.start_test_session(

lib/datadog/ci/contrib/rspec/runner.rb

@@ -15,7 +15,7 @@ def self.included(base)
 
           module InstanceMethods
             def run_specs(*args)
-              return super if ::RSpec.configuration.dry_run?
+              return super if ::RSpec.configuration.dry_run? && !datadog_configuration[:dry_run_enabled]
               return super unless datadog_configuration[:enabled]
 
               test_session = test_visibility_component.start_test_session(

lib/datadog/ci/test.rb

@@ -76,7 +76,7 @@ def is_retry?
       # - tests that read files from disk
       # - tests that make network requests
       # - tests that call external processes
-      # - tests that use forking or threading
+      # - tests that use forking
       #
       # @return [void]
       def itr_unskippable!