Merge pull request #109 from DataDog/anmarchenko/enable_tslv_by_default

[CIVIS-8594] Enable test suite level visibility by default (with killswitch)

README.md

@@ -6,7 +6,7 @@
 [![CircleCI](https://dl.circleci.com/status-badge/img/gh/DataDog/datadog-ci-rb/tree/main.svg?style=svg)](https://dl.circleci.com/status-badge/redirect/gh/DataDog/datadog-ci-rb/tree/main)
 
 Datadog's Ruby Library for instrumenting your test and continuous integration pipeline.
-Learn more on our [official website](https://docs.datadoghq.com/continuous_integration/tests/ruby/).
+Learn more on our [official website](https://docs.datadoghq.com/tests/) and check out our [documentation for this library](https://docs.datadoghq.com/tests/setup/ruby/?tab=cloudciprovideragentless).
 
 > [!IMPORTANT]
 > The `datadog-ci` gem is currently a component of [`ddtrace`](https://github.com/datadog/dd-trace-rb) and should not be used without it.
@@ -134,30 +134,6 @@ Datadog.configure { |c| c.ci.agentless_mode_enabled = true }
 
 ## Additional configuration
 
-### Experimental test suite level visibility support
-
-Test suite level visibility (TSLV) is available in experimental mode and disabled by default (yet).
-To enable it add the following line to Datadog configuration:
-
-```ruby
-Datadog.configure do |c|
-  #  ... usual ci configs ...
-  c.ci.experimental_test_suite_level_visibility_enabled = true
-end
-```
-
-This will enable visibility into how your test suites and test sessions behave, providing
-insights about performance and reliability on test suite/session level. Please submit your feedback
-and any issues encountered via Github issues.
-
-Example test suite trace for Ruby project:
-
-![test level visibility trace](./docs/screenshots/tslv-trace.jpeg)
-
-Example test suite level visibility dashboard for a set of Ruby projects:
-
-![test level visibility trace](./docs/screenshots/tslv-dashboard.jpeg)
-
 ### Add tracing instrumentations
 
 It can be useful to have rich tracing information about your tests that includes time spent performing database operations

lib/datadog/ci.rb

@@ -313,13 +313,13 @@ def trace(span_name, type: "span", tags: {}, &block)
       # ```
       # # start span
       # Datadog::CI.trace(
-      #   "step",
       #   "Given I have 42 cucumbers",
+      #   type: "step",
       #   tags: {}
       # )
       #
-      # # somewhere else, access the active "step" span
-      # step_span = Datadog::CI.active_span("step")
+      # # somewhere else, access the active step span
+      # step_span = Datadog::CI.active_span
       # step_span.finish()
       # ```
       #

lib/datadog/ci/configuration/components.rb

@@ -41,7 +41,7 @@ def activate_ci!(settings)
           elsif can_use_evp_proxy?(settings, agent_settings)
             test_visibility_transport = build_evp_proxy_transport(settings, agent_settings)
           else
-            settings.ci.experimental_test_suite_level_visibility_enabled = false
+            settings.ci.force_test_level_visibility = true
           end
 
           # Deactivate telemetry
@@ -72,7 +72,7 @@ def activate_ci!(settings)
           settings.tracing.test_mode.writer_options = writer_options
 
           @ci_recorder = TestVisibility::Recorder.new(
-            test_suite_level_visibility_enabled: settings.ci.experimental_test_suite_level_visibility_enabled
+            test_suite_level_visibility_enabled: !settings.ci.force_test_level_visibility
           )
         end
 
@@ -119,10 +119,10 @@ def build_evp_proxy_transport(settings, agent_settings)
         end
 
         def serializers_factory(settings)
-          if settings.ci.experimental_test_suite_level_visibility_enabled
-            Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel
-          else
+          if settings.ci.force_test_level_visibility
             Datadog::CI::TestVisibility::Serializers::Factories::TestLevel
+          else
+            Datadog::CI::TestVisibility::Serializers::Factories::TestSuiteLevel
           end
         end
 

lib/datadog/ci/configuration/settings.rb

@@ -34,10 +34,25 @@ def self.add_settings!(base)
                 o.env CI::Ext::Settings::ENV_AGENTLESS_URL
               end
 
+              option :force_test_level_visibility do |o|
+                o.type :bool
+                o.env CI::Ext::Settings::ENV_FORCE_TEST_LEVEL_VISIBILITY
+                o.default false
+              end
+
               option :experimental_test_suite_level_visibility_enabled do |o|
                 o.type :bool
                 o.env CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED
                 o.default false
+                o.after_set do |value|
+                  if value
+                    Datadog::Core.log_deprecation do
+                      "The experimental_test_suite_level_visibility_enabled setting has no effect and will be removed in 1.0. " \
+                        "Test suite level visibility is now enabled by default. " \
+                        "If you want to disable test suite level visibility set configuration.ci.force_test_level_visibility = true."
+                    end
+                  end
+                end
               end
 
               define_method(:instrument) do |integration_name, options = {}, &block|

lib/datadog/ci/ext/settings.rb

@@ -9,6 +9,7 @@ module Settings
         ENV_AGENTLESS_MODE_ENABLED = "DD_CIVISIBILITY_AGENTLESS_ENABLED"
         ENV_AGENTLESS_URL = "DD_CIVISIBILITY_AGENTLESS_URL"
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED = "DD_CIVISIBILITY_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED"
+        ENV_FORCE_TEST_LEVEL_VISIBILITY = "DD_CIVISIBILITY_FORCE_TEST_LEVEL_VISIBILITY"
 
         # Source: https://docs.datadoghq.com/getting_started/site/
         DD_SITE_ALLOWLIST = [

lib/datadog/ci/test.rb

@@ -37,6 +37,8 @@ def test_suite_id
         get_tag(Ext::Test::TAG_TEST_SUITE_ID)
       end
 
+      # Name of the running test suite this test belongs to.
+      # @return [String] the name of the test suite.
       def test_suite_name
         get_tag(Ext::Test::TAG_SUITE)
       end
@@ -47,7 +49,7 @@ def test_module_id
         get_tag(Ext::Test::TAG_TEST_MODULE_ID)
       end
 
-      # Span id of the running test module this test belongs to.
+      # Span id of the running test session this test belongs to.
       # @return [String] the span id of the test session.
       def test_session_id
         get_tag(Ext::Test::TAG_TEST_SESSION_ID)

sig/datadog/ci/ext/settings.rbs

@@ -6,6 +6,7 @@ module Datadog
         ENV_AGENTLESS_MODE_ENABLED: String
         ENV_AGENTLESS_URL: String
         ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED: String
+        ENV_FORCE_TEST_LEVEL_VISIBILITY: String
 
         DD_SITE_ALLOWLIST: Array[String]
       end

spec/datadog/ci/configuration/components_spec.rb

@@ -43,8 +43,8 @@
             .and_return(agentless_enabled)
 
           allow(settings.ci)
-            .to receive(:experimental_test_suite_level_visibility_enabled)
-            .and_return(experimental_test_suite_level_visibility_enabled)
+            .to receive(:force_test_level_visibility)
+            .and_return(force_test_level_visibility)
 
           allow(settings.ci)
             .to receive(:agentless_url)
@@ -85,7 +85,7 @@
             .to receive(:enabled=)
 
           allow(settings.ci)
-            .to receive(:experimental_test_suite_level_visibility_enabled=)
+            .to receive(:force_test_level_visibility=)
 
           allow(Datadog.logger)
             .to receive(:warn)
@@ -103,7 +103,7 @@
         let(:agentless_url) { nil }
         let(:dd_site) { nil }
         let(:agentless_enabled) { false }
-        let(:experimental_test_suite_level_visibility_enabled) { false }
+        let(:force_test_level_visibility) { false }
         let(:evp_proxy_supported) { false }
 
         context "is enabled" do
@@ -113,20 +113,20 @@
             expect(Datadog::CI::Ext::Environment).to have_received(:tags).with(ENV)
           end
 
-          context "when #experimental_test_suite_level_visibility_enabled" do
+          context "when #force_test_level_visibility" do
             context "is false" do
-              it "creates a CI recorder with test_suite_level_visibility_enabled=false" do
+              it "creates a CI recorder with test_suite_level_visibility_enabled=true" do
                 expect(components.ci_recorder).to be_kind_of(Datadog::CI::TestVisibility::Recorder)
-                expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(false)
+                expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(true)
               end
             end
 
             context "is true" do
-              let(:experimental_test_suite_level_visibility_enabled) { true }
+              let(:force_test_level_visibility) { true }
 
               it "creates a CI recorder with test_suite_level_visibility_enabled=false" do
                 expect(components.ci_recorder).to be_kind_of(Datadog::CI::TestVisibility::Recorder)
-                expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(true)
+                expect(components.ci_recorder.test_suite_level_visibility_enabled).to eq(false)
               end
             end
           end
@@ -164,8 +164,8 @@
                     .with(settings.ci.trace_flush || kind_of(Datadog::CI::TestVisibility::Flush::Partial))
 
                   expect(settings.ci)
-                    .to have_received(:experimental_test_suite_level_visibility_enabled=)
-                    .with(false)
+                    .to have_received(:force_test_level_visibility=)
+                    .with(true)
 
                   expect(settings.tracing.test_mode).to have_received(:writer_options=) do |options|
                     expect(options[:transport]).to be_nil

spec/datadog/ci/configuration/settings_spec.rb

@@ -172,17 +172,17 @@ def patcher
         end
       end
 
-      describe "#experimental_test_suite_level_visibility_enabled" do
-        subject(:experimental_test_suite_level_visibility_enabled) do
-          settings.ci.experimental_test_suite_level_visibility_enabled
+      describe "#force_test_level_visibility" do
+        subject(:force_test_level_visibility) do
+          settings.ci.force_test_level_visibility
         end
 
         it { is_expected.to be false }
 
-        context "when #{Datadog::CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED}" do
+        context "when #{Datadog::CI::Ext::Settings::ENV_FORCE_TEST_LEVEL_VISIBILITY}" do
           around do |example|
             ClimateControl.modify(
-              Datadog::CI::Ext::Settings::ENV_EXPERIMENTAL_TEST_SUITE_LEVEL_VISIBILITY_ENABLED => enable
+              Datadog::CI::Ext::Settings::ENV_FORCE_TEST_LEVEL_VISIBILITY => enable
             ) do
               example.run
             end
@@ -208,10 +208,10 @@ def patcher
         end
       end
 
-      describe "#experimental_test_suite_level_visibility_enabled=" do
+      describe "#force_test_level_visibility=" do
         it "updates the #enabled setting" do
-          expect { settings.ci.experimental_test_suite_level_visibility_enabled = true }
-            .to change { settings.ci.experimental_test_suite_level_visibility_enabled }
+          expect { settings.ci.force_test_level_visibility = true }
+            .to change { settings.ci.force_test_level_visibility }
             .from(false)
             .to(true)
         end

spec/datadog/ci/test_visibility/recorder_spec.rb

@@ -49,7 +49,7 @@
     let(:tags) { {"test.framework" => "my-framework", "my.tag" => "my_value"} }
 
     include_context "CI mode activated" do
-      let(:experimental_test_suite_level_visibility_enabled) { false }
+      let(:force_test_level_visibility) { true }
     end
 
     describe "#trace_test_session" do

spec/datadog/ci_spec.rb

@@ -207,7 +207,6 @@
   context "integration testing the manual API" do
     context "when CI mode is disabled" do
       include_context "CI mode activated" do
-        let(:experimental_test_suite_level_visibility_enabled) { false }
         let(:ci_enabled) { false }
       end
 
@@ -223,7 +222,6 @@
     context "when CI mode is enabled" do
       context "when test suite level visibility is enabled" do
         include_context "CI mode activated" do
-          let(:experimental_test_suite_level_visibility_enabled) { true }
           let(:ci_enabled) { true }
         end
 
@@ -239,7 +237,7 @@
 
       context "when test suite level visibility is disabled" do
         include_context "CI mode activated" do
-          let(:experimental_test_suite_level_visibility_enabled) { false }
+          let(:force_test_level_visibility) { true }
           let(:ci_enabled) { true }
         end
 

spec/support/ci_mode_helpers.rb

@@ -4,7 +4,7 @@
   let(:integration_options) { {} }
 
   let(:ci_enabled) { true }
-  let(:experimental_test_suite_level_visibility_enabled) { true }
+  let(:force_test_level_visibility) { false }
 
   let(:recorder) { Datadog.send(:components).ci_recorder }
 
@@ -17,7 +17,7 @@
 
     Datadog.configure do |c|
       c.ci.enabled = ci_enabled
-      c.ci.experimental_test_suite_level_visibility_enabled = experimental_test_suite_level_visibility_enabled
+      c.ci.force_test_level_visibility = force_test_level_visibility
       unless integration_name == :no_instrument
         c.ci.instrument integration_name, integration_options
       end