Merge pull request #204 from DataDog/anmarchenko/telemetry_metrics_itr

[SDTEST-160] implement ITR metrics for internal telemetry

lib/datadog/ci/test.rb

@@ -3,6 +3,7 @@
 require "json"
 
 require_relative "span"
+require_relative "test_optimisation/telemetry"
 require_relative "utils/test_run"
 
 module Datadog
@@ -80,9 +81,13 @@ def skipped_by_itr?
       #
       # @return [void]
       def itr_unskippable!
+        TestOptimisation::Telemetry.itr_unskippable
         set_tag(Ext::Test::TAG_ITR_UNSKIPPABLE, "true")
+
         if skipped_by_itr?
           clear_tag(Ext::Test::TAG_ITR_SKIPPED_BY_ITR)
+
+          TestOptimisation::Telemetry.itr_forced_run
           set_tag(Ext::Test::TAG_ITR_FORCED_RUN, "true")
         end
       end

lib/datadog/ci/test_optimisation/component.rb

@@ -170,6 +170,8 @@ def count_skipped_test(test)
           end
 
           @mutex.synchronize do
+            Telemetry.itr_skipped
+
             @skipped_tests_count += 1
           end
         end

lib/datadog/ci/test_optimisation/telemetry.rb

@@ -3,6 +3,7 @@
 require_relative "../ext/telemetry"
 require_relative "../ext/test"
 require_relative "../utils/telemetry"
+require_relative "../test_visibility/telemetry"
 
 module Datadog
   module CI
@@ -25,12 +26,30 @@ def self.code_coverage_files(count)
           Utils::Telemetry.distribution(Ext::Telemetry::METRIC_CODE_COVERAGE_FILES, count.to_f)
         end
 
+        def self.itr_skipped
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_SKIPPED, 1, tags_for_itr_metrics)
+        end
+
+        def self.itr_forced_run
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_FORCED_RUN, 1, tags_for_itr_metrics)
+        end
+
+        def self.itr_unskippable
+          Utils::Telemetry.inc(Ext::Telemetry::METRIC_ITR_UNSKIPPABLE, 1, tags_for_itr_metrics)
+        end
+
         def self.tags_for_test(test)
           {
             Ext::Telemetry::TAG_TEST_FRAMEWORK => test.get_tag(Ext::Test::TAG_FRAMEWORK),
             Ext::Telemetry::TAG_LIBRARY => Ext::Telemetry::Library::CUSTOM
           }
         end
+
+        def self.tags_for_itr_metrics
+          {
+            Ext::Telemetry::TAG_EVENT_TYPE => Ext::Telemetry::EventType::TEST
+          }
+        end
       end
     end
   end

sig/datadog/ci/test_optimisation/telemetry.rbs

@@ -10,7 +10,15 @@ module Datadog
 
         def self.code_coverage_files: (Integer count) -> void
 
+        def self.itr_skipped: () -> void
+
+        def self.itr_forced_run: () -> void
+
+        def self.itr_unskippable: () -> void
+
         def self.tags_for_test: (Datadog::CI::Test test) -> ::Hash[String, String]
+
+        def self.tags_for_itr_metrics: () -> ::Hash[String, String]
       end
     end
   end

spec/datadog/ci/test_optimisation/component_spec.rb

@@ -342,6 +342,8 @@
           .from(0)
           .to(1)
       end
+
+      it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_ITR_SKIPPED, 1
     end
 
     context "test is not skipped" do

spec/datadog/ci/test_optimisation/telemetry_spec.rb

@@ -82,4 +82,55 @@
 
     it { code_coverage_files }
   end
+
+  describe ".itr_skipped" do
+    subject(:itr_skipped) { described_class.itr_skipped }
+
+    before do
+      expect(Datadog::CI::Utils::Telemetry).to receive(:inc)
+        .with(Datadog::CI::Ext::Telemetry::METRIC_ITR_SKIPPED, 1, expected_tags)
+    end
+
+    let(:expected_tags) do
+      {
+        Datadog::CI::Ext::Telemetry::TAG_EVENT_TYPE => Datadog::CI::Ext::Telemetry::EventType::TEST
+      }
+    end
+
+    it { itr_skipped }
+  end
+
+  describe ".itr_forced_run" do
+    subject(:itr_forced_run) { described_class.itr_forced_run }
+
+    before do
+      expect(Datadog::CI::Utils::Telemetry).to receive(:inc)
+        .with(Datadog::CI::Ext::Telemetry::METRIC_ITR_FORCED_RUN, 1, expected_tags)
+    end
+
+    let(:expected_tags) do
+      {
+        Datadog::CI::Ext::Telemetry::TAG_EVENT_TYPE => Datadog::CI::Ext::Telemetry::EventType::TEST
+      }
+    end
+
+    it { itr_forced_run }
+  end
+
+  describe ".itr_unskippable" do
+    subject(:itr_unskippable) { described_class.itr_unskippable }
+
+    before do
+      expect(Datadog::CI::Utils::Telemetry).to receive(:inc)
+        .with(Datadog::CI::Ext::Telemetry::METRIC_ITR_UNSKIPPABLE, 1, expected_tags)
+    end
+
+    let(:expected_tags) do
+      {
+        Datadog::CI::Ext::Telemetry::TAG_EVENT_TYPE => Datadog::CI::Ext::Telemetry::EventType::TEST
+      }
+    end
+
+    it { itr_unskippable }
+  end
 end

spec/datadog/ci/test_spec.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
 RSpec.describe Datadog::CI::Test do
+  include_context "Telemetry spy"
+
   let(:tracer_span) { instance_double(Datadog::Tracing::SpanOperation, finish: true) }
   let(:test_visibility) { spy("test_visibility") }
   subject(:ci_test) { described_class.new(tracer_span) }
@@ -120,30 +122,35 @@
   end
 
   describe "#itr_unskippable!" do
+    subject { ci_test.itr_unskippable! }
+
     context "when test is not skipped by ITR" do
       before do
         allow(ci_test).to receive(:skipped_by_itr?).and_return(false)
+        expect(tracer_span).to receive(:set_tag).with(Datadog::CI::Ext::Test::TAG_ITR_UNSKIPPABLE, "true")
       end
 
       it "sets unskippable tag" do
-        expect(tracer_span).to receive(:set_tag).with(Datadog::CI::Ext::Test::TAG_ITR_UNSKIPPABLE, "true")
-
-        ci_test.itr_unskippable!
+        subject
       end
+
+      it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_ITR_UNSKIPPABLE, 1
     end
 
     context "when test is skipped by ITR" do
       before do
         allow(ci_test).to receive(:skipped_by_itr?).and_return(true)
-      end
-
-      it "sets unskippable tag, removes skipped by ITR tag, and sets forced run tag" do
         expect(tracer_span).to receive(:set_tag).with(Datadog::CI::Ext::Test::TAG_ITR_UNSKIPPABLE, "true")
         expect(tracer_span).to receive(:clear_tag).with(Datadog::CI::Ext::Test::TAG_ITR_SKIPPED_BY_ITR)
         expect(tracer_span).to receive(:set_tag).with(Datadog::CI::Ext::Test::TAG_ITR_FORCED_RUN, "true")
+      end
 
-        ci_test.itr_unskippable!
+      it "sets unskippable tag, removes skipped by ITR tag, and sets forced run tag" do
+        subject
       end
+
+      it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_ITR_UNSKIPPABLE, 1
+      it_behaves_like "emits telemetry metric", :inc, Datadog::CI::Ext::Telemetry::METRIC_ITR_FORCED_RUN, 1
     end
   end