diff --git a/.standard_todo.yml b/.standard_todo.yml index d72dcc59..63304fc5 100644 --- a/.standard_todo.yml +++ b/.standard_todo.yml @@ -16,3 +16,5 @@ ignore: - Performance/UnfreezeString - Appraisals: - Style/Alias + - spec/datadog/ci/contrib/minitest/instrumentation_spec.rb: + - Lint/ConstantDefinitionInBlock diff --git a/Steepfile b/Steepfile index e6cdc0c9..8707ad60 100644 --- a/Steepfile +++ b/Steepfile @@ -20,4 +20,5 @@ target :lib do library "rspec" library "cucumber" library "msgpack" + library "weakref" end diff --git a/lib/datadog/ci/configuration/components.rb b/lib/datadog/ci/configuration/components.rb index 5b8b4686..bb951d45 100644 --- a/lib/datadog/ci/configuration/components.rb +++ b/lib/datadog/ci/configuration/components.rb @@ -64,6 +64,7 @@ def activate_ci!(settings) if test_visibility_transport writer_options[:transport] = test_visibility_transport writer_options[:shutdown_timeout] = 60 + writer_options[:buffer_size] = 10_000 settings.tracing.test_mode.async = true end diff --git a/lib/datadog/ci/contrib/minitest/configuration/settings.rb b/lib/datadog/ci/contrib/minitest/configuration/settings.rb index 4a8e7c15..b3caa11c 100644 --- a/lib/datadog/ci/contrib/minitest/configuration/settings.rb +++ b/lib/datadog/ci/contrib/minitest/configuration/settings.rb @@ -19,7 +19,7 @@ class Settings < Datadog::CI::Contrib::Settings option :service_name do |o| o.type :string - o.default { Datadog.configuration.service_without_fallback || Ext::SERVICE_NAME } + o.default { Datadog.configuration.service_without_fallback || Ext::DEFAULT_SERVICE_NAME } end # @deprecated Will be removed in 1.0 diff --git a/lib/datadog/ci/contrib/minitest/ext.rb b/lib/datadog/ci/contrib/minitest/ext.rb index b45ab2a3..6fe194ce 100644 --- a/lib/datadog/ci/contrib/minitest/ext.rb +++ b/lib/datadog/ci/contrib/minitest/ext.rb @@ -7,13 +7,15 @@ module Minitest # Minitest integration constants # TODO: mark as `@public_api` when GA, to protect from resource and tag name changes. module Ext - APP = "minitest" ENV_ENABLED = "DD_TRACE_MINITEST_ENABLED" - ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME" + FRAMEWORK = "minitest" + + DEFAULT_SERVICE_NAME = "minitest" + + # TODO: remove in 1.0 + ENV_OPERATION_NAME = "DD_TRACE_MINITEST_OPERATION_NAME" OPERATION_NAME = "minitest.test" - SERVICE_NAME = "minitest" - TEST_TYPE = "test" end end end diff --git a/lib/datadog/ci/contrib/minitest/helpers.rb b/lib/datadog/ci/contrib/minitest/helpers.rb new file mode 100644 index 00000000..2837a3a5 --- /dev/null +++ b/lib/datadog/ci/contrib/minitest/helpers.rb @@ -0,0 +1,22 @@ +# frozen_string_literal: true + +module Datadog + module CI + module Contrib + module Minitest + module Helpers + def self.test_suite_name(klass, method_name) + source_location, = klass.instance_method(method_name).source_location + source_file_path = Pathname.new(source_location.to_s).relative_path_from(Pathname.pwd).to_s + + "#{klass.name} at #{source_file_path}" + end + + def self.parallel?(klass) + klass.ancestors.include?(::Minitest::Parallel::Test) + end + end + end + end + end +end diff --git a/lib/datadog/ci/contrib/minitest/hooks.rb b/lib/datadog/ci/contrib/minitest/hooks.rb index 570354a8..bef8591a 100644 --- a/lib/datadog/ci/contrib/minitest/hooks.rb +++ b/lib/datadog/ci/contrib/minitest/hooks.rb @@ -2,6 +2,7 @@ require_relative "../../ext/test" require_relative "ext" +require_relative "helpers" module Datadog module CI @@ -11,22 +12,27 @@ module Minitest module Hooks def before_setup super - return unless configuration[:enabled] + return unless datadog_configuration[:enabled] test_name = "#{class_name}##{name}" - path, = method(name).source_location - test_suite = Pathname.new(path.to_s).relative_path_from(Pathname.pwd).to_s + test_suite_name = Helpers.test_suite_name(self.class, name) + if Helpers.parallel?(self.class) + test_suite_name = "#{test_suite_name} (#{name} concurrently)" + + # for parallel execution we need to start a new test suite for each test + CI.start_test_suite(test_suite_name) + end CI.start_test( test_name, - test_suite, + test_suite_name, tags: { CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s, - CI::Ext::Test::TAG_TYPE => Ext::TEST_TYPE + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE }, - service: configuration[:service_name] + service: datadog_configuration[:service_name] ) end @@ -34,24 +40,46 @@ def after_teardown test_span = CI.active_test return super unless test_span - case result_code - when "." - test_span.passed! - when "E", "F" - test_span.failed!(exception: failure) - when "S" - test_span.skipped!(reason: failure.message) + finish_test(test_span, result_code) + if Helpers.parallel?(self.class) + finish_test_suite(test_span.test_suite, result_code) end - test_span.finish - super end private - def configuration - ::Datadog.configuration.ci[:minitest] + def finish_test(test_span, result_code) + finish_with_result(test_span, result_code) + + # mark test suite as failed if any test failed + if test_span.failed? + test_suite = test_span.test_suite + test_suite.failed! if test_suite + end + end + + def finish_test_suite(test_suite, result_code) + return unless test_suite + + finish_with_result(test_suite, result_code) + end + + def finish_with_result(span, result_code) + case result_code + when "." + span.passed! + when "E", "F" + span.failed!(exception: failure) + when "S" + span.skipped!(reason: failure.message) + end + span.finish + end + + def datadog_configuration + Datadog.configuration.ci[:minitest] end end end diff --git a/lib/datadog/ci/contrib/minitest/patcher.rb b/lib/datadog/ci/contrib/minitest/patcher.rb index b1b0d73b..6393870b 100644 --- a/lib/datadog/ci/contrib/minitest/patcher.rb +++ b/lib/datadog/ci/contrib/minitest/patcher.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative "hooks" +require_relative "runnable" module Datadog module CI @@ -17,7 +18,13 @@ def target_version end def patch + require_relative "plugin" + ::Minitest::Test.include(Hooks) + ::Minitest.include(Plugin) + ::Minitest::Runnable.include(Runnable) + + ::Minitest.extensions << "datadog_ci" end end end diff --git a/lib/datadog/ci/contrib/minitest/plugin.rb b/lib/datadog/ci/contrib/minitest/plugin.rb new file mode 100644 index 00000000..0df4892c --- /dev/null +++ b/lib/datadog/ci/contrib/minitest/plugin.rb @@ -0,0 +1,73 @@ +# frozen_string_literal: true + +require "weakref" + +require_relative "../../ext/test" +require_relative "ext" + +module Datadog + module CI + module Contrib + module Minitest + module Plugin + def self.included(base) + base.extend(ClassMethods) + end + + class DatadogReporter < ::Minitest::AbstractReporter + def initialize(minitest_reporter) + # This creates circular reference as minitest_reporter also holds reference to DatadogReporter. + # To make sure that minitest_reporter can be garbage collected, we use WeakRef. + @reporter = WeakRef.new(minitest_reporter) + end + + def report + active_test_session = CI.active_test_session + active_test_module = CI.active_test_module + + return unless @reporter.weakref_alive? + return if active_test_session.nil? || active_test_module.nil? + + if @reporter.passed? + active_test_module.passed! + active_test_session.passed! + else + active_test_module.failed! + active_test_session.failed! + end + + active_test_module.finish + active_test_session.finish + + nil + end + end + + module ClassMethods + def plugin_datadog_ci_init(*) + return unless datadog_configuration[:enabled] + + test_session = CI.start_test_session( + tags: { + CI::Ext::Test::TAG_FRAMEWORK => Ext::FRAMEWORK, + CI::Ext::Test::TAG_FRAMEWORK_VERSION => CI::Contrib::Minitest::Integration.version.to_s, + CI::Ext::Test::TAG_TYPE => CI::Ext::Test::TEST_TYPE + }, + service: datadog_configuration[:service_name] + ) + CI.start_test_module(test_session.name) + + reporter.reporters << DatadogReporter.new(reporter) + end + + private + + def datadog_configuration + Datadog.configuration.ci[:minitest] + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/contrib/minitest/runnable.rb b/lib/datadog/ci/contrib/minitest/runnable.rb new file mode 100644 index 00000000..cd239ae0 --- /dev/null +++ b/lib/datadog/ci/contrib/minitest/runnable.rb @@ -0,0 +1,42 @@ +require_relative "helpers" + +module Datadog + module CI + module Contrib + module Minitest + module Runnable + def self.included(base) + base.singleton_class.prepend(ClassMethods) + end + + module ClassMethods + def run(*) + return super unless datadog_configuration[:enabled] + return super if Helpers.parallel?(self) + + method = runnable_methods.first + return super if method.nil? + + test_suite_name = Helpers.test_suite_name(self, method) + + test_suite = Datadog::CI.start_test_suite(test_suite_name) + test_suite.passed! # will be overridden if any test fails + + results = super + + test_suite.finish + + results + end + + private + + def datadog_configuration + Datadog.configuration.ci[:minitest] + end + end + end + end + end + end +end diff --git a/lib/datadog/ci/contrib/rspec/integration.rb b/lib/datadog/ci/contrib/rspec/integration.rb index dbd625cc..339cf2be 100644 --- a/lib/datadog/ci/contrib/rspec/integration.rb +++ b/lib/datadog/ci/contrib/rspec/integration.rb @@ -22,7 +22,7 @@ def self.version end def self.loaded? - !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? && \ + !defined?(::RSpec).nil? && !defined?(::RSpec::Core).nil? && !defined?(::RSpec::Core::Example).nil? end diff --git a/lib/datadog/ci/contrib/settings.rb b/lib/datadog/ci/contrib/settings.rb index 2947c8c0..77097c22 100644 --- a/lib/datadog/ci/contrib/settings.rb +++ b/lib/datadog/ci/contrib/settings.rb @@ -27,7 +27,7 @@ def [](name) end def []=(name, value) - respond_to?("#{name}=") ? send("#{name}=", value) : set_option(name, value) + respond_to?(:"#{name}=") ? send(:"#{name}=", value) : set_option(name, value) end end end diff --git a/lib/datadog/ci/span.rb b/lib/datadog/ci/span.rb index 08f8cffe..b038611e 100644 --- a/lib/datadog/ci/span.rb +++ b/lib/datadog/ci/span.rb @@ -36,6 +36,30 @@ def span_type tracer_span.type end + # Checks whether span status is not set yet. + # @return [bool] true if span does not have status + def undefined? + tracer_span.get_tag(Ext::Test::TAG_STATUS).nil? + end + + # Checks whether span status is PASS. + # @return [bool] true if span status is PASS + def passed? + tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::PASS + end + + # Checks whether span status is FAIL. + # @return [bool] true if span status is FAIL + def failed? + tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::FAIL + end + + # Checks whether span status is SKIP. + # @return [bool] true if span status is SKIP + def skipped? + tracer_span.get_tag(Ext::Test::TAG_STATUS) == Ext::Test::Status::SKIP + end + # Sets the status of the span to "pass". # @return [void] def passed! diff --git a/lib/datadog/ci/test.rb b/lib/datadog/ci/test.rb index 4550b5be..d537c7fa 100644 --- a/lib/datadog/ci/test.rb +++ b/lib/datadog/ci/test.rb @@ -21,6 +21,14 @@ def finish CI.deactivate_test(self) end + # Running test suite that this test is part of (if any). + # @return [Datadog::CI::TestSuite] the test suite this test belongs to + # @return [nil] if the test suite is not found + def test_suite + suite_name = test_suite_name + CI.active_test_suite(suite_name) if suite_name + end + # Span id of the running test suite this test belongs to. # @return [String] the span id of the test suite. def test_suite_id diff --git a/sig/datadog/ci/contrib/minitest/ext.rbs b/sig/datadog/ci/contrib/minitest/ext.rbs index 9af544e4..8f4847c6 100644 --- a/sig/datadog/ci/contrib/minitest/ext.rbs +++ b/sig/datadog/ci/contrib/minitest/ext.rbs @@ -3,8 +3,6 @@ module Datadog module Contrib module Minitest module Ext - APP: String - ENV_ENABLED: String ENV_OPERATION_NAME: String @@ -13,9 +11,7 @@ module Datadog OPERATION_NAME: String - SERVICE_NAME: String - - TEST_TYPE: String + DEFAULT_SERVICE_NAME: String end end end diff --git a/sig/datadog/ci/contrib/minitest/helpers.rbs b/sig/datadog/ci/contrib/minitest/helpers.rbs new file mode 100644 index 00000000..4ceb76c5 --- /dev/null +++ b/sig/datadog/ci/contrib/minitest/helpers.rbs @@ -0,0 +1,13 @@ +module Datadog + module CI + module Contrib + module Minitest + module Helpers + def self.test_suite_name: (untyped klass, String? method_name) -> ::String + + def self.parallel?: (untyped klass) -> bool + end + end + end + end +end diff --git a/sig/datadog/ci/contrib/minitest/hooks.rbs b/sig/datadog/ci/contrib/minitest/hooks.rbs index 94e1fe62..4cad85e4 100644 --- a/sig/datadog/ci/contrib/minitest/hooks.rbs +++ b/sig/datadog/ci/contrib/minitest/hooks.rbs @@ -11,7 +11,15 @@ module Datadog private - def configuration: () -> untyped + def datadog_configuration: () -> untyped + + def finish_test: (Datadog::CI::Test test_span, String result_code) -> void + + def finish_test_suite: (Datadog::CI::TestSuite? test_suite, String result_code) -> void + + def finish_with_result: (Datadog::CI::Span span, String result_code) -> void + + def self.test_order: () -> (nil | :parallel | :sorted | :random | :alpha) end end end diff --git a/sig/datadog/ci/contrib/minitest/plugin.rbs b/sig/datadog/ci/contrib/minitest/plugin.rbs new file mode 100644 index 00000000..b2c73996 --- /dev/null +++ b/sig/datadog/ci/contrib/minitest/plugin.rbs @@ -0,0 +1,31 @@ +module Datadog + module CI + module Contrib + module Minitest + module Plugin + def self.included: (untyped base) -> untyped + + class DatadogReporter < ::Minitest::AbstractReporter + @reporter: WeakRef + + def initialize: (Minitest::AbstractReporter reporter) -> void + end + + module ClassMethods + def plugin_datadog_ci_init: (*untyped) -> (nil | untyped) + + def initialize: (untyped reporter) -> void + + def report: () -> (nil | untyped) + + def reporter: () -> Minitest::CompositeReporter + + private + + def datadog_configuration: () -> untyped + end + end + end + end + end +end diff --git a/sig/datadog/ci/contrib/minitest/runnable.rbs b/sig/datadog/ci/contrib/minitest/runnable.rbs new file mode 100644 index 00000000..a9e11f58 --- /dev/null +++ b/sig/datadog/ci/contrib/minitest/runnable.rbs @@ -0,0 +1,24 @@ +module Datadog + module CI + module Contrib + module Minitest + module Runnable + def self.included: (untyped base) -> untyped + + module ClassMethods : ::Minitest::Runnable + + def run: (*untyped) -> untyped + + private + + def datadog_configuration: () -> untyped + + def test_order: () -> (nil | :parallel | :random | :sorted | :alpha) + + def runnable_methods: () -> Array[String] + end + end + end + end + end +end diff --git a/sig/datadog/ci/span.rbs b/sig/datadog/ci/span.rbs index 6265f0d6..b3a7a5a0 100644 --- a/sig/datadog/ci/span.rbs +++ b/sig/datadog/ci/span.rbs @@ -13,6 +13,14 @@ module Datadog def service: () -> String + def undefined?: () -> bool + + def passed?: () -> bool + + def failed?: () -> bool + + def skipped?: () -> bool + def passed!: () -> void def failed!: (?exception: untyped?) -> void diff --git a/sig/datadog/ci/test.rbs b/sig/datadog/ci/test.rbs index d14485ec..0a685590 100644 --- a/sig/datadog/ci/test.rbs +++ b/sig/datadog/ci/test.rbs @@ -2,6 +2,7 @@ module Datadog module CI class Test < Span def finish: () -> void + def test_suite: () -> Datadog::CI::TestSuite? def test_suite_id: () -> String? def test_suite_name: () -> String? def test_module_id: () -> String? diff --git a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb index b875c90a..8444a110 100644 --- a/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb +++ b/spec/datadog/ci/contrib/minitest/instrumentation_spec.rb @@ -1,8 +1,15 @@ require "time" + require "minitest" require "minitest/spec" -RSpec.describe "Minitest hooks" do +# minitest adds `describe` method to Kernel, which conflicts with RSpec. +# here we define `minitest_describe` method to avoid this conflict. +module Kernel + alias_method :minitest_describe, :describe +end + +RSpec.describe "Minitest instrumentation" do include_context "CI mode activated" do let(:integration_name) { :minitest } let(:integration_options) { {service_name: "ltest"} } @@ -31,10 +38,10 @@ def test_foo expect(span.service).to eq("ltest") expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeTest#test_foo") expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( - "spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" + "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" ) expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST) - expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Contrib::Minitest::Ext::TEST_TYPE) + expect(span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq(Datadog::CI::Ext::Test::TEST_TYPE) expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq(Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK) expect(span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( Datadog::CI::Contrib::Minitest::Integration.version.to_s @@ -53,7 +60,7 @@ def self.name end num_tests.times do |i| - define_method("test_#{i}") {} + define_method(:"test_#{i}") {} end end @@ -82,7 +89,7 @@ def self.name expect(span.service).to eq("ltest") expect(span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeSpec##{method_name}") expect(span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( - "spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" + "SomeSpec at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" ) end @@ -352,4 +359,305 @@ def test_foo expect_skip end end + + context "run minitest suite" do + before do + Minitest.run([]) + end + + context "single test passed" do + before(:context) do + Minitest::Runnable.reset + + class SomeTest < Minitest::Test + def test_pass + assert true + end + + def test_pass_other + assert true + end + end + end + + it "creates a test session span" do + expect(test_session_span).not_to be_nil + expect(test_session_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SESSION) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Ext::Test::TEST_TYPE + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Minitest::Integration.version.to_s + ) + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::PASS + ) + end + + it "creates a test module span" do + expect(test_module_span).not_to be_nil + + expect(test_module_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_MODULE) + expect(test_module_span.name).to eq(test_command) + + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Ext::Test::TEST_TYPE + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Minitest::Integration.version.to_s + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::PASS + ) + end + + it "creates a test suite span" do + expect(test_suite_span).not_to be_nil + + expect(test_suite_span.span_type).to eq(Datadog::CI::Ext::AppTypes::TYPE_TEST_SUITE) + expect(test_suite_span.name).to eq("SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb") + + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_SPAN_KIND)).to eq( + Datadog::CI::Ext::AppTypes::TYPE_TEST + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_TYPE)).to eq( + Datadog::CI::Ext::Test::TEST_TYPE + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Minitest::Integration.version.to_s + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::PASS + ) + end + + it "creates test spans and connects them to the session, module, and suite" do + expect(test_spans.count).to eq(2) + + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE)).to eq( + "SomeTest at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" + ) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK)).to eq( + Datadog::CI::Contrib::Minitest::Ext::FRAMEWORK + ) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_FRAMEWORK_VERSION)).to eq( + Datadog::CI::Contrib::Minitest::Integration.version.to_s + ) + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::PASS + ) + + test_session_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID) }.uniq + test_module_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID) }.uniq + test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq + + expect(test_session_ids.count).to eq(1) + expect(test_session_ids.first).to eq(test_session_span.id.to_s) + + expect(test_module_ids.count).to eq(1) + expect(test_module_ids.first).to eq(test_module_span.id.to_s) + + expect(test_suite_ids.count).to eq(1) + expect(test_suite_ids.first).to eq(test_suite_span.id.to_s) + end + end + + context "single test failed" do + before(:context) do + Minitest::Runnable.reset + + class SomeFailedTest < Minitest::Test + def test_fail + assert false + end + end + end + + it "traces test, test session, test module with failed status" do + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_NAME)).to eq("SomeFailedTest#test_fail") + expect(first_test_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + + expect(test_session_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + expect(test_module_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + expect(test_suite_span.get_tag(Datadog::CI::Ext::Test::TAG_STATUS)).to eq( + Datadog::CI::Ext::Test::Status::FAIL + ) + end + end + + context "using Minitest::Spec" do + before(:context) do + Minitest::Runnable.reset + + class SomeSpec < Minitest::Spec + it "does not fail" do + end + + minitest_describe "in context" do + it "does not fail" do + end + + minitest_describe "deeper context" do + it "does not fail" do + end + end + end + + minitest_describe "in other context" do + it "does not fail" do + end + end + end + end + + it "traces tests with unique names" do + test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort + + expect(test_names).to eq( + [ + "SomeSpec#test_0001_does not fail", + "in context#test_0001_does not fail", + "in context::deeper context#test_0001_does not fail", + "in other context#test_0001_does not fail" + ] + ) + end + + it "connects tests to different test suites (one per spec context)" do + test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq + test_suite_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_SUITE) }.sort + + expect(test_suite_ids).to have(4).items + expect(test_suite_names).to eq( + [ + "SomeSpec at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb", + "in context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb", + "in context::deeper context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb", + "in other context at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb" + ] + ) + end + + it "connects tests to a single test session" do + test_session_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID) }.uniq + + expect(test_session_ids.count).to eq(1) + expect(test_session_ids.first).to eq(test_session_span.id.to_s) + end + end + + context "using parallel executor" do + before(:context) do + Minitest::Runnable.reset + + class ParallelTest < Minitest::Test + parallelize_me! + end + + class TestA < ParallelTest + def test_a_1 + Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id) + sleep 0.1 + end + + def test_a_2 + Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id) + sleep 0.1 + end + end + + class TestB < ParallelTest + def test_b_1 + Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id) + sleep 0.1 + end + + def test_b_2 + Datadog::CI.active_test.set_tag("minitest_thread", Thread.current.object_id) + sleep 0.1 + end + end + end + + it "traces all tests correctly, assigning a separate test suite to each of them" do + test_threads = test_spans.map { |span| span.get_tag("minitest_thread") }.uniq + + # make sure that tests were executed concurrently + # note that this test could be flaky + expect(test_threads.count).to be > 1 + + test_names = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_NAME) }.sort + expect(test_names).to eq( + [ + "TestA#test_a_1", + "TestA#test_a_2", + "TestB#test_b_1", + "TestB#test_b_2" + ] + ) + + test_suite_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SUITE_ID) }.uniq + expect(test_suite_ids).to have(4).items + end + + it "connects tests to a single test session and a single test module" do + test_session_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_SESSION_ID) }.uniq + test_module_ids = test_spans.map { |span| span.get_tag(Datadog::CI::Ext::Test::TAG_TEST_MODULE_ID) }.uniq + + expect(test_session_ids.count).to eq(1) + expect(test_session_ids.first).to eq(test_session_span.id.to_s) + + expect(test_module_ids.count).to eq(1) + expect(test_module_ids.first).to eq(test_module_span.id.to_s) + end + + it "correctly tracks test and session durations" do + test_session_duration = test_session_span.duration + + test_durations_sum = test_spans.map { |span| span.duration }.sum + # with parallel execution test durations sum should be greater than test session duration + expect(test_durations_sum).to be > test_session_duration + + # each individual test duration should be less than test session duration + test_spans.each do |span| + expect(span.duration).to be < test_session_duration + end + end + + it "creates test suite spans" do + expect(test_suite_spans).to have(4).items + + test_suite_names = test_suite_spans.map { |span| span.name }.sort + expect(test_suite_names).to eq( + [ + "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_a_1 concurrently)", + "TestA at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_a_2 concurrently)", + "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_b_1 concurrently)", + "TestB at spec/datadog/ci/contrib/minitest/instrumentation_spec.rb (test_b_2 concurrently)" + ] + ) + end + end + end end diff --git a/spec/datadog/ci/contrib/minitest/patcher_spec.rb b/spec/datadog/ci/contrib/minitest/patcher_spec.rb index 313c144f..1f47fc7a 100644 --- a/spec/datadog/ci/contrib/minitest/patcher_spec.rb +++ b/spec/datadog/ci/contrib/minitest/patcher_spec.rb @@ -4,12 +4,29 @@ describe ".patch" do subject!(:patch) { described_class.patch } - let(:test) { Minitest::Test } - - context "is patched" do + context "Minitest::Test is patched" do + let(:test) { Minitest::Test } it "has a custom bases" do expect(test.ancestors).to include(Datadog::CI::Contrib::Minitest::Hooks) end end + + context "Minitest::Runnable is patched" do + let(:runnable) { Minitest::Runnable } + it "has a custom bases" do + expect(runnable.ancestors).to include(Datadog::CI::Contrib::Minitest::Runnable) + end + end + + context "Minitest includes plugin" do + let(:minitest) { Minitest } + it "has a custom bases" do + expect(minitest.ancestors).to include(Datadog::CI::Contrib::Minitest::Plugin) + end + + it "has datadog_ci extension" do + expect(minitest.extensions).to include("datadog_ci") + end + end end end diff --git a/spec/datadog/ci/span_spec.rb b/spec/datadog/ci/span_spec.rb index 6519d90c..5b12e575 100644 --- a/spec/datadog/ci/span_spec.rb +++ b/spec/datadog/ci/span_spec.rb @@ -8,6 +8,94 @@ end end + describe "#passed?" do + context "when status is PASS" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass") + end + + it "returns true" do + expect(span.passed?).to eq(true) + end + end + + context "when status is not PASS" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return("fail") + end + + it "returns false" do + expect(span.passed?).to eq(false) + end + end + end + + describe "#failed?" do + context "when status is FAIL" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return("fail") + end + + it "returns true" do + expect(span.failed?).to eq(true) + end + end + + context "when status is not FAIL" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass") + end + + it "returns false" do + expect(span.failed?).to eq(false) + end + end + end + + describe "#skipped?" do + context "when status is SKIP" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return("skip") + end + + it "returns true" do + expect(span.skipped?).to eq(true) + end + end + + context "when status is not SKIP" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass") + end + + it "returns false" do + expect(span.skipped?).to eq(false) + end + end + end + + describe "#undefined?" do + context "when status is nil" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return(nil) + end + + it "returns true" do + expect(span.undefined?).to eq(true) + end + end + + context "when status is not nil" do + before do + allow(tracer_span).to receive(:get_tag).with("test.status").and_return("pass") + end + + it "returns false" do + expect(span.undefined?).to eq(false) + end + end + end + describe "#passed!" do it "sets the status to PASS" do expect(tracer_span).to receive(:set_tag).with("test.status", "pass") diff --git a/spec/datadog/ci/test_spec.rb b/spec/datadog/ci/test_spec.rb index 874127a5..75d5d642 100644 --- a/spec/datadog/ci/test_spec.rb +++ b/spec/datadog/ci/test_spec.rb @@ -47,6 +47,26 @@ it { is_expected.to eq("test suite name") } end + describe "#test_suite" do + subject(:test_suite) { ci_test.test_suite } + let(:ci_test) { described_class.new(tracer_span) } + + context "when test suite name is set" do + before do + allow(ci_test).to receive(:test_suite_name).and_return("test suite name") + allow(Datadog::CI).to receive(:active_test_suite).with("test suite name").and_return("test suite") + end + + it { is_expected.to eq("test suite") } + end + + context "when test suite name is not set" do + before { allow(ci_test).to receive(:test_suite_name).and_return(nil) } + + it { is_expected.to be_nil } + end + end + describe "#test_module_id" do subject(:test_module_id) { ci_test.test_module_id } let(:ci_test) { described_class.new(tracer_span) } diff --git a/vendor/rbs/weakref/0/weakref.rbs b/vendor/rbs/weakref/0/weakref.rbs new file mode 100644 index 00000000..f53b31a2 --- /dev/null +++ b/vendor/rbs/weakref/0/weakref.rbs @@ -0,0 +1,6 @@ +# as rbs misses delegator support, we make WeakRef a subclass of CompositeReporter +# for our current use case +class WeakRef < Minitest::CompositeReporter + def initialize: (untyped obj) -> untyped + def weakref_alive?: () -> bool +end