Do strict JSON comparison when running CI Vis instrumentation tests locally

dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityInstrumentationTest.groovy

@@ -341,13 +341,13 @@ abstract class CiVisibilityInstrumentationTest extends AgentTestRunner {
     ] + replacements
 
     // uncomment to generate expected data templates
-    //      def clazz = this.getClass()
-    //      def resourceName = clazz.name.replace('.', '/') + ".class"
-    //      def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
-    //      def modulePath = classfilePath.substring(0, classfilePath.indexOf("/build/classes"))
-    //      def baseTemplatesPath = modulePath + "/src/test/resources/" + testcaseName
-    //      CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
-    //      return [:]
+    //          def clazz = this.getClass()
+    //          def resourceName = "/" + clazz.name.replace('.', '/') + ".class"
+    //          def classfilePath = clazz.getResource(resourceName).toURI().schemeSpecificPart
+    //          def modulePath = classfilePath.substring(0, classfilePath.indexOf("/build/classes"))
+    //          def baseTemplatesPath = modulePath + "/src/test/resources/" + testcaseName
+    //          CiVisibilityTestUtils.generateTemplates(baseTemplatesPath, events, coverages, additionalReplacements)
+    //          return [:]
 
     return CiVisibilityTestUtils.assertData(testcaseName, events, coverages, additionalReplacements)
   }

dd-java-agent/agent-ci-visibility/src/testFixtures/groovy/datadog/trace/civisibility/CiVisibilityTestUtils.groovy

@@ -7,6 +7,12 @@ import com.jayway.jsonpath.JsonPath
 import com.jayway.jsonpath.Option
 import com.jayway.jsonpath.ReadContext
 import com.jayway.jsonpath.WriteContext
+import datadog.trace.api.Config
+import datadog.trace.civisibility.ci.CIProviderInfoFactory
+import datadog.trace.civisibility.ci.GitLabInfo
+import datadog.trace.civisibility.ci.GithubActionsInfo
+import datadog.trace.civisibility.ci.env.CiEnvironment
+import datadog.trace.civisibility.ci.env.CiEnvironmentImpl
 import freemarker.core.Environment
 import freemarker.core.InvalidReferenceException
 import freemarker.template.Template
@@ -87,18 +93,23 @@ abstract class CiVisibilityTestUtils {
       replacementMap.put(labelGenerator.forKey(e.key), "\"$e.value\"")
     }
 
+    def environment = System.getenv()
+    def ciRun = environment.get("GITHUB_ACTION") != null || environment.get("GITLAB_CI") != null
+    def comparisonMode = ciRun ? JSONCompareMode.LENIENT : JSONCompareMode.NON_EXTENSIBLE
+
     def expectedEvents = getFreemarkerTemplate(baseTemplatesPath + "/events.ftl", replacementMap, events)
     def actualEvents = JSON_MAPPER.writeValueAsString(events)
+
     try {
-      JSONAssert.assertEquals(expectedEvents, actualEvents, JSONCompareMode.LENIENT)
+      JSONAssert.assertEquals(expectedEvents, actualEvents, comparisonMode)
     } catch (AssertionError e) {
       throw new org.opentest4j.AssertionFailedError("Events mismatch", expectedEvents, actualEvents, e)
     }
 
     def expectedCoverages = getFreemarkerTemplate(baseTemplatesPath + "/coverages.ftl", replacementMap, coverages)
     def actualCoverages = JSON_MAPPER.writeValueAsString(coverages)
     try {
-      JSONAssert.assertEquals(expectedCoverages, actualCoverages, JSONCompareMode.LENIENT)
+      JSONAssert.assertEquals(expectedCoverages, actualCoverages, comparisonMode)
     } catch (AssertionError e) {
       throw new org.opentest4j.AssertionFailedError("Coverages mismatch", expectedCoverages, actualCoverages, e)
     }

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/groovy/CucumberTest.groovy

@@ -69,7 +69,7 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
 
     where:
     testcaseName                                      | success | features                                                                          | retriedTests
-    "test-failed"                                     | false   | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]               | []
+    "test-no-retry-failed"                            | false   | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]               | []
     "test-retry-failed"                               | false   | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"]               | [
       new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
     ]
@@ -148,13 +148,13 @@ class CucumberTest extends CiVisibilityInstrumentationTest {
     assertSpansData(testcaseName)
 
     where:
-    testcaseName                     | features                                                            | quarantined                 | known
+    testcaseName                    | features                                                            | quarantined                  | known
     "test-quarantined-failed-known" | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
       new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
     ]                                                                                                                                    | [
       new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
     ]
-    "test-quarantined-failed-efd"    | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
+    "test-quarantined-failed-efd"   | ["org/example/cucumber/calculator/basic_arithmetic_failed.feature"] | [
       new TestIdentifier("classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic", "Addition", null)
     ]                                                                                                                                    | []
   }

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/java/org/example/cucumber/calculator/CalculatorSteps.java

@@ -23,7 +23,7 @@ public void a_calculator_I_just_turned_on() {
 
   @Given("a flaky calculator I just turned on")
   public void a_flaky_calculator_I_just_turned_on() {
-    calc = ++flakyCounter >= 3 ? new Calculator() : null;
+    calc = new Calculator(++flakyCounter < 3);
   }
 
   @When("I add {int} and {int}")
@@ -94,6 +94,15 @@ public void setOperation(String operation) {
   static class Calculator {
     private static final List<String> OPS = asList("-", "+", "*", "/");
     private final Deque<Number> stack = new LinkedList<>();
+    private final boolean broken;
+
+    public Calculator() {
+      this(false);
+    }
+
+    public Calculator(boolean broken) {
+      this.broken = broken;
+    }
 
     public void push(Object arg) {
       if (OPS.contains(arg)) {
@@ -116,6 +125,9 @@ public void push(Object arg) {
     }
 
     public Number value() {
+      if (broken) {
+        throw new RuntimeException("This test is flaky");
+      }
       return stack.getLast();
     }
   }

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-failed-then-succeed/events.ftl

@@ -40,8 +40,9 @@
       "component" : "junit",
       "dummy_ci_tag" : "dummy_ci_tag_value",
       "env" : "none",
+      "error.message" : ${content_meta_error_message},
       "error.stack" : ${content_meta_error_stack},
-      "error.type" : "java.lang.NullPointerException",
+      "error.type" : "java.lang.RuntimeException",
       "language" : "jvm",
       "library_version" : ${content_meta_library_version},
       "runtime-id" : ${content_meta_runtime_id},
@@ -85,8 +86,9 @@
       "component" : "junit",
       "dummy_ci_tag" : "dummy_ci_tag_value",
       "env" : "none",
+      "error.message" : ${content_meta_error_message},
       "error.stack" : ${content_meta_error_stack_2},
-      "error.type" : "java.lang.NullPointerException",
+      "error.type" : "java.lang.RuntimeException",
       "language" : "jvm",
       "library_version" : ${content_meta_library_version},
       "runtime-id" : ${content_meta_runtime_id},

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-no-retry-failed/coverages.ftl

@@ -0,0 +1,8 @@
+[ {
+  "files" : [ {
+    "filename" : "org/example/cucumber/calculator/basic_arithmetic_failed.feature"
+  } ],
+  "span_id" : ${content_span_id},
+  "test_session_id" : ${content_test_session_id},
+  "test_suite_id" : ${content_test_suite_id}
+} ]

dd-java-agent/instrumentation/junit-5.3/cucumber-junit-5/src/test/resources/test-no-retry-failed/events.ftl

@@ -0,0 +1,145 @@
+[ {
+  "content" : {
+    "duration" : ${content_duration},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "library_version" : ${content_meta_library_version},
+      "span.kind" : "test_suite_end",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "cucumber-junit-5",
+      "test.status" : "fail",
+      "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count}
+    },
+    "name" : "junit.test_suite",
+    "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id},
+    "test_suite_id" : ${content_test_suite_id}
+  },
+  "type" : "test_suite_end",
+  "version" : 1
+}, {
+  "content" : {
+    "duration" : ${content_duration_2},
+    "error" : 1,
+    "meta" : {
+      "_dd.profiling.ctx" : "test",
+      "_dd.tracer_host" : ${content_meta__dd_tracer_host},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "error.message" : ${content_meta_error_message},
+      "error.stack" : ${content_meta_error_stack},
+      "error.type" : "java.lang.AssertionError",
+      "language" : "jvm",
+      "library_version" : ${content_meta_library_version},
+      "runtime-id" : ${content_meta_runtime_id},
+      "span.kind" : "test",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "cucumber-junit-5",
+      "test.name" : "Addition",
+      "test.status" : "fail",
+      "test.suite" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic",
+      "test.traits" : "{\"category\":[\"foo\"]}",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_2},
+      "_dd.profiling.enabled" : 0,
+      "_dd.trace_span_attribute_schema" : 0,
+      "process_id" : ${content_metrics_process_id}
+    },
+    "name" : "junit.test",
+    "parent_id" : ${content_parent_id},
+    "resource" : "classpath:org/example/cucumber/calculator/basic_arithmetic_failed.feature:Basic Arithmetic.Addition",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "span_id" : ${content_span_id},
+    "start" : ${content_start_2},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id},
+    "test_suite_id" : ${content_test_suite_id},
+    "trace_id" : ${content_trace_id}
+  },
+  "type" : "test",
+  "version" : 2
+}, {
+  "content" : {
+    "duration" : ${content_duration_3},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid_2},
+      "_dd.profiling.ctx" : "test",
+      "_dd.tracer_host" : ${content_meta__dd_tracer_host},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "language" : "jvm",
+      "library_version" : ${content_meta_library_version},
+      "runtime-id" : ${content_meta_runtime_id},
+      "span.kind" : "test_session_end",
+      "test.command" : "cucumber-junit-5",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.status" : "fail",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_3},
+      "_dd.profiling.enabled" : 0,
+      "_dd.trace_span_attribute_schema" : 0,
+      "process_id" : ${content_metrics_process_id}
+    },
+    "name" : "junit.test_session",
+    "resource" : "cucumber-junit-5",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start_3},
+    "test_session_id" : ${content_test_session_id}
+  },
+  "type" : "test_session_end",
+  "version" : 1
+}, {
+  "content" : {
+    "duration" : ${content_duration_4},
+    "error" : 0,
+    "meta" : {
+      "_dd.p.tid" : ${content_meta__dd_p_tid_3},
+      "component" : "junit",
+      "dummy_ci_tag" : "dummy_ci_tag_value",
+      "env" : "none",
+      "library_version" : ${content_meta_library_version},
+      "span.kind" : "test_module_end",
+      "test.framework" : "cucumber",
+      "test.framework_version" : ${content_meta_test_framework_version},
+      "test.module" : "cucumber-junit-5",
+      "test.status" : "fail",
+      "test.type" : "test",
+      "test_session.name" : "session-name"
+    },
+    "metrics" : {
+      "_dd.host.vcpu_count" : ${content_metrics__dd_host_vcpu_count_4}
+    },
+    "name" : "junit.test_module",
+    "resource" : "cucumber-junit-5",
+    "service" : "worker.org.gradle.process.internal.worker.gradleworkermain",
+    "start" : ${content_start_4},
+    "test_module_id" : ${content_test_module_id},
+    "test_session_id" : ${content_test_session_id}
+  },
+  "type" : "test_module_end",
+  "version" : 1
+} ]

dd-java-agent/instrumentation/selenium/src/test/resources/test-succeed-multiple/coverages.ftl

@@ -0,0 +1 @@
+[ ]