JENKINS-48583 Remove test result deduplication logic (#131)

Co-authored-by: Kevin Neal <Kevin_Neal@intuit.com> Co-authored-by: Tim Jacomb <timjacomb1+github@gmail.com>
jenkinsci · Jul 30, 2020 · 3a90162 · 3a90162
1 parent 67e9ddb
commit 3a90162
Show file tree

Hide file tree

Showing 6 changed files with 79 additions and 24 deletions.
diff --git a/src/main/java/hudson/tasks/junit/TestResult.java b/src/main/java/hudson/tasks/junit/TestResult.java
@@ -286,17 +286,10 @@ private void add(SuiteResult sr) {
         for (SuiteResult s : suites) {
             // JENKINS-12457: If a testsuite is distributed over multiple files, merge it into a single SuiteResult:
             if(s.getName().equals(sr.getName()) &&
-                    nullSafeEq(s.getId(),sr.getId()) &&
+                    eitherNullOrEq(s.getId(),sr.getId()) &&
                     nullSafeEq(s.getNodeId(),sr.getNodeId()) &&
                     nullSafeEq(s.getEnclosingBlocks(),sr.getEnclosingBlocks()) &&
                     nullSafeEq(s.getEnclosingBlockNames(),sr.getEnclosingBlockNames())) {
-
-                // However, a common problem is that people parse TEST-*.xml as well as TESTS-TestSuite.xml.
-                // In that case consider the result file as a duplicate and discard it.
-                // see http://jenkins.361315.n4.nabble.com/Problem-with-duplicate-build-execution-td371616.html for discussion.
-                if(strictEq(s.getTimestamp(),sr.getTimestamp())) {
-                    return;
-                }
 
                 duration += sr.getDuration();
                 s.merge(sr);
@@ -330,6 +323,13 @@ private boolean nullSafeEq(Object lhs, Object rhs) {
         return lhs.equals(rhs);
     }
 
+    private boolean eitherNullOrEq(Object lhs, Object rhs) {
+        // Merged testSuites may have attribute (ID) not preset in the original.
+        // If both have an ID, compare it.
+        // If either does not have an ID, then assume they are the same.
+        return lhs == null || rhs == null || lhs.equals(rhs);
+    }
+
     @Deprecated
     public void parse(File reportFile) throws IOException {
         parse(reportFile, null);

diff --git a/src/test/java/hudson/tasks/junit/TestResultTest.java b/src/test/java/hudson/tasks/junit/TestResultTest.java
@@ -136,7 +136,8 @@ public void testDuplicateTestMethods() throws IOException, URISyntaxException {
         assertEquals("Wrong number of test cases", 3, testResult.getTotalCount());
     }
 
-    @Bug(12457)
+    @Issue("JENKINS-12457")
+    @Test
     public void testTestSuiteDistributedOverMultipleFilesIsCountedAsOne() throws IOException, URISyntaxException {
         TestResult testResult = new TestResult();
         testResult.parse(getDataFile("JENKINS-12457/TestSuite_a1.xml"), null);
@@ -149,21 +150,6 @@ public void testTestSuiteDistributedOverMultipleFilesIsCountedAsOne() throws IOE
         // check duration: 157.980 (TestSuite_a1.xml) and 15.000 (TestSuite_a2.xml) = 172.98 
         assertEquals("Wrong duration for test result", 172.98, testResult.getDuration(), 0.1);
     }
-
-    /**
-     * A common problem is that people parse TEST-*.xml as well as TESTS-TestSuite.xml.
-     * See http://jenkins.361315.n4.nabble.com/Problem-with-duplicate-build-execution-td371616.html for discussion.
-     */
-    public void testDuplicatedTestSuiteIsNotCounted() throws IOException, URISyntaxException {
-        TestResult testResult = new TestResult();
-        testResult.parse(getDataFile("JENKINS-12457/TestSuite_b.xml"), null);
-        testResult.parse(getDataFile("JENKINS-12457/TestSuite_b_duplicate.xml"), null);
-        testResult.tally();
-
-        assertEquals("Wrong number of testsuites", 1, testResult.getSuites().size());
-        assertEquals("Wrong number of test cases", 1, testResult.getTotalCount());
-        assertEquals("Wrong duration for test result", 1.0, testResult.getDuration(), 0.01);
-    }
 
     @Issue("JENKINS-41134")
     @Test
@@ -246,6 +232,36 @@ public void testSuiteWithMultipleClasses() throws IOException, URISyntaxExceptio
         assertEquals("Wrong duration for test class", 93.0, class2.getDuration(), 0.1);
     }
 
+    @Issue("JENKINS-48583")
+    @Test
+    public void testMergeOriginalAntOutput() throws IOException, URISyntaxException {
+        TestResult testResult = new TestResult();
+        testResult.parse(getDataFile("JENKINS-48583/TEST-com.sample.test.TestMessage.xml"), null);
+        testResult.parse(getDataFile("JENKINS-48583/TEST-com.sample.test.TestMessage2.xml"), null);
+        testResult.parse(getDataFile("JENKINS-48583/TESTS-TestSuites.xml"), null);
+        testResult.parse(getDataFile("JENKINS-48583/TEST-com.sample.test.TestMessage.xml"), null);
+        testResult.tally();
+
+        assertEquals("Wrong number of testsuites", 2, testResult.getSuites().size());
+        assertEquals("Wrong number of test cases", 7, testResult.getTotalCount());
+    }
+
+    /**
+     * Sometimes legitimage test cases are split over multiple files with identical timestamps.
+     */
+    @Issue("JENKINS-48583")
+    @Test
+    public void testNonDuplicatedTestSuiteIsCounted() throws IOException, URISyntaxException {
+        TestResult testResult = new TestResult();
+        testResult.parse(getDataFile("JENKINS-12457/TestSuite_b.xml"), null);
+        testResult.parse(getDataFile("JENKINS-12457/TestSuite_b_duplicate.xml"), null);
+        testResult.parse(getDataFile("JENKINS-12457/TestSuite_b_nonduplicate.xml"), null);
+        testResult.tally();
+
+        assertEquals("Wrong number of testsuites", 1, testResult.getSuites().size());
+        assertEquals("Wrong number of test cases", 3, testResult.getTotalCount());
+    }
+
     @Issue("JENKINS-63113")
     @Test
     public void testTestcaseWithEmptyName() throws Exception {

diff --git a/src/test/resources/hudson/tasks/junit/JENKINS-12457/TestSuite_b_nonduplicate.xml b/src/test/resources/hudson/tasks/junit/JENKINS-12457/TestSuite_b_nonduplicate.xml
@@ -0,0 +1,4 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuite name="TestSuite_b" tests="1" errors="0" failures="0" skip="0" timestamp="2012-01-01T00:00:00">
+  <testcase classname="TestFoo" name="bar2" time="1" />
+</testsuite>
diff --git a/src/test/resources/hudson/tasks/junit/JENKINS-48583/TEST-com.sample.test.TestMessage.xml b/src/test/resources/hudson/tasks/junit/JENKINS-48583/TEST-com.sample.test.TestMessage.xml
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite errors="0" failures="0" hostname="myhost" name="com.sample.test.TestMessage" skipped="0" tests="1" time="0.063" timestamp="2015-01-13T07:23:07">
+  <properties>
+  </properties>
+  <testcase classname="com.sample.test.TestMessage" name="test_welcome_message" time="0.002" />
+  <system-out><![CDATA[]]></system-out>
+  <system-err><![CDATA[]]></system-err>
+</testsuite>
diff --git a/src/test/resources/hudson/tasks/junit/JENKINS-48583/TEST-com.sample.test.TestMessage2.xml b/src/test/resources/hudson/tasks/junit/JENKINS-48583/TEST-com.sample.test.TestMessage2.xml
@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite errors="0" failures="0" hostname="myhost" name="com.sample.test.TestMessage2" skipped="0" tests="2" time="0.06" timestamp="2015-01-13T07:23:08">
+  <properties>
+  </properties>
+  <testcase classname="com.sample.test.TestMessage2" name="test_welcome_message_2" time="0.001" />
+  <testcase classname="com.sample.test.TestMessage2" name="test_welcome_message_3" time="0.003" />
+  <system-out><![CDATA[]]></system-out>
+  <system-err><![CDATA[]]></system-err>
+</testsuite>
diff --git a/src/test/resources/hudson/tasks/junit/JENKINS-48583/TESTS-TestSuites.xml b/src/test/resources/hudson/tasks/junit/JENKINS-48583/TESTS-TestSuites.xml
@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuites>
+  <testsuite errors="0" failures="0" hostname="myhost" id="0" name="TestMessage" package="com.sample.test" skipped="0" tests="1" time="0.063" timestamp="2015-01-13T07:23:07">
+      <properties>
+      </properties>
+      <testcase classname="com.sample.test.TestMessage" name="test_welcome_message" time="0.002" />
+      <system-out><![CDATA[]]></system-out>
+      <system-err><![CDATA[]]></system-err>
+  </testsuite>
+  <testsuite errors="0" failures="0" hostname="myhost" id="1" name="TestMessage2" package="com.sample.test" skipped="0" tests="2" time="0.06" timestamp="2015-01-13T07:23:08">
+      <properties>
+      </properties>
+      <testcase classname="com.sample.test.TestMessage2" name="test_welcome_message_2" time="0.001" />
+      <testcase classname="com.sample.test.TestMessage2" name="test_welcome_message_3" time="0.003" />
+      <system-out><![CDATA[]]></system-out>
+      <system-err><![CDATA[]]></system-err>
+  </testsuite>
+</testsuites>