Test cases for parsing JUnit XML reports found in the wild

ingest_test.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
+
 	"github.com/stretchr/testify/require"
 )
 
@@ -59,3 +61,97 @@ func TestIngest(t *testing.T) {
 		})
 	}
 }
+
+func TestExamplesInTheWild(t *testing.T) {
+	tests := []struct {
+		title    string
+		filename string
+		origin   string
+		check    func(*testing.T, []Suite)
+	}{
+		{
+			title:    "catchsoftware example",
+			filename: "testdata/catchsoftware.xml",
+			origin:   "https://help.catchsoftware.com/display/ET/JUnit+Format",
+			check: func(t *testing.T, suites []Suite) {
+				assert.Len(t, suites, 2)
+				assert.Len(t, suites[0].Tests, 0)
+				assert.Len(t, suites[1].Tests, 3)
+				assert.EqualError(t, suites[1].Tests[0].Error, "Assertion failed")
+			},
+		},
+		{
+			title:    "cubic example",
+			filename: "testdata/cubic.xml",
+			origin:   "https://llg.cubic.org/docs/junit/",
+			check: func(t *testing.T, suites []Suite) {
+				assert.Len(t, suites, 1)
+				assert.Len(t, suites[0].Tests, 1)
+				assert.Equal(t, "STDOUT text", suites[0].SystemOut)
+				assert.Equal(t, "STDERR text", suites[0].SystemErr)
+				assert.Equal(t, "STDOUT text", suites[0].Tests[0].SystemOut)
+				assert.Equal(t, "STDERR text", suites[0].Tests[0].SystemErr)
+			},
+		},
+		{
+			title:    "go-junit-report example",
+			filename: "testdata/go-junit-report.xml",
+			origin:   "https://github.com/jstemmer/go-junit-report/blob/master/testdata/06-report.xml",
+			check: func(t *testing.T, suites []Suite) {
+				assert.Len(t, suites, 2)
+				assert.Len(t, suites[0].Tests, 2)
+				assert.Len(t, suites[1].Tests, 2)
+				assert.Equal(t, "1.0", suites[0].Properties["go.version"])
+				assert.Equal(t, "1.0", suites[1].Properties["go.version"])
+				assert.EqualError(t, suites[1].Tests[0].Error, "file_test.go:11: Error message\nfile_test.go:11: Longer\n\terror\n\tmessage.")
+			},
+		},
+		{
+			title:    "ibm example",
+			filename: "testdata/ibm.xml",
+			origin:   "https://www.ibm.com/support/knowledgecenter/en/SSQ2R2_14.2.0/com.ibm.rsar.analysis.codereview.cobol.doc/topics/cac_useresults_junit.html",
+			check: func(t *testing.T, suites []Suite) {
+			},
+		},
+		{
+			title:    "jenkinsci example",
+			filename: "testdata/jenkinsci.xml",
+			origin:   "https://github.com/jenkinsci/junit-plugin/blob/master/src/test/resources/hudson/tasks/junit/junit-report-1463.xml",
+			check: func(t *testing.T, suites []Suite) {
+			},
+		},
+		{
+			title:    "nose2 example",
+			filename: "testdata/nose2.xml",
+			origin:   "https://nose2.readthedocs.io/en/latest/plugins/junitxml.html",
+			check: func(t *testing.T, suites []Suite) {
+			},
+		},
+		{
+			title:    "python junit-xml example",
+			filename: "testdata/python-junit-xml.xml",
+			origin:   "https://pypi.org/project/junit-xml/",
+			check: func(t *testing.T, suites []Suite) {
+			},
+		},
+		{
+			title:    "surefire example",
+			filename: "testdata/surefire.xml",
+			origin:   "https://gist.github.com/rwbergstrom/6f0193b1a12dca9d358e6043ee6abba4",
+			check: func(t *testing.T, suites []Suite) {
+				assert.Equal(t, "Hello, World\n", suites[0].Tests[0].SystemOut)
+				assert.Equal(t, "I'm an error!\n", suites[0].Tests[0].SystemErr)
+			},
+		},
+	}
+
+	for index, test := range tests {
+		name := fmt.Sprintf("#%d - %s", index+1, test.title)
+
+		t.Run(name, func(t *testing.T) {
+			suites, err := IngestFile(test.filename)
+			require.NoError(t, err)
+			test.check(t, suites)
+		})
+	}
+}

testdata/catchsoftware.xml

@@ -0,0 +1,18 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<testsuites>
+    <testsuite name="JUnitXmlReporter" errors="0" tests="0" failures="0" time="0" timestamp="2013-05-24T10:23:58" />
+    <testsuite name="JUnitXmlReporter.constructor" errors="0" skipped="1" tests="3" failures="1" time="0.006" timestamp="2013-05-24T10:23:58">
+        <properties>
+            <property name="java.vendor" value="Sun Microsystems Inc." />
+            <property name="compiler.debug" value="on" />
+            <property name="project.jdk.classpath" value="jdk.classpath.1.6" />
+        </properties>
+        <testcase classname="JUnitXmlReporter.constructor" name="should default path to an empty string" time="0.006">
+            <failure message="test failure">Assertion failed</failure>
+        </testcase>
+        <testcase classname="JUnitXmlReporter.constructor" name="should default consolidate to true" time="0">
+            <skipped />
+        </testcase>
+        <testcase classname="JUnitXmlReporter.constructor" name="should default useDotNotation to true" time="0" />
+    </testsuite>
+</testsuites>

testdata/cubic.xml

@@ -0,0 +1,81 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!-- a description of the JUnit XML format and how Jenkins parses it. See also junit.xsd -->
+
+<!-- if only a single testsuite element is present, the testsuites
+     element can be omitted. All attributes are optional. -->
+<testsuites disabled=""
+            errors=""
+            failures=""
+            name=""
+            tests=""
+            time=""
+	    >
+
+  <!-- testsuite can appear multiple times, if contained in a testsuites element.
+       It can also be the root element. -->
+  <testsuite name=""
+	     tests=""
+	     disabled=""
+             errors=""
+             failures=""
+             hostname=""
+	     id=""
+	     package=""
+	     skipped=""
+	     time=""
+	     timestamp=""
+	     >
+
+    <!-- Properties (e.g., environment settings) set during test
+	 execution. The properties element can appear 0 or once. -->
+    <properties>
+      <!-- property can appear multiple times. The name and value attributres are required. -->
+      <property name="" value=""/>
+    </properties>
+
+    <!-- testcase can appear multiple times, see /testsuites/testsuite@tests -->
+    <testcase name=""
+	      assertions=""
+	      classname=""
+	      status=""
+	      time=""
+	      >
+
+      <!-- If the test was not executed or failed, you can specify one
+           the skipped, error or failure elements. -->
+
+      <!-- skipped can appear 0 or once. optional -->
+      <skipped message=""
+	  />
+
+      <!-- Indicates that the test errored. An errored test is one
+           that had an unanticipated problem. For example an unchecked
+           throwable or a problem with the implementation of the
+           test. Contains as a text node relevant data for the error,
+           for example a stack trace. optional -->
+      <error message=""
+	     type=""
+	     ></error>
+
+      <!-- Indicates that the test failed. A failure is a test which
+	   the code has explicitly failed by using the mechanisms for
+	   that purpose. For example via an assertEquals. Contains as
+	   a text node relevant data for the failure, e.g., a stack
+	   trace. optional -->
+      <failure message=""
+	       type=""
+	       ></failure>
+
+      <!-- Data that was written to standard out while the test was executed. optional -->
+      <system-out>STDOUT text</system-out>
+
+      <!-- Data that was written to standard error while the test was executed. optional -->
+      <system-err>STDERR text</system-err>
+    </testcase>
+
+    <!-- Data that was written to standard out while the test suite was executed. optional -->
+    <system-out>STDOUT text</system-out>
+    <!-- Data that was written to standard error while the test suite was executed. optional -->
+    <system-err>STDERR text</system-err>
+  </testsuite>
+</testsuites>

testdata/go-junit-report.xml

@@ -0,0 +1,18 @@
+<testsuites>
+	<testsuite tests="2" failures="0" time="0.160" name="package/name1">
+		<properties>
+			<property name="go.version" value="1.0"></property>
+		</properties>
+		<testcase classname="name1" name="TestOne" time="0.060"></testcase>
+		<testcase classname="name1" name="TestTwo" time="0.100"></testcase>
+	</testsuite>
+	<testsuite tests="2" failures="1" time="0.151" name="package/name2">
+		<properties>
+			<property name="go.version" value="1.0"></property>
+		</properties>
+		<testcase classname="name2" name="TestOne" time="0.020">
+			<failure message="Failed" type="">file_test.go:11: Error message&#xA;file_test.go:11: Longer&#xA;&#x9;error&#xA;&#x9;message.</failure>
+		</testcase>
+		<testcase classname="name2" name="TestTwo" time="0.130"></testcase>
+	</testsuite>
+</testsuites>

testdata/ibm.xml

@@ -0,0 +1,13 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+   <testsuites id="20140612_170519" name="New_configuration (14/06/12 17:05:19)" tests="225" failures="1262" time="0.001">
+      <testsuite id="codereview.cobol.analysisProvider" name="COBOL Code Review" tests="45" failures="17" time="0.001">
+         <testcase id="codereview.cobol.rules.ProgramIdRule" name="Use a program name that matches the source file name" time="0.001">
+            <failure message="PROGRAM.cbl:2 Use a program name that matches the source file name" type="WARNING">
+WARNING: Use a program name that matches the source file name
+Category: COBOL Code Review – Naming Conventions
+File: /project/PROGRAM.cbl
+Line: 2
+      </failure>
+    </testcase>
+  </testsuite>
+</testsuites>

testdata/jenkinsci.xml

@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+The MIT License
+
+Copyright (c) 2004-2009, Sun Microsystems, Inc., Kohsuke Kawaguchi, Erik Ramfelt
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+-->
+
+<testsuites>
+  <testsuite name="WLI-FI-Tests-Fake" tests="6" failures="0" errors="0" time="1426" package="IT-Interface-WLI-FI" id="1">
+    <properties>
+      <property name="java.runtime.name" value="Java(TM) SE Runtime Environment"/>
+      <property name="sun.boot.library.path" value="C:\Program Files\Java\jdk1.6.0_02\jre\bin"/>
+      <property name="java.vm.version" value="1.6.0_02-b06"/>
+      <property name="java.vm.vendor" value="Sun Microsystems Inc."/>
+      <property name="java.vendor.url" value="http://java.sun.com/"/>
+      <property name="path.separator" value=";"/>
+      <property name="java.vm.name" value="Java HotSpot(TM) Client VM"/>
+      <property name="file.encoding.pkg" value="sun.io"/>
+      <property name="sun.java.launcher" value="SUN_STANDARD"/>
+      <property name="user.country" value="NO"/>
+      <property name="sun.os.patch.level" value="Service Pack 2"/>
+      <property name="java.vm.specification.name" value="Java Virtual Machine Specification"/>
+      <property name="user.dir" value="C:\CI\hudson\jobs\Interface_WLI-FI-Fake\workspace"/>
+      <property name="java.runtime.version" value="1.6.0_02-b06"/>
+      <property name="java.awt.graphicsenv" value="sun.awt.Win32GraphicsEnvironment"/>
+      <property name="java.endorsed.dirs" value="C:\Program Files\Java\jdk1.6.0_02\jre\lib\endorsed"/>
+      <property name="os.arch" value="x86"/>
+      <property name="java.io.tmpdir" value="C:\Documents and Settings\e5274\Local Settings\Temp\"/>
+      <property name="line.separator" value="
+"/>
+      <property name="java.vm.specification.vendor" value="Sun Microsystems Inc."/>
+      <property name="user.variant" value=""/>
+      <property name="os.name" value="Windows XP"/>
+      <property name="sun.jnu.encoding" value="Cp1252"/>
+      <property name="java.library.path" value="C:\Program Files\Java\jdk1.6.0_02\bin;.;C:\WINDOWS\Sun\Java\bin;C:\WINDOWS\system32;C:\WINDOWS;C:\Program Files\Java\jdk1.6.0_02\bin;C:\PROGRA~1\COMPUW~1\FILE-A~1\Dme;C:\PROGRA~1\COMPUW~1\FILE-A~1;C:\WINDOWS\system32;C:\WINDOWS;C:\WINDOWS\System32\Wbem;C:\oracle\ora92\jre\1.4.2\bin\client\;C:\oracle\ora92\jre\1.4.2\bin;c:\oracle\ora92\bin;C:\Program Files\Oracle\jre\1.1.8\bin;C:\Program Files\IBM\Personal Communications\;C:\Program Files\IBM\Trace Facility\;C:\Groovy\groovy-1.1-RC1\bin;C:\Program Files\Subversion\bin;C:\Program Files\GnuWin32\bin;C:\Program Files\putty;C:\Program Files\Common Files\Compuware"/>
+      <property name="java.specification.name" value="Java Platform API Specification"/>
+      <property name="java.class.version" value="50.0"/>
+      <property name="sun.management.compiler" value="HotSpot Client Compiler"/>
+      <property name="os.version" value="5.1"/>
+      <property name="user.home" value="C:\Documents and Settings\e5274"/>
+      <property name="user.timezone" value="Europe/Berlin"/>
+      <property name="java.awt.printerjob" value="sun.awt.windows.WPrinterJob"/>
+      <property name="file.encoding" value="Cp1252"/>
+      <property name="java.specification.version" value="1.6"/>
+      <property name="java.class.path" value="C:\eviware\soapUI-Pro-1.7.6\bin\soapui-pro-1.7.6.jar;C:\eviware\soapUI-Pro-1.7.6\lib\activation-1.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\javamail-1.4.jar;C:\eviware\soapUI-Pro-1.7.6\lib\wsdl4j-1.6.2-fixed.jar;C:\eviware\soapUI-Pro-1.7.6\lib\junit-3.8.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\log4j-1.2.14.jar;C:\eviware\soapUI-Pro-1.7.6\lib\looks-2.1.2.jar;C:\eviware\soapUI-Pro-1.7.6\lib\forms-1.0.7.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jcalendar-1.3.2.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-logging-1.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\not-yet-commons-ssl-0.3.8.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-cli-1.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-beanutils-1.7.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-httpclient-3.0.1-soapui.jar;C:\eviware\soapUI-Pro-1.7.6\lib\swingx-SNAPSHOT.jar;C:\eviware\soapUI-Pro-1.7.6\lib\l2fprod-common-fontchooser-0.2-dev.jar;C:\eviware\soapUI-Pro-1.7.6\lib\commons-codec-1.3.jar;C:\eviware\soapUI-Pro-1.7.6\lib\groovy-all-1.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jetty-6.1.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jetty-util-6.1.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\servlet-api-2.5-6.1.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jxl-2.6.5.jar;C:\eviware\soapUI-Pro-1.7.6\lib\idw-1.5.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\ant-1.6.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xbean-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xbean_xpath-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xmlpublic-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\jsr173_1.0_api-xmlbeans-2.3.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\soapui-xmlbeans-1.7.6.jar;C:\eviware\soapUI-Pro-1.7.6\lib\license4j-1.3.jar;C:\eviware\soapUI-Pro-1.7.6\lib\soapui-1.7.6.jar;C:\eviware\soapUI-Pro-1.7.6\lib\soap-xmlbeans-1.2.jar;C:\eviware\soapUI-Pro-1.7.6\lib\j2ee-xmlbeans-1.4.jar;C:\eviware\soapUI-Pro-1.7.6\lib\ext-xmlbeans-1.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\saxon-8.8.jar;C:\eviware\soapUI-Pro-1.7.6\lib\saxon-dom-8.8.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xmlunit-1.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xmlsec-1.2.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\xalan-2.6.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\dtd-xercesImpl-2.9.1.jar;C:\eviware\soapUI-Pro-1.7.6\lib\wss4j-1.5.0.jar;C:\eviware\soapUI-Pro-1.7.6\lib\bcprov-jdk15-133.jar"/>
+      <property name="user.name" value="e5274"/>
+      <property name="java.vm.specification.version" value="1.0"/>
+      <property name="java.home" value="C:\Program Files\Java\jdk1.6.0_02\jre"/>
+      <property name="sun.arch.data.model" value="32"/>
+      <property name="user.language" value="no"/>
+      <property name="java.specification.vendor" value="Sun Microsystems Inc."/>
+      <property name="awt.toolkit" value="sun.awt.windows.WToolkit"/>
+      <property name="java.vm.info" value="mixed mode"/>
+      <property name="java.version" value="1.6.0_02"/>
+      <property name="java.ext.dirs" value="C:\Program Files\Java\jdk1.6.0_02\jre\lib\ext;C:\WINDOWS\Sun\Java\lib\ext"/>
+      <property name="sun.boot.class.path" value="C:\Program Files\Java\jdk1.6.0_02\jre\lib\resources.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\rt.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\sunrsasign.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\jsse.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\jce.jar;C:\Program Files\Java\jdk1.6.0_02\jre\lib\charsets.jar;C:\Program Files\Java\jdk1.6.0_02\jre\classes"/>
+      <property name="java.vendor" value="Sun Microsystems Inc."/>
+      <property name="file.separator" value="\"/>
+      <property name="java.vendor.url.bug" value="http://java.sun.com/cgi-bin/bugreport.cgi"/>
+      <property name="sun.io.unicode.encoding" value="UnicodeLittle"/>
+      <property name="sun.cpu.endian" value="little"/>
+      <property name="sun.desktop" value="windows"/>
+      <property name="sun.cpu.isalist" value=""/>
+    </properties>
+    <testcase name="IF_importTradeConfirmationToDwh" time="198.0"/>
+    <testcase name="IF_getAmartaDisbursements" time="119.0"/>
+    <testcase name="IF_importGLReconDataToDwh" time="423.0"/>
+    <testcase name="IF_importTradeInstructionsToDwh" time="278.0"/>
+    <testcase name="IF_getDeviationTradeInstructions" time="408.0"/>
+    <testcase name="IF_getDwhGLData" time="0.0"/>
+  </testsuite>
+</testsuites>