Process JUnit error field

If the test fails to run properly we need to record it as a failure for our purposes. Fixes #1025 Signed-off-by: John Schnake <jschnake@vmware.com>
vmware-tanzu · Dec 11, 2019 · ee12b94 · ee12b94
1 parent a7a8aa0
commit ee12b94
Show file tree

Hide file tree

Showing 4 changed files with 90 additions and 9 deletions.
diff --git a/pkg/client/results/junit.go b/pkg/client/results/junit.go
@@ -37,6 +37,9 @@ const (
 
  // JUnitFailureKey is the key in the Items.Details map for the failure output.
  JUnitFailureKey = "failure"
+
+ // JUnitErrorKey is the key in the Items.Details map for the error output.
+ JUnitErrorKey = "error"
 )
 
 // junitResult is a wrapper around the suite[s] which enable results to
@@ -89,14 +92,15 @@ type JUnitTestSuite struct {
 
 // JUnitTestCase is a single test case with its result.
 type JUnitTestCase struct {
- XMLName xml.Name `xml:"testcase"`
- Classname string `xml:"classname,attr"`
- Name string `xml:"name,attr"`
- Time string `xml:"time,attr"`
- SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
- Failure *JUnitFailureMessage `xml:"failure,omitempty"`
- SystemOut string `xml:"system-out,omitempty"`
- SystemErr string `xml:"system-err,omitempty"`
+ XMLName xml.Name `xml:"testcase"`
+ Classname string `xml:"classname,attr"`
+ Name string `xml:"name,attr"`
+ Time string `xml:"time,attr"`
+ SkipMessage *JUnitSkipMessage `xml:"skipped,omitempty"`
+ Failure *JUnitFailureMessage `xml:"failure,omitempty"`
+ ErrorMessage *JUnitErrorMessage `xml:"error,omitempty"`
+ SystemOut string `xml:"system-out,omitempty"`
+ SystemErr string `xml:"system-err,omitempty"`
 }
 
 // JUnitSkipMessage contains the reason why a testcase was skipped.
@@ -111,6 +115,13 @@ type JUnitFailureMessage struct {
  Contents string `xml:",chardata"`
 }
 
+// JUnitErrorMessage contains data related to a failed test.
+type JUnitErrorMessage struct {
+ Message string `xml:"message,attr"`
+ Type string `xml:"type,attr"`
+ Contents string `xml:",chardata"`
+}
+
 // JUnitProperty represents a key/value pair used to define properties.
 type JUnitProperty struct {
  Name string `xml:"name,attr"`
@@ -142,14 +153,19 @@ func JUnitSkipped(testCase JUnitTestCase) bool { return testCase.SkipMessage !=
 
 // JUnitPassed returns true if the test passed.
 func JUnitPassed(testCase JUnitTestCase) bool {
- return testCase.SkipMessage == nil && testCase.Failure == nil
+ return testCase.SkipMessage == nil && testCase.Failure == nil && testCase.ErrorMessage == nil
 }
 
 // JUnitFailed returns true if the test failed.
 func JUnitFailed(testCase JUnitTestCase) bool {
  return testCase.SkipMessage == nil && testCase.Failure != nil
 }
 
+// JUnitErrored returns true if the test errored.
+func JUnitErrored(testCase JUnitTestCase) bool {
+ return testCase.SkipMessage == nil && testCase.Failure == nil && testCase.ErrorMessage != nil
+}
+
 func junitProcessFile(pluginDir, currentFile string) (Item, error) {
  relPath, err := filepath.Rel(pluginDir, currentFile)
  if err != nil {
@@ -225,6 +241,10 @@ func junitProcessReader(r io.Reader, name string, metadata map[string]string) (I
  rootItem.Status = StatusFailed
  suiteItem.Status = StatusFailed
  status = StatusFailed
+ case JUnitErrored(t):
+ rootItem.Status = StatusFailed
+ suiteItem.Status = StatusFailed
+ status = StatusFailed
  case JUnitSkipped(t):
  status = StatusSkipped
  }
@@ -237,6 +257,11 @@ func junitProcessReader(r io.Reader, name string, metadata map[string]string) (I
  if hasFailureContents {
  testItem.Details[JUnitFailureKey] = strings.TrimSpace(t.Failure.Message + " " + t.Failure.Contents)
  }
+ hasErrorContents := (t.ErrorMessage != nil && (t.ErrorMessage.Message != "" || t.ErrorMessage.Contents != ""))
+ if hasErrorContents {
+ testItem.Details[JUnitErrorKey] = strings.TrimSpace(t.ErrorMessage.Message + " " + t.ErrorMessage.Contents)
+ }
+
  if t.SystemOut != "" {
  testItem.Details[JUnitStdoutKey] = t.SystemOut
  }

diff --git a/pkg/client/results/processing_test.go b/pkg/client/results/processing_test.go
@@ -157,6 +157,10 @@ func TestPostProcessPlugin(t *testing.T) {
  desc: "Errors can contain complex structured data",
  key: "job-complex-err",
  plugin: getPlugin("job-complex-err", "job", "junit", []string{}),
+ }, {
+ desc: "tmp name",
+ key: "job-junit-falsepositive",
+ plugin: getPlugin("job-junit-falsepositive", "job", "junit", []string{}),
  },
  }
  for _, tc := range testCases {

diff --git a/.../testdata/mockResults/plugins/job-junit-falsepositive/job-junit-falsepositive.golden.json b/.../testdata/mockResults/plugins/job-junit-falsepositive/job-junit-falsepositive.golden.json
@@ -0,0 +1,28 @@
+{
+"name": "job-junit-falsepositive",
+"status": "failed",
+"items": [
+{
+"name": "output.xml",
+"status": "failed",
+"meta": {
+"file": "results/global/output.xml"
+},
+"items": [
+{
+"name": "Alertmanager - Alertmanager statefuleset(executeK8sTests())",
+"status": "failed",
+"items": [
+{
+"name": "Alertmanager - Alertmanager statefuleset(executeK8sTests",
+"status": "failed",
+"details": {
+"error": "Alertmanager statefuleset - Connection refused (Connection refused) org.opentest4j.AssertionFailedError: Alertmanager statefuleset - Connection refused (Connection refused)\n\tat com.cg.k8s.tests.client.K8sPlatformTests.test(K8sPlatformTests.java:163)\n\tat com.cg.k8s.tests.client.K8sPlatformTests.lambda$0(K8sPlatformTests.java:94)\n\tat java.util.Optional.ifPresent(Optional.java:159)\n\tat java.util.ArrayList.forEach(ArrayList.java:1257)\n\tat java.util.ArrayList.forEach(ArrayList.java:1257)\n\tat com.cg.k8s.tests.client.RunTests.main(RunTests.java:41)"
+}
+}
+]
+}
+]
+}
+]
+}
diff --git a/...nt/results/testdata/mockResults/plugins/job-junit-falsepositive/results/global/output.xml b/...nt/results/testdata/mockResults/plugins/job-junit-falsepositive/results/global/output.xml
@@ -0,0 +1,24 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<testsuite errors="0" failures="1" hostname="cgogineni" name="Alertmanager - Alertmanager statefuleset(executeK8sTests())" skipped="0" tests="1" time="0.005" timestamp="2019-12-03T17:53:38">
+ <properties />
+ <testcase classname="com.cg.k8s.tests.reports.JUnitResultFormatterAsRunListener$DescriptionAsTest" name="Alertmanager - Alertmanager statefuleset(executeK8sTests" time="0.005">
+ <error message="Alertmanager statefuleset - Connection refused (Connection refused)" type="org.opentest4j.AssertionFailedError">org.opentest4j.AssertionFailedError: Alertmanager statefuleset - Connection refused (Connection refused)
+ at com.cg.k8s.tests.client.K8sPlatformTests.test(K8sPlatformTests.java:163)
+ at com.cg.k8s.tests.client.K8sPlatformTests.lambda$0(K8sPlatformTests.java:94)
+ at java.util.Optional.ifPresent(Optional.java:159)
+ at java.util.ArrayList.forEach(ArrayList.java:1257)
+ at java.util.ArrayList.forEach(ArrayList.java:1257)
+ at com.cg.k8s.tests.client.RunTests.main(RunTests.java:41)
+</error>
+ </testcase>
+ <system-out><![CDATA[Request method: GET
+Request URI: http://localhost:8001/apis/apps/v1/namespaces/monitoring/statefulsets/alertmanager-main/status
+Proxy: <none>
+...
+]]></system-out>
+ <system-err><![CDATA[java.net.ConnectException: Connection refused (Connection refused)
+ at java.net.PlainSocketImpl.socketConnect(Native Method)
+ ...
+ ...
+]]></system-err>
+</testsuite>