JUnitReportReporter should capture the test case output at the test c…

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+Fixed: GITHUB-2875: JUnitReportReporter should capture the test case output at the test case level
 Fixed: GITHUB-2771: After upgrading to TestNG 7.5.0, setting ITestResult.status to FAILURE doesn't fail the test anymore (Julien Herr & Krishnan Mahadevan)
 Fixed: GITHUB-2796: Option for onAfterClass to run after @AfterClass
 Fixed: GITHUB-2857: XmlTest index is not set for test suites invoked with YAML

testng-core/src/main/java/org/testng/reporters/JUnitReportReporter.java

@@ -130,22 +130,38 @@ public void generateReport(
 
  xsb.push(XMLConstants.TESTSUITE, p1);
  for (TestTag testTag : testCases) {
- if (putElement(xsb, XMLConstants.TESTCASE, testTag.properties, testTag.childTag != null)) {
- Properties p = new Properties();
- safeSetProperty(p, XMLConstants.ATTR_MESSAGE, testTag.message);
- safeSetProperty(p, XMLConstants.ATTR_TYPE, testTag.type);
-
- if (putElement(xsb, testTag.childTag, p, testTag.stackTrace != null)) {
- xsb.addCDATA(testTag.stackTrace);
- xsb.pop(testTag.childTag);
+ boolean testCaseHasChildElements = testTag.childTag != null || testTag.sysOut != null;
+ if (putElement(xsb, XMLConstants.TESTCASE, testTag.properties, testCaseHasChildElements)) {
+
+ if (testTag.childTag != null) {
+ Properties p = new Properties();
+ safeSetProperty(p, XMLConstants.ATTR_MESSAGE, testTag.message);
+ safeSetProperty(p, XMLConstants.ATTR_TYPE, testTag.type);
+
+ if (putElement(xsb, testTag.childTag, p, testTag.stackTrace != null)) {
+ xsb.addCDATA(testTag.stackTrace);
+ xsb.pop(testTag.childTag);
+ }
+ }
+
+ // Add reporter output for each test case as a child system-out element of testcase.
+ if (testTag.sysOut != null) {
+ putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), true);
+ xsb.addCDATA(testTag.sysOut);
+ xsb.pop(XMLConstants.SYSTEM_OUT);
  }
  xsb.pop(XMLConstants.TESTCASE);
  }
- if (putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), testTag.sysOut != null)) {
- xsb.addCDATA(testTag.sysOut);
- xsb.pop(XMLConstants.SYSTEM_OUT);
- }
  }
+
+ // Add the full reporter output once as a child system-out element of testsuite.
+ List<String> output = Reporter.getOutput();
+ if ((!output.isEmpty())) {
+ putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), true);
+ xsb.addCDATA(String.join("\n", output));
+ xsb.pop(XMLConstants.SYSTEM_OUT);
+ }
+
  xsb.pop(XMLConstants.TESTSUITE);
 
  String outputDirectory = defaultOutputDirectory + File.separator + "junitreports";

testng-core/src/test/java/test/junitreports/JUnitReportsTest.java

@@ -13,6 +13,7 @@
 import java.util.LinkedList;
 import java.util.List;
 import java.util.Map;
+import java.util.stream.Stream;
 import javax.xml.parsers.DocumentBuilder;
 import javax.xml.parsers.DocumentBuilderFactory;
 import javax.xml.parsers.ParserConfigurationException;
@@ -22,12 +23,16 @@
 import javax.xml.xpath.XPathConstants;
 import javax.xml.xpath.XPathFactory;
 import org.testng.ITestNGListener;
+import org.testng.Reporter;
 import org.testng.TestNG;
 import org.testng.annotations.Test;
 import org.testng.collections.Maps;
+import org.testng.reporters.XMLConstants;
 import org.testng.xml.XmlSuite;
 import org.testng.xml.XmlTest;
 import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.w3c.dom.NodeList;
 import org.xml.sax.SAXException;
 import org.xml.sax.SAXParseException;
 import org.xml.sax.helpers.DefaultHandler;
@@ -131,12 +136,165 @@ public void testEnsureTestnameDoesnotAcceptNullValues() throws IOException {
 
  @Test
  public void ensureTestReportContainsValidSysOutContent() throws Exception {
+ Class<?> testClass = TestClassSample.class;
  Path outputDir = TestHelper.createRandomDirectory();
- TestNG tng = createTests(outputDir, "suite", TestClassSample.class);
+ TestNG tng = createTests(outputDir, "suite", testClass);
  tng.setUseDefaultListeners(true);
+ Reporter.clear();
  tng.run();
+ Document doc = getJunitReport(outputDir, testClass);
+ XPath xPath = XPathFactory.newInstance().newXPath();
+
+ // Define a result for each test case in the test class that can be used to compare the expected
+ // and actual results.
+ class TestCaseResult {
+ private final String name;
+ private final String failureMessage;
+ private final String systemOut;
+
+ // Construct an expected result.
+ public TestCaseResult(String name, String failureMessage, String... systemOut) {
+ this.name = name;
+ this.failureMessage = failureMessage;
+ this.systemOut = systemOut.length > 0 ? String.join("\n", systemOut) : null;
+ }
+
+ // Construct from an actual testcase xml report element.
+ public TestCaseResult(Element actualTestCaseElement) {
+ name = actualTestCaseElement.getAttribute(XMLConstants.ATTR_NAME);
+
+ NodeList failureList = actualTestCaseElement.getElementsByTagName(XMLConstants.FAILURE);
+ NodeList systemOutList =
+ actualTestCaseElement.getElementsByTagName(XMLConstants.SYSTEM_OUT);
+ assertThat(failureList.getLength()).isLessThanOrEqualTo(1);
+ assertThat(systemOutList.getLength()).isLessThanOrEqualTo(1);
+
+ failureMessage =
+ 1 == failureList.getLength()
+ ? ((Element) failureList.item(0)).getAttribute(XMLConstants.ATTR_MESSAGE)
+ : null;
+ systemOut =
+ 1 == systemOutList.getLength() ? systemOutList.item(0).getTextContent().trim() : null;
+ }
+
+ public boolean matches(TestCaseResult testCaseResult) {
+ return toString().equals(testCaseResult.toString());
+ }
+
+ @Override
+ public String toString() {
+ return String.format(
+ "TestCaseResult{name='%s', failureMessage='%s', systemOut='%s'}",
+ name, failureMessage, systemOut);
+ }
+ }
+
+ // Build a list of expected test case results and describe their possible failure or system-out
+ // values.
+ List<TestCaseResult> expectedTestCaseResults =
+ Arrays.asList(
+ // Verify the system-out for the reporter testcase.
+ new TestCaseResult(
+ TestClassSample.TEST_METHOD_WITH_REPORTER,
+ null,
+ TestClassSample.MESSAGE_1,
+ TestClassSample.MESSAGE_2),
+ // Verify the system-out for multiple test cases of the same name from the data provider
+ // testcase.
+ new TestCaseResult(
+ TestClassSample.TEST_METHOD_WITH_DATA_PROVIDER_REPORTER,
+ null,
+ TestClassSample.MESSAGE_3),
+ // Verify the system-out for multiple test cases of the same name from the data provider
+ // testcase.
+ new TestCaseResult(
+ TestClassSample.TEST_METHOD_WITH_DATA_PROVIDER_REPORTER,
+ null,
+ TestClassSample.MESSAGE_4),
+ // Verify that a test case can include system-out and failure elements.
+ new TestCaseResult(
+ TestClassSample.TEST_METHOD_FAIL_WITH_REPORTER,
+ TestClassSample.MESSAGE_FAIL,
+ TestClassSample.MESSAGE_5),
+ // Verify that a test case can have a failure element without system-out.
+ new TestCaseResult(
+ TestClassSample.TEST_METHOD_FAIL_NO_REPORTER, TestClassSample.MESSAGE_FAIL),
+ // Verify that a test case without any Reporter logs does not include a system-out
+ // element.
+ new TestCaseResult(TestClassSample.TEST_METHOD_NO_REPORTER, null));
+
+ // Verify that the count of actual xml testcase elements matches the count of expected test case
+ // results.
+ NodeList actualTestcases = doc.getElementsByTagName(XMLConstants.TESTCASE);
+ assertThat(actualTestcases.getLength()).isEqualTo(expectedTestCaseResults.size());
+ // Verify that each actual xml testcase element matches exactly one expected test case result.
+ for (int i = 0; i < actualTestcases.getLength(); i++) {
+ Element actualTestCaseElement = (Element) actualTestcases.item(i);
+ TestCaseResult actualTestCaseResult = new TestCaseResult(actualTestCaseElement);
+ long actualMatchCount =
+ expectedTestCaseResults.stream()
+ .filter(
+ expectedTestCaseResult -> expectedTestCaseResult.matches(actualTestCaseResult))
+ .count();
+ assertThat(actualMatchCount)
+ .withFailMessage(
+ String.format(
+ "Could not find an expected result for actual test case result %s",
+ actualTestCaseResult))
+ .isEqualTo(1);
+ }
+
+ // Verify the actual full system-out for the testsuite which includes output lines from
+ // before, after, and test case methods.
+ String actualFullOutputExpression = "//testsuite/system-out";
+ String actualFullOutputData =
+ ((String) xPath.compile(actualFullOutputExpression).evaluate(doc, XPathConstants.STRING))
+ .trim();
+ List<String> actualFullOutputList = Arrays.asList(actualFullOutputData.split("\n"));
+
+ String[] expectedTestCaseOutputList =
+ expectedTestCaseResults.stream()
+ .flatMap(
+ result ->
+ null == result.systemOut
+ ? Stream.empty()
+ : Arrays.stream(result.systemOut.split("\n")))
+ .toArray(String[]::new);
+ int expectedFullOutputCount = expectedTestCaseOutputList.length + 2;
+
+ // Verify that the count of actual xml testsuite system-out lines matches the count of expected
+ // output lines.
+ assertThat(actualFullOutputList.size()).isEqualTo(expectedFullOutputCount);
+
+ // Verify that before and after messages are at the beginning and end of the testsuite
+ // system-out.
+ assertThat(actualFullOutputList.get(0)).isEqualTo(TestClassSample.MESSAGE_BEFORE);
+ assertThat(actualFullOutputList.get(expectedFullOutputCount - 1))
+ .isEqualTo(TestClassSample.MESSAGE_AFTER);
+
+ // The order of messages from the test cases depends on the order the test cases were run so
+ // allow the check to be in any order.
+ assertThat(actualFullOutputList.subList(1, expectedFullOutputCount - 1))
+ .containsExactlyInAnyOrder(expectedTestCaseOutputList);
+ }
+
+ // Test that a class that does not have any Reporter output does not add any system-out elements.
+ @Test
+ public void ensureTestReportContainsNoSysOutContent() throws Exception {
+ Class<?> testClass = SimpleTestSample.class;
+ Path outputDir = TestHelper.createRandomDirectory();
+ TestNG tng = createTests(outputDir, "suite", testClass);
+ tng.setUseDefaultListeners(true);
+ Reporter.clear();
+ tng.run();
+ Document doc = getJunitReport(outputDir, testClass);
+ assertThat(doc.getElementsByTagName("system-out").getLength()).isEqualTo(0);
+ }
+
+ private Document getJunitReport(Path outputDir, Class<?> testClass)
+ throws IOException, ParserConfigurationException, SAXException {
  DocumentBuilder builder = getJUnitDocumentBuilder();
- String name = "TEST-" + TestClassSample.class.getName();
+ String name = "TEST-" + testClass.getName();
  File file =
  new File(
  outputDir.toFile().getAbsolutePath()
@@ -145,11 +303,7 @@ public void ensureTestReportContainsValidSysOutContent() throws Exception {
  + File.separator
  + name
  + ".xml");
- Document doc = builder.parse(file);
- XPath xPath = XPathFactory.newInstance().newXPath();
- String expression = "//testsuite/system-out";
- String data = (String) xPath.compile(expression).evaluate(doc, XPathConstants.STRING);
- assertThat(data.trim()).isEqualTo(TestClassSample.MESSAGE_1 + "\n" + TestClassSample.MESSAGE_2);
+ return builder.parse(file);
  }
 
  private DocumentBuilder getJUnitDocumentBuilder()

testng-core/src/test/java/test/junitreports/issue2124/TestClassSample.java

@@ -1,16 +1,69 @@
 package test.junitreports.issue2124;
 
+import org.testng.Assert;
 import org.testng.Reporter;
+import org.testng.annotations.AfterSuite;
+import org.testng.annotations.BeforeSuite;
+import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 public class TestClassSample {
 
  public static final String MESSAGE_1 = "Teenage Mutant Ninja Turtles";
  public static final String MESSAGE_2 = "Teenage Mutant Ninja Turtles: <i>Out of the Shadows</i>";
+ public static final String MESSAGE_3 =
+ "Teenage Mutant Ninja Turtles: <i>The Secret of the Ooze</i>";
+ public static final String MESSAGE_4 = "Teenage Mutant Ninja Turtles: <i>Mutant Mayhem</i>";
+ public static final String MESSAGE_5 =
+ "Teenage Mutant Ninja Turtles: <i>Rise of the Teenage Mutant Ninja Turtles</i>";
+ public static final String MESSAGE_BEFORE = "Teenage Mutant Ninja Turtles Movies";
+ public static final String MESSAGE_AFTER = "To be continued";
+ public static final String MESSAGE_FAIL = "Cowabunga";
+
+ public static final String TEST_METHOD_WITH_REPORTER = "testReporter";
+ public static final String TEST_METHOD_WITH_DATA_PROVIDER_REPORTER =
+ "testReporterWithDataProvider";
+ public static final String TEST_METHOD_FAIL_WITH_REPORTER = "testFailWithReporter";
+ public static final String TEST_METHOD_FAIL_NO_REPORTER = "testFailNoReporter";
+ public static final String TEST_METHOD_NO_REPORTER = "testNoReporter";
+
+ @BeforeSuite
+ public void before() {
+ Reporter.log(MESSAGE_BEFORE, true);
+ }
+
+ @AfterSuite
+ public void after() {
+ Reporter.log(MESSAGE_AFTER, true);
+ }
 
  @Test
  public void testReporter() {
  Reporter.log(MESSAGE_1, true);
  Reporter.log(MESSAGE_2, true);
  }
+
+ @DataProvider(name = "testReporterDataProvider")
+ public Object[][] testReporterDataProvider() {
+ return new Object[][] {{MESSAGE_3}, {MESSAGE_4}};
+ }
+
+ @Test(dataProvider = "testReporterDataProvider")
+ public void testReporterWithDataProvider(String message) {
+ Reporter.log(message, true);
+ }
+
+ @Test
+ public void testFailWithReporter() {
+ Reporter.log(MESSAGE_5, true);
+ Assert.fail(MESSAGE_FAIL);
+ }
+
+ @Test
+ public void testFailNoReporter() {
+ Assert.fail(MESSAGE_FAIL);
+ }
+
+ @Test
+ public void testNoReporter() {}
 }