diff --git a/CHANGES.txt b/CHANGES.txt index 4929c7740c..15da78a1c9 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ Current +Fixed: GITHUB-2875: JUnitReportReporter should capture the test case output at the test case level Fixed: GITHUB-2771: After upgrading to TestNG 7.5.0, setting ITestResult.status to FAILURE doesn't fail the test anymore (Julien Herr & Krishnan Mahadevan) Fixed: GITHUB-2796: Option for onAfterClass to run after @AfterClass Fixed: GITHUB-2857: XmlTest index is not set for test suites invoked with YAML diff --git a/testng-core/src/main/java/org/testng/reporters/JUnitReportReporter.java b/testng-core/src/main/java/org/testng/reporters/JUnitReportReporter.java index fe9f81cdd0..e00eeadd50 100644 --- a/testng-core/src/main/java/org/testng/reporters/JUnitReportReporter.java +++ b/testng-core/src/main/java/org/testng/reporters/JUnitReportReporter.java @@ -130,22 +130,38 @@ public void generateReport( xsb.push(XMLConstants.TESTSUITE, p1); for (TestTag testTag : testCases) { - if (putElement(xsb, XMLConstants.TESTCASE, testTag.properties, testTag.childTag != null)) { - Properties p = new Properties(); - safeSetProperty(p, XMLConstants.ATTR_MESSAGE, testTag.message); - safeSetProperty(p, XMLConstants.ATTR_TYPE, testTag.type); - - if (putElement(xsb, testTag.childTag, p, testTag.stackTrace != null)) { - xsb.addCDATA(testTag.stackTrace); - xsb.pop(testTag.childTag); + boolean testCaseHasChildElements = testTag.childTag != null || testTag.sysOut != null; + if (putElement(xsb, XMLConstants.TESTCASE, testTag.properties, testCaseHasChildElements)) { + + if (testTag.childTag != null) { + Properties p = new Properties(); + safeSetProperty(p, XMLConstants.ATTR_MESSAGE, testTag.message); + safeSetProperty(p, XMLConstants.ATTR_TYPE, testTag.type); + + if (putElement(xsb, testTag.childTag, p, testTag.stackTrace != null)) { + xsb.addCDATA(testTag.stackTrace); + xsb.pop(testTag.childTag); + } + } + + // Add reporter output for each test case as a child system-out element of testcase. + if (testTag.sysOut != null) { + putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), true); + xsb.addCDATA(testTag.sysOut); + xsb.pop(XMLConstants.SYSTEM_OUT); } xsb.pop(XMLConstants.TESTCASE); } - if (putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), testTag.sysOut != null)) { - xsb.addCDATA(testTag.sysOut); - xsb.pop(XMLConstants.SYSTEM_OUT); - } } + + // Add the full reporter output once as a child system-out element of testsuite. + List output = Reporter.getOutput(); + if ((!output.isEmpty())) { + putElement(xsb, XMLConstants.SYSTEM_OUT, new Properties(), true); + xsb.addCDATA(String.join("\n", output)); + xsb.pop(XMLConstants.SYSTEM_OUT); + } + xsb.pop(XMLConstants.TESTSUITE); String outputDirectory = defaultOutputDirectory + File.separator + "junitreports"; diff --git a/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java b/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java index 1b29639817..bd3c0deff6 100644 --- a/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java +++ b/testng-core/src/test/java/test/junitreports/JUnitReportsTest.java @@ -13,6 +13,7 @@ import java.util.LinkedList; import java.util.List; import java.util.Map; +import java.util.stream.Stream; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.parsers.ParserConfigurationException; @@ -22,12 +23,16 @@ import javax.xml.xpath.XPathConstants; import javax.xml.xpath.XPathFactory; import org.testng.ITestNGListener; +import org.testng.Reporter; import org.testng.TestNG; import org.testng.annotations.Test; import org.testng.collections.Maps; +import org.testng.reporters.XMLConstants; import org.testng.xml.XmlSuite; import org.testng.xml.XmlTest; import org.w3c.dom.Document; +import org.w3c.dom.Element; +import org.w3c.dom.NodeList; import org.xml.sax.SAXException; import org.xml.sax.SAXParseException; import org.xml.sax.helpers.DefaultHandler; @@ -131,12 +136,165 @@ public void testEnsureTestnameDoesnotAcceptNullValues() throws IOException { @Test public void ensureTestReportContainsValidSysOutContent() throws Exception { + Class testClass = TestClassSample.class; Path outputDir = TestHelper.createRandomDirectory(); - TestNG tng = createTests(outputDir, "suite", TestClassSample.class); + TestNG tng = createTests(outputDir, "suite", testClass); tng.setUseDefaultListeners(true); + Reporter.clear(); tng.run(); + Document doc = getJunitReport(outputDir, testClass); + XPath xPath = XPathFactory.newInstance().newXPath(); + + // Define a result for each test case in the test class that can be used to compare the expected + // and actual results. + class TestCaseResult { + private final String name; + private final String failureMessage; + private final String systemOut; + + // Construct an expected result. + public TestCaseResult(String name, String failureMessage, String... systemOut) { + this.name = name; + this.failureMessage = failureMessage; + this.systemOut = systemOut.length > 0 ? String.join("\n", systemOut) : null; + } + + // Construct from an actual testcase xml report element. + public TestCaseResult(Element actualTestCaseElement) { + name = actualTestCaseElement.getAttribute(XMLConstants.ATTR_NAME); + + NodeList failureList = actualTestCaseElement.getElementsByTagName(XMLConstants.FAILURE); + NodeList systemOutList = + actualTestCaseElement.getElementsByTagName(XMLConstants.SYSTEM_OUT); + assertThat(failureList.getLength()).isLessThanOrEqualTo(1); + assertThat(systemOutList.getLength()).isLessThanOrEqualTo(1); + + failureMessage = + 1 == failureList.getLength() + ? ((Element) failureList.item(0)).getAttribute(XMLConstants.ATTR_MESSAGE) + : null; + systemOut = + 1 == systemOutList.getLength() ? systemOutList.item(0).getTextContent().trim() : null; + } + + public boolean matches(TestCaseResult testCaseResult) { + return toString().equals(testCaseResult.toString()); + } + + @Override + public String toString() { + return String.format( + "TestCaseResult{name='%s', failureMessage='%s', systemOut='%s'}", + name, failureMessage, systemOut); + } + } + + // Build a list of expected test case results and describe their possible failure or system-out + // values. + List expectedTestCaseResults = + Arrays.asList( + // Verify the system-out for the reporter testcase. + new TestCaseResult( + TestClassSample.TEST_METHOD_WITH_REPORTER, + null, + TestClassSample.MESSAGE_1, + TestClassSample.MESSAGE_2), + // Verify the system-out for multiple test cases of the same name from the data provider + // testcase. + new TestCaseResult( + TestClassSample.TEST_METHOD_WITH_DATA_PROVIDER_REPORTER, + null, + TestClassSample.MESSAGE_3), + // Verify the system-out for multiple test cases of the same name from the data provider + // testcase. + new TestCaseResult( + TestClassSample.TEST_METHOD_WITH_DATA_PROVIDER_REPORTER, + null, + TestClassSample.MESSAGE_4), + // Verify that a test case can include system-out and failure elements. + new TestCaseResult( + TestClassSample.TEST_METHOD_FAIL_WITH_REPORTER, + TestClassSample.MESSAGE_FAIL, + TestClassSample.MESSAGE_5), + // Verify that a test case can have a failure element without system-out. + new TestCaseResult( + TestClassSample.TEST_METHOD_FAIL_NO_REPORTER, TestClassSample.MESSAGE_FAIL), + // Verify that a test case without any Reporter logs does not include a system-out + // element. + new TestCaseResult(TestClassSample.TEST_METHOD_NO_REPORTER, null)); + + // Verify that the count of actual xml testcase elements matches the count of expected test case + // results. + NodeList actualTestcases = doc.getElementsByTagName(XMLConstants.TESTCASE); + assertThat(actualTestcases.getLength()).isEqualTo(expectedTestCaseResults.size()); + // Verify that each actual xml testcase element matches exactly one expected test case result. + for (int i = 0; i < actualTestcases.getLength(); i++) { + Element actualTestCaseElement = (Element) actualTestcases.item(i); + TestCaseResult actualTestCaseResult = new TestCaseResult(actualTestCaseElement); + long actualMatchCount = + expectedTestCaseResults.stream() + .filter( + expectedTestCaseResult -> expectedTestCaseResult.matches(actualTestCaseResult)) + .count(); + assertThat(actualMatchCount) + .withFailMessage( + String.format( + "Could not find an expected result for actual test case result %s", + actualTestCaseResult)) + .isEqualTo(1); + } + + // Verify the actual full system-out for the testsuite which includes output lines from + // before, after, and test case methods. + String actualFullOutputExpression = "//testsuite/system-out"; + String actualFullOutputData = + ((String) xPath.compile(actualFullOutputExpression).evaluate(doc, XPathConstants.STRING)) + .trim(); + List actualFullOutputList = Arrays.asList(actualFullOutputData.split("\n")); + + String[] expectedTestCaseOutputList = + expectedTestCaseResults.stream() + .flatMap( + result -> + null == result.systemOut + ? Stream.empty() + : Arrays.stream(result.systemOut.split("\n"))) + .toArray(String[]::new); + int expectedFullOutputCount = expectedTestCaseOutputList.length + 2; + + // Verify that the count of actual xml testsuite system-out lines matches the count of expected + // output lines. + assertThat(actualFullOutputList.size()).isEqualTo(expectedFullOutputCount); + + // Verify that before and after messages are at the beginning and end of the testsuite + // system-out. + assertThat(actualFullOutputList.get(0)).isEqualTo(TestClassSample.MESSAGE_BEFORE); + assertThat(actualFullOutputList.get(expectedFullOutputCount - 1)) + .isEqualTo(TestClassSample.MESSAGE_AFTER); + + // The order of messages from the test cases depends on the order the test cases were run so + // allow the check to be in any order. + assertThat(actualFullOutputList.subList(1, expectedFullOutputCount - 1)) + .containsExactlyInAnyOrder(expectedTestCaseOutputList); + } + + // Test that a class that does not have any Reporter output does not add any system-out elements. + @Test + public void ensureTestReportContainsNoSysOutContent() throws Exception { + Class testClass = SimpleTestSample.class; + Path outputDir = TestHelper.createRandomDirectory(); + TestNG tng = createTests(outputDir, "suite", testClass); + tng.setUseDefaultListeners(true); + Reporter.clear(); + tng.run(); + Document doc = getJunitReport(outputDir, testClass); + assertThat(doc.getElementsByTagName("system-out").getLength()).isEqualTo(0); + } + + private Document getJunitReport(Path outputDir, Class testClass) + throws IOException, ParserConfigurationException, SAXException { DocumentBuilder builder = getJUnitDocumentBuilder(); - String name = "TEST-" + TestClassSample.class.getName(); + String name = "TEST-" + testClass.getName(); File file = new File( outputDir.toFile().getAbsolutePath() @@ -145,11 +303,7 @@ public void ensureTestReportContainsValidSysOutContent() throws Exception { + File.separator + name + ".xml"); - Document doc = builder.parse(file); - XPath xPath = XPathFactory.newInstance().newXPath(); - String expression = "//testsuite/system-out"; - String data = (String) xPath.compile(expression).evaluate(doc, XPathConstants.STRING); - assertThat(data.trim()).isEqualTo(TestClassSample.MESSAGE_1 + "\n" + TestClassSample.MESSAGE_2); + return builder.parse(file); } private DocumentBuilder getJUnitDocumentBuilder() diff --git a/testng-core/src/test/java/test/junitreports/issue2124/TestClassSample.java b/testng-core/src/test/java/test/junitreports/issue2124/TestClassSample.java index ea246e0b0e..8219fb0449 100644 --- a/testng-core/src/test/java/test/junitreports/issue2124/TestClassSample.java +++ b/testng-core/src/test/java/test/junitreports/issue2124/TestClassSample.java @@ -1,16 +1,69 @@ package test.junitreports.issue2124; +import org.testng.Assert; import org.testng.Reporter; +import org.testng.annotations.AfterSuite; +import org.testng.annotations.BeforeSuite; +import org.testng.annotations.DataProvider; import org.testng.annotations.Test; public class TestClassSample { public static final String MESSAGE_1 = "Teenage Mutant Ninja Turtles"; public static final String MESSAGE_2 = "Teenage Mutant Ninja Turtles: Out of the Shadows"; + public static final String MESSAGE_3 = + "Teenage Mutant Ninja Turtles: The Secret of the Ooze"; + public static final String MESSAGE_4 = "Teenage Mutant Ninja Turtles: Mutant Mayhem"; + public static final String MESSAGE_5 = + "Teenage Mutant Ninja Turtles: Rise of the Teenage Mutant Ninja Turtles"; + public static final String MESSAGE_BEFORE = "Teenage Mutant Ninja Turtles Movies"; + public static final String MESSAGE_AFTER = "To be continued"; + public static final String MESSAGE_FAIL = "Cowabunga"; + + public static final String TEST_METHOD_WITH_REPORTER = "testReporter"; + public static final String TEST_METHOD_WITH_DATA_PROVIDER_REPORTER = + "testReporterWithDataProvider"; + public static final String TEST_METHOD_FAIL_WITH_REPORTER = "testFailWithReporter"; + public static final String TEST_METHOD_FAIL_NO_REPORTER = "testFailNoReporter"; + public static final String TEST_METHOD_NO_REPORTER = "testNoReporter"; + + @BeforeSuite + public void before() { + Reporter.log(MESSAGE_BEFORE, true); + } + + @AfterSuite + public void after() { + Reporter.log(MESSAGE_AFTER, true); + } @Test public void testReporter() { Reporter.log(MESSAGE_1, true); Reporter.log(MESSAGE_2, true); } + + @DataProvider(name = "testReporterDataProvider") + public Object[][] testReporterDataProvider() { + return new Object[][] {{MESSAGE_3}, {MESSAGE_4}}; + } + + @Test(dataProvider = "testReporterDataProvider") + public void testReporterWithDataProvider(String message) { + Reporter.log(message, true); + } + + @Test + public void testFailWithReporter() { + Reporter.log(MESSAGE_5, true); + Assert.fail(MESSAGE_FAIL); + } + + @Test + public void testFailNoReporter() { + Assert.fail(MESSAGE_FAIL); + } + + @Test + public void testNoReporter() {} }