From 3be910d15869cd974fee99c24915447d1f52df13 Mon Sep 17 00:00:00 2001 From: Krishnan Mahadevan Date: Mon, 20 Mar 2023 09:44:35 +0530 Subject: [PATCH] Ignore Config skips in xml reports Closes #2886 TestNG by design excludes suite level configs When there is inheritance involved and multiple Children of the same base class are included in a suite. But when reporting the results as xml TestNG is not excluding these configs from the reports. Fixed this discrepancy. --- CHANGES.txt | 1 + .../org/testng/reporters/XMLReporter.java | 2 +- .../java/test/reports/XmlReporterTest.java | 35 ++++++++++++++----- .../reports/issue2886/BaseClassSample.java | 9 +++++ .../reports/issue2886/HydeTestSample.java | 9 +++++ .../reports/issue2886/JekyllTestSample.java | 8 +++++ 6 files changed, 54 insertions(+), 10 deletions(-) create mode 100644 testng-core/src/test/java/test/reports/issue2886/BaseClassSample.java create mode 100644 testng-core/src/test/java/test/reports/issue2886/HydeTestSample.java create mode 100644 testng-core/src/test/java/test/reports/issue2886/JekyllTestSample.java diff --git a/CHANGES.txt b/CHANGES.txt index 97548b3288..9827a3c809 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -9,6 +9,7 @@ Fixed: GITHUB-2862: Allow test classes to define "configfailurepolicy" at a per Fixed: GITHUB-2796: Option for onAfterClass to run after @AfterClass (Oliver Hughes) Fixed: GITHUB-2857: XmlTest index is not set for test suites invoked with YAML (Sergei Baranov) Fixed: GITHUB-2880: Before configuration and before invocation set 'SKIP' when beforeMethod is 'skip' (Bob Shi) +Fixed: GITHUB-2886: testng-results xml reports config skips from base classes as ignored (Krishnan Mahadevan) 7.7.1 Fixed: GITHUB-2854: overloaded assertEquals methods do not work from Groovy (Krishnan Mahadevan) diff --git a/testng-core/src/main/java/org/testng/reporters/XMLReporter.java b/testng-core/src/main/java/org/testng/reporters/XMLReporter.java index dcb44fec7a..bc0d2cc685 100644 --- a/testng-core/src/main/java/org/testng/reporters/XMLReporter.java +++ b/testng-core/src/main/java/org/testng/reporters/XMLReporter.java @@ -55,7 +55,7 @@ public void generateReport( } skipped += skippedPerTest; retried += retriedPerTest; - ignored += testContext.getExcludedMethods().size(); + ignored += testContext.getExcludedMethods().stream().filter(ITestNGMethod::isTest).count(); } } diff --git a/testng-core/src/test/java/test/reports/XmlReporterTest.java b/testng-core/src/test/java/test/reports/XmlReporterTest.java index 4519582b35..13553f1be5 100644 --- a/testng-core/src/test/java/test/reports/XmlReporterTest.java +++ b/testng-core/src/test/java/test/reports/XmlReporterTest.java @@ -22,8 +22,11 @@ import org.testng.internal.WrappedTestNGMethod; import org.testng.reporters.RuntimeBehavior; import org.w3c.dom.Document; +import org.w3c.dom.Node; import test.SimpleBaseTest; import test.reports.issue2171.TestClassExample; +import test.reports.issue2886.HydeTestSample; +import test.reports.issue2886.JekyllTestSample; import test.simple.SimpleSample; public class XmlReporterTest extends SimpleBaseTest { @@ -60,7 +63,7 @@ public void ensureStackTraceHasLineFeedsTest() throws Exception { @Test(description = "GITHUB-2171") public void ensureCustomisationOfReportIsSupported() throws Exception { - File file = runTest(TestClassExample.class, "issue_2171.xml"); + File file = runTest("issue_2171.xml", null, TestClassExample.class); DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); DocumentBuilder builder = factory.newDocumentBuilder(); Document doc = builder.parse(file); @@ -74,7 +77,6 @@ public void ensureCustomisationOfReportIsSupported() throws Exception { public void ensureReportGenerationWhenTestMethodIsWrappedWithWrappedTestNGMethod() { File file = runTest( - SimpleSample.class, testng -> testng.setMethodInterceptor( (methods, context) -> @@ -88,19 +90,34 @@ public void ensureReportGenerationWhenTestMethodIsWrappedWithWrappedTestNGMethod assertThat(file.exists()).isTrue(); } - private static File runTest(Class clazz) { - return runTest(clazz, RuntimeBehavior.FILE_NAME, null); + @Test(description = "GITHUB-2886") + public void ensureConfigurationMethodsAreNotCountedAsSkippedInXmlReports() throws Exception { + File file = + runTest(RuntimeBehavior.FILE_NAME, null, JekyllTestSample.class, HydeTestSample.class); + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = factory.newDocumentBuilder(); + Document doc = builder.parse(file); + XPath xPath = XPathFactory.newInstance().newXPath(); + Node node = (Node) xPath.compile("/testng-results").evaluate(doc, XPathConstants.NODE); + int ignored = Integer.parseInt(node.getAttributes().getNamedItem("ignored").getNodeValue()); + int total = Integer.parseInt(node.getAttributes().getNamedItem("total").getNodeValue()); + int passed = Integer.parseInt(node.getAttributes().getNamedItem("passed").getNodeValue()); + int failed = Integer.parseInt(node.getAttributes().getNamedItem("failed").getNodeValue()); + assertThat(ignored).isZero(); + assertThat(total).isEqualTo(2); + assertThat(passed).isEqualTo(2); + assertThat(failed).isZero(); } - private static File runTest(Class clazz, Consumer customizer) { - return runTest(clazz, RuntimeBehavior.FILE_NAME, customizer); + private static File runTest(Class clazz) { + return runTest(RuntimeBehavior.FILE_NAME, null, clazz); } - private static File runTest(Class clazz, String fileName) { - return runTest(clazz, fileName, null); + private static File runTest(Consumer customizer) { + return runTest(RuntimeBehavior.FILE_NAME, customizer, SimpleSample.class); } - private static File runTest(Class clazz, String fileName, Consumer customizer) { + private static File runTest(String fileName, Consumer customizer, Class... clazz) { String suiteName = UUID.randomUUID().toString(); File fileLocation = createDirInTempDir(suiteName); TestNG testng = create(fileLocation.toPath(), clazz); diff --git a/testng-core/src/test/java/test/reports/issue2886/BaseClassSample.java b/testng-core/src/test/java/test/reports/issue2886/BaseClassSample.java new file mode 100644 index 0000000000..a1a0bdeae6 --- /dev/null +++ b/testng-core/src/test/java/test/reports/issue2886/BaseClassSample.java @@ -0,0 +1,9 @@ +package test.reports.issue2886; + +import org.testng.annotations.BeforeSuite; + +public class BaseClassSample { + + @BeforeSuite + public void beforeSuite() {} +} diff --git a/testng-core/src/test/java/test/reports/issue2886/HydeTestSample.java b/testng-core/src/test/java/test/reports/issue2886/HydeTestSample.java new file mode 100644 index 0000000000..738563229d --- /dev/null +++ b/testng-core/src/test/java/test/reports/issue2886/HydeTestSample.java @@ -0,0 +1,9 @@ +package test.reports.issue2886; + +import org.testng.annotations.Test; + +public class HydeTestSample extends BaseClassSample { + + @Test + public void rogueBehaviourTest() {} +} diff --git a/testng-core/src/test/java/test/reports/issue2886/JekyllTestSample.java b/testng-core/src/test/java/test/reports/issue2886/JekyllTestSample.java new file mode 100644 index 0000000000..f3474d935c --- /dev/null +++ b/testng-core/src/test/java/test/reports/issue2886/JekyllTestSample.java @@ -0,0 +1,8 @@ +package test.reports.issue2886; + +import org.testng.annotations.Test; + +public class JekyllTestSample extends BaseClassSample { + @Test + public void gentleBehaviourTest() {} +}