Segregate native & user listeners before ordering

Closes testng-team#2771

CHANGES.txt

@@ -1,4 +1,5 @@
 Current
+Fixed: GITHUB-2771: After upgrading to TestNG 7.5.0, setting ITestResult.status to FAILURE doesn't fail the test anymore (Julien Herr & Krishnan Mahadevan)
 Fixed: GITHUB-2796: Option for onAfterClass to run after @AfterClass
 Fixed: GITHUB-2857: XmlTest index is not set for test suites invoked with YAML
 

testng-core/src/main/java/org/testng/TestNG.java

@@ -777,7 +777,11 @@ public Set<IReporter> getReporters() {
   }
 
   public List<ITestListener> getTestListeners() {
-    return Lists.newArrayList(m_testListeners.values());
+    List<ITestListener> list = Lists.newArrayList(m_testListeners.values());
+    // Add the exit code listener as the last of the test listeners so that
+    // we can honour whatever changes to a test state that was done by a user's listener.
+    list.add(exitCodeListener);
+    return list;
   }
 
   public List<ISuiteListener> getSuiteListeners() {
@@ -924,7 +928,6 @@ private void initializeDefaultListeners() {
     if (m_failIfAllTestsSkipped) {
       this.exitCodeListener.failIfAllTestsSkipped();
     }
-    addListener(this.exitCodeListener);
     if (m_useDefaultListeners) {
       addReporter(SuiteHTMLReporter.class);
       addReporter(Main.class);
@@ -1111,12 +1114,16 @@ private void runExecutionListeners(boolean start) {
       for (IExecutionListener l : executionListeners) {
         l.onExecutionStart();
       }
+      // Invoke our exit code listener after all the user's listeners have run.
+      exitCodeListener.onExecutionStart();
     } else {
       List<IExecutionListener> executionListenersReversed =
           Lists.newReversedArrayList(executionListeners);
       for (IExecutionListener l : executionListenersReversed) {
         l.onExecutionFinish();
       }
+      // Invoke our exit code listener after all the user's listeners have run.
+      exitCodeListener.onExecutionFinish();
     }
   }
 
@@ -1128,7 +1135,11 @@ private static void usage() {
   }
 
   private void generateReports(List<ISuite> suiteRunners) {
-    for (IReporter reporter : m_reporters.values()) {
+    List<IReporter> reporters = new ArrayList<>(m_reporters.values());
+    // Add our Exit code listener as the last of the reporter so that we can still accommodate
+    // whatever changes were done by a user's reporting listener
+    reporters.add(exitCodeListener);
+    for (IReporter reporter : reporters) {
       try {
         long start = System.currentTimeMillis();
         reporter.generateReport(m_suites, suiteRunners, m_outputDir);
@@ -1343,7 +1354,7 @@ private SuiteRunner createSuiteRunner(XmlSuite xmlSuite) {
             m_useDefaultListeners,
             m_methodInterceptors,
             m_invokedMethodListeners.values(),
-            m_testListeners.values(),
+            getTestListeners(),
             m_classListeners.values(),
             holder,
             Systematiser.getComparator());

testng-core/src/test/java/test/listeners/ListenersTest.java

@@ -14,6 +14,7 @@
 import org.testng.TestNG;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
+import org.testng.internal.ExitCode;
 import org.testng.xml.XmlSuite;
 import test.SimpleBaseTest;
 import test.listeners.issue2638.DummyInvokedMethodListener;
@@ -23,6 +24,7 @@
 import test.listeners.issue2685.SampleTestFailureListener;
 import test.listeners.issue2752.ListenerSample;
 import test.listeners.issue2752.TestClassSample;
+import test.listeners.issue2771.TestCaseSample;
 
 public class ListenersTest extends SimpleBaseTest {
 
@@ -92,6 +94,13 @@ public void testWiringInOfListenersInMultipleTestTagsWithListenerInSuite() {
     assertThat(logs.get("Xml_Test_2")).containsAll(expected);
   }
 
+  @Test(description = "GITHUB-2771")
+  public void testEnsureNativeListenersAreRunAlwaysAtEnd() {
+    TestNG testng = create(TestCaseSample.class);
+    testng.run();
+    assertThat(testng.getStatus()).isEqualTo(ExitCode.FAILED);
+  }
+
   private void setupTest(boolean addExplicitListener) {
     TestNG testng = new TestNG();
     XmlSuite xmlSuite = createXmlSuite("Xml_Suite");

testng-core/src/test/java/test/listeners/issue2771/CustomSoftAssert.java

@@ -0,0 +1,12 @@
+package test.listeners.issue2771;
+
+import org.testng.ITestResult;
+import org.testng.reporters.ExitCodeListener;
+
+public class CustomSoftAssert extends ExitCodeListener {
+  @Override
+  public void onTestSuccess(ITestResult result) {
+    result.setStatus(ITestResult.FAILURE);
+    result.setThrowable(new AssertionError("There have been some failed soft asserts"));
+  }
+}

testng-core/src/test/java/test/listeners/issue2771/TestCaseSample.java

@@ -0,0 +1,10 @@
+package test.listeners.issue2771;
+
+import org.testng.annotations.Listeners;
+import org.testng.annotations.Test;
+
+@Listeners(CustomSoftAssert.class)
+public class TestCaseSample {
+  @Test
+  public void someCustomSoftAsserts() {}
+}

testng-core/src/test/java/test/reports/UniqueReporterInjectionTest.java

@@ -19,9 +19,8 @@ public void testPruningOfDuplicateReporter() {
     tng.addListener((ITestNGListener) new ReporterListenerForIssue1227());
     tng.run();
     // Since we have another reporting listener that is injected via the service loader file
-    // reporting listeners size will now have to be three (because the ExitCodeListener is also a
-    // reporter backed listener).
-    Assert.assertEquals(tng.getReporters().size(), 3);
+    // reporting listeners size will now have to be two
+    Assert.assertEquals(tng.getReporters().size(), 2);
     Assert.assertEquals(ReporterListenerForIssue1227.counter, 1);
   }