Merge pull request #2815 from krmahadevan/fix_2788

Streamline TestResult due to expectedExceptions
testng-team · Nov 1, 2022 · 728f480 · 728f480
2 parents bfdaf6d + afebb78
commit 728f480
Show file tree

Hide file tree

Showing 5 changed files with 63 additions and 3 deletions.
diff --git a/CHANGES.txt b/CHANGES.txt
@@ -1,4 +1,5 @@
 Current
+Fixed: GITHUB:2788: TestResult.isSuccess() is TRUE when test fails due to expectedExceptions (Krishnan Mahadevan)
 Fixed: GITHUB-2800: Running Test Classes with Inherited @Factory and @DataProvider Annotated Non-Static Methods Fail (Krishnan Mahadevan)
 New: Ability to provide custom error message for assertThrows\expectThrows methods (Anatolii Yuzhakov)
 Fixed: GITHUB-2780: Use SpotBugs instead of abandoned FindBugs

diff --git a/testng-core/src/main/java/org/testng/internal/ExitCode.java b/testng-core/src/main/java/org/testng/internal/ExitCode.java
@@ -19,7 +19,7 @@ public class ExitCode {
   public static final int HAS_NO_TEST = 8;
   private static final int FAILED_WITHIN_SUCCESS = 4;
   public static final int SKIPPED = 2;
-  private static final int FAILED = 1;
+  public static final int FAILED = 1;
   private static final int SIZE = 3;
 
   private final BitSet exitCodeBits;

diff --git a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java
@@ -726,6 +726,9 @@ private ITestResult invokeMethod(
       StatusHolder holder =
           considerExceptions(
               arguments.getTestMethod(), testResult, expectedExceptionClasses, failureContext);
+      // After considering exceptions, the test status may have gotten updated.
+      // So lets update our test status with the latest status obtained from StatusHolder
+      testResult.setStatus(holder.status);
       runInvokedMethodListeners(AFTER_INVOCATION, invokedMethod, testResult);
       updateStatusHolderAccordingToTestResult(testResult, holder);
       boolean willRetryMethod =

diff --git a/testng-core/src/test/java/test/expectedexceptions/ExpectedExceptionsTest.java b/testng-core/src/test/java/test/expectedexceptions/ExpectedExceptionsTest.java
@@ -1,12 +1,17 @@
 package test.expectedexceptions;
 
+import static org.assertj.core.api.Assertions.assertThat;
+
 import java.util.Collection;
 import java.util.List;
 import org.testng.Assert;
 import org.testng.ITestResult;
+import org.testng.TestNG;
 import org.testng.annotations.Test;
+import org.testng.internal.ExitCode;
 import test.BaseTest;
-import test.expectedexceptions.github1409.TestClassSample;
+import test.expectedexceptions.issue2788.TestClassSample;
+import test.expectedexceptions.issue2788.TestClassSample.Local;
 
 public class ExpectedExceptionsTest extends BaseTest {
 
@@ -19,6 +24,15 @@ public void expectedExceptionsDeprecatedSyntax() {
         new String[] {});
   }
 
+  @Test(description = "GITHUB-2788")
+  public void expectedExceptionsWithProperStatusPassedToListener() {
+    TestNG testng = new TestNG();
+    testng.setTestClasses(new Class[] {TestClassSample.class});
+    testng.run();
+    assertThat(testng.getStatus()).isEqualTo(ExitCode.FAILED);
+    assertThat(Local.getInstance().isPass()).isFalse();
+  }
+
   @Test
   public void expectedExceptions() {
     runTest(
@@ -31,7 +45,7 @@ public void expectedExceptions() {
   @Test
   public void expectedExceptionsMessage() {
     getFailedTests().clear();
-    addClass(TestClassSample.class);
+    addClass(test.expectedexceptions.github1409.TestClassSample.class);
     run();
     Collection<List<ITestResult>> failedTests = getFailedTests().values();
     Assert.assertFalse(failedTests.isEmpty());

diff --git a/testng-core/src/test/java/test/expectedexceptions/issue2788/TestClassSample.java b/testng-core/src/test/java/test/expectedexceptions/issue2788/TestClassSample.java
@@ -0,0 +1,42 @@
+package test.expectedexceptions.issue2788;
+
+import org.testng.IInvokedMethod;
+import org.testng.IInvokedMethodListener;
+import org.testng.ITestResult;
+import org.testng.annotations.Listeners;
+import org.testng.annotations.Test;
+import test.expectedexceptions.issue2788.TestClassSample.Local;
+
+@Listeners(Local.class)
+public class TestClassSample {
+
+  @Test(expectedExceptions = NullPointerException.class)
+  public void sampleTestMethod() {}
+
+  public static class Local implements IInvokedMethodListener {
+
+    public static Local instance;
+    private boolean pass;
+
+    private static void setInstance(Local localInstance) {
+      instance = localInstance;
+    }
+
+    public static Local getInstance() {
+      return instance;
+    }
+
+    public Local() {
+      setInstance(this);
+    }
+
+    public boolean isPass() {
+      return pass;
+    }
+
+    @Override
+    public void afterInvocation(IInvokedMethod method, ITestResult testResult) {
+      pass = testResult.isSuccess();
+    }
+  }
+}