Skip to content

Commit

Permalink
Segregate native & user listeners before ordering
Browse files Browse the repository at this point in the history
Closes #2771
  • Loading branch information
krmahadevan committed Jan 8, 2023
1 parent 8634720 commit 755cfd4
Show file tree
Hide file tree
Showing 13 changed files with 156 additions and 15 deletions.
1 change: 1 addition & 0 deletions CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
Current
Fixed: GITHUB-2796: Option for onAfterClass to run after @AfterClass
Fixed: GITHUB-2771: After upgrading to TestNG 7.5.0, setting ITestResult.status to FAILURE doesn't fail the test anymore (Krishnan Mahadevan)
Fixed: GITHUB-2857: XmlTest index is not set for test suites invoked with YAML

7.7.1
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,9 +16,14 @@ public final class RuntimeBehavior {
public static final String TESTNG_DEFAULT_VERBOSE = "testng.default.verbose";
public static final String IGNORE_CALLBACK_INVOCATION_SKIPS = "testng.ignore.callback.skip";
public static final String SYMMETRIC_LISTENER_EXECUTION = "testng.listener.execution.symmetric";
public static final String INTELLIJ_PACKAGE_NAME = "testng.intellij.package.name";

private RuntimeBehavior() {}

public static String getIntellijPackageName() {
return System.getProperty(INTELLIJ_PACKAGE_NAME, "com.intellij.rt.testng");
}

public static boolean ignoreCallbackInvocationSkips() {
return Boolean.getBoolean(IGNORE_CALLBACK_INVOCATION_SKIPS);
}
Expand Down
5 changes: 3 additions & 2 deletions testng-core/src/main/java/org/testng/SuiteRunner.java
Original file line number Diff line number Diff line change
Expand Up @@ -260,11 +260,12 @@ public void setReportResults(boolean reportResults) {

private void invokeListeners(boolean start) {
if (start) {
for (ISuiteListener sl : Lists.newArrayList(listeners.values())) {
for (ISuiteListener sl : ListenerOrderDeterminer.order(listeners.values())) {
sl.onStart(this);
}
} else {
List<ISuiteListener> suiteListenersReversed = Lists.newReversedArrayList(listeners.values());
List<ISuiteListener> suiteListenersReversed =
ListenerOrderDeterminer.reversedOrder(listeners.values());
for (ISuiteListener sl : suiteListenersReversed) {
sl.onFinish(this);
}
Expand Down
6 changes: 4 additions & 2 deletions testng-core/src/main/java/org/testng/TestNG.java
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@
import org.testng.internal.DynamicGraph;
import org.testng.internal.ExitCode;
import org.testng.internal.IConfiguration;
import org.testng.internal.ListenerOrderDeterminer;
import org.testng.internal.OverrideProcessor;
import org.testng.internal.ReporterConfig;
import org.testng.internal.RuntimeBehavior;
Expand Down Expand Up @@ -1106,14 +1107,15 @@ private void runSuiteAlterationListeners() {
}

private void runExecutionListeners(boolean start) {
List<IExecutionListener> executionListeners = m_configuration.getExecutionListeners();
List<IExecutionListener> executionListeners =
ListenerOrderDeterminer.order(m_configuration.getExecutionListeners());
if (start) {
for (IExecutionListener l : executionListeners) {
l.onExecutionStart();
}
} else {
List<IExecutionListener> executionListenersReversed =
Lists.newReversedArrayList(executionListeners);
ListenerOrderDeterminer.reversedOrder(executionListeners);
for (IExecutionListener l : executionListenersReversed) {
l.onExecutionFinish();
}
Expand Down
6 changes: 4 additions & 2 deletions testng-core/src/main/java/org/testng/TestRunner.java
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@
import org.testng.internal.IContainer;
import org.testng.internal.ITestClassConfigInfo;
import org.testng.internal.ITestResultNotifier;
import org.testng.internal.ListenerOrderDeterminer;
import org.testng.internal.MethodGroupsHelper;
import org.testng.internal.MethodHelper;
import org.testng.internal.ResultMap;
Expand Down Expand Up @@ -955,12 +956,13 @@ private void logStart() {
*/
private void fireEvent(boolean isStart) {
if (isStart) {
for (ITestListener itl : m_testListeners) {
for (ITestListener itl : ListenerOrderDeterminer.order(m_testListeners)) {
itl.onStart(this);
}

} else {
List<ITestListener> testListenersReversed = Lists.newReversedArrayList(m_testListeners);
List<ITestListener> testListenersReversed =
ListenerOrderDeterminer.reversedOrder(m_testListeners);
for (ITestListener itl : testListenersReversed) {
itl.onFinish(this);
}
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,94 @@
package org.testng.internal;

import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
import java.util.function.Predicate;
import org.testng.collections.Lists;

/**
* A Utility that helps us differentiate between a user's listener and TestNG's native listeners(
* which also includes IntelliJ IDEA Listener)
*/
public final class ListenerOrderDeterminer {

private ListenerOrderDeterminer() {
// Defeat instantiation
}

private static final List<String> KNOWN =
Arrays.asList(
"org.testng.internal.ExitCodeListener",
"org.testng.SuiteRunner",
"org.testng.TestRunner.ConfigurationListener",
"org.testng.reporters.DotTestListener",
"org.testng.reporters.EmailableReporter",
"org.testng.reporters.EmailableReporter2",
"org.testng.reporters.ExitCodeListener",
"org.testng.reporters.FailedReporter",
"org.testng.reporters.JUnitReportReporter",
"org.testng.reporters.JUnitXMLReporter",
"org.testng.reporters.SuiteHTMLReporter",
"org.testng.reporters.TestHTMLReporter",
"org.testng.reporters.TextReporter",
"org.testng.reporters.VerboseReporter",
"org.testng.reporters.XMLReporter",
"org.testng.reporters.jq.Main");

private static final Predicate<Class<?>> INTELLIJ_IDE_LISTENER =
clazz -> clazz.getName().contains(RuntimeBehavior.getIntellijPackageName());

/**
* @param original - The original collection of listeners
* @return - A re-ordered collection wherein TestNG's known listeners are added at the end
*/
public static <T> List<T> order(Collection<T> original) {
List<T> nativeListeners = new ArrayList<>();
List<T> foreignListeners = new ArrayList<>();
original.stream()
.filter(Objects::nonNull)
.forEach(
each -> {
if (isNativeListener(each.getClass())) {
nativeListeners.add(each);
} else {
foreignListeners.add(each);
}
});
return Lists.merge(foreignListeners, nativeListeners);
}

/**
* @param original - The original collection of listeners
* @return - A reversed ordered list wherein the user listeners are found in reverse order
* followed by TestNG known listeners also in reverse order.
*/
public static <T> List<T> reversedOrder(Collection<T> original) {
List<T> nativeListeners = new ArrayList<>();
List<T> foreignListeners = new ArrayList<>();
original.stream()
.filter(Objects::nonNull)
.forEach(
each -> {
if (isNativeListener(each.getClass())) {
nativeListeners.add(each);
} else {
foreignListeners.add(each);
}
});
Collections.reverse(foreignListeners);
Collections.reverse(nativeListeners);
return Lists.merge(foreignListeners, nativeListeners);
}

private static boolean isNativeListener(Class<?> clazz) {
return KNOWN.stream()
.anyMatch(
each ->
each.equalsIgnoreCase(clazz.getCanonicalName())
|| INTELLIJ_IDE_LISTENER.test(clazz));
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ private TestListenerHelper() {

public static void runPreConfigurationListeners(
ITestResult tr, ITestNGMethod tm, List<IConfigurationListener> listeners) {
for (IConfigurationListener icl : listeners) {
for (IConfigurationListener icl : ListenerOrderDeterminer.order(listeners)) {
icl.beforeConfiguration(tr);
try {
icl.beforeConfiguration(tr, tm);
Expand All @@ -35,7 +35,8 @@ public static void runPreConfigurationListeners(

public static void runPostConfigurationListeners(
ITestResult tr, ITestNGMethod tm, List<IConfigurationListener> listeners) {
List<IConfigurationListener> listenersreversed = Lists.newReversedArrayList(listeners);
List<IConfigurationListener> listenersreversed =
ListenerOrderDeterminer.reversedOrder(listeners);
for (IConfigurationListener icl : listenersreversed) {
switch (tr.getStatus()) {
case ITestResult.SKIP:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,10 +10,10 @@
import org.testng.ITestResult;
import org.testng.SkipException;
import org.testng.SuiteRunState;
import org.testng.collections.Lists;
import org.testng.collections.Maps;
import org.testng.internal.IConfiguration;
import org.testng.internal.ITestResultNotifier;
import org.testng.internal.ListenerOrderDeterminer;
import org.testng.internal.Utils;
import org.testng.internal.annotations.IAnnotationFinder;

Expand Down Expand Up @@ -60,8 +60,8 @@ protected void runInvokedMethodListeners(
boolean isAfterInvocation = InvokedMethodListenerMethod.AFTER_INVOCATION == listenerMethod;
Collection<IInvokedMethodListener> listeners =
isAfterInvocation
? Lists.newReversedArrayList(m_invokedMethodListeners)
: m_invokedMethodListeners;
? ListenerOrderDeterminer.reversedOrder(m_invokedMethodListeners)
: ListenerOrderDeterminer.order(m_invokedMethodListeners);
for (IInvokedMethodListener currentListener : listeners) {
try {
invoker.invokeListener(currentListener, invokedMethod);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -272,8 +272,8 @@ public void runTestResultListener(ITestResult tr) {
boolean isFinished = tr.getStatus() != ITestResult.STARTED;
List<ITestListener> listeners =
isFinished
? Lists.newReversedArrayList(m_notifier.getTestListeners())
: m_notifier.getTestListeners();
? ListenerOrderDeterminer.reversedOrder(m_notifier.getTestListeners())
: ListenerOrderDeterminer.order(m_notifier.getTestListeners());
TestListenerHelper.runTestListeners(tr, listeners);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
import org.testng.*;
import org.testng.collections.Lists;
import org.testng.internal.ITestResultNotifier;
import org.testng.internal.ListenerOrderDeterminer;
import org.testng.internal.TestListenerHelper;
import org.testng.internal.invokers.InvokedMethod;

Expand Down Expand Up @@ -94,8 +95,8 @@ public void endTest(Test test) {
boolean isFinished = tr.getStatus() != ITestResult.STARTED;
List<ITestListener> listeners =
isFinished
? Lists.newReversedArrayList(m_parentRunner.getTestListeners())
: m_parentRunner.getTestListeners();
? ListenerOrderDeterminer.reversedOrder(m_parentRunner.getTestListeners())
: ListenerOrderDeterminer.order(m_parentRunner.getTestListeners());
TestListenerHelper.runTestListeners(tr, listeners);
}

Expand Down
9 changes: 9 additions & 0 deletions testng-core/src/test/java/test/listeners/ListenersTest.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import org.testng.TestNG;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import org.testng.internal.ExitCode;
import org.testng.xml.XmlSuite;
import test.SimpleBaseTest;
import test.listeners.issue2638.DummyInvokedMethodListener;
Expand All @@ -23,6 +24,7 @@
import test.listeners.issue2685.SampleTestFailureListener;
import test.listeners.issue2752.ListenerSample;
import test.listeners.issue2752.TestClassSample;
import test.listeners.issue2771.TestCaseSample;

public class ListenersTest extends SimpleBaseTest {

Expand Down Expand Up @@ -92,6 +94,13 @@ public void testWiringInOfListenersInMultipleTestTagsWithListenerInSuite() {
assertThat(logs.get("Xml_Test_2")).containsAll(expected);
}

@Test(description = "GITHUB-2771")
public void testEnsureNativeListenersAreRunAlwaysAtEnd() {
TestNG testng = create(TestCaseSample.class);
testng.run();
assertThat(testng.getStatus()).isEqualTo(ExitCode.FAILED);
}

private void setupTest(boolean addExplicitListener) {
TestNG testng = new TestNG();
XmlSuite xmlSuite = createXmlSuite("Xml_Suite");
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package test.listeners.issue2771;

import org.testng.ITestResult;
import org.testng.reporters.ExitCodeListener;

public class CustomSoftAssert extends ExitCodeListener {
@Override
public void onTestSuccess(ITestResult result) {
result.setStatus(ITestResult.FAILURE);
result.setThrowable(new AssertionError("There have been some failed soft asserts"));
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package test.listeners.issue2771;

import org.testng.annotations.Listeners;
import org.testng.annotations.Test;

@Listeners(CustomSoftAssert.class)
public class TestCaseSample {
@Test
public void someCustomSoftAsserts() {
// doing custom soft asserts
// which will trigger test failure in listener
}
}

0 comments on commit 755cfd4

Please sign in to comment.