From cc37588194b04f81a062d3c123dd7fe33d67230f Mon Sep 17 00:00:00 2001 From: Krishnan Mahadevan Date: Sun, 25 Feb 2024 19:55:54 +0530 Subject: [PATCH] Allow data providers to be non cacheable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #3041 We can now configure TestNG to enable/disable caching of test data produced by a data provider when TestNG is retrying a failed test method using the attribute “cacheDataForTestRetries” on the “@DataProvider” annotation. Below is a sample, which forces TestNG to re-invoke the data provider when a test method fails and it needs to be retried. ``` @DataProvider(name = "dp", cacheDataForTestRetries = false) public Object[][] getData() { return new Object[][] {{1}, {2}}; } ``` --- CHANGES.txt | 1 + .../java/org/testng/IDataProviderMethod.java | 8 ++++ .../org/testng/annotations/DataProvider.java | 6 +++ .../annotations/IDataProviderAnnotation.java | 12 ++++++ .../testng/internal/DataProviderMethod.java | 5 +++ .../java/org/testng/internal/Parameters.java | 2 +- .../annotations/DataProviderAnnotation.java | 11 +++++ .../internal/annotations/JDK15TagFactory.java | 1 + .../testng/internal/invokers/TestInvoker.java | 19 +++++++++ .../test/dataprovider/DataProviderTest.java | 9 ++++ .../issue3041/SampleTestCase.java | 41 +++++++++++++++++++ 11 files changed, 114 insertions(+), 1 deletion(-) create mode 100644 testng-core/src/test/java/test/dataprovider/issue3041/SampleTestCase.java diff --git a/CHANGES.txt b/CHANGES.txt index bcd59d2ed1..71cc122c51 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ Current (7.10.0) +Fixed: GITHUB-3041: TestNG 7.x DataProvider works in opposite to TestNG 6.x when retrying tests. (Krishnan Mahadevan) Fixed: GITHUB-3066: How to dynamically adjust the number of TestNG threads after IExecutorFactory is deprecated? (Krishnan Mahadevan) New: GITHUB-2874: Allow users to define ordering for TestNG listeners (Krishnan Mahadevan) Fixed: GITHUB-3033: Moved ant support under own repository https://github.com/testng-team/testng-ant (Julien Herr) diff --git a/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java b/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java index 4011bdd5c6..fb972de800 100644 --- a/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java +++ b/testng-core-api/src/main/java/org/testng/IDataProviderMethod.java @@ -38,4 +38,12 @@ default boolean propagateFailureAsTestFailure() { default Class retryUsing() { return IRetryDataProvider.DisableDataProviderRetries.class; } + + /** + * @return - true if TestNG should use data returned by the original data provider + * invocation, when a test method fails and is configured to be retried. + */ + default boolean cacheDataForTestRetries() { + return true; + } } diff --git a/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java b/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java index 9d8124dd8d..d3223c68d2 100644 --- a/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java +++ b/testng-core-api/src/main/java/org/testng/annotations/DataProvider.java @@ -57,6 +57,12 @@ */ boolean propagateFailureAsTestFailure() default false; + /** + * @return - true if TestNG should use data returned by the original data provider + * invocation, when a test method fails and is configured to be retried. + */ + boolean cacheDataForTestRetries() default true; + /** * @return - An Class which implements {@link IRetryDataProvider} and which can be used to retry a * data provider. diff --git a/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java b/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java index a13a198bd2..5705499707 100644 --- a/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java +++ b/testng-core-api/src/main/java/org/testng/annotations/IDataProviderAnnotation.java @@ -40,4 +40,16 @@ public interface IDataProviderAnnotation extends IAnnotation { * data provider. */ Class retryUsing(); + + /** + * @param cache - when set to true, TestNG does not invoke the data provider again + * when retrying failed tests using a retry analyzer. + */ + void cacheDataForTestRetries(boolean cache); + + /** + * @return - true if TestNG should use data returned by the original data provider + * invocation, when a test method fails and is configured to be retried. + */ + boolean isCacheDataForTestRetries(); } diff --git a/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java b/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java index bce2a1e426..2fca52ad42 100644 --- a/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java +++ b/testng-core/src/main/java/org/testng/internal/DataProviderMethod.java @@ -53,4 +53,9 @@ public boolean propagateFailureAsTestFailure() { public Class retryUsing() { return annotation.retryUsing(); } + + @Override + public boolean cacheDataForTestRetries() { + return annotation.isCacheDataForTestRetries(); + } } diff --git a/testng-core/src/main/java/org/testng/internal/Parameters.java b/testng-core/src/main/java/org/testng/internal/Parameters.java index b08af6223b..14460e2e6e 100644 --- a/testng-core/src/main/java/org/testng/internal/Parameters.java +++ b/testng-core/src/main/java/org/testng/internal/Parameters.java @@ -442,7 +442,7 @@ private static void checkParameterTypes( throw new TestNGException( errPrefix + ".\nFor more information on native dependency injection please refer to " - + "https://testng.org/doc/documentation-main.html#native-dependency-injection"); + + "https://testng.org/#_dependency_injection"); } } diff --git a/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java b/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java index e793e72f41..680bcb6fb2 100644 --- a/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java +++ b/testng-core/src/main/java/org/testng/internal/annotations/DataProviderAnnotation.java @@ -13,6 +13,8 @@ public class DataProviderAnnotation extends BaseAnnotation implements IDataProvi private boolean m_bubbleUpFailures = false; private Class retryUsing; + private boolean cachedDataForTestRetries = true; + @Override public boolean isParallel() { return m_parallel; @@ -62,4 +64,13 @@ public void setRetryUsing(Class retry) { public Class retryUsing() { return retryUsing; } + + public void cacheDataForTestRetries(boolean cache) { + this.cachedDataForTestRetries = cache; + } + + @Override + public boolean isCacheDataForTestRetries() { + return cachedDataForTestRetries; + } } diff --git a/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java b/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java index 000180101d..ffb6e74ed1 100644 --- a/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java +++ b/testng-core/src/main/java/org/testng/internal/annotations/JDK15TagFactory.java @@ -483,6 +483,7 @@ private IAnnotation createDataProviderTag(Method method, Annotation a) { result.propagateFailureAsTestFailure(); } result.setRetryUsing(c.retryUsing()); + result.cacheDataForTestRetries(c.cacheDataForTestRetries()); return result; } diff --git a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java index 1d4adafe32..bcf0a4ab90 100644 --- a/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java +++ b/testng-core/src/main/java/org/testng/internal/invokers/TestInvoker.java @@ -23,6 +23,7 @@ import org.testng.DataProviderHolder; import org.testng.IClassListener; import org.testng.IDataProviderListener; +import org.testng.IDataProviderMethod; import org.testng.IHookable; import org.testng.IInvokedMethod; import org.testng.IInvokedMethodListener; @@ -238,6 +239,24 @@ public FailureContext retryFailed( failure.representsRetriedMethod.set(true); do { failure.instances = Lists.newArrayList(); + boolean cacheData = + Optional.ofNullable(arguments.getTestMethod().getDataProviderMethod()) + .map(IDataProviderMethod::cacheDataForTestRetries) + .orElse(false); + if (!cacheData) { + Map allParameters = Maps.newHashMap(); + int verbose = testContext.getCurrentXmlTest().getVerbose(); + ParameterHandler handler = + new ParameterHandler( + m_configuration.getObjectFactory(), annotationFinder(), this.holder, verbose); + + ParameterBag bag = + handler.createParameters( + arguments.getTestMethod(), arguments.getParameters(), allParameters, testContext); + if (bag.hasErrors()) { + continue; + } + } Object[] parameterValues = arguments.getParameterValues(); TestMethodArguments tma = new TestMethodArguments.Builder() diff --git a/testng-core/src/test/java/test/dataprovider/DataProviderTest.java b/testng-core/src/test/java/test/dataprovider/DataProviderTest.java index 63a2c626ea..80cebc0c8d 100644 --- a/testng-core/src/test/java/test/dataprovider/DataProviderTest.java +++ b/testng-core/src/test/java/test/dataprovider/DataProviderTest.java @@ -55,12 +55,21 @@ import test.dataprovider.issue2934.TestCaseSample.CoreListener; import test.dataprovider.issue2934.TestCaseSample.ToggleDataProvider; import test.dataprovider.issue2980.LoggingListener; +import test.dataprovider.issue3041.SampleTestCase; import test.dataprovider.issue3045.DataProviderListener; import test.dataprovider.issue3045.DataProviderTestClassSample; import test.dataprovider.issue3045.DataProviderWithoutListenerTestClassSample; public class DataProviderTest extends SimpleBaseTest { + @Test(description = "GITHUB-3041") + public void ensureDataProvidersCanBeInstructedNotToCacheDataForFailedTestRetries() { + TestNG testng = create(SampleTestCase.class); + testng.setVerbose(2); + testng.run(); + assertThat(SampleTestCase.invocationCount.get()).isEqualTo(2); + } + @Test(description = "GITHUB-2819") public void testDataProviderCanBeRetriedOnFailures() { TestNG testng = create(TestClassUsingDataProviderRetrySample.class); diff --git a/testng-core/src/test/java/test/dataprovider/issue3041/SampleTestCase.java b/testng-core/src/test/java/test/dataprovider/issue3041/SampleTestCase.java new file mode 100644 index 0000000000..e42e64d766 --- /dev/null +++ b/testng-core/src/test/java/test/dataprovider/issue3041/SampleTestCase.java @@ -0,0 +1,41 @@ +package test.dataprovider.issue3041; + +import java.util.Random; +import java.util.concurrent.atomic.AtomicInteger; +import org.testng.IRetryAnalyzer; +import org.testng.ITestResult; +import org.testng.annotations.DataProvider; +import org.testng.annotations.Test; + +public class SampleTestCase { + + public static final AtomicInteger invocationCount = new AtomicInteger(0); + private static final Random random = new Random(); + + @Test(dataProvider = "dp", retryAnalyzer = MyRetry.class) + public void testMethod(int i) { + if (invocationCount.get() != 2) { + throw new RuntimeException("Failed for " + i); + } + } + + @DataProvider(name = "dp", cacheDataForTestRetries = false) + public Object[][] getData() { + invocationCount.incrementAndGet(); + return new Object[][] {{next()}, {next()}}; + } + + private static int next() { + return random.nextInt(); + } + + public static class MyRetry implements IRetryAnalyzer { + + private final AtomicInteger counter = new AtomicInteger(1); + + @Override + public boolean retry(ITestResult result) { + return counter.getAndIncrement() != 2; + } + } +}