Skip to content

Commit

Permalink
Allow data providers to be non cacheable
Browse files Browse the repository at this point in the history
Closes #3041

We can now configure TestNG to enable/disable 
caching of test data produced by a data provider
when TestNG is retrying a failed test method using
the attribute “cacheDataForTestRetries” on the 
“@dataProvider” annotation.

Below is a sample, which forces TestNG to re-invoke
the data provider when a test method fails and it 
needs to be retried.

```
@dataProvider(name = "dp", cacheDataForTestRetries = false)
public Object[][] getData() {
  return new Object[][] {{1}, {2}};
}
```
  • Loading branch information
krmahadevan committed Feb 26, 2024
1 parent 2cc332d commit 823799c
Show file tree
Hide file tree
Showing 11 changed files with 114 additions and 1 deletion.
1 change: 1 addition & 0 deletions CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
Current (7.10.0)
Fixed: GITHUB-3041: TestNG 7.x DataProvider works in opposite to TestNG 6.x when retrying tests. (Krishnan Mahadevan)
Fixed: GITHUB-3066: How to dynamically adjust the number of TestNG threads after IExecutorFactory is deprecated? (Krishnan Mahadevan)
New: GITHUB-2874: Allow users to define ordering for TestNG listeners (Krishnan Mahadevan)
Fixed: GITHUB-3033: Moved ant support under own repository https://github.com/testng-team/testng-ant (Julien Herr)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,12 @@ default boolean propagateFailureAsTestFailure() {
default Class<? extends IRetryDataProvider> retryUsing() {
return IRetryDataProvider.DisableDataProviderRetries.class;
}

/**
* @return - <code>true</code> if TestNG should use data returned by the original data provider
* invocation, when a test method fails and is configured to be retried.
*/
default boolean cacheDataForTestRetries() {
return true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@
*/
boolean propagateFailureAsTestFailure() default false;

/**
* @return - <code>true</code> if TestNG should use data returned by the original data provider
* invocation, when a test method fails and is configured to be retried.
*/
boolean cacheDataForTestRetries() default true;

/**
* @return - An Class which implements {@link IRetryDataProvider} and which can be used to retry a
* data provider.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,16 @@ public interface IDataProviderAnnotation extends IAnnotation {
* data provider.
*/
Class<? extends IRetryDataProvider> retryUsing();

/**
* @param cache - when set to <code>true</code>, TestNG does not invoke the data provider again
* when retrying failed tests using a retry analyzer.
*/
void cacheDataForTestRetries(boolean cache);

/**
* @return - <code>true</code> if TestNG should use data returned by the original data provider
* invocation, when a test method fails and is configured to be retried.
*/
boolean isCacheDataForTestRetries();
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,9 @@ public boolean propagateFailureAsTestFailure() {
public Class<? extends IRetryDataProvider> retryUsing() {
return annotation.retryUsing();
}

@Override
public boolean cacheDataForTestRetries() {
return annotation.isCacheDataForTestRetries();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ private static void checkParameterTypes(
throw new TestNGException(
errPrefix
+ ".\nFor more information on native dependency injection please refer to "
+ "https://testng.org/doc/documentation-main.html#native-dependency-injection");
+ "https://testng.org/#_dependency_injection");
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public class DataProviderAnnotation extends BaseAnnotation implements IDataProvi
private boolean m_bubbleUpFailures = false;
private Class<? extends IRetryDataProvider> retryUsing;

private boolean cachedDataForTestRetries = true;

@Override
public boolean isParallel() {
return m_parallel;
Expand Down Expand Up @@ -62,4 +64,13 @@ public void setRetryUsing(Class<? extends IRetryDataProvider> retry) {
public Class<? extends IRetryDataProvider> retryUsing() {
return retryUsing;
}

public void cacheDataForTestRetries(boolean cache) {
this.cachedDataForTestRetries = cache;
}

@Override
public boolean isCacheDataForTestRetries() {
return cachedDataForTestRetries;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,7 @@ private IAnnotation createDataProviderTag(Method method, Annotation a) {
result.propagateFailureAsTestFailure();
}
result.setRetryUsing(c.retryUsing());
result.cacheDataForTestRetries(c.cacheDataForTestRetries());
return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.testng.DataProviderHolder;
import org.testng.IClassListener;
import org.testng.IDataProviderListener;
import org.testng.IDataProviderMethod;
import org.testng.IHookable;
import org.testng.IInvokedMethod;
import org.testng.IInvokedMethodListener;
Expand Down Expand Up @@ -238,6 +239,24 @@ public FailureContext retryFailed(
failure.representsRetriedMethod.set(true);
do {
failure.instances = Lists.newArrayList();
boolean cacheData =
Optional.ofNullable(arguments.getTestMethod().getDataProviderMethod())
.map(IDataProviderMethod::cacheDataForTestRetries)
.orElse(false);
if (!cacheData) {
Map<String, String> allParameters = Maps.newHashMap();
int verbose = testContext.getCurrentXmlTest().getVerbose();
ParameterHandler handler =
new ParameterHandler(
m_configuration.getObjectFactory(), annotationFinder(), this.holder, verbose);

ParameterBag bag =
handler.createParameters(
arguments.getTestMethod(), arguments.getParameters(), allParameters, testContext);
if (bag.hasErrors()) {
continue;
}
}
Object[] parameterValues = arguments.getParameterValues();
TestMethodArguments tma =
new TestMethodArguments.Builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,21 @@
import test.dataprovider.issue2934.TestCaseSample.CoreListener;
import test.dataprovider.issue2934.TestCaseSample.ToggleDataProvider;
import test.dataprovider.issue2980.LoggingListener;
import test.dataprovider.issue3041.SampleTestCase;
import test.dataprovider.issue3045.DataProviderListener;
import test.dataprovider.issue3045.DataProviderTestClassSample;
import test.dataprovider.issue3045.DataProviderWithoutListenerTestClassSample;

public class DataProviderTest extends SimpleBaseTest {

@Test(description = "GITHUB-3041")
public void ensureDataProvidersCanBeInstructedNotToCacheDataForFailedTestRetries() {
TestNG testng = create(SampleTestCase.class);
testng.setVerbose(2);
testng.run();
assertThat(SampleTestCase.invocationCount.get()).isEqualTo(2);
}

@Test(description = "GITHUB-2819")
public void testDataProviderCanBeRetriedOnFailures() {
TestNG testng = create(TestClassUsingDataProviderRetrySample.class);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package test.dataprovider.issue3041;

import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import org.testng.IRetryAnalyzer;
import org.testng.ITestResult;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

public class SampleTestCase {

public static final AtomicInteger invocationCount = new AtomicInteger(0);
private static final Random random = new Random();

@Test(dataProvider = "dp", retryAnalyzer = MyRetry.class)
public void testMethod(int i) {
if (invocationCount.get() != 2) {
throw new RuntimeException("Failed for " + i);
}
}

@DataProvider(name = "dp", cacheDataForTestRetries = false)
public Object[][] getData() {
invocationCount.incrementAndGet();
return new Object[][] {{next()}, {next()}};
}

private static int next() {
return random.nextInt();
}

public static class MyRetry implements IRetryAnalyzer {

private final AtomicInteger counter = new AtomicInteger(1);

@Override
public boolean retry(ITestResult result) {
return counter.getAndIncrement() != 2;
}
}
}

0 comments on commit 823799c

Please sign in to comment.