Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow data providers to be non cacheable #3076

Merged
merged 1 commit into from
Feb 26, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES.txt
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
Current (7.10.0)
Fixed: GITHUB-3041: TestNG 7.x DataProvider works in opposite to TestNG 6.x when retrying tests. (Krishnan Mahadevan)
Fixed: GITHUB-3066: How to dynamically adjust the number of TestNG threads after IExecutorFactory is deprecated? (Krishnan Mahadevan)
New: GITHUB-2874: Allow users to define ordering for TestNG listeners (Krishnan Mahadevan)
Fixed: GITHUB-3033: Moved ant support under own repository https://github.com/testng-team/testng-ant (Julien Herr)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -38,4 +38,12 @@ default boolean propagateFailureAsTestFailure() {
default Class<? extends IRetryDataProvider> retryUsing() {
return IRetryDataProvider.DisableDataProviderRetries.class;
}

/**
* @return - <code>true</code> if TestNG should use data returned by the original data provider
* invocation, when a test method fails and is configured to be retried.
*/
default boolean cacheDataForTestRetries() {
return true;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -57,6 +57,12 @@
*/
boolean propagateFailureAsTestFailure() default false;

/**
* @return - <code>true</code> if TestNG should use data returned by the original data provider
* invocation, when a test method fails and is configured to be retried.
*/
boolean cacheDataForTestRetries() default true;

/**
* @return - An Class which implements {@link IRetryDataProvider} and which can be used to retry a
* data provider.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -40,4 +40,16 @@ public interface IDataProviderAnnotation extends IAnnotation {
* data provider.
*/
Class<? extends IRetryDataProvider> retryUsing();

/**
* @param cache - when set to <code>true</code>, TestNG does not invoke the data provider again
* when retrying failed tests using a retry analyzer.
*/
void cacheDataForTestRetries(boolean cache);

/**
* @return - <code>true</code> if TestNG should use data returned by the original data provider
* invocation, when a test method fails and is configured to be retried.
*/
boolean isCacheDataForTestRetries();
}
Original file line number Diff line number Diff line change
Expand Up @@ -53,4 +53,9 @@ public boolean propagateFailureAsTestFailure() {
public Class<? extends IRetryDataProvider> retryUsing() {
return annotation.retryUsing();
}

@Override
public boolean cacheDataForTestRetries() {
return annotation.isCacheDataForTestRetries();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -442,7 +442,7 @@ private static void checkParameterTypes(
throw new TestNGException(
errPrefix
+ ".\nFor more information on native dependency injection please refer to "
+ "https://testng.org/doc/documentation-main.html#native-dependency-injection");
+ "https://testng.org/#_dependency_injection");
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ public class DataProviderAnnotation extends BaseAnnotation implements IDataProvi
private boolean m_bubbleUpFailures = false;
private Class<? extends IRetryDataProvider> retryUsing;

private boolean cachedDataForTestRetries = true;

@Override
public boolean isParallel() {
return m_parallel;
Expand Down Expand Up @@ -62,4 +64,13 @@ public void setRetryUsing(Class<? extends IRetryDataProvider> retry) {
public Class<? extends IRetryDataProvider> retryUsing() {
return retryUsing;
}

public void cacheDataForTestRetries(boolean cache) {
this.cachedDataForTestRetries = cache;
}

@Override
public boolean isCacheDataForTestRetries() {
return cachedDataForTestRetries;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -483,6 +483,7 @@ private IAnnotation createDataProviderTag(Method method, Annotation a) {
result.propagateFailureAsTestFailure();
}
result.setRetryUsing(c.retryUsing());
result.cacheDataForTestRetries(c.cacheDataForTestRetries());
return result;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@
import org.testng.DataProviderHolder;
import org.testng.IClassListener;
import org.testng.IDataProviderListener;
import org.testng.IDataProviderMethod;
import org.testng.IHookable;
import org.testng.IInvokedMethod;
import org.testng.IInvokedMethodListener;
Expand Down Expand Up @@ -238,6 +239,24 @@ public FailureContext retryFailed(
failure.representsRetriedMethod.set(true);
do {
failure.instances = Lists.newArrayList();
boolean cacheData =
Optional.ofNullable(arguments.getTestMethod().getDataProviderMethod())
.map(IDataProviderMethod::cacheDataForTestRetries)
.orElse(false);
if (!cacheData) {
Map<String, String> allParameters = Maps.newHashMap();
int verbose = testContext.getCurrentXmlTest().getVerbose();
ParameterHandler handler =
new ParameterHandler(
m_configuration.getObjectFactory(), annotationFinder(), this.holder, verbose);

ParameterBag bag =
handler.createParameters(
arguments.getTestMethod(), arguments.getParameters(), allParameters, testContext);
if (bag.hasErrors()) {
continue;
}
}
Object[] parameterValues = arguments.getParameterValues();
TestMethodArguments tma =
new TestMethodArguments.Builder()
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,21 @@
import test.dataprovider.issue2934.TestCaseSample.CoreListener;
import test.dataprovider.issue2934.TestCaseSample.ToggleDataProvider;
import test.dataprovider.issue2980.LoggingListener;
import test.dataprovider.issue3041.SampleTestCase;
import test.dataprovider.issue3045.DataProviderListener;
import test.dataprovider.issue3045.DataProviderTestClassSample;
import test.dataprovider.issue3045.DataProviderWithoutListenerTestClassSample;

public class DataProviderTest extends SimpleBaseTest {

@Test(description = "GITHUB-3041")
public void ensureDataProvidersCanBeInstructedNotToCacheDataForFailedTestRetries() {
TestNG testng = create(SampleTestCase.class);
testng.setVerbose(2);
testng.run();
assertThat(SampleTestCase.invocationCount.get()).isEqualTo(2);
}

@Test(description = "GITHUB-2819")
public void testDataProviderCanBeRetriedOnFailures() {
TestNG testng = create(TestClassUsingDataProviderRetrySample.class);
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package test.dataprovider.issue3041;

import java.util.Random;
import java.util.concurrent.atomic.AtomicInteger;
import org.testng.IRetryAnalyzer;
import org.testng.ITestResult;
import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;

public class SampleTestCase {

public static final AtomicInteger invocationCount = new AtomicInteger(0);
private static final Random random = new Random();

@Test(dataProvider = "dp", retryAnalyzer = MyRetry.class)
public void testMethod(int i) {
if (invocationCount.get() != 2) {
throw new RuntimeException("Failed for " + i);
}
}

@DataProvider(name = "dp", cacheDataForTestRetries = false)
public Object[][] getData() {
invocationCount.incrementAndGet();
return new Object[][] {{next()}, {next()}};
}

private static int next() {
return random.nextInt();
}

public static class MyRetry implements IRetryAnalyzer {

private final AtomicInteger counter = new AtomicInteger(1);

@Override
public boolean retry(ITestResult result) {
return counter.getAndIncrement() != 2;
}
}
}
Loading