Skip to content

[SUREFIRE-1710] skipAfterFailureCount in JUnit5 provider #503

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 1 commit into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
@@ -3054,30 +3054,43 @@ else if ( skipAfterFailureCount > 0 )
{
try
{
Artifact testng = getTestNgArtifact();
if ( testng != null )
Artifact junit5 = getJUnit5Artifact();
if ( junit5 != null )
{
VersionRange range = VersionRange.createFromVersionSpec( "[5.10,)" );
if ( !range.containsVersion( new DefaultArtifactVersion( testng.getVersion() ) ) )
VersionRange range = VersionRange.createFromVersionSpec( "[1.4.0,)" );
if ( !range.containsVersion( new DefaultArtifactVersion( junit5.getVersion() ) ) )
{
throw new MojoFailureException(
"Parameter \"skipAfterFailureCount\" expects TestNG Version 5.10 or higher. "
+ "java.lang.NoClassDefFoundError: org/testng/IInvokedMethodListener" );
"Parameter \"skipAfterFailureCount\" expects JUnit5 Version 5.4.0 or higher." );
}
}
else
{
// TestNG is dependent on JUnit
Artifact junit = getJunitArtifact();
if ( junit != null )
Artifact testng = getTestNgArtifact();
if ( testng != null )
{
VersionRange range = VersionRange.createFromVersionSpec( "[4.0,)" );
if ( !range.containsVersion( new DefaultArtifactVersion( junit.getVersion() ) ) )
VersionRange range = VersionRange.createFromVersionSpec( "[5.10,)" );
if ( !range.containsVersion( new DefaultArtifactVersion( testng.getVersion() ) ) )
{
throw new MojoFailureException(
"Parameter \"skipAfterFailureCount\" expects JUnit Version 4.0 or higher. "
+ "java.lang.NoSuchMethodError: "
+ "org.junit.runner.notification.RunNotifier.pleaseStop()V" );
"Parameter \"skipAfterFailureCount\" expects TestNG Version 5.10 or higher. "
+ "java.lang.NoClassDefFoundError: org/testng/IInvokedMethodListener" );
}
}
else
{
// TestNG is dependent on JUnit
Artifact junit = getJunitArtifact();
if ( junit != null )
{
VersionRange range = VersionRange.createFromVersionSpec( "[4.0,)" );
if ( !range.containsVersion( new DefaultArtifactVersion( junit.getVersion() ) ) )
{
throw new MojoFailureException(
"Parameter \"skipAfterFailureCount\" expects JUnit Version 4.0 or higher. "
+ "java.lang.NoSuchMethodError: "
+ "org.junit.runner.notification.RunNotifier.pleaseStop()V" );
}
}
}
}
Original file line number Diff line number Diff line change
@@ -42,7 +42,7 @@ Skipping Tests After the Nth Failure or Error
Prerequisite

Use ${project.artifactId} 2.19 or higher, JUnit 4.0 or higher, or
TestNG 5.10 or higher.
TestNG 5.10 or higher, JUnit5 5.4.0 or higher.

If version of TestNG is lower than 5.10 while the parameter
<<<skipAfterFailureCount>>> is set, the plugin fails with error:
6 changes: 0 additions & 6 deletions pom.xml
Original file line number Diff line number Diff line change
@@ -296,12 +296,6 @@
<groupId>org.powermock</groupId>
<artifactId>powermock-reflect</artifactId>
<version>${powermockVersion}</version>
<exclusions>
<exclusion>
<groupId>org.objenesis</groupId>
<artifactId>objenesis</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.javassist</groupId>
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package org.apache.maven.surefire.its;

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import java.util.ArrayList;

import org.junit.runners.Parameterized.Parameters;

/**
*
*/
public class FailFastJUnit5IT
extends AbstractFailFastIT
{

@Parameters( name = "{0}" )
@SuppressWarnings( "checkstyle:linelength" )
public static Iterable<Object[]> data()
{
/*
* reuseForks=false is not used because of race conditions between forks.
*/
ArrayList<Object[]> args = new ArrayList<>();
// description
// profile
// forkCount,
// fail-fast-count,
// reuseForks
// total
// failures
// errors
// skipped
// pipes
args.add( new Object[] { "fc1", null, props( 1, 3, true ), 5, 0, 3, 0, true } );
args.add( new Object[] { "fc2", null, props( 2, 3, true ), 5, 0, 3, 0, true } );
return args;
}

@Override
protected String withProvider()
{
return "junit5";
}
}
56 changes: 56 additions & 0 deletions surefire-its/src/test/resources/fail-fast-junit5/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<groupId>org.apache.maven.plugins.surefire</groupId>
<artifactId>fail-fast-junit5</artifactId>
<version>1.0</version>

<properties>
<maven.compiler.source>${java.specification.version}</maven.compiler.source>
<maven.compiler.target>${java.specification.version}</maven.compiler.target>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
</properties>

<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-engine</artifactId>
<version>5.8.2</version>
<scope>test</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-surefire-plugin</artifactId>
<version>${surefire.version}</version>
<configuration>
<runOrder>alphabetical</runOrder>
</configuration>
</plugin>
</plugins>
</build>
</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,18 @@
package pkg;

import org.junit.jupiter.api.Test;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

public class ATest
{
@Test
public void someMethod()
throws InterruptedException
{
MILLISECONDS.sleep( 1000L );
throw new RuntimeException( "assert \"foo\" == \"bar\"\n" +
" |\n"
+ " false" );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package pkg;

import org.junit.jupiter.api.Test;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

public class BTest
{
@Test
public void test()
throws InterruptedException
{
MILLISECONDS.sleep( 1000L );
throw new RuntimeException();
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package pkg;

import org.junit.jupiter.api.Test;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

public class CTest
{
@Test
public void test()
throws InterruptedException
{
MILLISECONDS.sleep( 1000L );
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package pkg;

import org.junit.jupiter.api.Test;

import static java.util.concurrent.TimeUnit.MILLISECONDS;

public class DTest
{
@Test
public void test()
throws InterruptedException
{
MILLISECONDS.sleep( 1000L );
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
package pkg;

import org.junit.jupiter.api.Test;

public class ETest
{
@Test
public void test()
{
}

}
55 changes: 55 additions & 0 deletions surefire-providers/common-junit5/pom.xml
Original file line number Diff line number Diff line change
@@ -0,0 +1,55 @@
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ Licensed to the Apache Software Foundation (ASF) under one
~ or more contributor license agreements. See the NOTICE file
~ distributed with this work for additional information
~ regarding copyright ownership. The ASF licenses this file
~ to you under the Apache License, Version 2.0 (the
~ "License"); you may not use this file except in compliance
~ with the License. You may obtain a copy of the License at
~
~ http://www.apache.org/licenses/LICENSE-2.0
~
~ Unless required by applicable law or agreed to in writing,
~ software distributed under the License is distributed on an
~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
~ KIND, either express or implied. See the License for the
~ specific language governing permissions and limitations
~ under the License.
-->

<project xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>

<parent>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>surefire-providers</artifactId>
<version>3.0.0-M6-SNAPSHOT</version>
</parent>

<artifactId>common-junit5</artifactId>
<name>Shared JUnit5 Provider Code</name>
<description>Shared JUnit5 Provider Code</description>

<dependencies>
<dependency>
<groupId>org.junit.jupiter</groupId>
<artifactId>junit-jupiter-api</artifactId>
<version>5.4.0</version>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>com.google.code.findbugs</groupId>
<artifactId>jsr305</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>org.powermock</groupId>
<artifactId>powermock-reflect</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

</project>
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
package org.apache.maven.surefire.common.junit5;

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import javax.annotation.Nonnegative;
import javax.annotation.Nonnull;

import java.util.concurrent.atomic.AtomicInteger;

import static org.apache.maven.surefire.api.util.internal.ConcurrencyUtils.runIfZeroCountDown;

/**
*
*/
public final class SkipTestsAfterFailureSingleton
{
private static final SkipTestsAfterFailureSingleton SINGLETON = new SkipTestsAfterFailureSingleton();

public static SkipTestsAfterFailureSingleton getSingleton()
{
return SINGLETON;
}

private final AtomicInteger skipAfterFailureCount = new AtomicInteger();
private volatile boolean active;
private volatile Runnable onThresholdReached;
private volatile boolean disableTests;

private SkipTestsAfterFailureSingleton()
{
}

void setDisableTests( boolean disableTests )
{
this.disableTests = disableTests;
}

public void setDisableTests()
{
setDisableTests( true );
}

public boolean isDisableTests()
{
return disableTests;
}

public void setReRunMode()
{
active = false;
}

/**
* Sets the threshold of failure count in current JVM.
*
* @param count threshold
*/
public void init( @Nonnegative int count, @Nonnull Runnable onThresholdReached )
{
this.onThresholdReached = onThresholdReached;
active = count > 0;
skipAfterFailureCount.set( count );
}

public void runIfFailureCountReachedThreshold()
{
if ( active )
{
runIfZeroCountDown( () -> disableTests = true, skipAfterFailureCount );
onThresholdReached.run();
}
}

}
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
package org.apache.maven.surefire.common.junit5;

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import java.util.Optional;

import org.junit.jupiter.api.extension.ConditionEvaluationResult;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.junit.jupiter.api.extension.ExecutionCondition;
import org.junit.jupiter.api.extension.TestWatcher;

import static org.junit.jupiter.api.extension.ConditionEvaluationResult.enabled;
import static org.junit.jupiter.api.extension.ConditionEvaluationResult.disabled;

/**
* Supports the parameter <code>skipAfterFailureCount</code>.
* <br>
* Disables the execution if the number of failures has exceeded
* the threshold defined in <code>skipAfterFailureCount</code>.
*
* @since 3.0.0-M5
*/
public class TestExecutionWatcherSPI
implements ExecutionCondition, TestWatcher
{
private static final SkipTestsAfterFailureSingleton FAILURES_COUNTER =
SkipTestsAfterFailureSingleton.getSingleton();

@Override
public ConditionEvaluationResult evaluateExecutionCondition( ExtensionContext context )
{
return FAILURES_COUNTER.isDisableTests() ? disabled( "" ) : enabled( "" );
}

@Override
public void testDisabled( ExtensionContext extensionContext, Optional<String> optional )
{

}

@Override
public void testSuccessful( ExtensionContext extensionContext )
{

}

@Override
public void testAborted( ExtensionContext extensionContext, Throwable throwable )
{

}

@Override
public void testFailed( ExtensionContext extensionContext, Throwable throwable )
{
FAILURES_COUNTER.runIfFailureCountReachedThreshold();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,89 @@
package org.apache.maven.surefire.common.junit5;

/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/

import org.junit.Test;

import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
import static org.powermock.reflect.Whitebox.getInternalState;

/**
*
*/
public class SkipTestsAfterFailureSingletonTest
{
@Test
public void shouldBeActive()
{
boolean[] onThresholdReached = {false};
SkipTestsAfterFailureSingleton singleton = SkipTestsAfterFailureSingleton.getSingleton();
singleton.setDisableTests( false );
singleton.init( 2, () -> onThresholdReached[0] = true );

TestExecutionWatcherSPI spi = new TestExecutionWatcherSPI();

spi.testFailed( null, null );
assertFalse( singleton.isDisableTests() );
assertFalse( spi.evaluateExecutionCondition( null ).isDisabled() );
assertTrue( onThresholdReached[0] );

spi.testFailed( null, null );
assertTrue( singleton.isDisableTests() );
assertTrue( spi.evaluateExecutionCondition( null ).isDisabled() );
assertTrue( onThresholdReached[0] );
}

@Test
public void shouldNotBeActive()
{
boolean[] onThresholdReached = {false};
SkipTestsAfterFailureSingleton singleton = SkipTestsAfterFailureSingleton.getSingleton();
singleton.setDisableTests( false );
singleton.init( 0, () -> onThresholdReached[0] = true );

TestExecutionWatcherSPI spi = new TestExecutionWatcherSPI();

for ( int i = 0; i < 10; i++ )
{
spi.testFailed( null, null );
assertFalse( singleton.isDisableTests() );
assertFalse( spi.evaluateExecutionCondition( null ).isDisabled() );
assertFalse( onThresholdReached[0] );
}
}

@Test
public void shouldNotBeActiveIfRerun()
{
boolean[] onThresholdReached = {false};
SkipTestsAfterFailureSingleton singleton = SkipTestsAfterFailureSingleton.getSingleton();
singleton.setDisableTests( false );
singleton.init( 1, () -> onThresholdReached[0] = true );
assertTrue( getInternalState( singleton, "active" ) );
singleton.setReRunMode();
assertFalse( getInternalState( singleton, "active" ) );
assertFalse( singleton.isDisableTests() );
assertFalse( onThresholdReached[0] );
singleton.runIfFailureCountReachedThreshold();
assertFalse( singleton.isDisableTests() );
assertFalse( onThresholdReached[0] );
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
org.apache.maven.surefire.common.junit5.TestExecutionWatcherSPI
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
org.apache.maven.surefire.common.junit5.TestExecutionWatcherSPI
1 change: 1 addition & 0 deletions surefire-providers/pom.xml
Original file line number Diff line number Diff line change
@@ -38,6 +38,7 @@
<module>common-java5</module>
<module>common-junit4</module>
<module>common-junit48</module>
<module>common-junit5</module>
<module>surefire-junit3</module>
<module>surefire-junit4</module>
<module>surefire-junit47</module>
5 changes: 5 additions & 0 deletions surefire-providers/surefire-junit-platform/pom.xml
Original file line number Diff line number Diff line change
@@ -82,6 +82,11 @@
<artifactId>common-java5</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.maven.surefire</groupId>
<artifactId>common-junit5</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.junit.platform</groupId>
<artifactId>junit-platform-launcher</artifactId>
Original file line number Diff line number Diff line change
@@ -52,14 +52,17 @@
import java.util.logging.Logger;

import org.apache.maven.surefire.api.provider.AbstractProvider;
import org.apache.maven.surefire.api.provider.CommandChainReader;
import org.apache.maven.surefire.api.provider.ProviderParameters;
import org.apache.maven.surefire.api.report.ReporterException;
import org.apache.maven.surefire.api.report.ReporterFactory;
import org.apache.maven.surefire.api.report.TestReportListener;
import org.apache.maven.surefire.api.suite.RunResult;
import org.apache.maven.surefire.api.testset.TestSetFailedException;
import org.apache.maven.surefire.api.util.ScanResult;
import org.apache.maven.surefire.api.util.SurefireReflectionException;
import org.apache.maven.surefire.api.util.TestsToRun;
import org.apache.maven.surefire.common.junit5.SkipTestsAfterFailureSingleton;
import org.apache.maven.surefire.shared.utils.StringUtils;
import org.junit.platform.engine.DiscoverySelector;
import org.junit.platform.engine.Filter;
@@ -88,6 +91,8 @@ public class JUnitPlatformProvider

private final Map<String, String> configurationParameters;

private final CommandChainReader commandsReader;

public JUnitPlatformProvider( ProviderParameters parameters )
{
this( parameters, new LazyLauncher() );
@@ -99,6 +104,8 @@ public JUnitPlatformProvider( ProviderParameters parameters )
this.launcher = launcher;
filters = newFilters();
configurationParameters = newConfigurationParameters();
// don't start a thread in CommandReader while we are in in-plugin process
commandsReader = parameters.isInsideFork() ? parameters.getCommandReader() : null;
}

@Override
@@ -122,10 +129,23 @@ public RunResult invoke( Object forkTestSet )
final RunResult runResult;
try
{
RunListenerAdapter adapter = new RunListenerAdapter( reporterFactory.createTestReportListener() );
TestReportListener listener = reporterFactory.createTestReportListener();

SkipTestsAfterFailureSingleton.getSingleton()
.init( getSkipAfterFailureCount(), listener::testExecutionSkippedByUser );

RunListenerAdapter adapter = new RunListenerAdapter( listener );

adapter.setRunMode( NORMAL_RUN );
startCapture( adapter );

setupJunitLogger();

if ( isFailFast() && commandsReader != null )
{
registerPleaseStopJUnit();
}

if ( forkTestSet instanceof TestsToRun )
{
invokeAllTests( (TestsToRun) forkTestSet, adapter );
@@ -151,6 +171,27 @@ else if ( forkTestSet == null )
return runResult;
}

private boolean isFailFast()
{
return parameters.getSkipAfterFailureCount() > 0;
}

private int getSkipAfterFailureCount()
{
return isFailFast() ? parameters.getSkipAfterFailureCount() : 0;
}

private void registerShutdownListener( final TestsToRun testsToRun )
{
commandsReader.addShutdownListener( command -> testsToRun.markTestSetFinished() );
}

private void registerPleaseStopJUnit()
{
commandsReader.addSkipNextTestsListener( command ->
SkipTestsAfterFailureSingleton.getSingleton().setDisableTests() );
}

private static void setupJunitLogger()
{
Logger logger = Logger.getLogger( "org.junit" );
@@ -169,7 +210,14 @@ private TestsToRun scanClasspath()
}

private void invokeAllTests( TestsToRun testsToRun, RunListenerAdapter adapter )
throws TestSetFailedException
{
if ( commandsReader != null )
{
registerShutdownListener( testsToRun );
commandsReader.awaitStarted();
}

try
{
execute( testsToRun, adapter );
@@ -179,6 +227,7 @@ private void invokeAllTests( TestsToRun testsToRun, RunListenerAdapter adapter )
closeLauncher();
}
// Rerun failing tests if requested
SkipTestsAfterFailureSingleton.getSingleton().setReRunMode();
int count = parameters.getTestRequest().getRerunFailingTestsCount();
if ( count > 0 && adapter.hasFailingTests() )
{