From 649113768555b310779be46411298aadc650aa87 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20H=C3=B6f?= Date: Mon, 9 Dec 2024 17:56:59 +0100 Subject: [PATCH] provide name of test class to re-run in failure reproduction info, if test class hierarchy of failed test contains Lifecycle PER_CLASS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Höf --- README.md | 15 +- .../valueprovider/ValueProviderException.java | 32 +++- .../ValueProviderExceptionTest.java | 20 ++- .../valueprovider/ValueProviderExtension.java | 25 ++- .../ValueProviderExtensionFailureTest.java | 164 +++++++++++++++--- 5 files changed, 211 insertions(+), 45 deletions(-) diff --git a/README.md b/README.md index c4109d0..7211b79 100644 --- a/README.md +++ b/README.md @@ -499,7 +499,7 @@ at com.tngtech.valueprovider.ValueProviderRule$1.evaluate(ValueProviderRule.java ``` If the failure is related to random data, you can easily reproduce it. Just specify the above shown JVM system -properties in the command line when you re-run the test, e.g., in your IDE: +properties in the command line when you re-run the failed test, e.g., in your IDE: ``` -Dvalue.provider.factory.test.class.seed=0 @@ -524,8 +524,17 @@ for details). The nesting may be arbitrarily deep, i.e. `@Nested` classes may co test lifecycle may be chosen individually for the main test class as well as for each `@Nested` test class. As long as the main test class and all `@Nested` test classes use the default test lifecycle `PER_METHOD`, it is again sufficient to rerun the individual test method, regardless if it is in the main class or any nested class. As soon as the lifecycle -`PER_CLASS` is used for one of the classes in the nesting hierarchy where the failure occured, you have to rerun __all -test methods of this hierarchy__ of test classes to reproduce the failure. +`PER_CLASS` is used for one or more classes in the nesting hierarchy where the failure occured, you have to re-run __all +test methods of this hierarchy__ of test classes to reproduce the failure. For convenience, the failure message +generated by the infrastructure provides the name of the root test class of this hierarchy in addition to the seed +values as shown in the following example: + +``` +"If the failure is related to random ValueProviders, re-run all tests of 'com.tngtech.valueprovider.ValueProviderExceptionTest' and specify the following system properties for the JVM to reproduce: +-Dvalue.provider.factory.test.class.seed=0 +-Dvalue.provider.factory.test.method.seed=-5385145878463633929 +-Dvalue.provider.factory.reference.date.time=2024-12-09T17:02:50.109" +``` ### Reproducible ValueProviders diff --git a/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java b/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java index 95c1a0f..083df54 100644 --- a/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java +++ b/core/src/main/java/com/tngtech/valueprovider/ValueProviderException.java @@ -1,29 +1,43 @@ package com.tngtech.valueprovider; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_CLASS_SEED_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY; -import static com.tngtech.valueprovider.ValueProviderFactory.getFormattedReferenceDateTime; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestClassSeed; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestMethodSeed; +import java.util.Optional; + +import com.google.common.annotations.VisibleForTesting; + +import static com.tngtech.valueprovider.InitializationCreator.*; +import static com.tngtech.valueprovider.ValueProviderFactory.*; import static java.lang.String.format; +import static java.util.Optional.empty; +@SuppressWarnings("OptionalUsedAsFieldOrParameterType") public class ValueProviderException extends RuntimeException { ValueProviderException() { - super(provideFailureReproductionInfo()); + this(empty()); + } + + ValueProviderException(Optional> testClassToReRunForReproduction) { + super(provideFailureReproductionInfo(testClassToReRunForReproduction)); } - static String provideFailureReproductionInfo() { + @VisibleForTesting + static String provideFailureReproductionInfo(Optional> testClassToReRunForReproduction) { long testClassSeed = getTestClassSeed(); long testMethodSeed = getTestMethodSeed(); String referenceDateTime = getFormattedReferenceDateTime(); return format( - "If the failure is related to random ValueProviders, specify the following system properties for the JVM to reproduce:%n" + + "If the failure is related to random ValueProviders, %sspecify the following system properties for the JVM to reproduce:%n" + "-D%s=%d%n" + "-D%s=%d%n" + "-D%s=%s", + formatReRunMessageFor(testClassToReRunForReproduction), VALUE_PROVIDER_FACTORY_TEST_CLASS_SEED_PROPERTY, testClassSeed, VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY, testMethodSeed, VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY, referenceDateTime); } + + private static String formatReRunMessageFor(Optional> testClassToReRunForReproduction) { + return testClassToReRunForReproduction.map(testClass -> + format("re-run all tests of '%s' and ", testClass.getName())) + .orElse(""); + } } diff --git a/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java b/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java index d5334d1..a41474e 100644 --- a/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java +++ b/core/src/test/java/com/tngtech/valueprovider/ValueProviderExceptionTest.java @@ -1,13 +1,11 @@ package com.tngtech.valueprovider; +import java.util.Optional; + import org.junit.jupiter.api.Test; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_CLASS_SEED_PROPERTY; -import static com.tngtech.valueprovider.InitializationCreator.VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY; -import static com.tngtech.valueprovider.ValueProviderFactory.getFormattedReferenceDateTime; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestClassSeed; -import static com.tngtech.valueprovider.ValueProviderFactory.getTestMethodSeed; +import static com.tngtech.valueprovider.InitializationCreator.*; +import static com.tngtech.valueprovider.ValueProviderFactory.*; import static org.assertj.core.api.Assertions.assertThat; class ValueProviderExceptionTest { @@ -26,4 +24,14 @@ void should_show_seed_values_reference_date_time_and_respective_system_propertie VALUE_PROVIDER_FACTORY_TEST_METHOD_SEED_PROPERTY, VALUE_PROVIDER_FACTORY_REFERENCE_DATE_TIME_PROPERTY); } + + @Test + void should_show_test_class_to_re_run_for_failure_reproduction_if_provided() { + Class testClassToReRun = this.getClass(); + ValueProviderException exception = new ValueProviderException(Optional.of(testClassToReRun)); + + String message = exception.getMessage(); + + assertThat(message).contains(testClassToReRun.getName()); + } } \ No newline at end of file diff --git a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java index 95a6c9c..e5a0014 100644 --- a/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java +++ b/junit5/src/main/java/com/tngtech/valueprovider/ValueProviderExtension.java @@ -13,6 +13,7 @@ import static com.tngtech.valueprovider.ValueProviderExtension.TestMethodCycleState.*; import static java.lang.System.identityHashCode; +import static java.util.Optional.empty; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; @@ -78,9 +79,13 @@ public void beforeEach(ExtensionContext context) { public void handleTestExecutionException(ExtensionContext context, Throwable throwable) throws Throwable { logger.debug("{} handleTestExecutionException {}", identityHashCode(this), buildQualifiedTestMethodName(context)); + // If the test class hierarchy of the failed test method contains any class(es) with Lifecycle PER_CLASS, + // all test methods of this hierarchy must be re-run to reproduce the failure. + // The root test class of the hierarchy must therefore be shown in the failure reproduction info. + Optional> testClassToReRunForReproduction = getRootClassInHierarchyWithLifecyclePerClass(context); // Note: handleTestExecutionException() is invoked BEFORE afterEach, i.e. BEFORE seed is reset, // so that the correct seed values appear in the failure message - throwable.addSuppressed(new ValueProviderException()); + throwable.addSuppressed(new ValueProviderException(testClassToReRunForReproduction)); throw throwable; } @@ -179,6 +184,24 @@ private static boolean isLastTestClassInHierarchyWithLifecyclePerClass(Extension return remainingLifecyclesInHierarchy.isEmpty() || containsOnlyLifecyclePerMethod(remainingLifecyclesInHierarchy); } + private static Optional> getRootClassInHierarchyWithLifecyclePerClass(ExtensionContext startContext) { + List> testClassesInHierarchyWithLifecyclePerClass = new ArrayList<>(); + traverseContextHierarchy(startContext, context -> + addTestClassAtBeginningIfLifecyclePerClass(context, testClassesInHierarchyWithLifecyclePerClass)); + if (testClassesInHierarchyWithLifecyclePerClass.isEmpty()) { + return empty(); + } + return Optional.of(testClassesInHierarchyWithLifecyclePerClass.get(0)); + } + + private static void addTestClassAtBeginningIfLifecyclePerClass(ExtensionContext context, List> addTo) { + if (!isLifecycle(context, PER_CLASS)) { + return; + } + context.getTestClass().ifPresent(testClass -> + addTo.add(0, testClass)); + } + private static boolean testClassHierarchyHasOnlyLifecyclePerMethod(ExtensionContext context) { Set lifecyclesInHierarchy = determineLifecyclesInTestClassHierarchy(Optional.of(context)); return containsOnlyLifecyclePerMethod(lifecyclesInHierarchy); diff --git a/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java b/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java index 4ea6d7d..1f8e911 100644 --- a/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java +++ b/junit5/src/test/java/com/tngtech/valueprovider/ValueProviderExtensionFailureTest.java @@ -2,25 +2,29 @@ import java.util.HashMap; import java.util.Map; +import java.util.Optional; import java.util.Set; import org.assertj.core.api.Condition; import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Nested; import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; import org.junit.jupiter.api.condition.EnabledIf; import org.junit.jupiter.api.extension.ExtendWith; import org.junit.platform.testkit.engine.EngineTestKit; import org.junit.platform.testkit.engine.Events; +import static com.tngtech.valueprovider.ValueProviderExtensionFailureTest.FailureInformation.expectRootClassOfFailingTestInReproductionInfo; +import static com.tngtech.valueprovider.ValueProviderExtensionFailureTest.FailureInformation.noTestClassInReproductionInfo; +import static java.util.Optional.empty; import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Fail.fail; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_CLASS; +import static org.junit.jupiter.api.TestInstance.Lifecycle.PER_METHOD; import static org.junit.platform.engine.discovery.DiscoverySelectors.selectClass; -import static org.junit.platform.testkit.engine.EventConditions.event; -import static org.junit.platform.testkit.engine.EventConditions.finishedWithFailure; -import static org.junit.platform.testkit.engine.EventConditions.test; -import static org.junit.platform.testkit.engine.TestExecutionResultConditions.instanceOf; -import static org.junit.platform.testkit.engine.TestExecutionResultConditions.message; -import static org.junit.platform.testkit.engine.TestExecutionResultConditions.suppressed; +import static org.junit.platform.testkit.engine.EventConditions.*; +import static org.junit.platform.testkit.engine.TestExecutionResultConditions.*; class ValueProviderExtensionFailureTest { private static final Map test2FailureInformation = new HashMap<>(); @@ -48,6 +52,23 @@ void extension_should_provide_failure_reproduction_info() { assertExpectedTestFailure(testEvents, testName)); } + @Test + void extension_should_provide_class_name_to_reproduce_failure_if_test_class_hierarchy_contains_lifecyle_per_class() { + MainFailureTest.enabled = true; + + Events testEvents = EngineTestKit.engine("junit-jupiter") + .selectors(selectClass(MainFailureTest.class)) + .execute() + .testEvents(); + + Set testNames = test2FailureInformation.keySet(); + assertThat(testNames).as("executed tests").isNotEmpty(); + testEvents.started().assertThatEvents() + .as("number of started tests").hasSameSizeAs(testNames); + testNames.forEach(testName -> + assertExpectedTestFailure(testEvents, testName)); + } + private void assertExpectedTestFailure(Events testEvents, String testName) { FailureInformation failureInformation = getFailureInformation(testName); Condition expectedException = instanceOf(AssertionError.class); @@ -57,17 +78,27 @@ private void assertExpectedTestFailure(Events testEvents, String testName) { suppressed(0, instanceOf(ValueProviderException.class)); Condition expectedFailureReproductionInfo = suppressed(0, message(x -> x.contains(failureInformation.expectedFailureReproductionInfo))); - testEvents.assertThatEvents() - .haveExactly(1, - event(test(testName), - finishedWithFailure( - expectedException, - expectedTestFailureMessage, - expectedSuppressedException, - expectedFailureReproductionInfo - ))); + Condition expectedRootClassOfFailingTestInReproductionInfo = + suppressed(0, message(x -> + { + boolean containsRootClassOfFailingTest = x.contains(failureInformation.classOfFailingTest.getName()); + boolean expectRootClassOfFailingTest = failureInformation.expectRootClassOfFailingTestInReproductionInfo; + return containsRootClassOfFailingTest == expectRootClassOfFailingTest; + })); + if (failureInformation.expectRootClassOfFailingTestInReproductionInfo) + testEvents.assertThatEvents() + .haveExactly(1, + event(test(testName), + finishedWithFailure( + expectedException, + expectedTestFailureMessage, + expectedSuppressedException, + expectedFailureReproductionInfo, + expectedRootClassOfFailingTestInReproductionInfo + ))); } + @SuppressWarnings("JUnitMalformedDeclaration") // false positive, class is executed programmatically @ExtendWith(ValueProviderExtension.class) @EnabledIf("enabled") static class FailureTest { @@ -83,34 +114,115 @@ static boolean enabled() { @Test void simply_failing() { - addFailureInformation("simply_failing", "failing intentionally"); - //noinspection ResultOfMethodCallIgnored - fail("failing intentionally"); + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("simply_failing", failureInformation); + fail(failureMessage); } @Test void failing_with_assertion_error() { - addFailureInformation("failing_with_assertion_error", "My specific assertion description"); - //noinspection ConstantConditions - assertThat(true).as("My specific assertion description").isFalse(); + String failureMessage = "My specific assertion description"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_with_assertion_error", failureInformation); + assertThat(enabled).as(failureMessage).isFalse(); } } - private static void addFailureInformation(String testName, String expectedFailureMessage) { - test2FailureInformation.put(testName, new FailureInformation(expectedFailureMessage)); + @SuppressWarnings("JUnitMalformedDeclaration") // false positive, class is executed programmatically + @ExtendWith(ValueProviderExtension.class) + @EnabledIf("enabled") + static class MainFailureTest { + /** + * To control test execution and esp. avoid running the test as part of the gradle/CI-build. + */ + private static boolean enabled = false; + + @SuppressWarnings("unused") // used by JUnit @EnabledIf + static boolean enabled() { + return enabled; + } + + @Test + void failing_in_main() { + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_in_main", failureInformation); + fail(failureMessage); + } + + @Nested + @TestInstance(PER_CLASS) + class NestedFailureTestLifecyclePerClass { + + @Test + void failing_in_nested_with_lifecycle_per_class() { + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = expectRootClassOfFailingTestInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_in_nested_with_lifecycle_per_class", failureInformation); + fail(failureMessage); + } + + @Nested + @TestInstance(PER_METHOD) + class NestedFailureTestChildOfLifecyclePerClass { + + @Test + void failing_in_child_with_lifecycle_per_method_of_parent_with_lifecycle_per_class() { + String failureMessage = "failing intentionally"; + Class parentWithLifecyclePerClass = NestedFailureTestLifecyclePerClass.class; + FailureInformation failureInformation = expectRootClassOfFailingTestInReproductionInfo(failureMessage, parentWithLifecyclePerClass); + addFailureInformation("failing_in_child_with_lifecycle_per_method_of_parent_with_lifecycle_per_class", + failureInformation); + fail(failureMessage); + } + } + } + + @Nested + class NestedFailureTestLifecyclePerMethod { + + @Test + void failing_in_nested_with_lifecycle_per_method() { + String failureMessage = "failing intentionally"; + FailureInformation failureInformation = noTestClassInReproductionInfo(failureMessage, this.getClass()); + addFailureInformation("failing_in_nested_with_lifecycle_per_method", failureInformation); + fail(failureMessage); + } + } + } + + private static void addFailureInformation(String testName, FailureInformation failureInformation) { + test2FailureInformation.put(testName, failureInformation); } private static FailureInformation getFailureInformation(String testName) { return test2FailureInformation.get(testName); } - private static class FailureInformation { + static class FailureInformation { private final String expectedTestFailureMessage; private final String expectedFailureReproductionInfo; + private final Class classOfFailingTest; + private final boolean expectRootClassOfFailingTestInReproductionInfo; + + static FailureInformation noTestClassInReproductionInfo(String expectedTestFailureMessage, Class rootClassOfFailingTest) { + return new FailureInformation(expectedTestFailureMessage, rootClassOfFailingTest, false); + } + + static FailureInformation expectRootClassOfFailingTestInReproductionInfo(String expectedTestFailureMessage, Class rootClassOfFailingTest) { + return new FailureInformation(expectedTestFailureMessage, rootClassOfFailingTest, true); + } - private FailureInformation(String expectedTestFailureMessage) { + private FailureInformation(String expectedTestFailureMessage, + Class rootClassOfFailingTest, boolean expectRootClassOfFailingTestInReproductionInfo) { this.expectedTestFailureMessage = expectedTestFailureMessage; - this.expectedFailureReproductionInfo = ValueProviderException.provideFailureReproductionInfo(); + Optional> testClassToReRunForReproduction = expectRootClassOfFailingTestInReproductionInfo + ? Optional.of(rootClassOfFailingTest) + : empty(); + this.expectedFailureReproductionInfo = ValueProviderException.provideFailureReproductionInfo(testClassToReRunForReproduction); + this.classOfFailingTest = rootClassOfFailingTest; + this.expectRootClassOfFailingTestInReproductionInfo = expectRootClassOfFailingTestInReproductionInfo; } } }