diff --git a/src/main/java/com/xceptance/neodymium/module/order/DefaultStatementRunOrder.java b/src/main/java/com/xceptance/neodymium/module/order/DefaultStatementRunOrder.java index e4200f20e..79290fc30 100644 --- a/src/main/java/com/xceptance/neodymium/module/order/DefaultStatementRunOrder.java +++ b/src/main/java/com/xceptance/neodymium/module/order/DefaultStatementRunOrder.java @@ -2,6 +2,7 @@ import com.xceptance.neodymium.module.statement.browser.BrowserStatement; import com.xceptance.neodymium.module.statement.parameter.ParameterStatement; +import com.xceptance.neodymium.module.statement.repeat.RepeatStatement; import com.xceptance.neodymium.module.statement.testdata.TestdataStatement; public class DefaultStatementRunOrder extends StatementRunOrder @@ -11,5 +12,6 @@ public DefaultStatementRunOrder() runOrder.add(BrowserStatement.class); runOrder.add(ParameterStatement.class); runOrder.add(TestdataStatement.class); + runOrder.add(RepeatStatement.class); } } diff --git a/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailure.java b/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailure.java new file mode 100644 index 000000000..0ba82396b --- /dev/null +++ b/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailure.java @@ -0,0 +1,18 @@ +package com.xceptance.neodymium.module.statement.repeat; + +import static java.lang.annotation.ElementType.METHOD; +import static java.lang.annotation.ElementType.TYPE; +import static java.lang.annotation.RetentionPolicy.RUNTIME; + +import java.lang.annotation.Retention; +import java.lang.annotation.Target; + +@Retention(RUNTIME) +@Target( +{ + TYPE, METHOD +}) +public @interface RepeatOnFailure +{ + int value() default 1; +} diff --git a/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailureData.java b/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailureData.java new file mode 100644 index 000000000..ebac7a255 --- /dev/null +++ b/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailureData.java @@ -0,0 +1,24 @@ +package com.xceptance.neodymium.module.statement.repeat; + +public class RepeatOnFailureData +{ + private int iterationNumber; + + private int maxNumber; + + public RepeatOnFailureData(int iterationNumber, int maxNumber) + { + this.iterationNumber = iterationNumber; + this.maxNumber = maxNumber; + } + + public int getIterationNumber() + { + return iterationNumber; + } + + public int getMaxNumber() + { + return maxNumber; + } +} diff --git a/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatStatement.java b/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatStatement.java new file mode 100644 index 000000000..c688a37a7 --- /dev/null +++ b/src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatStatement.java @@ -0,0 +1,109 @@ +package com.xceptance.neodymium.module.statement.repeat; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.WeakHashMap; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + +import org.junit.Assume; +import org.junit.runners.model.FrameworkMethod; +import org.junit.runners.model.Statement; +import org.junit.runners.model.TestClass; + +import com.xceptance.neodymium.module.StatementBuilder; + +public class RepeatStatement extends StatementBuilder +{ + private static final Map> CONTEXTS = Collections.synchronizedMap(new WeakHashMap<>()); + + private Statement next; + + private int iterationIndex; + + private int maxRepetitionNumber; + + public RepeatStatement(Statement next, RepeatOnFailureData data) + { + this.next = next; + this.iterationIndex = data.getIterationNumber(); + this.maxRepetitionNumber = data.getMaxNumber(); + } + + public RepeatStatement() + { + } + + private static List getContext() + { + return CONTEXTS.computeIfAbsent(Thread.currentThread(), key -> { + return new ArrayList<>(); + }); + } + + @Override + public List createIterationData(TestClass testClass, FrameworkMethod method) throws Throwable + { + RepeatOnFailure classRepeatOnFailure = testClass.getAnnotation(RepeatOnFailure.class); + List methodRepeatOnFailures = getAnnotations(method.getMethod(), RepeatOnFailure.class); + int numberOfRepetitions = 0; + if (classRepeatOnFailure != null) + { + numberOfRepetitions = classRepeatOnFailure.value(); + + } + if (!methodRepeatOnFailures.isEmpty()) + { + numberOfRepetitions = methodRepeatOnFailures.get(0).value(); + } + final int maxRepetitionNumber = numberOfRepetitions; + return IntStream.rangeClosed(1, numberOfRepetitions) + .boxed().map(x -> new RepeatOnFailureData(x, maxRepetitionNumber)).collect(Collectors.toList()); + } + + @Override + public StatementBuilder createStatement(Object testClassInstance, Statement next, Object parameter) + { + return new RepeatStatement(next, (RepeatOnFailureData) parameter); + } + + @Override + public String getTestName(Object data) + { + RepeatOnFailureData d = (RepeatOnFailureData) data; + return "Run number " + d.getIterationNumber(); + } + + @Override + public String getCategoryName(Object data) + { + return "Repeat on failure test"; + } + + @Override + public void evaluate() throws Throwable + { + getContext().add(true); + boolean toExecute = getContext().size() < 2 || + !getContext().get(iterationIndex - 2); + if (iterationIndex == maxRepetitionNumber) + { + CONTEXTS.remove(Thread.currentThread()); + } + Assume.assumeTrue("Original test succeeded", toExecute); + try + { + next.evaluate(); + } + catch (Throwable t) + { + if (iterationIndex != maxRepetitionNumber) + { + getContext().set(iterationIndex - 1, false); + } + throw t; + } + } +} diff --git a/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureBrowserCombinationTest.java b/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureBrowserCombinationTest.java new file mode 100644 index 000000000..911724bd4 --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureBrowserCombinationTest.java @@ -0,0 +1,31 @@ +package com.xceptance.neodymium.testclasses.repeat.classlevel; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.xceptance.neodymium.NeodymiumRunner; +import com.xceptance.neodymium.module.statement.browser.multibrowser.Browser; +import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure; + +@RepeatOnFailure(10) +@Browser("Chrome_headless") +@Browser("Chrome_1500x1000_headless") +@RunWith(NeodymiumRunner.class) +public class ClassRepeatOnFailureBrowserCombinationTest +{ + public static AtomicInteger val = new AtomicInteger(0); + + @Test + public void testVisitingHomepage() + { + int i = val.getAndIncrement(); + if (val.get() == 6) + { + val.set(0); + } + Assert.assertFalse("Produce test failure number " + i, i < 5); + } +} diff --git a/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTest.java b/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTest.java new file mode 100644 index 000000000..415002eec --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTest.java @@ -0,0 +1,28 @@ +package com.xceptance.neodymium.testclasses.repeat.classlevel; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.xceptance.neodymium.NeodymiumRunner; +import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure; + +@RepeatOnFailure(10) +@RunWith(NeodymiumRunner.class) +public class ClassRepeatOnFailureTest +{ + public static AtomicInteger val = new AtomicInteger(0); + + @Test + public void testVisitingHomepage() + { + int i = val.getAndIncrement(); + if (val.get() == 6) + { + val.set(0); + } + Assert.assertFalse("Produce test failure number " + i, i < 5); + } +} diff --git a/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTestdataCombinationTest.java b/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTestdataCombinationTest.java new file mode 100644 index 000000000..4db0788ad --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTestdataCombinationTest.java @@ -0,0 +1,28 @@ +package com.xceptance.neodymium.testclasses.repeat.classlevel; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.xceptance.neodymium.NeodymiumRunner; +import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure; + +@RepeatOnFailure(10) +@RunWith(NeodymiumRunner.class) +public class ClassRepeatOnFailureTestdataCombinationTest +{ + public static AtomicInteger val = new AtomicInteger(0); + + @Test + public void testVisitingHomepage() + { + int i = val.getAndIncrement(); + if (val.get() == 6) + { + val.set(0); + } + Assert.assertFalse("Produce test failure number " + i, i < 5); + } +} diff --git a/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureBrowserCombinationTest.java b/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureBrowserCombinationTest.java new file mode 100644 index 000000000..83eed16ff --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureBrowserCombinationTest.java @@ -0,0 +1,31 @@ +package com.xceptance.neodymium.testclasses.repeat.methodlevel; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.xceptance.neodymium.NeodymiumRunner; +import com.xceptance.neodymium.module.statement.browser.multibrowser.Browser; +import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure; + +@Browser("Chrome_headless") +@Browser("Chrome_1500x1000_headless") +@RunWith(NeodymiumRunner.class) +public class MethodRepeatOnFailureBrowserCombinationTest +{ + public static AtomicInteger val = new AtomicInteger(0); + + @Test + @RepeatOnFailure(10) + public void testVisitingHomepage() + { + int i = val.getAndIncrement(); + if (val.get() == 6) + { + val.set(0); + } + Assert.assertFalse("Produce test failure number " + i, i < 5); + } +} diff --git a/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTest.java b/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTest.java new file mode 100644 index 000000000..d10d9e3e4 --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTest.java @@ -0,0 +1,29 @@ +package com.xceptance.neodymium.testclasses.repeat.methodlevel; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.xceptance.neodymium.NeodymiumRunner; +import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure; + +@RepeatOnFailure(3) +@RunWith(NeodymiumRunner.class) +public class MethodRepeatOnFailureTest +{ + public static AtomicInteger val = new AtomicInteger(0); + + @Test + @RepeatOnFailure(10) + public void testVisitingHomepage() + { + int i = val.getAndIncrement(); + if (val.get() == 6) + { + val.set(0); + } + Assert.assertFalse("Produce test failure number " + i, i < 5); + } +} diff --git a/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTestdataCombinationTest.java b/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTestdataCombinationTest.java new file mode 100644 index 000000000..2b74e913a --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTestdataCombinationTest.java @@ -0,0 +1,28 @@ +package com.xceptance.neodymium.testclasses.repeat.methodlevel; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.xceptance.neodymium.NeodymiumRunner; +import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure; + +@RunWith(NeodymiumRunner.class) +public class MethodRepeatOnFailureTestdataCombinationTest +{ + public static AtomicInteger val = new AtomicInteger(0); + + @Test + @RepeatOnFailure(10) + public void testVisitingHomepage() + { + int i = val.getAndIncrement(); + if (val.get() == 6) + { + val.set(0); + } + Assert.assertFalse("Produce test failure number " + i, i < 5); + } +} diff --git a/src/test/java/com/xceptance/neodymium/testclasses/repeat/mix/OverridingRepeatOnFailureTest.java b/src/test/java/com/xceptance/neodymium/testclasses/repeat/mix/OverridingRepeatOnFailureTest.java new file mode 100644 index 000000000..8168eb276 --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/testclasses/repeat/mix/OverridingRepeatOnFailureTest.java @@ -0,0 +1,28 @@ +package com.xceptance.neodymium.testclasses.repeat.mix; + +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.Assert; +import org.junit.Test; +import org.junit.runner.RunWith; + +import com.xceptance.neodymium.NeodymiumRunner; +import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure; + +@RunWith(NeodymiumRunner.class) +public class OverridingRepeatOnFailureTest +{ + public static AtomicInteger val = new AtomicInteger(0); + + @Test + @RepeatOnFailure(10) + public void testVisitingHomepage() + { + int i = val.getAndIncrement(); + if (val.get() == 6) + { + val.set(0); + } + Assert.assertFalse("Produce test failure number " + i, i < 5); + } +} diff --git a/src/test/java/com/xceptance/neodymium/tests/NeodymiumTest.java b/src/test/java/com/xceptance/neodymium/tests/NeodymiumTest.java index f09faf8ce..61cd8cae3 100644 --- a/src/test/java/com/xceptance/neodymium/tests/NeodymiumTest.java +++ b/src/test/java/com/xceptance/neodymium/tests/NeodymiumTest.java @@ -106,6 +106,13 @@ public void check(final Result result, final boolean expectSuccessful, final int } } + public void checkAssumptionFailure(final Result result, final boolean expectSuccessful, final int expectedRunCount, final int expectedIgnoreCount, + final int expectedFailCount, final int assumtionFailureCount, final Map expectedFailureMessages) + { + check(result, expectSuccessful, expectedRunCount, expectedIgnoreCount, expectedFailCount, expectedFailureMessages); + Assert.assertEquals("Method assumption failure count", assumtionFailureCount, result.getAssumptionFailureCount()); + } + /** * Assert that all tests have passed. * diff --git a/src/test/java/com/xceptance/neodymium/tests/RepeatOnFailureAnnotationTest.java b/src/test/java/com/xceptance/neodymium/tests/RepeatOnFailureAnnotationTest.java new file mode 100644 index 000000000..942750080 --- /dev/null +++ b/src/test/java/com/xceptance/neodymium/tests/RepeatOnFailureAnnotationTest.java @@ -0,0 +1,65 @@ +package com.xceptance.neodymium.tests; + +import org.junit.Test; +import org.junit.runner.JUnitCore; +import org.junit.runner.Result; + +import com.xceptance.neodymium.testclasses.repeat.classlevel.ClassRepeatOnFailureBrowserCombinationTest; +import com.xceptance.neodymium.testclasses.repeat.classlevel.ClassRepeatOnFailureTest; +import com.xceptance.neodymium.testclasses.repeat.classlevel.ClassRepeatOnFailureTestdataCombinationTest; +import com.xceptance.neodymium.testclasses.repeat.methodlevel.MethodRepeatOnFailureBrowserCombinationTest; +import com.xceptance.neodymium.testclasses.repeat.methodlevel.MethodRepeatOnFailureTest; +import com.xceptance.neodymium.testclasses.repeat.methodlevel.MethodRepeatOnFailureTestdataCombinationTest; +import com.xceptance.neodymium.testclasses.repeat.mix.OverridingRepeatOnFailureTest; + +public class RepeatOnFailureAnnotationTest extends NeodymiumTest +{ + @Test + public void testClassRepeatOnFailure() + { + Result result = JUnitCore.runClasses(ClassRepeatOnFailureTest.class); + checkAssumptionFailure(result, false, 10, 0, 5, 4, null); + } + + @Test + public void testClassRepeatOnFailureBrowserCombination() + { + Result result = JUnitCore.runClasses(ClassRepeatOnFailureBrowserCombinationTest.class); + checkAssumptionFailure(result, false, 20, 0, 10, 8, null); + } + + @Test + public void testClassRepeatOnFailureTestdataCombination() + { + Result result = JUnitCore.runClasses(ClassRepeatOnFailureTestdataCombinationTest.class); + checkAssumptionFailure(result, false, 30, 0, 15, 12, null); + } + + @Test + public void testMethodRepeatOnFailure() + { + Result result = JUnitCore.runClasses(MethodRepeatOnFailureTest.class); + checkAssumptionFailure(result, false, 10, 0, 5, 4, null); + } + + @Test + public void testMethodRepeatOnFailureBrowserCombination() + { + Result result = JUnitCore.runClasses(MethodRepeatOnFailureBrowserCombinationTest.class); + checkAssumptionFailure(result, false, 20, 0, 10, 8, null); + } + + @Test + public void testMethodRepeatOnFailureTestdataCombination() + { + Result result = JUnitCore.runClasses(MethodRepeatOnFailureTestdataCombinationTest.class); + checkAssumptionFailure(result, false, 30, 0, 15, 12, null); + } + + @Test + public void testOverridingRepeatOnFailure() + { + Result result = JUnitCore.runClasses(OverridingRepeatOnFailureTest.class); + checkAssumptionFailure(result, false, 10, 0, 5, 4, null); + } +} diff --git a/src/test/resources/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTestdataCombinationTest.json b/src/test/resources/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTestdataCombinationTest.json new file mode 100644 index 000000000..b4dfb0a6b --- /dev/null +++ b/src/test/resources/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTestdataCombinationTest.json @@ -0,0 +1,11 @@ +[ + { + "testId": "data set 1" + }, + { + "testId": "data set 2" + }, + { + "testId": "data set 3" + } +] diff --git a/src/test/resources/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTestdataCombinationTest.json b/src/test/resources/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTestdataCombinationTest.json new file mode 100644 index 000000000..b4dfb0a6b --- /dev/null +++ b/src/test/resources/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTestdataCombinationTest.json @@ -0,0 +1,11 @@ +[ + { + "testId": "data set 1" + }, + { + "testId": "data set 2" + }, + { + "testId": "data set 3" + } +]