Repeat on failure #53

src/main/java/com/xceptance/neodymium/module/order/DefaultStatementRunOrder.java

@@ -2,6 +2,7 @@
 
 import com.xceptance.neodymium.module.statement.browser.BrowserStatement;
 import com.xceptance.neodymium.module.statement.parameter.ParameterStatement;
+import com.xceptance.neodymium.module.statement.repeat.RepeatStatement;
 import com.xceptance.neodymium.module.statement.testdata.TestdataStatement;
 
 public class DefaultStatementRunOrder extends StatementRunOrder
@@ -11,5 +12,6 @@ public DefaultStatementRunOrder()
         runOrder.add(BrowserStatement.class);
         runOrder.add(ParameterStatement.class);
         runOrder.add(TestdataStatement.class);
+        runOrder.add(RepeatStatement.class);
     }
 }

src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailure.java

@@ -0,0 +1,18 @@
+package com.xceptance.neodymium.module.statement.repeat;
+
+import static java.lang.annotation.ElementType.METHOD;
+import static java.lang.annotation.ElementType.TYPE;
+import static java.lang.annotation.RetentionPolicy.RUNTIME;
+
+import java.lang.annotation.Retention;
+import java.lang.annotation.Target;
+
+@Retention(RUNTIME)
+@Target(
+{
+  TYPE, METHOD
+})
+public @interface RepeatOnFailure
+{
+    int value() default 1;
+}

src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatOnFailureData.java

@@ -0,0 +1,24 @@
+package com.xceptance.neodymium.module.statement.repeat;
+
+public class RepeatOnFailureData
+{
+    private int iterationNumber;
+
+    private int maxNumber;
+
+    public RepeatOnFailureData(int iterationNumber, int maxNumber)
+    {
+        this.iterationNumber = iterationNumber;
+        this.maxNumber = maxNumber;
+    }
+
+    public int getIterationNumber()
+    {
+        return iterationNumber;
+    }
+
+    public int getMaxNumber()
+    {
+        return maxNumber;
+    }
+}

src/main/java/com/xceptance/neodymium/module/statement/repeat/RepeatStatement.java

@@ -0,0 +1,109 @@
+package com.xceptance.neodymium.module.statement.repeat;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.WeakHashMap;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
+
+import org.junit.Assume;
+import org.junit.runners.model.FrameworkMethod;
+import org.junit.runners.model.Statement;
+import org.junit.runners.model.TestClass;
+
+import com.xceptance.neodymium.module.StatementBuilder;
+
+public class RepeatStatement extends StatementBuilder
+{
+    private static final Map<Thread, List<Boolean>> CONTEXTS = Collections.synchronizedMap(new WeakHashMap<>());
+
+    private Statement next;
+
+    private int iterationIndex;
+
+    private int maxRepetitionNumber;
+
+    public RepeatStatement(Statement next, RepeatOnFailureData data)
+    {
+        this.next = next;
+        this.iterationIndex = data.getIterationNumber();
+        this.maxRepetitionNumber = data.getMaxNumber();
+    }
+
+    public RepeatStatement()
+    {
+    }
+
+    private static List<Boolean> getContext()
+    {
+        return CONTEXTS.computeIfAbsent(Thread.currentThread(), key -> {
+            return new ArrayList<>();
+        });
+    }
+
+    @Override
+    public List<Object> createIterationData(TestClass testClass, FrameworkMethod method) throws Throwable
+    {
+        RepeatOnFailure classRepeatOnFailure = testClass.getAnnotation(RepeatOnFailure.class);
+        List<RepeatOnFailure> methodRepeatOnFailures = getAnnotations(method.getMethod(), RepeatOnFailure.class);
+        int numberOfRepetitions = 0;
+        if (classRepeatOnFailure != null)
+        {
+            numberOfRepetitions = classRepeatOnFailure.value();
+
+        }
+        if (!methodRepeatOnFailures.isEmpty())
+        {
+            numberOfRepetitions = methodRepeatOnFailures.get(0).value();
+        }
+        final int maxRepetitionNumber = numberOfRepetitions;
+        return IntStream.rangeClosed(1, numberOfRepetitions)
+                        .boxed().map(x -> new RepeatOnFailureData(x, maxRepetitionNumber)).collect(Collectors.toList());
+    }
+
+    @Override
+    public StatementBuilder createStatement(Object testClassInstance, Statement next, Object parameter)
+    {
+        return new RepeatStatement(next, (RepeatOnFailureData) parameter);
+    }
+
+    @Override
+    public String getTestName(Object data)
+    {
+        RepeatOnFailureData d = (RepeatOnFailureData) data;
+        return "Run number " + d.getIterationNumber();
+    }
+
+    @Override
+    public String getCategoryName(Object data)
+    {
+        return "Repeat on failure test";
+    }
+
+    @Override
+    public void evaluate() throws Throwable
+    {
+        getContext().add(true);
+        boolean toExecute = getContext().size() < 2 ||
+                            !getContext().get(iterationIndex - 2);
+        if (iterationIndex == maxRepetitionNumber)
+        {
+            CONTEXTS.remove(Thread.currentThread());
+        }
+        Assume.assumeTrue("Original test succeeded", toExecute);
+        try
+        {
+            next.evaluate();
+        }
+        catch (Throwable t)
+        {
+            if (iterationIndex != maxRepetitionNumber)
+            {
+                getContext().set(iterationIndex - 1, false);
+            }
+            throw t;
+        }
+    }
+}

src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureBrowserCombinationTest.java

@@ -0,0 +1,31 @@
+package com.xceptance.neodymium.testclasses.repeat.classlevel;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.xceptance.neodymium.NeodymiumRunner;
+import com.xceptance.neodymium.module.statement.browser.multibrowser.Browser;
+import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure;
+
+@RepeatOnFailure(10)
+@Browser("Chrome_headless")
+@Browser("Chrome_1500x1000_headless")
+@RunWith(NeodymiumRunner.class)
+public class ClassRepeatOnFailureBrowserCombinationTest
+{
+    public static AtomicInteger val = new AtomicInteger(0);
+
+    @Test
+    public void testVisitingHomepage()
+    {
+        int i = val.getAndIncrement();
+        if (val.get() == 6)
+        {
+            val.set(0);
+        }
+        Assert.assertFalse("Produce test failure number " + i, i < 5);
+    }
+}

src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTest.java

@@ -0,0 +1,28 @@
+package com.xceptance.neodymium.testclasses.repeat.classlevel;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.xceptance.neodymium.NeodymiumRunner;
+import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure;
+
+@RepeatOnFailure(10)
+@RunWith(NeodymiumRunner.class)
+public class ClassRepeatOnFailureTest
+{
+    public static AtomicInteger val = new AtomicInteger(0);
+
+    @Test
+    public void testVisitingHomepage()
+    {
+        int i = val.getAndIncrement();
+        if (val.get() == 6)
+        {
+            val.set(0);
+        }
+        Assert.assertFalse("Produce test failure number " + i, i < 5);
+    }
+}

src/test/java/com/xceptance/neodymium/testclasses/repeat/classlevel/ClassRepeatOnFailureTestdataCombinationTest.java

@@ -0,0 +1,28 @@
+package com.xceptance.neodymium.testclasses.repeat.classlevel;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.xceptance.neodymium.NeodymiumRunner;
+import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure;
+
+@RepeatOnFailure(10)
+@RunWith(NeodymiumRunner.class)
+public class ClassRepeatOnFailureTestdataCombinationTest
+{
+    public static AtomicInteger val = new AtomicInteger(0);
+
+    @Test
+    public void testVisitingHomepage()
+    {
+        int i = val.getAndIncrement();
+        if (val.get() == 6)
+        {
+            val.set(0);
+        }
+        Assert.assertFalse("Produce test failure number " + i, i < 5);
+    }
+}

src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureBrowserCombinationTest.java

@@ -0,0 +1,31 @@
+package com.xceptance.neodymium.testclasses.repeat.methodlevel;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.xceptance.neodymium.NeodymiumRunner;
+import com.xceptance.neodymium.module.statement.browser.multibrowser.Browser;
+import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure;
+
+@Browser("Chrome_headless")
+@Browser("Chrome_1500x1000_headless")
+@RunWith(NeodymiumRunner.class)
+public class MethodRepeatOnFailureBrowserCombinationTest
+{
+    public static AtomicInteger val = new AtomicInteger(0);
+
+    @Test
+    @RepeatOnFailure(10)
+    public void testVisitingHomepage()
+    {
+        int i = val.getAndIncrement();
+        if (val.get() == 6)
+        {
+            val.set(0);
+        }
+        Assert.assertFalse("Produce test failure number " + i, i < 5);
+    }
+}

src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTest.java

@@ -0,0 +1,29 @@
+package com.xceptance.neodymium.testclasses.repeat.methodlevel;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.xceptance.neodymium.NeodymiumRunner;
+import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure;
+
+@RepeatOnFailure(3)
+@RunWith(NeodymiumRunner.class)
+public class MethodRepeatOnFailureTest
+{
+    public static AtomicInteger val = new AtomicInteger(0);
+
+    @Test
+    @RepeatOnFailure(10)
+    public void testVisitingHomepage()
+    {
+        int i = val.getAndIncrement();
+        if (val.get() == 6)
+        {
+            val.set(0);
+        }
+        Assert.assertFalse("Produce test failure number " + i, i < 5);
+    }
+}

src/test/java/com/xceptance/neodymium/testclasses/repeat/methodlevel/MethodRepeatOnFailureTestdataCombinationTest.java

@@ -0,0 +1,28 @@
+package com.xceptance.neodymium.testclasses.repeat.methodlevel;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.xceptance.neodymium.NeodymiumRunner;
+import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure;
+
+@RunWith(NeodymiumRunner.class)
+public class MethodRepeatOnFailureTestdataCombinationTest
+{
+    public static AtomicInteger val = new AtomicInteger(0);
+
+    @Test
+    @RepeatOnFailure(10)
+    public void testVisitingHomepage()
+    {
+        int i = val.getAndIncrement();
+        if (val.get() == 6)
+        {
+            val.set(0);
+        }
+        Assert.assertFalse("Produce test failure number " + i, i < 5);
+    }
+}

src/test/java/com/xceptance/neodymium/testclasses/repeat/mix/OverridingRepeatOnFailureTest.java

@@ -0,0 +1,28 @@
+package com.xceptance.neodymium.testclasses.repeat.mix;
+
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+
+import com.xceptance.neodymium.NeodymiumRunner;
+import com.xceptance.neodymium.module.statement.repeat.RepeatOnFailure;
+
+@RunWith(NeodymiumRunner.class)
+public class OverridingRepeatOnFailureTest
+{
+    public static AtomicInteger val = new AtomicInteger(0);
+
+    @Test
+    @RepeatOnFailure(10)
+    public void testVisitingHomepage()
+    {
+        int i = val.getAndIncrement();
+        if (val.get() == 6)
+        {
+            val.set(0);
+        }
+        Assert.assertFalse("Produce test failure number " + i, i < 5);
+    }
+}

src/test/java/com/xceptance/neodymium/tests/NeodymiumTest.java

@@ -106,6 +106,13 @@ public void check(final Result result, final boolean expectSuccessful, final int
         }
     }
 
+    public void checkAssumptionFailure(final Result result, final boolean expectSuccessful, final int expectedRunCount, final int expectedIgnoreCount,
+                                       final int expectedFailCount, final int assumtionFailureCount, final Map<String, String> expectedFailureMessages)
+    {
+        check(result, expectSuccessful, expectedRunCount, expectedIgnoreCount, expectedFailCount, expectedFailureMessages);
+        Assert.assertEquals("Method assumption failure count", assumtionFailureCount, result.getAssumptionFailureCount());
+    }
+
     /**
      * Assert that all tests have passed.
      *