The RerunFormatter does not handle failures in background and scenario outline examples correctly

core/src/main/java/cucumber/runtime/formatter/RerunFormatter.java

@@ -24,16 +24,12 @@
  * Failed means: (failed, undefined, pending) test result
  */
 class RerunFormatter implements Formatter, Reporter {
-
     private final NiceAppendable out;
-
     private String featureLocation;
-
-    private Step step;
-
+    private Scenario scenario;
+    private boolean isTestFailed = false;
     private Map<String, LinkedHashSet<Integer>> featureAndFailedLinesMapping = new HashMap<String, LinkedHashSet<Integer>>();
 
-
     public RerunFormatter(Appendable out) {
         this.out = new NiceAppendable(out);
     }
@@ -53,6 +49,7 @@ public void background(Background background) {
 
     @Override
     public void scenario(Scenario scenario) {
+        this.scenario = scenario;
     }
 
     @Override
@@ -65,7 +62,6 @@ public void examples(Examples examples) {
 
     @Override
     public void step(Step step) {
-        this.step = step;
     }
 
     @Override
@@ -78,10 +74,10 @@ public void syntaxError(String state, String event, List<String> legalEvents, St
 
     @Override
     public void done() {
-        reportFailedSteps();
+        reportFailedScenarios();
     }
 
-    private void reportFailedSteps() {
+    private void reportFailedScenarios() {
         Set<Map.Entry<String, LinkedHashSet<Integer>>> entries = featureAndFailedLinesMapping.entrySet();
         boolean firstFeature = true;
         for (Map.Entry<String, LinkedHashSet<Integer>> entry : entries) {
@@ -105,23 +101,27 @@ public void close() {
 
     @Override
     public void startOfScenarioLifeCycle(Scenario scenario) {
-        // NoOp
+        isTestFailed = false;
     }
 
     @Override
     public void endOfScenarioLifeCycle(Scenario scenario) {
-        // NoOp
+        if (isTestFailed) {
+            recordTestFailed();
+        }
     }
 
     @Override
     public void before(Match match, Result result) {
-
+        if (isTestFailed(result)) {
+            isTestFailed = true;
+        }
     }
 
     @Override
     public void result(Result result) {
         if (isTestFailed(result)) {
-            recordTestFailed();
+            isTestFailed = true;
         }
     }
 
@@ -131,17 +131,20 @@ private boolean isTestFailed(Result result) {
     }
 
     private void recordTestFailed() {
-        LinkedHashSet<Integer> failedSteps = this.featureAndFailedLinesMapping.get(featureLocation);
-        if (failedSteps == null) {
-            failedSteps = new LinkedHashSet<Integer>();
-            this.featureAndFailedLinesMapping.put(featureLocation, failedSteps);
+        LinkedHashSet<Integer> failedScenarios = this.featureAndFailedLinesMapping.get(featureLocation);
+        if (failedScenarios == null) {
+            failedScenarios = new LinkedHashSet<Integer>();
+            this.featureAndFailedLinesMapping.put(featureLocation, failedScenarios);
         }
 
-        failedSteps.add(step.getLine());
+        failedScenarios.add(scenario.getLine());
     }
 
     @Override
     public void after(Match match, Result result) {
+        if (isTestFailed(result)) {
+            isTestFailed = true;
+        }
     }
 
     @Override

core/src/test/java/cucumber/runtime/TestHelper.java

@@ -1,14 +1,46 @@
 package cucumber.runtime;
 
+import cucumber.runtime.Env;
+import cucumber.runtime.io.ClasspathResourceLoader;
+import gherkin.formatter.Formatter;
+import gherkin.formatter.Reporter;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
+import cucumber.runtime.formatter.StepMatcher;
+import gherkin.formatter.model.Step;
+import gherkin.formatter.model.Tag;
+import cucumber.api.PendingException;
+import gherkin.I18n;
+import junit.framework.AssertionFailedError;
 import cucumber.runtime.io.Resource;
 import cucumber.runtime.model.CucumberFeature;
 import org.junit.Ignore;
 
 import java.io.ByteArrayInputStream;
+import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.PrintWriter;
 import java.io.UnsupportedEncodingException;
+import java.util.AbstractMap.SimpleEntry;
 import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static java.util.Arrays.asList;
+import static org.mockito.Mockito.doAnswer;
+import static org.mockito.Matchers.argThat;
+import static org.mockito.Matchers.anyCollectionOf;
+import static org.junit.Assert.fail;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.doThrow;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.mock;
 
 @Ignore
 public class TestHelper {
@@ -37,4 +69,138 @@ public String getClassName() {
         }, new ArrayList<Object>());
         return cucumberFeatures.get(0);
     }
+
+    public static void runFeatureWithFormatter(final CucumberFeature feature, final Map<String, String> stepsToResult, final List<SimpleEntry<String, String>> hooks,
+            final long stepHookDuration, final Formatter formatter, final Reporter reporter) throws Throwable, FileNotFoundException {
+        runFeaturesWithFormatter(Arrays.asList(feature), stepsToResult, Collections.<String,String>emptyMap(), hooks, stepHookDuration, formatter, reporter);
+    }
+
+    public static void runFeaturesWithFormatter(final List<CucumberFeature> features, final Map<String, String> stepsToResult,
+            final List<SimpleEntry<String, String>> hooks, final long stepHookDuration, final Formatter formatter, final Reporter reporter) throws Throwable {
+        runFeaturesWithFormatter(features, stepsToResult, Collections.<String,String>emptyMap(), hooks, stepHookDuration, formatter, reporter);
+    }
+
+    public static void runFeatureWithFormatter(final CucumberFeature feature, final Map<String, String> stepsToLocation,
+            final Formatter formatter, final Reporter reporter) throws Throwable {
+        runFeaturesWithFormatter(Arrays.asList(feature), Collections.<String,String>emptyMap(), stepsToLocation,
+                Collections.<SimpleEntry<String, String>>emptyList(), 0L, formatter, reporter);
+    }
+
+    private static void runFeaturesWithFormatter(final List<CucumberFeature> features, final Map<String, String> stepsToResult, final Map<String, String> stepsToLocation,
+            final List<SimpleEntry<String, String>> hooks, final long stepHookDuration, final Formatter formatter, final Reporter reporter) throws Throwable {
+        final RuntimeOptions runtimeOptions = new RuntimeOptions(new Env());
+        final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
+        final ClasspathResourceLoader resourceLoader = new ClasspathResourceLoader(classLoader);
+        final RuntimeGlue glue = createMockedRuntimeGlueThatMatchesTheSteps(stepsToResult, stepsToLocation, hooks);
+        final Runtime runtime = new Runtime(resourceLoader, classLoader, asList(mock(Backend.class)), runtimeOptions, new StopWatch.Stub(stepHookDuration), glue);
+
+        for (CucumberFeature feature : features) {
+            feature.run(formatter, reporter, runtime);
+        }
+        formatter.done();
+        formatter.close();
+    }
+
+    private static RuntimeGlue createMockedRuntimeGlueThatMatchesTheSteps(Map<String, String> stepsToResult, Map<String, String> stepsToLocation,
+            final List<SimpleEntry<String, String>> hooks) throws Throwable {
+        RuntimeGlue glue = mock(RuntimeGlue.class);
+        TestHelper.mockSteps(glue, stepsToResult, stepsToLocation);
+        TestHelper.mockHooks(glue, hooks);
+        return glue;
+    }
+
+    private static void mockSteps(RuntimeGlue glue, Map<String, String> stepsToResult, Map<String, String> stepsToLocation) throws Throwable {
+        for (String stepName : mergeStepSets(stepsToResult, stepsToLocation)) {
+            String stepResult = getResultWithDefaultPassed(stepsToResult, stepName);
+            if (!"undefined".equals(stepResult)) {
+                StepDefinitionMatch matchStep = mock(StepDefinitionMatch.class);
+                when(glue.stepDefinitionMatch(anyString(), TestHelper.stepWithName(stepName), (I18n) any())).thenReturn(matchStep);
+                mockStepResult(stepResult, stepName, matchStep);
+                mockStepLocation(getLocationWithDefaultEmptyString(stepsToLocation, stepName), stepName, matchStep);
+            }
+        }
+    }
+
+    private static void mockStepResult(String stepResult, String stepName, StepDefinitionMatch matchStep) throws Throwable {
+        if ("pending".equals(stepResult)) {
+            doThrow(new PendingException()).when(matchStep).runStep((I18n) any());
+        } else if ("failed".equals(stepResult)) {
+            AssertionFailedError error = TestHelper.mockAssertionFailedError();
+            doThrow(error).when(matchStep).runStep((I18n) any());
+        } else if (!"passed".equals(stepResult) &&
+                !"skipped".equals(stepResult)) {
+            fail("Cannot mock step to the result: " + stepResult);
+        }
+    }
+
+    private static void mockStepLocation(String stepLocation, String stepName, StepDefinitionMatch matchStep) {
+        when(matchStep.getLocation()).thenReturn(stepLocation);
+    }
+
+    private static void mockHooks(RuntimeGlue glue, final List<SimpleEntry<String, String>> hooks) throws Throwable {
+        List<HookDefinition> beforeHooks = new ArrayList<HookDefinition>();
+        List<HookDefinition> afterHooks = new ArrayList<HookDefinition>();
+        for (SimpleEntry<String, String> hookEntry : hooks) {
+            TestHelper.mockHook(hookEntry, beforeHooks, afterHooks);
+        }
+        if (beforeHooks.size() != 0) {
+            when(glue.getBeforeHooks()).thenReturn(beforeHooks);
+        }
+        if (afterHooks.size() != 0) {
+            when(glue.getAfterHooks()).thenReturn(afterHooks);
+        }
+    }
+
+    private static void mockHook(SimpleEntry<String, String> hookEntry, List<HookDefinition> beforeHooks,
+            List<HookDefinition> afterHooks) throws Throwable {
+        HookDefinition hook = mock(HookDefinition.class);
+        when(hook.matches(anyCollectionOf(Tag.class))).thenReturn(true);
+        if (hookEntry.getValue().equals("failed")) {
+            AssertionFailedError error = TestHelper.mockAssertionFailedError();
+            doThrow(error).when(hook).execute((cucumber.api.Scenario) any());
+        }
+        if ("before".equals(hookEntry.getKey())) {
+            beforeHooks.add(hook);
+        } else if ("after".equals(hookEntry.getKey())) {
+            afterHooks.add(hook);
+        } else {
+            fail("Only before and after hooks are allowed, hook type found was: " + hookEntry.getKey());
+        }
+    }
+
+    private static Step stepWithName(String name) {
+        return argThat(new StepMatcher(name));
+    }
+
+    private static AssertionFailedError mockAssertionFailedError() {
+        AssertionFailedError error = mock(AssertionFailedError.class);
+        Answer<Object> printStackTraceHandler = new Answer<Object>() {
+            @Override
+            public Object answer(InvocationOnMock invocation) throws Throwable {
+                PrintWriter writer = (PrintWriter) invocation.getArguments()[0];
+                writer.print("the stack trace");
+                return null;
+            }
+        };
+        doAnswer(printStackTraceHandler).when(error).printStackTrace((PrintWriter) any());
+        return error;
+    }
+
+    public static SimpleEntry<String, String> hookEntry(String type, String result) {
+        return new SimpleEntry<String, String>(type, result);
+    }
+
+    private static Set<String> mergeStepSets(Map<String, String> stepsToResult, Map<String, String> stepsToLocation) {
+        Set<String> steps = new HashSet<String>(stepsToResult.keySet());
+        steps.addAll(stepsToLocation.keySet());
+        return steps;
+    }
+
+    private static String getResultWithDefaultPassed(Map<String, String> stepsToResult, String step) {
+        return stepsToResult.containsKey(step) ? stepsToResult.get(step) : "passed";
+    }
+
+    private static String getLocationWithDefaultEmptyString(Map<String, String> stepsToLocation, String step) {
+        return stepsToLocation.containsKey(step) ? stepsToLocation.get(step) : "";
+    }
 }

core/src/test/java/cucumber/runtime/formatter/CucumberPrettyFormatterTest.java

@@ -1,35 +1,20 @@
 package cucumber.runtime.formatter;
 
-import cucumber.runtime.Backend;
-import cucumber.runtime.Env;
-import cucumber.runtime.Runtime;
-import cucumber.runtime.RuntimeGlue;
-import cucumber.runtime.RuntimeOptions;
-import cucumber.runtime.StepDefinitionMatch;
-import cucumber.runtime.io.ClasspathResourceLoader;
+import cucumber.runtime.TestHelper;
 import cucumber.runtime.model.CucumberFeature;
-import gherkin.I18n;
-import gherkin.formatter.model.Step;
 import org.junit.Test;
 
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
 
 import static cucumber.runtime.TestHelper.feature;
-import static java.util.Arrays.asList;
 import static org.hamcrest.CoreMatchers.containsString;
 import static org.junit.Assert.assertThat;
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyString;
-import static org.mockito.Matchers.argThat;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
 
 public class CucumberPrettyFormatterTest {
 
     @Test
-    public void should_align_the_indentation_of_location_strings() throws IOException {
+    public void should_align_the_indentation_of_location_strings() throws Throwable {
         CucumberFeature feature = feature("path/test.feature",
                 "Feature: feature name\n" +
                         "  Scenario: scenario name\n" +
@@ -50,32 +35,12 @@ public void should_align_the_indentation_of_location_strings() throws IOExceptio
                         "    Then third step       # path/step_definitions.java:11\n"));
     }
 
-    private String runFeatureWithPrettyFormatter(final CucumberFeature feature, final Map<String, String> stepsToLocation) throws IOException {
-        final RuntimeOptions runtimeOptions = new RuntimeOptions(new Env());
-        final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
-        final ClasspathResourceLoader resourceLoader = new ClasspathResourceLoader(classLoader);
-        final RuntimeGlue glue = createMockedRuntimeGlueThatMatchesTheSteps(stepsToLocation);
-        final Runtime runtime = new Runtime(resourceLoader, classLoader, asList(mock(Backend.class)), runtimeOptions, glue);
+    private String runFeatureWithPrettyFormatter(final CucumberFeature feature, final Map<String, String> stepsToLocation) throws Throwable {
         final StringBuilder out = new StringBuilder();
         final CucumberPrettyFormatter prettyFormatter = new CucumberPrettyFormatter(out);
         prettyFormatter.setMonochrome(true);
-
-        feature.run(prettyFormatter, prettyFormatter, runtime);
-
+        TestHelper.runFeatureWithFormatter(feature, stepsToLocation, prettyFormatter, prettyFormatter);
         return out.toString();
     }
 
-    private RuntimeGlue createMockedRuntimeGlueThatMatchesTheSteps(Map<String, String> stepsToLocation) {
-        RuntimeGlue glue = mock(RuntimeGlue.class);
-        for (String stepName : stepsToLocation.keySet()) {
-            StepDefinitionMatch matchStep = mock(StepDefinitionMatch.class);
-            when(matchStep.getLocation()).thenReturn(stepsToLocation.get(stepName));
-            when(glue.stepDefinitionMatch(anyString(), stepWithName(stepName), (I18n) any())).thenReturn(matchStep);
-        }
-        return glue;
-    }
-
-    private Step stepWithName(String name) {
-        return argThat(new StepMatcher(name));
-    }
 }