Let the JUnitFormatter mark skipped tests as failures in strict mode

core/src/main/java/cucumber/runtime/RuntimeOptions.java

@@ -2,6 +2,7 @@
 
 import cucumber.runtime.formatter.ColorAware;
 import cucumber.runtime.formatter.FormatterFactory;
+import cucumber.runtime.formatter.StrictAware;
 import cucumber.runtime.io.ResourceLoader;
 import cucumber.runtime.model.CucumberFeature;
 import gherkin.formatter.Formatter;
@@ -31,14 +32,19 @@ public class RuntimeOptions {
     public final List<Object> filters = new ArrayList<Object>();
     public final List<Formatter> formatters = new ArrayList<Formatter>();
     public final List<String> featurePaths = new ArrayList<String>();
-    private final FormatterFactory formatterFactory = new FormatterFactory();
+    private final FormatterFactory formatterFactory;
     public URL dotCucumber;
     public boolean dryRun;
     public boolean strict = false;
     public boolean monochrome = false;
 
     public RuntimeOptions(Properties properties, String... argv) {
         /* IMPORTANT! Make sure USAGE.txt is always uptodate if this class changes */
+        this(properties, new FormatterFactory(), argv);
+    }
+
+    RuntimeOptions(Properties properties, FormatterFactory formatterFactory, String... argv) {
+        this.formatterFactory = formatterFactory;
 
         parse(new ArrayList<String>(asList(argv)));
         if (properties.containsKey("cucumber.options")) {
@@ -48,12 +54,7 @@ public RuntimeOptions(Properties properties, String... argv) {
         if (formatters.isEmpty()) {
             formatters.add(formatterFactory.create("progress"));
         }
-        for (Formatter formatter : formatters) {
-            if (formatter instanceof ColorAware) {
-                ColorAware colorAware = (ColorAware) formatter;
-                colorAware.setMonochrome(monochrome);
-            }
-        }
+        setFormatterOptions();
     }
 
     private List<String> cucumberOptionsSplit(String property) {
@@ -149,4 +150,25 @@ public Object invoke(Object target, Method method, Object[] args) throws Throwab
             }
         });
     }
+
+    private void setFormatterOptions() {
+        for (Formatter formatter : formatters) {
+            setMonochromeOnColorAwareFormatters(formatter);
+            setStrictOnStrictAwareFormatters(formatter);
+        }
+    }
+
+    private void setMonochromeOnColorAwareFormatters(Formatter formatter) {
+        if (formatter instanceof ColorAware) {
+            ColorAware colorAware = (ColorAware) formatter;
+            colorAware.setMonochrome(monochrome);
+        }
+    }
+
+    private void setStrictOnStrictAwareFormatters(Formatter formatter) {
+        if (formatter instanceof StrictAware) {
+            StrictAware strictAware = (StrictAware) formatter;
+            strictAware.setStrict(strict);
+        }
+    }
 }

core/src/main/java/cucumber/runtime/formatter/JUnitFormatter.java

@@ -36,7 +36,7 @@
 import java.util.List;
 import java.util.Locale;
 
-class JUnitFormatter implements Formatter, Reporter {
+class JUnitFormatter implements Formatter, Reporter, StrictAware {
     private final Writer out;
     private final Document doc;
     private final Element rootElement;
@@ -170,6 +170,11 @@ public void eof() {
     public void syntaxError(String state, String event, List<String> legalEvents, String uri, Integer line) {
     }
 
+    @Override
+    public void setStrict(boolean strict) {
+        TestCase.treatSkippedAsFailure = strict;
+    }
+
     private static class TestCase {
         private static final DecimalFormat NUMBER_FORMAT = (DecimalFormat) NumberFormat.getNumberInstance(Locale.US);
 
@@ -187,6 +192,7 @@ private TestCase() {
         Scenario scenario;
         static Feature feature;
         static int examples = 0;
+        static boolean treatSkippedAsFailure = false;
         final List<Step> steps = new ArrayList<Step>();
         final List<Result> results = new ArrayList<Result>();
 
@@ -232,7 +238,13 @@ public void updateElement(Document doc, Element tc) {
                 child.setAttribute("message", failed.getErrorMessage());
                 child.appendChild(doc.createCDATASection(sb.toString()));
             } else if (skipped != null) {
-                child = doc.createElement("skipped");
+                if (treatSkippedAsFailure) {
+                    child = doc.createElement("failure");
+                    child.setAttribute("message", "The scenario has pending or undefined step(s)");
+                }
+                else {
+                    child = doc.createElement("skipped");
+                }
                 child.appendChild(doc.createCDATASection(sb.toString()));
             } else {
                 child = doc.createElement("system-out");

core/src/main/java/cucumber/runtime/formatter/StrictAware.java

@@ -0,0 +1,5 @@
+package cucumber.runtime.formatter;
+
+public interface StrictAware {
+    public void setStrict(boolean strict);
+}

core/src/test/java/cucumber/runtime/RuntimeOptionsTest.java

@@ -2,6 +2,12 @@
 
 import org.junit.Test;
 
+import cucumber.runtime.formatter.ColorAware;
+import cucumber.runtime.formatter.FormatterFactory;
+import cucumber.runtime.formatter.StrictAware;
+
+import gherkin.formatter.Formatter;
+
 import java.net.MalformedURLException;
 import java.net.URL;
 import java.util.Properties;
@@ -12,6 +18,10 @@
 import static org.junit.Assert.assertFalse;
 import static org.junit.Assert.assertTrue;
 import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
 
 public class RuntimeOptionsTest {
     @Test
@@ -165,4 +175,25 @@ public void fail_on_unsupported_options() {
         }
     }
 
+    @Test
+    public void set_monochrome_on_color_aware_formatters() throws Exception {
+        FormatterFactory factory = mock(FormatterFactory.class);
+        Formatter colorAwareFormatter = mock(Formatter.class, withSettings().extraInterfaces(ColorAware.class));
+        when(factory.create("progress")).thenReturn(colorAwareFormatter);
+
+        new RuntimeOptions(new Properties(), factory, "--monochrome", "--format", "progress");
+
+        verify((ColorAware)colorAwareFormatter).setMonochrome(true);
+    }
+
+    @Test
+    public void set_strict_on_strict_aware_formatters() throws Exception {
+        FormatterFactory factory = mock(FormatterFactory.class);
+        Formatter strictAwareFormatter = mock(Formatter.class, withSettings().extraInterfaces(StrictAware.class));
+        when(factory.create("junit:out/dir")).thenReturn(strictAwareFormatter);
+
+        new RuntimeOptions(new Properties(), factory, "--strict", "--format", "junit:out/dir");
+
+        verify((StrictAware)strictAwareFormatter).setStrict(true);
+    }
 }

core/src/test/java/cucumber/runtime/formatter/JUnitFormatterTest.java

@@ -45,12 +45,26 @@ public void featureWithOutlineTest() throws Exception {
         assertXmlEqual("cucumber/runtime/formatter/JUnitFormatterTest_3.report.xml", report);
     }
 
+    @Test
+    public void featureSimpleStrictTest() throws Exception {
+        boolean strict = true;
+        File report = runFeaturesWithJunitFormatter(asList("cucumber/runtime/formatter/JUnitFormatterTest_1.feature"), strict);
+        assertXmlEqual("cucumber/runtime/formatter/JUnitFormatterTest_1_strict.report.xml", report);
+    }
+
     private File runFeaturesWithJunitFormatter(final List<String> featurePaths) throws IOException {
+        return runFeaturesWithJunitFormatter(featurePaths, false);
+    }
+
+    private File runFeaturesWithJunitFormatter(final List<String> featurePaths, boolean strict) throws IOException {
         File report = File.createTempFile("cucumber-jvm-junit", "xml");
         final ClassLoader classLoader = Thread.currentThread().getContextClassLoader();
         final ClasspathResourceLoader resourceLoader = new ClasspathResourceLoader(classLoader);
 
         List<String> args = new ArrayList<String>();
+        if (strict) {
+            args.add("--strict");
+        }
         args.add("--format");
         args.add("junit:" + report.getAbsolutePath());
         args.addAll(featurePaths);

core/src/test/resources/cucumber/runtime/formatter/JUnitFormatterTest_1_strict.report.xml

@@ -0,0 +1,15 @@
+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<testsuite tests="2" failures="2">
+    <testcase classname="Feature_1" name="Scenario_1" time="0">
+        <failure message="The scenario has pending or undefined step(s)"><![CDATA[Given step_1................................................................undefined
+When step_2.................................................................undefined
+Then step_3.................................................................undefined
+]]></failure>
+    </testcase>
+    <testcase classname="Feature_1" name="Scenario_2" time="0">
+        <failure message="The scenario has pending or undefined step(s)"><![CDATA[Given step_1................................................................undefined
+When step_2.................................................................undefined
+Then step_3.................................................................undefined
+]]></failure>
+    </testcase>
+</testsuite>