diff --git a/core/src/main/java/cucumber/runtime/Runtime.java b/core/src/main/java/cucumber/runtime/Runtime.java index 7d95006b96..23fccb4541 100644 --- a/core/src/main/java/cucumber/runtime/Runtime.java +++ b/core/src/main/java/cucumber/runtime/Runtime.java @@ -31,7 +31,7 @@ public class Runtime implements UnreportedStepExecutor { private static final Object DUMMY_ARG = new Object(); private static final byte ERRORS = 0x1; - private final UndefinedStepsTracker undefinedStepsTracker = new UndefinedStepsTracker(); + final UndefinedStepsTracker undefinedStepsTracker = new UndefinedStepsTracker(); private final Glue glue; private final RuntimeOptions runtimeOptions; @@ -121,12 +121,44 @@ public List getErrors() { public byte exitStatus() { byte result = 0x0; - if (!errors.isEmpty()) { + if (hasErrors() || hasUndefinedOrPendingStepsAndIsStrict()) { result |= ERRORS; } return result; } + private boolean hasUndefinedOrPendingStepsAndIsStrict() + { + return runtimeOptions.strict && hasUndefinedOrPendingSteps(); + } + + private boolean hasUndefinedOrPendingSteps() + { + return hasUndefinedSteps() || hasPendingSteps(); + } + + private boolean hasUndefinedSteps() + { + return undefinedStepsTracker.hasUndefinedSteps(); + } + + private boolean hasPendingSteps() + { + return !errors.isEmpty() && !hasErrors(); + } + + private boolean hasErrors() + { + for (Throwable error : errors) + { + if (!(error instanceof PendingException)) + { + return true; + } + } + return false; + } + public List getSnippets() { return undefinedStepsTracker.getSnippets(backends); } diff --git a/core/src/main/java/cucumber/runtime/RuntimeOptions.java b/core/src/main/java/cucumber/runtime/RuntimeOptions.java index b5fde26108..fd0f09bc3b 100644 --- a/core/src/main/java/cucumber/runtime/RuntimeOptions.java +++ b/core/src/main/java/cucumber/runtime/RuntimeOptions.java @@ -27,6 +27,7 @@ public class RuntimeOptions { public List glue = new ArrayList(); public File dotCucumber; public boolean dryRun; + public boolean strict = false; public List tags = new ArrayList(); public List formatters = new ArrayList(); public List featurePaths = new ArrayList(); @@ -69,6 +70,8 @@ private void parse(ArrayList args) { dotCucumber = new File(args.remove(0)); } else if (arg.equals("--dry-run") || arg.equals("-d")) { dryRun = true; + } else if (arg.equals("--strict") || arg.equals("-s")) { + strict = true; } else if (arg.equals("--monochrome") || arg.equals("-m")) { monochrome = true; } else { diff --git a/core/src/main/java/cucumber/runtime/UndefinedStepsTracker.java b/core/src/main/java/cucumber/runtime/UndefinedStepsTracker.java index 26c4dff29f..61ef58fad9 100644 --- a/core/src/main/java/cucumber/runtime/UndefinedStepsTracker.java +++ b/core/src/main/java/cucumber/runtime/UndefinedStepsTracker.java @@ -72,4 +72,9 @@ private Step givenWhenThenStep(Step step, I18n i18n) { return new Step(step.getComments(), lastGivenWhenThenStepKeyword, step.getName(), step.getLine(), step.getRows(), step.getDocString()); } } + + public boolean hasUndefinedSteps() + { + return !undefinedSteps.isEmpty(); + } } diff --git a/core/src/test/java/cucumber/runtime/RuntimeOptionsTest.java b/core/src/test/java/cucumber/runtime/RuntimeOptionsTest.java index 151fa205b6..2e408e93ed 100644 --- a/core/src/test/java/cucumber/runtime/RuntimeOptionsTest.java +++ b/core/src/test/java/cucumber/runtime/RuntimeOptionsTest.java @@ -7,6 +7,7 @@ import static java.util.Arrays.asList; import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertFalse; import static org.junit.Assert.assertTrue; public class RuntimeOptionsTest { @@ -43,4 +44,22 @@ public void creates_formatter() { RuntimeOptions options = new RuntimeOptions("--format", "html:some/dir", "--glue", "somewhere"); assertEquals(HTMLFormatter.class, options.formatters.get(0).getClass()); } + + @Test + public void assigns_strict() { + RuntimeOptions options = new RuntimeOptions("--strict", "--glue", "somewhere"); + assertTrue(options.strict); + } + + @Test + public void assigns_strict_short() { + RuntimeOptions options = new RuntimeOptions("-s", "--glue", "somewhere"); + assertTrue(options.strict); + } + + @Test + public void default_strict() { + RuntimeOptions options = new RuntimeOptions("--glue", "somewhere"); + assertFalse(options.strict); + } } diff --git a/core/src/test/java/cucumber/runtime/RuntimeTest.java b/core/src/test/java/cucumber/runtime/RuntimeTest.java index 67384ba97a..ea4f3e22fe 100644 --- a/core/src/test/java/cucumber/runtime/RuntimeTest.java +++ b/core/src/test/java/cucumber/runtime/RuntimeTest.java @@ -1,18 +1,29 @@ package cucumber.runtime; import cucumber.io.ClasspathResourceLoader; +import cucumber.io.ResourceLoader; import cucumber.runtime.model.CucumberFeature; +import gherkin.I18n; import gherkin.formatter.JSONPrettyFormatter; +import gherkin.formatter.model.Step; + import org.junit.Test; +import java.util.Arrays; +import java.util.Collection; import java.util.List; import static cucumber.runtime.TestHelper.feature; import static java.util.Arrays.asList; import static org.junit.Assert.assertEquals; import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.when; public class RuntimeTest { + + private static final I18n ENGLISH = new I18n("en"); + @Test public void runs_feature_with_json_formatter() throws Exception { CucumberFeature feature = feature("test.feature", "" + @@ -81,4 +92,93 @@ public void runs_feature_with_json_formatter() throws Exception { "]"; assertEquals(expected, out.toString()); } + + @Test + public void strict_without_pending_steps_or_errors() + { + Runtime runtime = createStrictRuntime(); + + assertEquals(0x0, runtime.exitStatus()); + } + + @Test + public void non_strict_without_pending_steps_or_errors() + { + Runtime runtime = createNonStrictRuntime(); + + assertEquals(0x0, runtime.exitStatus()); + } + + @Test + public void non_strict_with_undefined_steps() + { + Runtime runtime = createNonStrictRuntime(); + runtime.undefinedStepsTracker.addUndefinedStep(new Step(null, "Given ", "A", 1, null, null), ENGLISH); + assertEquals(0x0, runtime.exitStatus()); + } + + @Test + public void strict_with_undefined_steps() + { + Runtime runtime = createStrictRuntime(); + runtime.undefinedStepsTracker.addUndefinedStep(new Step(null, "Given ", "A", 1, null, null), ENGLISH); + assertEquals(0x1, runtime.exitStatus()); + } + + @Test + public void strict_with_pending_steps_and_no_errors() + { + Runtime runtime = createStrictRuntime(); + runtime.addError(new PendingException()); + + assertEquals(0x1, runtime.exitStatus()); + } + + @Test + public void non_strict_with_pending_steps() + { + Runtime runtime = createNonStrictRuntime(); + runtime.addError(new PendingException()); + + assertEquals(0x0, runtime.exitStatus()); + } + + @Test + public void non_strict_with_errors() + { + Runtime runtime = createNonStrictRuntime(); + runtime.addError(new RuntimeException()); + + assertEquals(0x1, runtime.exitStatus()); + } + + @Test + public void strict_with_errors() + { + Runtime runtime = createStrictRuntime(); + runtime.addError(new RuntimeException()); + + assertEquals(0x1, runtime.exitStatus()); + } + + private Runtime createStrictRuntime() + { + return createRuntime("-g anything", "--strict"); + } + + private Runtime createNonStrictRuntime() + { + return createRuntime("-g anything"); + } + + private Runtime createRuntime(String ... runtimeArgs) + { + ResourceLoader resourceLoader = mock(ResourceLoader.class); + ClassLoader classLoader = mock(ClassLoader.class); + RuntimeOptions runtimeOptions = new RuntimeOptions(runtimeArgs); + Backend backend = mock(Backend.class); + Collection backends = Arrays.asList(backend); + + return new Runtime(resourceLoader, classLoader, backends, runtimeOptions); + } } diff --git a/core/src/test/java/cucumber/runtime/UndefinedStepsTrackerTest.java b/core/src/test/java/cucumber/runtime/UndefinedStepsTrackerTest.java index e50aeb91b2..928341830f 100644 --- a/core/src/test/java/cucumber/runtime/UndefinedStepsTrackerTest.java +++ b/core/src/test/java/cucumber/runtime/UndefinedStepsTrackerTest.java @@ -10,11 +10,28 @@ import static java.util.Arrays.asList; import static junit.framework.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; public class UndefinedStepsTrackerTest { private static final I18n ENGLISH = new I18n("en"); + @Test + public void has_undefined_steps() + { + UndefinedStepsTracker undefinedStepsTracker = new UndefinedStepsTracker(); + undefinedStepsTracker.addUndefinedStep(new Step(null, "Given ", "A", 1, null, null), ENGLISH); + assertTrue(undefinedStepsTracker.hasUndefinedSteps()); + } + + @Test + public void has_no_undefined_steps() + { + UndefinedStepsTracker undefinedStepsTracker = new UndefinedStepsTracker(); + assertFalse(undefinedStepsTracker.hasUndefinedSteps()); + } + @Test public void removes_duplicates() { Backend backend = new TestBackend(); diff --git a/junit/src/main/java/cucumber/junit/Cucumber.java b/junit/src/main/java/cucumber/junit/Cucumber.java index ac23f285ef..8ed50a4330 100644 --- a/junit/src/main/java/cucumber/junit/Cucumber.java +++ b/junit/src/main/java/cucumber/junit/Cucumber.java @@ -56,7 +56,8 @@ public Cucumber(Class clazz) throws InitializationError, IOException { RuntimeOptions runtimeOptions = runtimeOptionsFactory.create(); runtime = new Runtime(resourceLoader, classLoader, runtimeOptions); - jUnitReporter = new JUnitReporter(runtimeOptions.reporter(classLoader), runtimeOptions.formatter(classLoader)); + jUnitReporter = new JUnitReporter(runtimeOptions.reporter(classLoader), runtimeOptions.formatter(classLoader) + , runtimeOptions.strict); addChildren(runtimeOptions.cucumberFeatures(resourceLoader)); } @@ -118,6 +119,12 @@ private void addChildren(List cucumberFeatures) throws Initiali */ boolean dryRun() default false; + /** + * Scenarios fail if + * @return + */ + boolean strict() default false; + /** * @return the paths to the feature(s) */ @@ -142,5 +149,6 @@ private void addChildren(List cucumberFeatures) throws Initiali * @return whether or not to use monochrome output */ boolean monochrome() default false; + } } \ No newline at end of file diff --git a/junit/src/main/java/cucumber/junit/JUnitReporter.java b/junit/src/main/java/cucumber/junit/JUnitReporter.java index 4b17179b3f..e2c18b3c9f 100644 --- a/junit/src/main/java/cucumber/junit/JUnitReporter.java +++ b/junit/src/main/java/cucumber/junit/JUnitReporter.java @@ -1,5 +1,13 @@ package cucumber.junit; +import java.io.InputStream; +import java.util.ArrayList; +import java.util.List; + +import org.junit.internal.runners.model.EachTestNotifier; +import org.junit.runner.Description; +import org.junit.runner.notification.RunNotifier; + import cucumber.runtime.PendingException; import gherkin.formatter.Formatter; import gherkin.formatter.Reporter; @@ -10,28 +18,23 @@ import gherkin.formatter.model.Scenario; import gherkin.formatter.model.ScenarioOutline; import gherkin.formatter.model.Step; -import org.junit.internal.runners.model.EachTestNotifier; -import org.junit.runner.Description; -import org.junit.runner.notification.RunNotifier; - -import java.io.InputStream; -import java.util.ArrayList; -import java.util.List; class JUnitReporter implements Reporter, Formatter { private final List steps = new ArrayList(); private final Reporter reporter; private final Formatter formatter; + private final boolean strict; - private EachTestNotifier stepNotifier; + EachTestNotifier stepNotifier; private ExecutionUnitRunner executionUnitRunner; private RunNotifier runNotifier; - private EachTestNotifier executionUnitNotifier; + EachTestNotifier executionUnitNotifier; - public JUnitReporter(Reporter reporter, Formatter formatter) { + public JUnitReporter(Reporter reporter, Formatter formatter, boolean strict) { this.reporter = reporter; this.formatter = formatter; + this.strict = strict; } public void startExecutionUnit(ExecutionUnitRunner executionUnitRunner, RunNotifier runNotifier) { @@ -66,18 +69,23 @@ public void write(String text) { public void result(Result result) { Throwable error = result.getError(); - if (Result.SKIPPED == result || Result.UNDEFINED == result || error instanceof PendingException) { + if (Result.SKIPPED == result) { stepNotifier.fireTestIgnored(); + } else if (isPendingOrUndefined(result)) { + addFailureOrIgnoreStep(result); } else { - if (stepNotifier != null) { + if (stepNotifier != null) + { //Should only fireTestStarted if not ignored stepNotifier.fireTestStarted(); - if (error != null) { + if (error != null) + { stepNotifier.addFailure(error); } stepNotifier.fireTestFinished(); } - if (error != null) { + if (error != null) + { executionUnitNotifier.addFailure(error); } } @@ -90,6 +98,33 @@ public void result(Result result) { reporter.result(result); } + private boolean isPendingOrUndefined(Result result) + { + Throwable error = result.getError(); + return Result.UNDEFINED == result || error instanceof PendingException; + } + + private void addFailureOrIgnoreStep(Result result) + { + if (strict) { + addFailure(result); + } else { + stepNotifier.fireTestIgnored(); + } + } + + private void addFailure(Result result) + { + + Throwable error = result.getError(); + if (error == null) + { + error = new PendingException(); + } + stepNotifier.addFailure(error); + executionUnitNotifier.addFailure(error); + } + @Override public void uri(String uri) { formatter.uri(uri); diff --git a/junit/src/main/java/cucumber/junit/RuntimeOptionsFactory.java b/junit/src/main/java/cucumber/junit/RuntimeOptionsFactory.java index cee8726f19..47eb4ec3e0 100644 --- a/junit/src/main/java/cucumber/junit/RuntimeOptionsFactory.java +++ b/junit/src/main/java/cucumber/junit/RuntimeOptionsFactory.java @@ -25,6 +25,7 @@ public RuntimeOptions create() { addTags(options, args); addFormats(options, args); addFeatures(options, clazz, args); + addStrict(options, args); return new RuntimeOptions(args.toArray(new String[args.size()])); @@ -96,4 +97,12 @@ private void addFeatures(Cucumber.Options options, Class clazz, List arg args.add(packagePath(clazz)); } } + + private void addStrict(Cucumber.Options options, List args) + { + if (options != null && options.strict()) + { + args.add("--strict"); + } + } } diff --git a/junit/src/test/java/cucumber/junit/JUnitReporterTest.java b/junit/src/test/java/cucumber/junit/JUnitReporterTest.java index 68d2c48170..1a5373ea77 100644 --- a/junit/src/test/java/cucumber/junit/JUnitReporterTest.java +++ b/junit/src/test/java/cucumber/junit/JUnitReporterTest.java @@ -1,43 +1,39 @@ package cucumber.junit; -import gherkin.formatter.Formatter; -import gherkin.formatter.Reporter; -import gherkin.formatter.model.Result; -import org.junit.Before; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + import org.junit.Test; +import org.junit.internal.runners.model.EachTestNotifier; import org.junit.runner.Description; import org.junit.runner.notification.Failure; import org.junit.runner.notification.RunNotifier; import org.mockito.ArgumentCaptor; +import org.mockito.Matchers; -import static org.junit.Assert.assertEquals; -import static org.mockito.Mockito.mock; -import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; +import cucumber.runtime.PendingException; +import gherkin.formatter.Formatter; +import gherkin.formatter.Reporter; +import gherkin.formatter.model.Result; public class JUnitReporterTest { private JUnitReporter jUnitReporter; - - @Before - public void setUp() { - Formatter formatter = mock(Formatter.class); - Reporter reporter = mock(Reporter.class); - - jUnitReporter = new JUnitReporter(reporter, formatter); - } + private RunNotifier runNotifier; @Test public void resultWithError() { + createNonStrictReporter(); Result result = mock(Result.class); Throwable exception = mock(Throwable.class); when(result.getError()).thenReturn(exception); - RunNotifier runNotifier = mock(RunNotifier.class); - ExecutionUnitRunner executionUnitRunner = mock(ExecutionUnitRunner.class); Description description = mock(Description.class); - when(executionUnitRunner.getDescription()).thenReturn(description); - jUnitReporter.startExecutionUnit(executionUnitRunner, runNotifier); + createRunNotifier(description); jUnitReporter.result(result); @@ -48,4 +44,145 @@ public void resultWithError() { assertEquals(description, failure.getDescription()); assertEquals(exception, failure.getException()); } + + @Test + public void result_with_undefined_step_non_strict() { + createNonStrictReporter(); + EachTestNotifier stepNotifier = mock(EachTestNotifier.class); + jUnitReporter.stepNotifier = stepNotifier; + + jUnitReporter.result(Result.UNDEFINED); + + verify(stepNotifier, times(0)).fireTestStarted(); + verify(stepNotifier, times(0)).fireTestFinished(); + verify(stepNotifier, times(0)).addFailure(Matchers.any(Throwable.class)); + verify(stepNotifier).fireTestIgnored(); + } + + @Test + public void result_with_undefined_step_strict() { + createStrictReporter(); + createDefaultRunNotifier(); + EachTestNotifier stepNotifier = mock(EachTestNotifier.class); + jUnitReporter.stepNotifier = stepNotifier; + EachTestNotifier executionUnitNotifier = mock(EachTestNotifier.class); + jUnitReporter.executionUnitNotifier = executionUnitNotifier; + + jUnitReporter.result(Result.UNDEFINED); + + verify(stepNotifier, times(0)).fireTestStarted(); + verify(stepNotifier, times(0)).fireTestFinished(); + verifyAddFailureWithPendingException(stepNotifier); + verifyAddFailureWithPendingException(executionUnitNotifier); + verify(stepNotifier, times(0)).fireTestIgnored(); + } + + private void verifyAddFailureWithPendingException(EachTestNotifier stepNotifier) + { + ArgumentCaptor captor = ArgumentCaptor.forClass(Throwable.class); + verify(stepNotifier).addFailure(captor.capture()); + Throwable error = captor.getValue(); + assertTrue(error instanceof PendingException); + } + + @Test + public void result_with_pending_step_non_strict() { + createNonStrictReporter(); + Result result = mock(Result.class); + when(result.getError()).thenReturn(new PendingException()); + + EachTestNotifier stepNotifier = mock(EachTestNotifier.class); + jUnitReporter.stepNotifier = stepNotifier; + + jUnitReporter.result(result); + + verify(stepNotifier, times(0)).fireTestStarted(); + verify(stepNotifier, times(0)).fireTestFinished(); + verify(stepNotifier, times(0)).addFailure(Matchers.any(Throwable.class)); + verify(stepNotifier).fireTestIgnored(); + } + + @Test + public void result_with_pending_step_strict() { + createStrictReporter(); + createDefaultRunNotifier(); + Result result = mock(Result.class); + when(result.getError()).thenReturn(new PendingException()); + + EachTestNotifier stepNotifier = mock(EachTestNotifier.class); + jUnitReporter.stepNotifier = stepNotifier; + EachTestNotifier executionUnitNotifier = mock(EachTestNotifier.class); + jUnitReporter.executionUnitNotifier = executionUnitNotifier; + + jUnitReporter.result(result); + + verify(stepNotifier, times(0)).fireTestStarted(); + verify(stepNotifier, times(0)).fireTestFinished(); + verifyAddFailureWithPendingException(stepNotifier); + verifyAddFailureWithPendingException(executionUnitNotifier); + verify(stepNotifier, times(0)).fireTestIgnored(); + } + + @Test + public void result_without_error_non_strict() { + createNonStrictReporter(); + Result result = mock(Result.class); + + EachTestNotifier stepNotifier = mock(EachTestNotifier.class); + jUnitReporter.stepNotifier = stepNotifier; + + jUnitReporter.result(result); + + verify(stepNotifier).fireTestStarted(); + verify(stepNotifier).fireTestFinished(); + verify(stepNotifier, times(0)).addFailure(Matchers.any(Throwable.class)); + verify(stepNotifier, times(0)).fireTestIgnored(); + } + + @Test + public void result_without_error_strict() { + createStrictReporter(); + Result result = mock(Result.class); + + EachTestNotifier stepNotifier = mock(EachTestNotifier.class); + jUnitReporter.stepNotifier = stepNotifier; + + jUnitReporter.result(result); + + verify(stepNotifier).fireTestStarted(); + verify(stepNotifier).fireTestFinished(); + verify(stepNotifier, times(0)).addFailure(Matchers.any(Throwable.class)); + verify(stepNotifier, times(0)).fireTestIgnored(); + } + + private void createDefaultRunNotifier() + { + createRunNotifier(mock(Description.class)); + } + + private void createRunNotifier(Description description) + { + runNotifier = mock(RunNotifier.class); + ExecutionUnitRunner executionUnitRunner = mock(ExecutionUnitRunner.class); + when(executionUnitRunner.getDescription()).thenReturn(description); + jUnitReporter.startExecutionUnit(executionUnitRunner, runNotifier); + } + + private void createStrictReporter() + { + createReporter(true); + } + + private void createNonStrictReporter() + { + createReporter(false); + } + + private void createReporter(boolean strict) { + Formatter formatter = mock(Formatter.class); + Reporter reporter = mock(Reporter.class); + + jUnitReporter = new JUnitReporter(reporter, formatter, strict); + } + } diff --git a/junit/src/test/java/cucumber/junit/RuntimeOptionsFactoryTest.java b/junit/src/test/java/cucumber/junit/RuntimeOptionsFactoryTest.java new file mode 100644 index 0000000000..3ef0cd2174 --- /dev/null +++ b/junit/src/test/java/cucumber/junit/RuntimeOptionsFactoryTest.java @@ -0,0 +1,50 @@ +package cucumber.junit; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertTrue; + +import org.junit.Test; + +import cucumber.runtime.RuntimeOptions; + +public class RuntimeOptionsFactoryTest +{ + @Test + public void create_strict() throws Exception + { + RuntimeOptionsFactory runtimeOptionsFactory = new RuntimeOptionsFactory(Strict.class); + RuntimeOptions runtimeOptions = runtimeOptionsFactory.create(); + assertTrue(runtimeOptions.strict); + } + + @Test + public void create_non_strict() throws Exception + { + RuntimeOptionsFactory runtimeOptionsFactory = new RuntimeOptionsFactory(NotStrict.class); + RuntimeOptions runtimeOptions = runtimeOptionsFactory.create(); + assertFalse(runtimeOptions.strict); + } + + @Test + public void create_without_options() throws Exception + { + RuntimeOptionsFactory runtimeOptionsFactory = new RuntimeOptionsFactory(WithoutOptions.class); + RuntimeOptions runtimeOptions = runtimeOptionsFactory.create(); + assertFalse(runtimeOptions.strict); + } + + @Cucumber.Options(strict=true) + public static class Strict { + // empty + } + + @Cucumber.Options() + public static class NotStrict { + // empty + } + + public static class WithoutOptions { + // empty + } + +}