[Core] Implement TeamCity output format plugin (#1842)

This plugin inserts markers that can be picked up by IDEA and
Teamcity. These in turn can then use this information to display a
test tree and show the output grouped by scenario.

This is plugin is nessesary because Intelij uses reflection
to extract more information from the plugin system then what would
normally be available. By substituting the `CucumberJvm[1-5]SMFormatter`
with this plugin we avoid runtime errors when our internals inevitably change.

Unfortunately the teamcity format is poorly documented[1]. Reverse
engingeering[2][3] the format from the plugin yields some information
but the exact function is rather opaque.

1. https://confluence.jetbrains.com/display/TCD9/Build+Script+Interaction+with+TeamCity
2. https://github.com/JetBrains/intellij-community/tree/master/plugins/cucumber-jvm-formatter5/src/org/jetbrains/plugins/cucumber/java/run
3. https://github.com/JetBrains/intellij-community/tree/master/plugins/cucumber-jvm-formatter/src/org/jetbrains/plugins/cucumber/java/run

core/src/main/java/io/cucumber/core/options/PluginOption.java

@@ -12,6 +12,7 @@
 import io.cucumber.core.plugin.PrettyFormatter;
 import io.cucumber.core.plugin.ProgressFormatter;
 import io.cucumber.core.plugin.RerunFormatter;
+import io.cucumber.core.plugin.TeamCityPlugin;
 import io.cucumber.core.plugin.TestNGFormatter;
 import io.cucumber.core.plugin.TimelineFormatter;
 import io.cucumber.core.plugin.UnusedStepsSummaryPrinter;
@@ -44,15 +45,16 @@ public class PluginOption implements Options.Plugin {
         put("timeline", TimelineFormatter.class);
         put("unused", UnusedStepsSummaryPrinter.class);
         put("usage", UsageFormatter.class);
+        put("teamcity", UsageFormatter.class);
     }};
 
     // Refuse plugins known to implement the old API
-    private static final HashMap<String, Class<? extends Plugin>> OLD_INTELLIJ_IDEA_PLUGIN_CLASSES = new HashMap<String, Class<? extends Plugin>>() {{
-        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvmSMFormatter", PrettyFormatter.class);
-        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm2SMFormatter", PrettyFormatter.class);
-        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm3SMFormatter", PrettyFormatter.class);
-        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm4SMFormatter", PrettyFormatter.class);
-        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm5SMFormatter", PrettyFormatter.class);
+    private static final HashMap<String, Class<? extends Plugin>> INCOMPATIBLE_INTELLIJ_IDEA_PLUGIN_CLASSES = new HashMap<String, Class<? extends Plugin>>() {{
+        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvmSMFormatter", TeamCityPlugin.class);
+        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm2SMFormatter", TeamCityPlugin.class);
+        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm3SMFormatter", TeamCityPlugin.class);
+        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm4SMFormatter", TeamCityPlugin.class);
+        put("org.jetbrains.plugins.cucumber.java.run.CucumberJvm5SMFormatter", TeamCityPlugin.class);
     }};
 
     private final String pluginString;
@@ -76,9 +78,9 @@ public static PluginOption parse(String pluginArgumentPattern) {
     }
 
     private static Class<? extends Plugin> parsePluginName(String pluginName) {
-        Class<? extends Plugin> oldApiPlugin = OLD_INTELLIJ_IDEA_PLUGIN_CLASSES.get(pluginName);
+        Class<? extends Plugin> oldApiPlugin = INCOMPATIBLE_INTELLIJ_IDEA_PLUGIN_CLASSES.get(pluginName);
         if (oldApiPlugin != null) {
-            log.warn(() -> "Incompatible IntelliJ IDEA Plugin detected. Falling back to pretty formatter");
+            log.debug(() -> "Incompatible IntelliJ IDEA Plugin detected. Falling back to teamcity plugin");
             return oldApiPlugin;
         }
 

core/src/main/java/io/cucumber/core/runner/PickleStepDefinitionMatch.java

@@ -157,7 +157,7 @@ public String getPattern() {
     }
 
     private StackTraceElement getStepLocation() {
-        return new StackTraceElement("✽", step.getText(), uri.getSchemeSpecificPart(), step.getLine());
+        return new StackTraceElement("✽", step.getText(), uri.toString(), step.getLine());
     }
 
     StepDefinition getStepDefinition() {

core/src/main/java/io/cucumber/core/runner/Runner.java

@@ -119,7 +119,7 @@ private PickleStepDefinitionMatch matchStepToStepDefinition(Pickle pickle, Step
             }
             List<String> snippets = generateSnippetsForStep(step);
             if (!snippets.isEmpty()) {
-                bus.send(new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), step.getLine(), snippets));
+                bus.send(new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), pickle.getScenarioLocation().getLine(), step.getLine(), snippets));
             }
             return new UndefinedPickleStepDefinitionMatch(pickle.getUri(), step);
         } catch (AmbiguousStepDefinitionsException e) {

core/src/test/java/io/cucumber/core/plugin/CanonicalEventOrderTest.java

@@ -43,7 +43,7 @@ private static Event createTestCaseEvent(final URI uri, final int line) {
 
     private Event runStarted = new TestRunStarted(getInstant());
     private Event testRead = new TestSourceRead(getInstant(), URI.create("file:path/to.feature"), "source");
-    private Event suggested = new SnippetsSuggestedEvent(getInstant(), URI.create("file:path/to/1.feature"), 0, Collections.emptyList());
+    private Event suggested = new SnippetsSuggestedEvent(getInstant(), URI.create("file:path/to/1.feature"), 0, 0, Collections.emptyList());
     private Event feature1Case1Started = createTestCaseEvent(URI.create("file:path/to/1.feature"), 1);
     private Event feature1Case2Started = createTestCaseEvent(URI.create("file:path/to/1.feature"), 9);
     private Event feature1Case3Started = createTestCaseEvent(URI.create("file:path/to/1.feature"), 11);

core/src/test/java/io/cucumber/core/plugin/TeamCityPluginTest.java

@@ -0,0 +1,140 @@
+package io.cucumber.core.plugin;
+
+import io.cucumber.core.feature.TestFeatureParser;
+import io.cucumber.core.gherkin.Feature;
+import io.cucumber.core.runner.TestHelper;
+import io.cucumber.plugin.event.Result;
+import org.junit.jupiter.api.Test;
+import org.mockito.stubbing.Answer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.PrintStream;
+import java.util.AbstractMap.SimpleEntry;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static io.cucumber.core.runner.TestHelper.createWriteHookAction;
+import static io.cucumber.core.runner.TestHelper.result;
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.hamcrest.CoreMatchers.containsString;
+import static org.hamcrest.MatcherAssert.assertThat;
+
+class TeamCityPluginTest {
+
+    private final List<Feature> features = new ArrayList<>();
+    private final Map<String, Result> stepsToResult = new HashMap<>();
+    private final Map<String, String> stepsToLocation = new HashMap<>();
+    private final List<SimpleEntry<String, Result>> hooks = new ArrayList<>();
+    private final List<String> hookLocations = new ArrayList<>();
+    private final List<Answer<Object>> hookActions = new ArrayList<>();
+    private final String location = new File("").toURI().toString();
+
+    @Test
+    void should_handle_scenario_outline() {
+        Feature feature = TestFeatureParser.parse("path/test.feature", "" +
+            "Feature: feature name\n" +
+            "  Scenario Outline: <name>\n" +
+            "    Given first step\n" +
+            "    Then <arg> step\n" +
+            "    Examples: examples name\n" +
+            "      |  name  |  arg   |\n" +
+            "      | name 1 | second |\n" +
+            "      | name 2 | third  |\n");
+        features.add(feature);
+        stepsToLocation.put("first step", "path/step_definitions.java:3");
+        stepsToLocation.put("second step", "path/step_definitions.java:7");
+        stepsToLocation.put("third step", "path/step_definitions.java:11");
+
+        String formatterOutput = runFeaturesWithFormatter();
+
+        assertThat(formatterOutput, containsString("" +
+            "##teamcity[enteredTheMatrix timestamp = '1970-01-01T12:00:00.000+0000']\n" +
+            "##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' name = 'Cucumber']\n" +
+            "##teamcity[customProgressStatus testsCategory = 'Scenarios' count = '0' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
+            "##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:1' name = 'feature name']\n" +
+            "##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:2' name = '<name>']\n" +
+            "##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:5' name = 'examples name']\n" +
+            "##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:7' name = 'Example #1']\n" +
+            "##teamcity[customProgressStatus type = 'testStarted' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
+            "##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:3' captureStandardOutput = 'true' name = 'first step']\n" +
+            "##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'first step']\n" +
+            "##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:4' captureStandardOutput = 'true' name = 'second step']\n" +
+            "##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'second step']\n" +
+            "##teamcity[customProgressStatus type = 'testFinished' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
+            "##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'Example #1']\n" +
+            "##teamcity[testSuiteStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:8' name = 'Example #2']\n" +
+            "##teamcity[customProgressStatus type = 'testStarted' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
+            "##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:3' captureStandardOutput = 'true' name = 'first step']\n" +
+            "##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'first step']\n" +
+            "##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = '" + location + "path/test.feature:4' captureStandardOutput = 'true' name = 'third step']\n" +
+            "##teamcity[testFinished timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' name = 'third step']\n" +
+            "##teamcity[customProgressStatus type = 'testFinished' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
+            "##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'Example #2']\n" +
+            "##teamcity[customProgressStatus testsCategory = '' count = '0' timestamp = '1970-01-01T12:00:00.000+0000']\n" +
+            "##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'examples name']\n" +
+            "##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = '<name>']\n" +
+            "##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'feature name']\n" +
+            "##teamcity[testSuiteFinished timestamp = '1970-01-01T12:00:00.000+0000' name = 'Cucumber']\n"
+        ));
+    }
+
+    @Test
+    void should_print_error_message_for_failed_steps() {
+        Feature feature = TestFeatureParser.parse("path/test.feature", "" +
+            "Feature: feature name\n" +
+            "  Scenario: scenario name\n" +
+            "    Given first step\n");
+        features.add(feature);
+        stepsToLocation.put("first step", "path/step_definitions.java:3");
+        stepsToResult.put("first step", result("failed"));
+
+        String formatterOutput = runFeaturesWithFormatter();
+
+        assertThat(formatterOutput, containsString("" +
+            "##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step failed' details = 'the stack trace' name = 'first step']\n"
+        ));
+    }
+
+    @Test
+    void should_print_error_message_for_before_hooks() {
+        Feature feature = TestFeatureParser.parse("path/test.feature", "" +
+            "Feature: feature name\n" +
+            "  Scenario: scenario name\n" +
+            "    Given first step\n");
+        features.add(feature);
+        stepsToLocation.put("first step", "path/step_definitions.java:3");
+        stepsToResult.put("first step", result("passed"));
+        hooks.add(TestHelper.hookEntry("before", result("failed")));
+        hookLocations.add("HookDefinition.java:3");
+
+        String formatterOutput = runFeaturesWithFormatter();
+
+        assertThat(formatterOutput, containsString("" +
+            "##teamcity[testStarted timestamp = '1970-01-01T12:00:00.000+0000' locationHint = 'java:test://HookDefinition.java:3' captureStandardOutput = 'true' name = 'Before']\n" +
+            "##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step failed' details = 'the stack trace' name = 'Before']\n"
+        ));
+    }
+
+    private String runFeaturesWithFormatter() {
+        ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream();
+        PrintStream printStream = new PrintStream(byteArrayOutputStream);
+        final TeamCityPlugin formatter = new TeamCityPlugin(printStream);
+
+        TestHelper.builder()
+            .withFormatterUnderTest(formatter)
+            .withFeatures(features)
+            .withStepsToResult(stepsToResult)
+            .withStepsToLocation(stepsToLocation)
+            .withHooks(hooks)
+            .withHookLocations(hookLocations)
+            .withHookActions(hookActions)
+            .build()
+            .run();
+
+        return new String(byteArrayOutputStream.toByteArray(), UTF_8);
+    }
+
+}

gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesExamples.java

@@ -4,22 +4,28 @@
 import io.cucumber.core.gherkin.Examples;
 import io.cucumber.core.gherkin.Location;
 
+import java.util.Collection;
+import java.util.List;
 import java.util.concurrent.atomic.AtomicInteger;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
 
 final class GherkinMessagesExamples implements Examples {
 
     private final io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario.Examples examples;
+    private final List<Example> children;
 
     GherkinMessagesExamples(io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario.Examples examples) {
         this.examples = examples;
+
+        AtomicInteger row = new AtomicInteger(1);
+        this.children = examples.getTableBodyList().stream()
+            .map(tableRow -> new GherkinMessagesExample(tableRow, row.getAndIncrement()))
+            .collect(Collectors.toList());
     }
 
     @Override
-    public Stream<Example> children() {
-        AtomicInteger row = new AtomicInteger(1);
-        return examples.getTableBodyList().stream()
-            .map(tableRow -> new GherkinMessagesExample(tableRow, row.getAndIncrement()));
+    public Collection<Example> children() {
+        return children;
     }
 
     @Override

gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesFeature.java

@@ -3,49 +3,52 @@
 import io.cucumber.core.gherkin.Feature;
 import io.cucumber.core.gherkin.Located;
 import io.cucumber.core.gherkin.Location;
-import io.cucumber.core.gherkin.Pickle;
 import io.cucumber.core.gherkin.Node;
+import io.cucumber.core.gherkin.Pickle;
 import io.cucumber.messages.Messages;
 import io.cucumber.messages.Messages.GherkinDocument;
 import io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario;
 
 import java.net.URI;
+import java.util.Collection;
 import java.util.List;
 import java.util.Objects;
 import java.util.Optional;
-import java.util.stream.Stream;
+import java.util.stream.Collectors;
 
 final class GherkinMessagesFeature implements Feature {
     private final URI uri;
     private final List<Pickle> pickles;
     private final List<Messages.Envelope> envelopes;
     private final GherkinDocument gherkinDocument;
     private final String gherkinSource;
+    private final List<Node> children;
 
     GherkinMessagesFeature(GherkinDocument gherkinDocument, URI uri, String gherkinSource, List<Pickle> pickles, List<Messages.Envelope> envelopes) {
         this.gherkinDocument = gherkinDocument;
         this.uri = uri;
         this.gherkinSource = gherkinSource;
         this.pickles = pickles;
         this.envelopes = envelopes;
-    }
-
-    @Override
-    public Stream<Node> children() {
-        return gherkinDocument.getFeature().getChildrenList().stream()
+        this.children = gherkinDocument.getFeature().getChildrenList().stream()
             .filter(featureChild -> featureChild.hasRule() || featureChild.hasScenario())
             .map(featureChild -> {
                 if (featureChild.hasRule()) {
                     return new GherkinMessagesRule(featureChild.getRule());
                 }
-
                 Scenario scenario = featureChild.getScenario();
                 if (scenario.getExamplesCount() > 0) {
                     return new GherkinMessagesScenarioOutline(scenario);
                 } else {
                     return new GherkinMessagesScenario(scenario);
                 }
-            });
+            })
+            .collect(Collectors.toList());
+    }
+
+    @Override
+    public Collection<Node> children() {
+        return children;
     }
 
     @Override

gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesRule.java

@@ -1,24 +1,23 @@
 package io.cucumber.core.gherkin.messages;
 
 import io.cucumber.core.gherkin.Location;
-import io.cucumber.core.gherkin.Rule;
 import io.cucumber.core.gherkin.Node;
+import io.cucumber.core.gherkin.Rule;
 import io.cucumber.messages.Messages;
 import io.cucumber.messages.Messages.GherkinDocument.Feature.FeatureChild.RuleChild;
 
-import java.util.stream.Stream;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
 
 final class GherkinMessagesRule implements Rule {
 
     private final Messages.GherkinDocument.Feature.FeatureChild.Rule rule;
+    private final List<Node> children;
 
     GherkinMessagesRule(Messages.GherkinDocument.Feature.FeatureChild.Rule rule) {
         this.rule = rule;
-    }
-
-    @Override
-    public Stream<Node> children() {
-        return rule.getChildrenList().stream()
+        this.children = rule.getChildrenList().stream()
             .filter(RuleChild::hasScenario)
             .map(ruleChild -> {
                 Messages.GherkinDocument.Feature.Scenario scenario = ruleChild.getScenario();
@@ -27,7 +26,13 @@ public Stream<Node> children() {
                 } else {
                     return new GherkinMessagesScenario(scenario);
                 }
-            });
+            })
+            .collect(Collectors.toList());
+    }
+
+    @Override
+    public Collection<Node> children() {
+        return children;
     }
 
     @Override

gherkin-messages/src/main/java/io/cucumber/core/gherkin/messages/GherkinMessagesScenarioOutline.java

@@ -5,21 +5,26 @@
 import io.cucumber.core.gherkin.ScenarioOutline;
 import io.cucumber.messages.Messages.GherkinDocument.Feature.Scenario;
 
-import java.util.stream.Stream;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
 
 final class GherkinMessagesScenarioOutline implements ScenarioOutline {
 
     private final Scenario scenario;
+    private final List<Examples> children;
 
     GherkinMessagesScenarioOutline(Scenario scenario) {
         this.scenario = scenario;
+        this.children = scenario.getExamplesList().stream()
+            .map(GherkinMessagesExamples::new)
+            .collect(Collectors.toList());
     }
 
 
     @Override
-    public Stream<Examples> children() {
-        return scenario.getExamplesList().stream()
-            .map(GherkinMessagesExamples::new);
+    public Collection<Examples> children() {
+        return children;
     }
 
     @Override