Skip to content

Commit

Permalink
[Core] Improve undefined step reporting
Browse files Browse the repository at this point in the history 
The various ways to execute Cucumber report undefined steps inconsistently.
Partially because different tools have different reporting needs. Nevertheless:

When reporting per scenario the suggestion should include snippets for all
undefined steps in that scenario. This applies to:
 - JUnit4
 - TestNG
 - JUnit5
 - Teamcity Plugin

When reporting per test run the suggestion should include snippets for all
undefined steps in the execution. This applies to:
 - CLI

When printing snippets they should be copy-pasted into an IDE without further
editing. This means individual snippets or groups of snippets should not be
separated by spacers, text or anything else.

Fixes: #2024
  • Loading branch information
@mpkorstanje
mpkorstanje committed Feb 13, 2021
1 parent e0ad566 commit 558a17d
Show file tree
Hide file tree
Showing 19 changed files with 497 additions and 266 deletions.
2 changes: 1 addition & 1 deletion core/src/main/java/io/cucumber/core/plugin/DefaultSummaryPrinter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ public void setEventPublisher(EventPublisher publisher) {
}

private void handleSnippetsSuggestedEvent(SnippetsSuggestedEvent event) {
this.snippets.addAll(event.getSnippets());
this.snippets.addAll(event.getSuggestion().getSnippets());
}

private void print() {
Expand Down
56 changes: 35 additions & 21 deletions core/src/main/java/io/cucumber/core/plugin/TeamCityPlugin.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import io.cucumber.plugin.event.PickleStepTestStep;
import io.cucumber.plugin.event.Result;
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;
import io.cucumber.plugin.event.Status;
import io.cucumber.plugin.event.TestCase;
import io.cucumber.plugin.event.TestCaseFinished;
Expand Down Expand Up @@ -42,8 +43,10 @@
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;

import static java.util.Collections.emptyList;
import static java.util.stream.Collectors.joining;

public class TeamCityPlugin implements EventListener {

Expand Down Expand Up @@ -86,9 +89,10 @@ public class TeamCityPlugin implements EventListener {
private static final Pattern LAMBDA_GLUE_CODE_LOCATION_PATTERN = Pattern.compile("^(.*)\\.(.*)\\(.*:.*\\)");

private final PrintStream out;
private final List<SnippetsSuggestedEvent> snippets = new ArrayList<>();
private final List<SnippetsSuggestedEvent> suggestions = new ArrayList<>();
private final Map<URI, Collection<Node>> parsedTestSources = new HashMap<>();
private List<Node> currentStack = new ArrayList<>();
private TestCase currentTestCase;

@SuppressWarnings("unused") // Used by PluginFactory
public TeamCityPlugin() {
Expand Down Expand Up @@ -150,6 +154,7 @@ private void printTestCaseStarted(TestCaseStarted event) {
poppedNodes(path).forEach(node -> finishNode(timestamp, node));
pushedNodes(path).forEach(node -> startNode(uri, timestamp, node));
this.currentStack = path;
this.currentTestCase = testCase;

print(TEMPLATE_PROGRESS_TEST_STARTED, timestamp);
}
Expand Down Expand Up @@ -254,8 +259,7 @@ private void printTestStepFinished(TestStepFinished event) {
name);
break;
case UNDEFINED:
PickleStepTestStep testStep = (PickleStepTestStep) event.getTestStep();
print(TEMPLATE_TEST_FAILED, timeStamp, duration, "Step undefined", getSnippet(testStep), name);
print(TEMPLATE_TEST_FAILED, timeStamp, duration, "Step undefined", getSnippets(currentTestCase), name);
break;
case AMBIGUOUS:
case FAILED:
Expand Down Expand Up @@ -299,31 +303,41 @@ private String extractName(TestStep step) {
return "Unknown step";
}

private String getSnippet(PickleStepTestStep testStep) {
StringBuilder builder = new StringBuilder();
private String getSnippets(TestCase testCase) {
URI uri = testCase.getUri();
Location location = testCase.getLocation();
List<Suggestion> suggestionForTestCase = suggestions.stream()
.filter(suggestions -> suggestions.getUri().equals(uri) &&
suggestions.getTestCaseLocation().equals(location))
.map(SnippetsSuggestedEvent::getSuggestion)
.collect(Collectors.toList());
return createMessage(suggestionForTestCase);
}

if (snippets.isEmpty()) {
return builder.toString();
private static String createMessage(Collection<Suggestion> suggestions) {
if (suggestions.isEmpty()) {
return "";
}

snippets.stream()
.filter(snippet -> snippet.getStepLocation().equals(testStep.getStep().getLocation()) &&
snippet.getUri().equals(testStep.getUri()))
.findFirst()
.ifPresent(event -> {
builder.append("You can implement missing steps with the snippets below:\n");
event.getSnippets().forEach(snippet -> {
builder.append(snippet);
builder.append("\n");
});
});
return builder.toString();
StringBuilder sb = new StringBuilder("You can implement this step");
if (suggestions.size() > 1) {
sb.append(" and ").append(suggestions.size() - 1).append(" other step(s)");
}
sb.append("using the snippet(s) below:\n\n");
String snippets = suggestions
.stream()
.map(Suggestion::getSnippets)
.flatMap(Collection::stream)
.distinct()
.collect(joining("\n", "", "\n"));
sb.append(snippets);
return sb.toString();
}

private void printTestCaseFinished(TestCaseFinished event) {
String timestamp = extractTimeStamp(event);
print(TEMPLATE_PROGRESS_TEST_FINISHED, timestamp);
finishNode(timestamp, currentStack.remove(currentStack.size() - 1));
this.currentTestCase = null;
}

private long extractDuration(Result result) {
Expand All @@ -342,7 +356,7 @@ private void printTestRunFinished(TestRunFinished event) {
}

private void handleSnippetSuggested(SnippetsSuggestedEvent event) {
snippets.add(event);
suggestions.add(event);
}

private void handleEmbedEvent(EmbedEvent event) {
Expand Down
21 changes: 16 additions & 5 deletions core/src/main/java/io/cucumber/core/runner/Runner.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,9 @@
import io.cucumber.core.snippets.SnippetGenerator;
import io.cucumber.core.stepexpression.StepTypeRegistry;
import io.cucumber.plugin.event.HookType;
import io.cucumber.plugin.event.Location;
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;

import java.net.URI;
import java.util.ArrayList;
Expand Down Expand Up @@ -147,17 +149,26 @@ private PickleStepDefinitionMatch matchStepToStepDefinition(Pickle pickle, Step
if (match != null) {
return match;
}
List<String> snippets = generateSnippetsForStep(step);
if (!snippets.isEmpty()) {
bus.send(new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), pickle.getScenarioLocation(),
step.getLocation(), snippets));
}
emitSnippetSuggestedEvent(pickle, step);
return new UndefinedPickleStepDefinitionMatch(pickle.getUri(), step);
} catch (AmbiguousStepDefinitionsException e) {
return new AmbiguousPickleStepDefinitionsMatch(pickle.getUri(), step, e);
}
}

private void emitSnippetSuggestedEvent(Pickle pickle, Step step) {
List<String> snippets = generateSnippetsForStep(step);
if (snippets.isEmpty()) {
return;
}
Suggestion suggestion = new Suggestion(step.getText(), snippets);
Location scenarioLocation = pickle.getLocation();
Location stepLocation = step.getLocation();
SnippetsSuggestedEvent event = new SnippetsSuggestedEvent(bus.getInstant(), pickle.getUri(), scenarioLocation,
stepLocation, suggestion);
bus.send(event);
}

private List<HookTestStep> createAfterStepHooks(List<String> tags) {
return createTestStepsForHooks(tags, glue.getAfterStepHooks(), HookType.AFTER_STEP);
}
Expand Down
61 changes: 5 additions & 56 deletions core/src/main/java/io/cucumber/core/runtime/TestCaseResultObserver.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,79 +2,46 @@

import io.cucumber.plugin.event.EventHandler;
import io.cucumber.plugin.event.EventPublisher;
import io.cucumber.plugin.event.PickleStepTestStep;
import io.cucumber.plugin.event.Result;
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
import io.cucumber.plugin.event.Status;
import io.cucumber.plugin.event.TestCaseFinished;
import io.cucumber.plugin.event.TestStep;
import io.cucumber.plugin.event.TestStepFinished;

import java.net.URI;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.TreeMap;
import java.util.function.Function;
import java.util.function.Supplier;

import static io.cucumber.plugin.event.Status.PASSED;
import static io.cucumber.plugin.event.Status.PENDING;
import static io.cucumber.plugin.event.Status.SKIPPED;
import static io.cucumber.plugin.event.Status.UNDEFINED;
import static java.util.Collections.unmodifiableList;
import static java.util.Objects.requireNonNull;

public final class TestCaseResultObserver implements AutoCloseable {

private final EventPublisher bus;
private final Map<StepLocation, List<String>> snippetsPerStep = new TreeMap<>();
private final List<Suggestion> suggestions = new ArrayList<>();
private final EventHandler<SnippetsSuggestedEvent> snippetsSuggested = this::handleSnippetSuggestedEvent;
private final EventHandler<TestStepFinished> testStepFinished = this::handleTestStepFinished;
private Result result;
private final EventHandler<TestCaseFinished> testCaseFinished = this::handleTestCaseFinished;

public TestCaseResultObserver(EventPublisher bus) {
this.bus = bus;
bus.registerHandlerFor(SnippetsSuggestedEvent.class, snippetsSuggested);
bus.registerHandlerFor(TestStepFinished.class, testStepFinished);
bus.registerHandlerFor(TestCaseFinished.class, testCaseFinished);
}

@Override
public void close() {
bus.removeHandlerFor(SnippetsSuggestedEvent.class, snippetsSuggested);
bus.removeHandlerFor(TestStepFinished.class, testStepFinished);
bus.removeHandlerFor(TestCaseFinished.class, testCaseFinished);
}

private void handleSnippetSuggestedEvent(SnippetsSuggestedEvent event) {
snippetsPerStep.putIfAbsent(new StepLocation(
event.getUri(),
event.getStepLine()),
event.getSnippets());
}

private void handleTestStepFinished(TestStepFinished event) {
Result result = event.getResult();
Status status = result.getStatus();
if (!status.is(UNDEFINED)) {
return;
}

TestStep testStep = event.getTestStep();
if (!(testStep instanceof PickleStepTestStep)) {
return;
}

PickleStepTestStep pickleStepTestStep = (PickleStepTestStep) testStep;
String stepText = pickleStepTestStep.getStepText();

List<String> snippets = snippetsPerStep.get(
new StepLocation(
pickleStepTestStep.getUri(),
pickleStepTestStep.getStepLine()));
suggestions.add(new Suggestion(stepText, snippets));
SnippetsSuggestedEvent.Suggestion s = event.getSuggestion();
suggestions.add(new Suggestion(s.getStep(), s.getSnippets()));
}

private void handleTestCaseFinished(TestCaseFinished event) {
Expand Down Expand Up @@ -117,32 +84,14 @@ public TestCaseFailed(Throwable throwable) {

}

private static final class StepLocation implements Comparable<StepLocation> {

private final URI uri;
private final int line;

private StepLocation(URI uri, int line) {
this.uri = uri;
this.line = line;
}

@Override
public int compareTo(StepLocation o) {
int order = uri.compareTo(o.uri);
return order != 0 ? order : Integer.compare(line, o.line);
}

}

public static final class Suggestion {

final String step;
final List<String> snippets;

public Suggestion(String step, List<String> snippets) {
this.step = step;
this.snippets = snippets;
this.step = requireNonNull(step);
this.snippets = unmodifiableList(requireNonNull(snippets));
}

public String getStep() {
Expand Down
5 changes: 3 additions & 2 deletions core/src/test/java/io/cucumber/core/plugin/CanonicalEventOrderTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
import io.cucumber.plugin.event.Location;
import io.cucumber.plugin.event.Result;
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;
import io.cucumber.plugin.event.Status;
import io.cucumber.plugin.event.TestCase;
import io.cucumber.plugin.event.TestCaseStarted;
Expand Down Expand Up @@ -48,13 +49,13 @@ class CanonicalEventOrderTest {
URI.create("file:path/to/1.feature"),
new Location(0, -1),
new Location(0, -1),
Collections.emptyList());
new Suggestion("", Collections.emptyList()));
private final Event suggested2 = new SnippetsSuggestedEvent(
ofEpochMilli(5),
URI.create("file:path/to/1.feature"),
new Location(0, -1),
new Location(0, -1),
Collections.emptyList());
new Suggestion("", Collections.emptyList()));
private final Event feature1Case1Started = createTestCaseEvent(
ofEpochMilli(5),
URI.create("file:path/to/1.feature"),
Expand Down
5 changes: 3 additions & 2 deletions core/src/test/java/io/cucumber/core/plugin/DefaultSummaryPrinterTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
import io.cucumber.plugin.event.Location;
import io.cucumber.plugin.event.Result;
import io.cucumber.plugin.event.SnippetsSuggestedEvent;
import io.cucumber.plugin.event.SnippetsSuggestedEvent.Suggestion;
import io.cucumber.plugin.event.Status;
import io.cucumber.plugin.event.TestRunFinished;
import org.junit.jupiter.api.BeforeEach;
Expand Down Expand Up @@ -44,14 +45,14 @@ void does_not_print_duplicate_snippets() {
URI.create("classpath:com/example.feature"),
new Location(12, -1),
new Location(13, -1),
singletonList("snippet")));
new Suggestion("", singletonList("snippet"))));

bus.send(new SnippetsSuggestedEvent(
bus.getInstant(),
URI.create("classpath:com/example.feature"),
new Location(12, -1),
new Location(14, -1),
singletonList("snippet")));
new Suggestion("", singletonList("snippet"))));

bus.send(new TestRunFinished(bus.getInstant(), new Result(Status.PASSED, Duration.ZERO, null)));

Expand Down
5 changes: 3 additions & 2 deletions core/src/test/java/io/cucumber/core/plugin/TeamCityPluginTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -199,7 +199,8 @@ void should_print_error_message_for_undefined_steps() {
Feature feature = TestFeatureParser.parse("path/test.feature", "" +
"Feature: feature name\n" +
" Scenario: scenario name\n" +
" Given first step\n");
" Given first step\n" +
" Given second step\n");

ByteArrayOutputStream out = new ByteArrayOutputStream();
Runtime.builder()
Expand All @@ -211,7 +212,7 @@ void should_print_error_message_for_undefined_steps() {
.run();

assertThat(out, bytesContainsString("" +
"##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step undefined' details = 'You can implement missing steps with the snippets below:|n|n' name = 'first step']\n"));
"##teamcity[testFailed timestamp = '1970-01-01T12:00:00.000+0000' duration = '0' message = 'Step undefined' details = 'You can implement this step and 1 other step(s)using the snippet(s) below:|n|ntest snippet 0|ntest snippet 1|n' name = 'first step']"));
}

@Test
Expand Down
4 changes: 3 additions & 1 deletion core/src/test/java/io/cucumber/core/snippets/TestSnippet.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,11 @@

public class TestSnippet implements Snippet {

private int i;

@Override
public MessageFormat template() {
return new MessageFormat("");
return new MessageFormat("test snippet " + i++);
}

@Override
Expand Down
Loading

0 comments on commit 558a17d

Please sign in to comment.