1369

.github/workflows/gradle.yml

@@ -421,6 +421,14 @@ jobs:
  name: BrowserStack - System Tests - Allure report - Android
  path: vividus-tests/output/reports/bs-android-allure
 
+ - name: Validate fail fast
+ if: matrix.platform == 'macos-latest'
+ shell: bash
+ run: |
+ ./gradlew :vividus-tests:validateRunStatistics \
+ -Pvividus.configuration.suites=failFast \
+ -PexpectedRunStatistics="resources/main/failFast/expected.json"
+
  - name: Publish snapshot artifacts
  if: github.ref == 'refs/heads/master' && matrix.platform == 'ubuntu-latest'
  env:

docs/modules/ROOT/pages/tests-configuration.adoc

@@ -50,13 +50,25 @@ NOTE: The properties marked with *bold* are mandatory.
 |suite meta-filter
 |The meta-filter used to filter the batch stories and scenarios
 
-|`bdd.batch-<batch-number>.ignore-failure`
-|`true`
-|If `false` and any failure is occurred during the batch execution, the subsequent batches are not executed.
+|`bdd.batch-<batch-number>.fail-fast`
+|`false`
+|If `true` and any failure is occurred during the batch execution, the subsequent batches will not be executed.
 
 |`bdd.batch-<batch-number>.story-execution-timeout`
 |`PT3H`
 |The max duration of the single story in the batch.
+
+|`bdd.batch.fail-fast`
+|`false`
+|If set to `true` the subsequent batches execution will be stopped after first failed assertion
+
+|`bdd.story.fail-fast`
+|`false`
+|If set to `true` the story execution will be stopped after first failed assertion
+
+|`bdd.scenario.fail-fast`
+|`false`
+|If set to `true` the scenario execution will be stopped after first failed assertion
 |===
 
 == Known Issues

vividus-allure-adaptor/build.gradle

@@ -6,6 +6,9 @@ dependencies {
  implementation project(':vividus-reporter')
  implementation project(':vividus-util')
 
+ api(group: 'io.qameta.allure', name: 'allure-model', version: versions.allure)
+ api(group: 'io.qameta.allure', name: 'allure-plugin-api', version: versions.allure)
+
  implementation(group: 'io.qameta.allure', name: 'allure-java-commons', version: versions.allure)
  implementation(group: 'io.qameta.allure', name: 'allure-generator', version: versions.allure)
  implementation(group: 'io.qameta.allure', name: 'allure-plugin-api', version: versions.allure)

vividus-allure-adaptor/src/main/java/org/vividus/bdd/report/allure/AllureStoryReporter.java

@@ -527,7 +527,7 @@ private void modifyStepTitle(String stepTitle)
 
  private boolean isStatusUpdateNeeded(WithStatus item, Status overrideStatus)
  {
- return StatusPriority.fromStatus(item.getStatus()).getPriority() > StatusPriority.fromStatus(overrideStatus)
+ return StatusPriority.from(item.getStatus()).getPriority() > StatusPriority.from(overrideStatus)
  .getPriority();
  }
 

vividus-allure-adaptor/src/main/java/org/vividus/bdd/report/allure/listener/AllureAssertionFailureListener.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,7 +21,6 @@
 import org.vividus.bdd.report.allure.IAllureStepReporter;
 import org.vividus.bdd.report.allure.model.StatusPriority;
 import org.vividus.softassert.event.AssertionFailedEvent;
-import org.vividus.softassert.model.SoftAssertionError;
 
 import io.qameta.allure.model.Status;
 
@@ -37,9 +36,7 @@ public AllureAssertionFailureListener(IAllureStepReporter allureStepReporter)
  @Subscribe
  public void onAssertionFailure(AssertionFailedEvent event)
  {
- SoftAssertionError softAssertionError = event.getSoftAssertionError();
- Status status = (softAssertionError.isKnownIssue() && !softAssertionError.getKnownIssue().isFixed()
- ? StatusPriority.KNOWN_ISSUES_ONLY : StatusPriority.FAILED).getStatusModel();
+ Status status = StatusPriority.from(event).getStatusModel();
  allureStepReporter.updateStepStatus(status);
  }
 }

vividus-allure-adaptor/src/main/java/org/vividus/bdd/report/allure/model/StatusPriority.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -18,6 +18,9 @@
 
 import java.util.function.Function;
 
+import org.vividus.softassert.event.AssertionFailedEvent;
+import org.vividus.softassert.model.SoftAssertionError;
+
 import io.qameta.allure.model.Status;
 
 public enum StatusPriority
@@ -43,16 +46,27 @@ public enum StatusPriority
  this.priority = priority;
  }
 
- public static StatusPriority fromStatus(Status status)
+ public static StatusPriority from(Status status)
  {
  return getStatusPriority(status, StatusPriority::getStatusModel);
  }
 
- public static StatusPriority fromStatus(io.qameta.allure.entity.Status status)
+ public static StatusPriority from(io.qameta.allure.entity.Status status)
  {
  return getStatusPriority(status, StatusPriority::getStatusEntity);
  }
 
+ public static StatusPriority from(AssertionFailedEvent event)
+ {
+ SoftAssertionError softAssertionError = event.getSoftAssertionError();
+ return isNotFixedKnownIssue(softAssertionError) ? KNOWN_ISSUES_ONLY : FAILED;
+ }
+
+ private static boolean isNotFixedKnownIssue(SoftAssertionError softAssertionError)
+ {
+ return softAssertionError.isKnownIssue() && !softAssertionError.getKnownIssue().isFixed();
+ }
+
  private static <T> StatusPriority getStatusPriority(T status, Function<StatusPriority, T> statusFromPriority)
  {
  for (StatusPriority statusPriority : values())

vividus-allure-adaptor/src/test/java/org/vividus/bdd/report/allure/model/StatusPriorityTests.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -17,8 +17,16 @@
 package org.vividus.bdd.report.allure.model;
 
 import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+import static org.mockito.Mockito.withSettings;
 
 import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import org.vividus.softassert.event.AssertionFailedEvent;
+import org.vividus.softassert.model.KnownIssue;
+import org.vividus.softassert.model.SoftAssertionError;
 
 import io.qameta.allure.model.Status;
 
@@ -33,6 +41,20 @@ void testGetLowest()
  @Test
  void testFromStatus()
  {
- assertEquals(StatusPriority.PASSED, StatusPriority.fromStatus(Status.PASSED));
+ assertEquals(StatusPriority.PASSED, StatusPriority.from(Status.PASSED));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"false,false,FAILED", "true,false,KNOWN_ISSUES_ONLY", "true,true,FAILED"})
+ void shoudCreateStatusPriorityFromAssertionFailedEvent(boolean known, boolean fixed, StatusPriority expected)
+ {
+ AssertionFailedEvent event = mock(AssertionFailedEvent.class);
+ SoftAssertionError softAssertionError = mock(SoftAssertionError.class, withSettings().lenient());
+ KnownIssue knownIssue = mock(KnownIssue.class, withSettings().lenient());
+ when(softAssertionError.isKnownIssue()).thenReturn(known);
+ when(softAssertionError.getKnownIssue()).thenReturn(knownIssue);
+ when(knownIssue.isFixed()).thenReturn(fixed);
+ when(event.getSoftAssertionError()).thenReturn(softAssertionError);
+ assertEquals(expected, StatusPriority.from(event));
  }
 }

vividus-analytics/src/main/java/org/vividus/analytics/AnalyticsStoryReporter.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -25,13 +25,13 @@
 
 import org.jbehave.core.model.Scenario;
 import org.jbehave.core.model.Story;
-import org.jbehave.core.reporters.NullStoryReporter;
 import org.vividus.analytics.model.AnalyticsEvent;
 import org.vividus.analytics.model.CustomDefinitions;
+import org.vividus.bdd.SystemStoriesAwareStoryReporter;
 import org.vividus.reporter.environment.EnvironmentConfigurer;
 import org.vividus.reporter.environment.PropertyCategory;
 
-public class AnalyticsStoryReporter extends NullStoryReporter
+public class AnalyticsStoryReporter extends SystemStoriesAwareStoryReporter
 {
  private static final AtomicLong STORIES = new AtomicLong();
  private static final AtomicLong SCENARIOS = new AtomicLong();
@@ -70,7 +70,8 @@ public void beforeStep(String step)
  STEPS.incrementAndGet();
  }
 
- private void beforeStories()
+ @Override
+ protected void beforeStories()
  {
  Map<String, String> payload = new HashMap<>();
  Map<String, String> configuration = getEnvironmentProperties(PropertyCategory.CONFIGURATION);
@@ -100,7 +101,8 @@ private void postPluginsAnalytic(Map<String, String> modules)
  });
  }
 
- public void afterStories()
+ @Override
+ protected void afterStories()
  {
  long duration = stopwatch.elapsed().toSeconds();
  Map<String, String> payload = new HashMap<>();
@@ -117,21 +119,4 @@ private Map<String, String> getEnvironmentProperties(PropertyCategory propertyCa
  {
  return EnvironmentConfigurer.ENVIRONMENT_CONFIGURATION.get(propertyCategory);
  }
-
- private boolean processSystemStory(String story)
- {
- if (story.startsWith("BeforeStories"))
- {
- beforeStories();
- }
- else if (story.startsWith("AfterStories"))
- {
- afterStories();
- }
- else
- {
- return false;
- }
- return true;
- }
 }

vividus-bdd-engine/src/main/java/org/vividus/bdd/BatchedEmbedder.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -50,7 +50,7 @@ public class BatchedEmbedder extends Embedder
  private boolean generateViewAfterBatches;
 
  private String batch;
- private boolean ignoreFailure;
+ private boolean failFast;
 
  public BatchedEmbedder(BddRunContext bddRunContext, IBddVariableContext bddVariableContext,
  BatchStorage batchStorage)
@@ -80,7 +80,7 @@ public void runStoriesAsPaths(Map<String, List<String>> storyPathsBatches)
  useEmbedderControls(createEmbedderControls(batchExecutionConfiguration));
  useMetaFilters(batchExecutionConfiguration.getMetaFilters());
 
- ignoreFailure = batchExecutionConfiguration.isIgnoreFailure();
+ failFast = batchExecutionConfiguration.isFailFast();
 
  EmbedderControls embedderControls = embedderControls();
  embedderMonitor.usingControls(embedderControls);
@@ -108,7 +108,7 @@ public void runStoriesAsPaths(Map<String, List<String>> storyPathsBatches)
  storyManager().runStoriesAsPaths(storyPaths, filter, failures);
 
  handleFailures(failures);
- if (!ignoreFailure && !failures.isEmpty())
+ if (failFast && !failures.isEmpty())
  {
  break;
  }
@@ -145,7 +145,7 @@ public PerformableTree performableTree()
  BatchedPerformableTree performableTree = (BatchedPerformableTree) super.performableTree();
  performableTree.setReportBeforeStories(reportBeforeStories);
  performableTree.setReportAfterStories(reportAfterStories);
- performableTree.setIgnoreFailureInBatches(ignoreFailure);
+ performableTree.setFailFast(failFast);
  return performableTree;
  }
 

vividus-bdd-engine/src/main/java/org/vividus/bdd/BatchedPerformableTree.java

@@ -1,5 +1,5 @@
 /*
- * Copyright 2019-2020 the original author or authors.
+ * Copyright 2019-2021 the original author or authors.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -21,15 +21,15 @@
 
 public class BatchedPerformableTree extends PerformableTree
 {
- private boolean ignoreFailureInBatches;
+ private boolean failFast;
  private boolean reportBeforeStories;
  private boolean reportAfterStories;
 
  @Override
  public void performBeforeOrAfterStories(RunContext context, Stage stage)
  {
  if (reportBeforeStories && Stage.BEFORE.equals(stage) || (Stage.AFTER.equals(stage)
- && (reportAfterStories || !ignoreFailureInBatches && !context.getFailures().isEmpty())))
+ && (reportAfterStories || failFast && !context.getFailures().isEmpty())))
  {
  super.performBeforeOrAfterStories(context, stage);
  }
@@ -45,8 +45,8 @@ public void setReportAfterStories(boolean reportAfterStories)
  this.reportAfterStories = reportAfterStories;
  }
 
- public void setIgnoreFailureInBatches(boolean ignoreFailureInBatches)
+ public void setFailFast(boolean failFast)
  {
- this.ignoreFailureInBatches = ignoreFailureInBatches;
+ this.failFast = failFast;
  }
 }

vividus-bdd-engine/src/main/java/org/vividus/bdd/SystemStoriesAwareStoryReporter.java

@@ -0,0 +1,49 @@
+/*
+ * Copyright 2021 the original author or authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.vividus.bdd;
+
+import org.jbehave.core.reporters.NullStoryReporter;
+
+public class SystemStoriesAwareStoryReporter extends NullStoryReporter
+{
+ protected void beforeStories()
+ {
+ //no default realization
+ }
+
+ protected void afterStories()
+ {
+ //no default realization
+ }
+
+ protected boolean processSystemStory(String story)
+ {
+ if (story.startsWith("BeforeStories"))
+ {
+ beforeStories();
+ }
+ else if (story.startsWith("AfterStories"))
+ {
+ afterStories();
+ }
+ else
+ {
+ return false;
+ }
+ return true;
+ }
+}