new configure abortSuiteOnFailure implemented #2090

karatelabs · Oct 31, 2022 · 03a8408 · 03a8408
1 parent 88fca8f
commit 03a8408
Show file tree

Hide file tree

Showing 5 changed files with 49 additions and 18 deletions.
diff --git a/README.md b/README.md
@@ -2238,6 +2238,7 @@ You can adjust configuration settings for the HTTP client used by Karate using t
 `driverTarget` | JSON / Java Object | See [`configure driverTarget`](karate-core#configure-drivertarget)
 `pauseIfNotPerf` | boolean | defaults to `false`, relevant only for performance-testing, see [`karate.pause()`](#karate-pause) and [`karate-gatling`](karate-gatling#think-time)
 `xmlNamespaceAware` | boolean | defaults to `false`, to handle XML namespaces in [some special circumstances](https://github.com/karatelabs/karate/issues/1587)
+`abortSuiteOnFailure` | boolean | defaults to `false`, to not attempt to run any more tests upon a failure
 
 Examples:
 ```cucumber

diff --git a/karate-core/src/main/java/com/intuit/karate/Suite.java b/karate-core/src/main/java/com/intuit/karate/Suite.java
@@ -23,7 +23,6 @@
  */
 package com.intuit.karate;
 
-import com.intuit.karate.core.Feature;
 import com.intuit.karate.core.FeatureCall;
 import com.intuit.karate.core.FeatureResult;
 import com.intuit.karate.core.FeatureRuntime;
@@ -52,6 +51,7 @@
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.locks.ReentrantLock;
 import java.util.stream.Stream;
 import org.slf4j.LoggerFactory;
@@ -67,6 +67,7 @@ public class Suite implements Runnable {
     public final long startTime;
     protected long endTime;
     protected int skippedCount;
+    private AtomicBoolean abort = new AtomicBoolean(false);
 
     public final String env;
     public final String tagSelector;
@@ -259,6 +260,14 @@ public void run() {
             hooks.forEach(h -> h.afterSuite(this));
         }
     }
+
+    public void abort() {
+        abort.set(true);
+    }
+
+    public boolean isAborted() {
+        return abort.get();
+    }
 
     public void saveFeatureResults(FeatureResult fr) {
         File file = ReportUtils.saveKarateJson(reportDir, fr, null);

diff --git a/karate-core/src/main/java/com/intuit/karate/core/Config.java b/karate-core/src/main/java/com/intuit/karate/core/Config.java
@@ -90,6 +90,7 @@ public class Config {
     private Variable responseHeaders = Variable.NULL;
     private List<Method> continueOnStepFailureMethods = new ArrayList<>();
     private boolean continueAfterContinueOnStepFailure;
+    private boolean abortSuiteOnFailure;
 
     // retry config
     private int retryInterval = DEFAULT_RETRY_INTERVAL;
@@ -201,6 +202,8 @@ public boolean configure(String key, Variable value) { // TODO use enum
                 return false;
             case "abortedStepsShouldPass":
                 abortedStepsShouldPass = value.isTrue();
+            case "abortSuiteOnFailure":
+                abortSuiteOnFailure = value.isTrue();
                 return false;
             case "callSingleCache":
                 if (value.isMap()) {
@@ -344,6 +347,7 @@ public Config(Config parent) {
         afterFeature = parent.afterFeature;
         continueOnStepFailureMethods = parent.continueOnStepFailureMethods;
         continueAfterContinueOnStepFailure = parent.continueAfterContinueOnStepFailure;
+        abortSuiteOnFailure = parent.abortSuiteOnFailure;
         imageComparisonOptions = parent.imageComparisonOptions;
     }
 
@@ -563,6 +567,14 @@ public void setContinueAfterContinueOnStepFailure(boolean continueAfterContinueO
         this.continueAfterContinueOnStepFailure = continueAfterContinueOnStepFailure;
     }
 
+    public void setAbortSuiteOnFailure(boolean abortSuiteOnFailure) {
+        this.abortSuiteOnFailure = abortSuiteOnFailure;
+    }
+
+    public boolean isAbortSuiteOnFailure() {
+        return abortSuiteOnFailure;
+    }        
+
     public Map<String, Object> getImageComparisonOptions() {
         return imageComparisonOptions;
     }

diff --git a/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java b/karate-core/src/main/java/com/intuit/karate/core/ScenarioRuntime.java
@@ -59,8 +59,8 @@ public class ScenarioRuntime implements Runnable {
     public final boolean perfMode;
     public final boolean dryRun;
     public final LogAppender logAppender;
-    
-    private boolean skipBackground; 
+
+    private boolean skipBackground;
     private boolean ignoringFailureSteps;
 
     public ScenarioRuntime(FeatureRuntime featureRuntime, Scenario scenario) {
@@ -83,19 +83,19 @@ public ScenarioRuntime(FeatureRuntime featureRuntime, Scenario scenario) {
         actions = new ScenarioActions(engine);
         this.scenario = scenario;
         if (scenario.isDynamic() && !scenario.isOutlineExample()) { // from dynamic scenario iterator
-            steps = Collections.emptyList();             
+            steps = Collections.emptyList();
             skipped = true; // ensures run() is a no-op
             magicVariables = Collections.emptyMap();
         } else {
             magicVariables = initMagicVariables();
-        }        
-        result = new ScenarioResult(scenario);        
+        }
+        result = new ScenarioResult(scenario);
         dryRun = featureRuntime.suite.dryRun;
         tags = scenario.getTagsEffective();
         reportDisabled = perfMode ? true : tags.valuesFor("report").isAnyOf("false");
-        selectedForExecution = isSelectedForExecution(featureRuntime, scenario, tags);        
+        selectedForExecution = isSelectedForExecution(featureRuntime, scenario, tags);
     }
-    
+
     private Map<String, Object> initMagicVariables() {
         // magic variables are only in the JS engine - [ see ScenarioEngine.init() ]
         // and not "visible" and tracked in ScenarioEngine.vars
@@ -142,7 +142,7 @@ public boolean isSkipBackground() {
 
     public void setSkipBackground(boolean skipBackground) {
         this.skipBackground = skipBackground;
-    }    
+    }
 
     public String getEmbedFileName(ResourceType resourceType) {
         String extension = resourceType == null ? null : resourceType.getExtension();
@@ -319,16 +319,16 @@ private static boolean isSelectedForExecution(FeatureRuntime fr, Scenario scenar
             logger.trace("skipping scenario at line: {} - {}, needed: {}", scenario.getLine(), scenario.getName(), callName);
             return false;
         }
-        String callTag = fr.featureCall.callTag;       
+        String callTag = fr.featureCall.callTag;
         if (callTag != null && (!fr.caller.isNone() || fr.perfHook != null)) {
-                // only if this is a legit "call" or a gatling "call by tag"
-                if (tags.contains(callTag)) {
-                    logger.info("{} - call by tag at line {}: {}", fr, scenario.getLine(), callTag);
-                    return true;
-                }
-                logger.trace("skipping scenario at line: {} with call by tag effective: {}", scenario.getLine(), callTag);
-                return false;                
+            // only if this is a legit "call" or a gatling "call by tag"
+            if (tags.contains(callTag)) {
+                logger.info("{} - call by tag at line {}: {}", fr, scenario.getLine(), callTag);
+                return true;
             }
+            logger.trace("skipping scenario at line: {} with call by tag effective: {}", scenario.getLine(), callTag);
+            return false;
+        }
         if (fr.caller.isNone()) {
             if (tags.evaluate(fr.suite.tagSelector, fr.suite.env)) {
                 logger.trace("matched scenario at line: {} with tags effective: {}", scenario.getLine(), tags.getTags());
@@ -343,7 +343,11 @@ private static boolean isSelectedForExecution(FeatureRuntime fr, Scenario scenar
 
     //==========================================================================
     //
-    public void beforeRun() {        
+    public void beforeRun() {
+        if (featureRuntime.caller.isNone() && featureRuntime.suite.isAborted()) {
+            skipped = true;
+            return;
+        }
         steps = skipBackground ? scenario.getSteps() : scenario.getStepsIncludingBackground();
         ScenarioEngine.set(engine);
         engine.init();
@@ -395,6 +399,9 @@ public void run() {
         } finally {
             if (!skipped) {
                 afterRun();
+                if (isFailed() && engine.getConfig().isAbortSuiteOnFailure()) {
+                    featureRuntime.suite.abort();
+                }
             }
             if (caller.isNone()) {
                 logAppender.close(); // reclaim memory
@@ -493,6 +500,7 @@ public void afterRun() {
             }
             addStepLogEmbedsAndCallResults();
         } catch (Exception e) {
+            error = e;
             logError("scenario [cleanup] failed\n" + e.getMessage());
             currentStepResult = result.addFakeStepResult("scenario [cleanup] failed", e);
         }

diff --git a/karate-demo/src/test/java/karate-config.js b/karate-demo/src/test/java/karate-config.js
@@ -1,6 +1,7 @@
 function fn() {
   karate.configure('connectTimeout', 5000);
   karate.configure('readTimeout', 5000);  
+  // karate.configure('abortSuiteOnFailure', true);
   var port = karate.properties['demo.server.port'] || '8080';
   var protocol = 'http';
   if (karate.properties['demo.server.https'] === 'true') {