test_runner: bail

doc/api/cli.md

@@ -1507,6 +1507,18 @@ Starts the Node.js command line test runner. This flag cannot be combined with
 See the documentation on [running tests from the command line][]
 for more details.
 
+### `--test-bail`
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+Instructs the test runner to bail out if a test failure occurs.
+See the documentation on [test bailout][] for more details.
+
 ### `--test-name-pattern`
 
 <!-- YAML
@@ -2711,6 +2723,7 @@ done
 [security warning]: #warning-binding-inspector-to-a-public-ipport-combination-is-insecure
 [semi-space]: https://www.memorymanagement.org/glossary/s.html#semi.space
 [single executable application]: single-executable-applications.md
+[test bailout]: test.md#bailing-out
 [test reporters]: test.md#test-reporters
 [timezone IDs]: https://en.wikipedia.org/wiki/List_of_tz_database_time_zones
 [tracking issue for user-land snapshots]: https://github.com/nodejs/node/issues/44014

doc/api/test.md

@@ -602,6 +602,22 @@ test('mocks setTimeout to be executed synchronously without having to actually w
 });
 ```
 
+## Bailing out
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+> Stability: 1 - Experimental
+
+The `--test-bail` flag provides a way to stop the test execution
+as soon as a test fails.
+By enabling this flag, the test runner will exit the test suite early
+when it encounters the first failing test, preventing
+the execution of subsequent tests.
+**Default:** `false`.
+
 ## Test reporters
 
 <!-- YAML
@@ -870,6 +886,11 @@ changes:
 
 * `options` {Object} Configuration options for running tests. The following
  properties are supported:
+ * `bail` {boolean}
+ If `true`, it will exit the test suite early
+ when it encounters the first failing test, preventing
+ the execution of subsequent tests.
+ **Default:** `false`.
  * `concurrency` {number|boolean} If a number is provided,
  then that many test processes would run in parallel, where each process
  corresponds to one test file.

doc/node.1

@@ -410,6 +410,9 @@ Specify the minimum allocation from the OpenSSL secure heap. The default is 2. T
 .It Fl -test
 Starts the Node.js command line test runner.
 .
+.It Fl -test-bail
+Instructs the test runner to bail out if a test failure occurs.
+.
 .It Fl -test-name-pattern
 A regular expression that configures the test runner to only execute tests
 whose name matches the provided pattern.

lib/internal/main/test_runner.js

@@ -57,7 +57,12 @@ if (shardOption) {
  };
 }
 
-run({ concurrency, inspectPort, watch: getOptionValue('--watch'), setup: setupTestReporters, shard })
+run({ concurrency,
+ inspectPort,
+ watch: getOptionValue('--watch'),
+ setup: setupTestReporters,
+ shard,
+ bail: getOptionValue('--test-bail') })
 .on('test:fail', (data) => {
  if (data.todo === undefined || data.todo === false) {
  process.exitCode = kGenericUserError;

lib/internal/test_runner/reporter/tap.js

@@ -14,6 +14,7 @@ const {
 const { inspectWithNoCustomRetry } = require('internal/errors');
 const { isError, kEmptyObject } = require('internal/util');
 const { getCoverageReport } = require('internal/test_runner/utils');
+const { kBailedOut } = require('internal/test_runner/test');
 const kDefaultIndent = ' '; // 4 spaces
 const kFrameStartRegExp = /^ {4}at /;
 const kLineBreakRegExp = /\n|\r\n/;
@@ -32,6 +33,8 @@ async function * tapReporter(source) {
  for await (const { type, data } of source) {
  switch (type) {
  case 'test:fail': {
+ // If test bailed out do not report
+ if (data.details?.error?.failureType === kBailedOut) break;
  yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo);
  const location = `${data.file}:${data.line}:${data.column}`;
  yield reportDetails(data.nesting, data.details, location);

lib/internal/test_runner/runner.js

@@ -110,14 +110,17 @@ function filterExecArgv(arg, i, arr) {
  !ArrayPrototypeSome(kFilterArgValues, (p) => arg === p || (i > 0 && arr[i - 1] === p) || StringPrototypeStartsWith(arg, `${p}=`));
 }
 
-function getRunArgs({ path, inspectPort, testNamePatterns }) {
+function getRunArgs({ path, inspectPort, testNamePatterns, bail }) {
  const argv = ArrayPrototypeFilter(process.execArgv, filterExecArgv);
  if (isUsingInspector()) {
  ArrayPrototypePush(argv, `--inspect-port=${getInspectPort(inspectPort)}`);
  }
  if (testNamePatterns) {
  ArrayPrototypeForEach(testNamePatterns, (pattern) => ArrayPrototypePush(argv, `--test-name-pattern=${pattern}`));
  }
+ if (bail) {
+ ArrayPrototypePush(argv, '--test-bail');
+ }
  ArrayPrototypePush(argv, path);
 
  return argv;
@@ -301,10 +304,10 @@ class FileTest extends Test {
  }
 }
 
-function runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns) {
+function runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns, bail) {
  const watchMode = filesWatcher != null;
  const subtest = root.createSubtest(FileTest, path, async (t) => {
- const args = getRunArgs({ __proto__: null, path, inspectPort, testNamePatterns });
+ const args = getRunArgs({ __proto__: null, path, inspectPort, testNamePatterns, bail });
  const stdio = ['pipe', 'pipe', 'pipe'];
  const env = { __proto__: null, ...process.env, NODE_TEST_CONTEXT: 'child-v8' };
  if (watchMode) {
@@ -381,7 +384,7 @@ function runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns) {
  return subtest.start();
 }
 
-function watchFiles(testFiles, root, inspectPort, signal, testNamePatterns) {
+function watchFiles(testFiles, root, inspectPort, signal, testNamePatterns, bail) {
  const runningProcesses = new SafeMap();
  const runningSubtests = new SafeMap();
  const watcher = new FilesWatcher({ __proto__: null, debounce: 200, mode: 'filter', signal });
@@ -403,7 +406,7 @@ function watchFiles(testFiles, root, inspectPort, signal, testNamePatterns) {
  root.harness.counters.topLevel = 0;
  }
  await runningSubtests.get(file);
- runningSubtests.set(file, runTestFile(file, root, inspectPort, filesWatcher, testNamePatterns));
+ runningSubtests.set(file, runTestFile(file, root, inspectPort, filesWatcher, testNamePatterns, bail));
  }, undefined, (error) => {
  triggerUncaughtException(error, true /* fromPromise */);
  }));
@@ -425,14 +428,17 @@ function run(options) {
  options = kEmptyObject;
  }
  let { testNamePatterns, shard } = options;
- const { concurrency, timeout, signal, files, inspectPort, watch, setup } = options;
+ const { concurrency, timeout, signal, files, inspectPort, watch, setup, bail } = options;
 
  if (files != null) {
  validateArray(files, 'options.files');
  }
  if (watch != null) {
  validateBoolean(watch, 'options.watch');
  }
+ if (bail != null) {
+ validateBoolean(bail, 'options.bail');
+ }
  if (shard != null) {
  validateObject(shard, 'options.shard');
  // Avoid re-evaluating the shard object in case it's a getter
@@ -479,13 +485,13 @@ function run(options) {
  let postRun = () => root.postRun();
  let filesWatcher;
  if (watch) {
- filesWatcher = watchFiles(testFiles, root, inspectPort, signal, testNamePatterns);
+ filesWatcher = watchFiles(testFiles, root, inspectPort, signal, testNamePatterns, bail);
  postRun = undefined;
  }
  const runFiles = () => {
  root.harness.bootstrapComplete = true;
  return SafePromiseAllSettledReturnVoid(testFiles, (path) => {
- const subtest = runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns);
+ const subtest = runTestFile(path, root, inspectPort, filesWatcher, testNamePatterns, bail);
  filesWatcher?.runningSubtests.set(path, subtest);
  return subtest;
  });

lib/internal/test_runner/test.js

@@ -66,6 +66,7 @@ const kParentAlreadyFinished = 'parentAlreadyFinished';
 const kSubtestsFailed = 'subtestsFailed';
 const kTestCodeFailure = 'testCodeFailure';
 const kTestTimeoutFailure = 'testTimeoutFailure';
+const kBailedOut = 'testBailedOut';
 const kHookFailure = 'hookFailed';
 const kDefaultTimeout = null;
 const noop = FunctionPrototype;
@@ -75,7 +76,7 @@ const kHookNames = ObjectSeal(['before', 'after', 'beforeEach', 'afterEach']);
 const kUnwrapErrors = new SafeSet()
  .add(kTestCodeFailure).add(kHookFailure)
  .add('uncaughtException').add('unhandledRejection');
-const { testNamePatterns, testOnlyFlag } = parseCommandLine();
+const { testNamePatterns, testOnlyFlag, bail } = parseCommandLine();
 let kResistStopPropagation;
 
 function stopTest(timeout, signal) {
@@ -204,6 +205,7 @@ class Test extends AsyncResource {
  abortController;
  outerSignal;
  #reportedSubtest;
+ bailedOut;
 
  constructor(options) {
  super('Test');
@@ -469,13 +471,14 @@ class Test extends AsyncResource {
  if (this.endTime !== null) {
  return;
  }
-
- this.fail(error ||
- new ERR_TEST_FAILURE(
-  'test did not finish before its parent and was cancelled',
-  kCancelledByParent,
- ),
+ const cancelledError = this.root.bailedOut ? new ERR_TEST_FAILURE(
+  'test bailed out',
+ kBailedOut,
+ ) : new ERR_TEST_FAILURE(
+ 'test did not finish before its parent and was cancelled',
+ kCancelledByParent,
  );
+ this.fail(error || cancelledError);
  this.startTime = this.startTime || this.endTime; // If a test was canceled before it was started, e.g inside a hook
  this.cancelled = true;
  this.abortController.abort();
@@ -500,6 +503,11 @@ class Test extends AsyncResource {
  this.endTime = hrtime();
  this.passed = false;
  this.error = err;
+ if (bail && !this.root.bailedOut) {
+ this.bailedOut = true;
+ this.root.bailedOut = true;
+ this.root.postRun();
+ }
  }
 
  pass() {
@@ -811,7 +819,7 @@ class Test extends AsyncResource {
 
  if (this.passed) {
  this.reporter.ok(this.nesting, this.loc, this.testNumber, this.name, details, directive);
- } else {
+ } else if (this.error.failureType !== kBailedOut) {
  details.error = this.error;
  this.reporter.fail(this.nesting, this.loc, this.testNumber, this.name, details, directive);
  }
@@ -822,7 +830,7 @@ class Test extends AsyncResource {
  }
 
  reportStarted() {
- if (this.#reportedSubtest || this.parent === null) {
+ if (this.#reportedSubtest || this.parent === null || this.root.bailedOut) {
  return;
  }
  this.#reportedSubtest = true;
@@ -968,6 +976,7 @@ module.exports = {
  kSubtestsFailed,
  kTestCodeFailure,
  kTestTimeoutFailure,
+ kBailedOut,
  kAborted,
  kUnwrapErrors,
  Suite,

lib/internal/test_runner/utils.js

@@ -179,6 +179,7 @@ function parseCommandLine() {
  }
 
  const isTestRunner = getOptionValue('--test');
+ const bail = getOptionValue('--test-bail');
  const coverage = getOptionValue('--experimental-test-coverage');
  const isChildProcess = process.env.NODE_TEST_CONTEXT === 'child';
  const isChildProcessV8 = process.env.NODE_TEST_CONTEXT === 'child-v8';
@@ -230,6 +231,7 @@ function parseCommandLine() {
  globalTestOptions = {
  __proto__: null,
  isTestRunner,
+ bail,
  coverage,
  testOnlyFlag,
  testNamePatterns,

src/node_options.cc

@@ -606,6 +606,9 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() {
  "run test at specific shard",
  &EnvironmentOptions::test_shard,
  kAllowedInEnvvar);
+ AddOption("--test-bail",
+ "stop test runner execution on the first test failure",
+ &EnvironmentOptions::test_bail);
  AddOption("--test-udp-no-try-send", "", // For testing only.
  &EnvironmentOptions::test_udp_no_try_send);
  AddOption("--throw-deprecation",

src/node_options.h

@@ -161,6 +161,7 @@ class EnvironmentOptions : public Options {
  std::string env_file;
  bool has_env_file_string = false;
  bool test_runner = false;
+ bool test_bail = false;
  bool test_runner_coverage = false;
  std::vector<std::string> test_name_pattern;
  std::vector<std::string> test_reporter;

test/fixtures/test-runner/bail/bail.js

@@ -0,0 +1,18 @@
+'use strict';
+const common = require('../../../common');
+const test = require('node:test');
+
+test('nested', (t) => {
+ t.test('first', () => {});
+ t.test('second', () => {
+ throw new Error();
+ });
+ t.test('third', common.mustNotCall(() => {}));
+});
+
+test('top level', (t) => {
+ t.test('forth', () => {});
+ t.test('fifth', () => {
+ throw new Error();
+ });
+});

test/fixtures/test-runner/bail/error.js

@@ -0,0 +1,12 @@
+'use strict';
+const common = require('../../../common');
+const assert = require('assert');
+const test = require('node:test');
+
+test('keep error', (t) => {
+ assert.strictEqual(0, 1);
+});
+
+test('dont show', common.mustNotCall((t) => {
+ assert.strictEqual(0, 2);
+}));

test/fixtures/test-runner/output/bail-error.js

@@ -0,0 +1,13 @@
+// Flags: --test-bail --test-reporter=tap
+'use strict';
+const common = require('../../../common');
+const assert = require('assert');
+const test = require('node:test');
+
+test('keep error', (t) => {
+ assert.strictEqual(0, 1);
+});
+
+test('dont show', common.mustNotCall((t) => {
+ assert.strictEqual(0, 2);
+}));

test/fixtures/test-runner/output/bail-error.snapshot

@@ -0,0 +1,32 @@
+TAP version 13
+not ok 1 - keep error
+ ---
+ duration_ms: *
+ location: '/test/fixtures/test-runner/output/bail-error.js:(LINE):1'
+ failureType: 'testCodeFailure'
+ error: |-
+ Expected values to be strictly equal:
+
+ 0 !== 1
+
+ code: 'ERR_ASSERTION'
+ name: 'AssertionError'
+ expected: 1
+ actual: 0
+ operator: 'strictEqual'
+ stack: |-
+ *
+ *
+ *
+ *
+ *
+ ...
+1..2
+# tests 2
+# suites 0
+# pass 0
+# fail 1
+# cancelled 1
+# skipped 0
+# todo 0
+# duration_ms *