test_runner: support expecting a test-case to fail

Co-Authored-By: Alejandro Espa <98526766+vespa7@users.noreply.github.com>
PR-URL: #60669
Reviewed-By: Aviv Keller <me@aviv.sh>
Reviewed-By: Pietro Marchini <pietro.marchini94@gmail.com>
Reviewed-By: Moshe Atlow <moshe@atlow.co.il>
Reviewed-By: Jordan Harband <ljharb@gmail.com>

Author: 2 people

doc/api/test.md

@@ -224,6 +224,57 @@ test('todo() method with message', (t) => {
 });
 ```
 
+## Expecting tests to fail
+
+<!-- YAML
+added:
+ - REPLACEME
+-->
+
+This flips the pass/fail reporting for a specific test or suite: A flagged test/test-case must throw
+in order to "pass"; a test/test-case that does not throw, fails.
+
+In the following, `doTheThing()` returns _currently_ `false` (`false` does not equal `true`, causing
+`strictEqual` to throw, so the test-case passes).
+
+```js
+it.expectFailure('should do the thing', () => {
+  assert.strictEqual(doTheThing(), true);
+});
+
+it('should do the thing', { expectFailure: true }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+```
+
+`skip` and/or `todo` are mutually exclusive to `expectFailure`, and `skip` or `todo`
+will "win" when both are applied (`skip` wins against both, and `todo` wins
+against `expectFailure`).
+
+These tests will be skipped (and not run):
+
+```js
+it.expectFailure('should do the thing', { skip: true }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+
+it.skip('should do the thing', { expectFailure: true }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+```
+
+These tests will be marked "todo" (silencing errors):
+
+```js
+it.expectFailure('should do the thing', { todo: true }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+
+it.todo('should do the thing', { expectFailure: true }, () => {
+  assert.strictEqual(doTheThing(), true);
+});
+```
+
 ## `describe()` and `it()` aliases
 
 Suites and tests can also be written using the `describe()` and `it()`

lib/internal/test_runner/harness.js

@@ -377,7 +377,7 @@ function runInParentContext(Factory) {
 
     return run(name, options, fn, overrides);
   };
-  ArrayPrototypeForEach(['skip', 'todo', 'only'], (keyword) => {
+  ArrayPrototypeForEach(['expectFailure', 'skip', 'todo', 'only'], (keyword) => {
     test[keyword] = (name, options, fn) => {
       const overrides = {
         __proto__: null,

lib/internal/test_runner/reporter/tap.js

@@ -33,12 +33,12 @@ async function * tapReporter(source) {
   for await (const { type, data } of source) {
     switch (type) {
       case 'test:fail': {
-        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo);
+        yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo, data.expectFailure);
         const location = data.file ? `${data.file}:${data.line}:${data.column}` : null;
         yield reportDetails(data.nesting, data.details, location);
         break;
       } case 'test:pass':
-        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo);
+        yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo, data.expectFailure);
         yield reportDetails(data.nesting, data.details, null);
         break;
       case 'test:plan':
@@ -65,7 +65,7 @@ async function * tapReporter(source) {
   }
 }
 
-function reportTest(nesting, testNumber, status, name, skip, todo) {
+function reportTest(nesting, testNumber, status, name, skip, todo, expectFailure) {
   let line = `${indent(nesting)}${status} ${testNumber}`;
 
   if (name) {
@@ -76,6 +76,8 @@ function reportTest(nesting, testNumber, status, name, skip, todo) {
     line += ` # SKIP${typeof skip === 'string' && skip.length ? ` ${tapEscape(skip)}` : ''}`;
   } else if (todo !== undefined) {
     line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`;
+  } else if (expectFailure !== undefined) {
+    line += ' # EXPECTED FAILURE';
   }
 
   line += '\n';

lib/internal/test_runner/reporter/utils.js

@@ -71,14 +71,16 @@ function formatError(error, indent) {
 function formatTestReport(type, data, prefix = '', indent = '', hasChildren = false, showErrorDetails = true) {
   let color = reporterColorMap[type] ?? colors.white;
   let symbol = reporterUnicodeSymbolMap[type] ?? ' ';
-  const { skip, todo } = data;
+  const { skip, todo, expectFailure } = data;
   const duration_ms = data.details?.duration_ms ? ` ${colors.gray}(${data.details.duration_ms}ms)${colors.white}` : '';
   let title = `${data.name}${duration_ms}`;
 
   if (skip !== undefined) {
     title += ` # ${typeof skip === 'string' && skip.length ? skip : 'SKIP'}`;
   } else if (todo !== undefined) {
     title += ` # ${typeof todo === 'string' && todo.length ? todo : 'TODO'}`;
+  } else if (expectFailure !== undefined) {
+    title += ` # EXPECTED FAILURE`;
   }
 
   const error = showErrorDetails ? formatError(data.details?.error, indent) : '';

lib/internal/test_runner/test.js

@@ -496,7 +496,7 @@ class Test extends AsyncResource {
     super('Test');
 
     let { fn, name, parent } = options;
-    const { concurrency, entryFile, loc, only, timeout, todo, skip, signal, plan } = options;
+    const { concurrency, entryFile, expectFailure, loc, only, timeout, todo, skip, signal, plan } = options;
 
     if (typeof fn !== 'function') {
       fn = noop;
@@ -635,6 +635,7 @@ class Test extends AsyncResource {
     this.plan = null;
     this.expectedAssertions = plan;
     this.cancelled = false;
+    this.expectFailure = expectFailure !== undefined && expectFailure !== false;
     this.skipped = skip !== undefined && skip !== false;
     this.isTodo = (todo !== undefined && todo !== false) || this.parent?.isTodo;
     this.startTime = null;
@@ -938,7 +939,12 @@ class Test extends AsyncResource {
       return;
     }
 
-    this.passed = false;
+    if (this.expectFailure === true) {
+      this.passed = true;
+    } else {
+      this.passed = false;
+    }
+
     this.error = err;
   }
 
@@ -1335,6 +1341,8 @@ class Test extends AsyncResource {
       directive = this.reporter.getSkip(this.message);
     } else if (this.isTodo) {
       directive = this.reporter.getTodo(this.message);
+    } else if (this.expectFailure) {
+      directive = this.reporter.getXFail(this.expectFailure); // TODO(@JakobJingleheimer): support specifying failure
     }
 
     if (this.reportedType) {
@@ -1349,6 +1357,7 @@ class Test extends AsyncResource {
     if (this.passedAttempt !== undefined) {
       details.passed_on_attempt = this.passedAttempt;
     }
+
     return { __proto__: null, details, directive };
   }
 

lib/internal/test_runner/tests_stream.js

@@ -87,6 +87,10 @@ class TestsStream extends Readable {
     return { __proto__: null, todo: reason ?? true };
   }
 
+  getXFail(expectation = undefined) {
+    return { __proto__: null, expectFailure: expectation ?? true };
+  }
+
   enqueue(nesting, loc, name, type) {
     this[kEmitMessage]('test:enqueue', {
       __proto__: null,

test/fixtures/test-runner/output/describe_it.js

@@ -5,8 +5,23 @@ const { describe, it, test } = require('node:test');
 const util = require('util');
 
 
-it.todo('sync pass todo', () => {
+it.expectFailure('sync expect fail (method)', () => {
+  throw new Error('should pass');
+});
+
+it('sync expect fail (options)', { expectFailure: true }, () => {
+  throw new Error('should pass');
+});
 
+it.expectFailure('async expect fail (method)', async () => {
+  throw new Error('should pass');
+});
+
+it('async expect fail (options)', { expectFailure: true }, async () => {
+  throw new Error('should pass');
+});
+
+it.todo('sync pass todo', () => {
 });
 
 it('sync pass todo with message', { todo: 'this is a passing todo' }, () => {
@@ -16,13 +31,21 @@ it.todo('sync todo', () => {
   throw new Error('should not count as a failure');
 });
 
+it.todo('sync todo with expect fail', { expectFailure: true }, () => {
+  throw new Error('should not count as an expected failure');
+});
+
 it('sync todo with message', { todo: 'this is a failing todo' }, () => {
   throw new Error('should not count as a failure');
 });
 
 it.skip('sync skip pass', () => {
 });
 
+it.skip('sync skip expect fail', { expectFailure: true }, () => {
+  throw new Error('should not fail');
+});
+
 it('sync skip pass with message', { skip: 'this is skipped' }, () => {
 });