From 4d50db14b3293e4bbc12f4e26b67df5cdc4e6cbf Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Mon, 19 Dec 2022 19:35:57 +0200 Subject: [PATCH] test_runner: add reporters PR-URL: https://github.com/nodejs/node/pull/45712 Backport-PR-URL: https://github.com/nodejs/node/pull/46839 Fixes: https://github.com/nodejs/node/issues/45648 Reviewed-By: Antoine du Hamel Reviewed-By: Colin Ihrig Reviewed-By: Benjamin Gruenbaum --- doc/api/cli.md | 19 ++ doc/api/test.md | 247 ++++++++++++--- lib/internal/main/test_runner.js | 7 +- lib/internal/modules/run_main.js | 25 +- lib/internal/modules/utils.js | 54 ++++ lib/internal/test_runner/harness.js | 4 +- lib/internal/test_runner/runner.js | 33 ++- lib/internal/test_runner/test.js | 50 ++-- lib/internal/test_runner/tests_stream.js | 74 +++++ lib/internal/test_runner/utils.js | 75 ++++- .../{yaml_parser.js => yaml_to_js.js} | 0 lib/internal/util/colors.js | 2 + lib/test/reporter/dot.js | 17 ++ lib/test/reporter/spec.js | 107 +++++++ .../tap_stream.js => test/reporter/tap.js} | 146 ++++----- src/node_options.cc | 6 + src/node_options.h | 2 + .../test-runner/custom_reporters/custom.cjs | 17 ++ .../test-runner/custom_reporters/custom.js | 8 + .../test-runner/custom_reporters/custom.mjs | 8 + test/fixtures/test-runner/reporters.js | 11 + test/message/test_runner_describe_it.out | 7 - test/message/test_runner_hooks.out | 3 - test/message/test_runner_output.js | 14 +- test/message/test_runner_output.out | 7 - test/message/test_runner_output_cli.js | 3 +- test/message/test_runner_output_cli.out | 7 - .../test_runner_output_dot_reporter.js | 6 + .../test_runner_output_dot_reporter.out | 4 + .../test_runner_output_spec_reporter.js | 10 + .../test_runner_output_spec_reporter.out | 280 ++++++++++++++++++ test/parallel/test-bootstrap-modules.js | 1 + test/parallel/test-runner-exit-code.js | 3 +- test/parallel/test-runner-reporters.js | 95 ++++++ test/parallel/test-runner-run.mjs | 1 - tools/doc/type-parser.mjs | 2 +- 36 files changed, 1124 insertions(+), 231 deletions(-) create mode 100644 lib/internal/modules/utils.js create mode 100644 lib/internal/test_runner/tests_stream.js rename lib/internal/test_runner/{yaml_parser.js => yaml_to_js.js} (100%) create mode 100644 lib/test/reporter/dot.js create mode 100644 lib/test/reporter/spec.js rename lib/{internal/test_runner/tap_stream.js => test/reporter/tap.js} (63%) create mode 100644 test/fixtures/test-runner/custom_reporters/custom.cjs create mode 100644 test/fixtures/test-runner/custom_reporters/custom.js create mode 100644 test/fixtures/test-runner/custom_reporters/custom.mjs create mode 100644 test/fixtures/test-runner/reporters.js create mode 100644 test/message/test_runner_output_dot_reporter.js create mode 100644 test/message/test_runner_output_dot_reporter.out create mode 100644 test/message/test_runner_output_spec_reporter.js create mode 100644 test/message/test_runner_output_spec_reporter.out create mode 100644 test/parallel/test-runner-reporters.js diff --git a/doc/api/cli.md b/doc/api/cli.md index ecfff40f137a1b..c876f9431b6e89 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -1219,6 +1219,24 @@ A regular expression that configures the test runner to only execute tests whose name matches the provided pattern. See the documentation on [filtering tests by name][] for more details. +### `--test-reporter` + + + +A test reporter to use when running tests. See the documentation on +[test reporters][] for more details. + +### `--test-reporter-destination` + + + +The destination for the corresponding test reporter. See the documentation on +[test reporters][] for more details. + ### `--test-only` -The `node:test` module facilitates the creation of JavaScript tests that -report results in [TAP][] format. To access it: +The `node:test` module facilitates the creation of JavaScript tests. +To access it: ```mjs import test from 'node:test'; @@ -91,9 +91,7 @@ test('callback failing test', (t, done) => { }); ``` -As a test file executes, TAP is written to the standard output of the Node.js -process. This output can be interpreted by any test harness that understands -the TAP format. If any tests fail, the process exit code is set to `1`. +If any tests fail, the process exit code is set to `1`. ## Subtests @@ -122,8 +120,7 @@ test to fail. ## Skipping tests Individual tests can be skipped by passing the `skip` option to the test, or by -calling the test context's `skip()` method. Both of these options support -including a message that is displayed in the TAP output as shown in the +calling the test context's `skip()` method as shown in the following example. ```js @@ -258,7 +255,7 @@ Test name patterns do not change the set of files that the test runner executes. ## Extraneous asynchronous activity -Once a test function finishes executing, the TAP results are output as quickly +Once a test function finishes executing, the results are reported as quickly as possible while maintaining the order of the tests. However, it is possible for the test function to generate asynchronous activity that outlives the test itself. The test runner handles this type of activity, but does not delay the @@ -267,13 +264,13 @@ reporting of test results in order to accommodate it. In the following example, a test completes with two `setImmediate()` operations still outstanding. The first `setImmediate()` attempts to create a new subtest. Because the parent test has already finished and output its -results, the new subtest is immediately marked as failed, and reported in the -top level of the file's TAP output. +results, the new subtest is immediately marked as failed, and reported later +to the {TestsStream}. The second `setImmediate()` creates an `uncaughtException` event. `uncaughtException` and `unhandledRejection` events originating from a completed test are marked as failed by the `test` module and reported as diagnostic -warnings in the top level of the file's TAP output. +warnings at the top level by the {TestsStream}. ```js test('a test that creates asynchronous activity', (t) => { @@ -454,6 +451,166 @@ test('spies on an object method', (t) => { }); ``` +## Test reporters + + + +The `node:test` module supports passing [`--test-reporter`][] +flags for the test runner to use a specific reporter. + +The following built-reporters are supported: + +* `tap` + The `tap` reporter is the default reporter used by the test runner. It outputs + the test results in the [TAP][] format. + +* `spec` + The `spec` reporter outputs the test results in a human-readable format. + +* `dot` + The `dot` reporter outputs the test results in a comact format, + where each passing test is represented by a `.`, + and each failing test is represented by a `X`. + +### Custom reporters + +[`--test-reporter`][] can be used to specify a path to custom reporter. +a custom reporter is a module that exports a value +accepted by [stream.compose][]. +Reporters should transform events emitted by a {TestsStream} + +Example of a custom reporter using {stream.Transform}: + +```mjs +import { Transform } from 'node:stream'; + +const customReporter = new Transform({ + writableObjectMode: true, + transform(event, encoding, callback) { + switch (event.type) { + case 'test:start': + callback(null, `test ${event.data.name} started`); + break; + case 'test:pass': + callback(null, `test ${event.data.name} passed`); + break; + case 'test:fail': + callback(null, `test ${event.data.name} failed`); + break; + case 'test:plan': + callback(null, 'test plan'); + break; + case 'test:diagnostic': + callback(null, event.data.message); + break; + } + }, +}); + +export default customReporter; +``` + +```cjs +const { Transform } = require('node:stream'); + +const customReporter = new Transform({ + writableObjectMode: true, + transform(event, encoding, callback) { + switch (event.type) { + case 'test:start': + callback(null, `test ${event.data.name} started`); + break; + case 'test:pass': + callback(null, `test ${event.data.name} passed`); + break; + case 'test:fail': + callback(null, `test ${event.data.name} failed`); + break; + case 'test:plan': + callback(null, 'test plan'); + break; + case 'test:diagnostic': + callback(null, event.data.message); + break; + } + }, +}); + +module.exports = customReporter; +``` + +Example of a custom reporter using a generator function: + +```mjs +export default async function * customReporter(source) { + for await (const event of source) { + switch (event.type) { + case 'test:start': + yield `test ${event.data.name} started\n`; + break; + case 'test:pass': + yield `test ${event.data.name} passed\n`; + break; + case 'test:fail': + yield `test ${event.data.name} failed\n`; + break; + case 'test:plan': + yield 'test plan'; + break; + case 'test:diagnostic': + yield `${event.data.message}\n`; + break; + } + } +} +``` + +```cjs +module.exports = async function * customReporter(source) { + for await (const event of source) { + switch (event.type) { + case 'test:start': + yield `test ${event.data.name} started\n`; + break; + case 'test:pass': + yield `test ${event.data.name} passed\n`; + break; + case 'test:fail': + yield `test ${event.data.name} failed\n`; + break; + case 'test:plan': + yield 'test plan\n'; + break; + case 'test:diagnostic': + yield `${event.data.message}\n`; + break; + } + } +}; +``` + +### Multiple reporters + +The [`--test-reporter`][] flag can be specified multiple times to report test +results in several formats. In this situation +it is required to specify a destination for each reporter +using [`--test-reporter-destination`][]. +Destination can be `stdout`, `stderr`, or a file path. +Reporters and destinations are paired according +to the order they were specified. + +In the following example, the `spec` reporter will output to `stdout`, +and the `dot` reporter will output to `file.txt`: + +```bash +node --test-reporter=spec --test-reporter=dot --test-reporter-destination=stdout --test-reporter-destination=file.txt +``` + +When a single reporter is specified, the destination will default to `stdout`, +unless a destination is explicitly provided. + ## `run([options])` -* `message` {string} Message to be displayed as a TAP diagnostic. +* `message` {string} Message to be reported. -This function is used to write TAP diagnostics to the output. Any diagnostic +This function is used to write diagnostics to the output. Any diagnostic information is included at the end of the test's results. This function does not return a value. @@ -1245,10 +1421,10 @@ test('top level test', async (t) => { added: v18.0.0 --> -* `message` {string} Optional skip message to be displayed in TAP output. +* `message` {string} Optional skip message. This function causes the test's output to indicate the test as skipped. If -`message` is provided, it is included in the TAP output. Calling `skip()` does +`message` is provided, it is included in the output. Calling `skip()` does not terminate execution of the test function. This function does not return a value. @@ -1265,10 +1441,10 @@ test('top level test', (t) => { added: v18.0.0 --> -* `message` {string} Optional `TODO` message to be displayed in TAP output. +* `message` {string} Optional `TODO` message. This function adds a `TODO` directive to the test's output. If `message` is -provided, it is included in the TAP output. Calling `todo()` does not terminate +provided, it is included in the output. Calling `todo()` does not terminate execution of the test function. This function does not return a value. ```js @@ -1363,6 +1539,8 @@ added: v18.7.0 [TAP]: https://testanything.org/ [`--test-name-pattern`]: cli.md#--test-name-pattern [`--test-only`]: cli.md#--test-only +[`--test-reporter-destination`]: cli.md#--test-reporter-destination +[`--test-reporter`]: cli.md#--test-reporter [`--test`]: cli.md#--test [`MockFunctionContext`]: #class-mockfunctioncontext [`MockTracker.method`]: #mockmethodobject-methodname-implementation-options @@ -1376,4 +1554,5 @@ added: v18.7.0 [`test()`]: #testname-options-fn [describe options]: #describename-options-fn [it options]: #testname-options-fn +[stream.compose]: stream.md#streamcomposestreams [test runner execution model]: #test-runner-execution-model diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js index 3ad34562a94b0e..cefe2f9fad0ce8 100644 --- a/lib/internal/main/test_runner.js +++ b/lib/internal/main/test_runner.js @@ -6,6 +6,7 @@ const { const { getOptionValue } = require('internal/options'); const { isUsingInspector } = require('internal/util/inspector'); const { run } = require('internal/test_runner/runner'); +const { setupTestReporters } = require('internal/test_runner/utils'); prepareMainThreadExecution(false); markBootstrapComplete(); @@ -20,8 +21,8 @@ if (isUsingInspector()) { inspectPort = process.debugPort; } -const tapStream = run({ concurrency, inspectPort, watch: getOptionValue('--watch') }); -tapStream.pipe(process.stdout); -tapStream.once('test:fail', () => { +const testsStream = run({ concurrency, inspectPort, watch: getOptionValue('--watch') }); +testsStream.once('test:fail', () => { process.exitCode = 1; }); +setupTestReporters(testsStream); diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 5a50d5d6afab6e..a9ec80b4c92adc 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -2,12 +2,12 @@ const { ObjectCreate, - StringPrototypeEndsWith, } = primordials; const CJSLoader = require('internal/modules/cjs/loader'); -const { Module, toRealPath, readPackageScope } = CJSLoader; +const { Module, toRealPath } = CJSLoader; const { getOptionValue } = require('internal/options'); const path = require('path'); +const { shouldUseESMLoader } = require('internal/modules/utils'); const { handleProcessExit, } = require('internal/modules/esm/handle_process_exit'); @@ -27,27 +27,6 @@ function resolveMainPath(main) { return mainPath; } -function shouldUseESMLoader(mainPath) { - /** - * @type {string[]} userLoaders A list of custom loaders registered by the user - * (or an empty list when none have been registered). - */ - const userLoaders = getOptionValue('--experimental-loader'); - if (userLoaders.length > 0) - return true; - const esModuleSpecifierResolution = - getOptionValue('--experimental-specifier-resolution'); - if (esModuleSpecifierResolution === 'node') - return true; - // Determine the module format of the main - if (mainPath && StringPrototypeEndsWith(mainPath, '.mjs')) - return true; - if (!mainPath || StringPrototypeEndsWith(mainPath, '.cjs')) - return false; - const pkg = readPackageScope(mainPath); - return pkg && pkg.data.type === 'module'; -} - function runMainESM(mainPath) { const { loadESM } = require('internal/process/esm_loader'); const { pathToFileURL } = require('internal/url'); diff --git a/lib/internal/modules/utils.js b/lib/internal/modules/utils.js new file mode 100644 index 00000000000000..d2ba0a352f7571 --- /dev/null +++ b/lib/internal/modules/utils.js @@ -0,0 +1,54 @@ +'use strict'; + +const { + ObjectCreate, + StringPrototypeEndsWith, +} = primordials; +const { getOptionValue } = require('internal/options'); + + +function shouldUseESMLoader(mainPath) { + /** + * @type {string[]} userLoaders A list of custom loaders registered by the user + * (or an empty list when none have been registered). + */ + const userLoaders = getOptionValue('--experimental-loader'); + if (userLoaders.length > 0) + return true; + const esModuleSpecifierResolution = + getOptionValue('--experimental-specifier-resolution'); + if (esModuleSpecifierResolution === 'node') + return true; + // Determine the module format of the main + if (mainPath && StringPrototypeEndsWith(mainPath, '.mjs')) + return true; + if (!mainPath || StringPrototypeEndsWith(mainPath, '.cjs')) + return false; + const { readPackageScope } = require('internal/modules/cjs/loader'); + const pkg = readPackageScope(mainPath); + return pkg && pkg.data.type === 'module'; +} + +/** + * @param {string} filePath + * @returns {any} + * requireOrImport imports a module if the file is an ES module, otherwise it requires it. + */ +function requireOrImport(filePath) { + const useESMLoader = shouldUseESMLoader(filePath); + if (useESMLoader) { + const { esmLoader } = require('internal/process/esm_loader'); + const { pathToFileURL } = require('internal/url'); + const { isAbsolute } = require('path'); + const file = isAbsolute(filePath) ? pathToFileURL(filePath).href : filePath; + return esmLoader.import(file, undefined, ObjectCreate(null)); + } + const { Module } = require('internal/modules/cjs/loader'); + + return new Module._load(filePath, null, false); +} + +module.exports = { + shouldUseESMLoader, + requireOrImport, +}; diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index e6af310f9ea623..0bae82d8e7417c 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -16,6 +16,7 @@ const { const { kEmptyObject } = require('internal/util'); const { getOptionValue } = require('internal/options'); const { kCancelledByParent, Test, ItTest, Suite } = require('internal/test_runner/test'); +const { setupTestReporters } = require('internal/test_runner/utils'); const { bigint: hrtime } = process.hrtime; const isTestRunnerCli = getOptionValue('--test'); @@ -107,7 +108,6 @@ function setup(root) { } root.startTime = hrtime(); - root.reporter.version(); wasRootSetup.add(root); return root; @@ -117,10 +117,10 @@ let globalRoot; function getGlobalRoot() { if (!globalRoot) { globalRoot = createTestTree(); - globalRoot.reporter.pipe(process.stdout); globalRoot.reporter.once('test:fail', () => { process.exitCode = 1; }); + setupTestReporters(globalRoot.reporter); } return globalRoot; } diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 1e5c0dda0fabd9..4659073bfb61b4 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -6,6 +6,7 @@ const { ArrayPrototypeIncludes, ArrayPrototypePush, ArrayPrototypeSlice, + ArrayPrototypeSome, ArrayPrototypeSort, ObjectAssign, PromisePrototypeThen, @@ -14,7 +15,7 @@ const { SafePromiseAllSettledReturnVoid, SafeMap, SafeSet, - StringPrototypeRepeat, + StringPrototypeStartsWith, } = primordials; const { spawn } = require('child_process'); @@ -32,9 +33,9 @@ const { validateArray, validateBoolean } = require('internal/validators'); const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector'); const { kEmptyObject } = require('internal/util'); const { createTestTree } = require('internal/test_runner/harness'); -const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test'); +const { kSubtestsFailed, Test } = require('internal/test_runner/test'); const { TapParser } = require('internal/test_runner/tap_parser'); -const { YAMLToJs } = require('internal/test_runner/yaml_parser'); +const { YAMLToJs } = require('internal/test_runner/yaml_to_js'); const { TokenKind } = require('internal/test_runner/tap_lexer'); const { @@ -48,6 +49,7 @@ const { } = internalBinding('errors'); const kFilterArgs = ['--test', '--watch']; +const kFilterArgValues = ['--test-reporter', '--test-reporter-destination']; // TODO(cjihrig): Replace this with recursive readdir once it lands. function processPath(path, testFiles, options) { @@ -111,8 +113,9 @@ function createTestFileList() { return ArrayPrototypeSort(ArrayFrom(testFiles)); } -function filterExecArgv(arg) { - return !ArrayPrototypeIncludes(kFilterArgs, arg); +function filterExecArgv(arg, i, arr) { + return !ArrayPrototypeIncludes(kFilterArgs, arg) && + !ArrayPrototypeSome(kFilterArgValues, (p) => arg === p || (i > 0 && arr[i - 1] === p) || StringPrototypeStartsWith(arg, `${p}=`)); } function getRunArgs({ path, inspectPort }) { @@ -127,7 +130,7 @@ function getRunArgs({ path, inspectPort }) { class FileTest extends Test { #buffer = []; #handleReportItem({ kind, node, nesting = 0 }) { - const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1); + nesting += 1; switch (kind) { case TokenKind.TAP_VERSION: @@ -136,11 +139,11 @@ class FileTest extends Test { break; case TokenKind.TAP_PLAN: - this.reporter.plan(indent, node.end - node.start + 1); + this.reporter.plan(nesting, node.end - node.start + 1); break; case TokenKind.TAP_SUBTEST_POINT: - this.reporter.subtest(indent, node.name); + this.reporter.start(nesting, node.name); break; case TokenKind.TAP_TEST_POINT: @@ -159,7 +162,7 @@ class FileTest extends Test { if (pass) { this.reporter.ok( - indent, + nesting, node.id, node.description, YAMLToJs(node.diagnostics), @@ -167,7 +170,7 @@ class FileTest extends Test { ); } else { this.reporter.fail( - indent, + nesting, node.id, node.description, YAMLToJs(node.diagnostics), @@ -177,15 +180,15 @@ class FileTest extends Test { break; case TokenKind.COMMENT: - if (indent === kDefaultIndent) { + if (nesting === 1) { // Ignore file top level diagnostics break; } - this.reporter.diagnostic(indent, node.comment); + this.reporter.diagnostic(nesting, node.comment); break; case TokenKind.UNKNOWN: - this.reporter.diagnostic(indent, node.value); + this.reporter.diagnostic(nesting, node.value); break; } } @@ -194,11 +197,11 @@ class FileTest extends Test { ArrayPrototypePush(this.#buffer, ast); return; } - this.reportSubtest(); + this.reportStarted(); this.#handleReportItem(ast); } report() { - this.reportSubtest(); + this.reportStarted(); ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast)); super.report(); } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index 8b0ba16f1a6a79..14ddb96d1155be 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -33,7 +33,7 @@ const { } = require('internal/errors'); const { getOptionValue } = require('internal/options'); const { MockTracker } = require('internal/test_runner/mock'); -const { TapStream } = require('internal/test_runner/tap_stream'); +const { TestsStream } = require('internal/test_runner/tests_stream'); const { convertStringToRegExp, createDeferredCallback, @@ -63,7 +63,6 @@ const kTestCodeFailure = 'testCodeFailure'; const kTestTimeoutFailure = 'testTimeoutFailure'; const kHookFailure = 'hookFailed'; const kDefaultTimeout = null; -const kDefaultIndent = ' '; // 4 spaces const noop = FunctionPrototype; const isTestRunner = getOptionValue('--test'); const testOnlyFlag = !isTestRunner && getOptionValue('--test-only'); @@ -190,18 +189,18 @@ class Test extends AsyncResource { if (parent === null) { this.concurrency = 1; - this.indent = ''; + this.nesting = 0; this.only = testOnlyFlag; - this.reporter = new TapStream(); + this.reporter = new TestsStream(); this.runOnlySubtests = this.only; this.testNumber = 0; this.timeout = kDefaultTimeout; } else { - const indent = parent.parent === null ? parent.indent : - parent.indent + kDefaultIndent; + const nesting = parent.parent === null ? parent.nesting : + parent.nesting + 1; this.concurrency = parent.concurrency; - this.indent = indent; + this.nesting = nesting; this.only = only ?? !parent.runOnlySubtests; this.reporter = parent.reporter; this.runOnlySubtests = !this.only; @@ -334,7 +333,7 @@ class Test extends AsyncResource { } if (i === 1 && this.parent !== null) { - this.reportSubtest(); + this.reportStarted(); } // Report the subtest's results and remove it from the ready map. @@ -633,19 +632,19 @@ class Test extends AsyncResource { this.parent.processPendingSubtests(); } else if (!this.reported) { this.reported = true; - this.reporter.plan(this.indent, this.subtests.length); + this.reporter.plan(this.nesting, this.subtests.length); for (let i = 0; i < this.diagnostics.length; i++) { - this.reporter.diagnostic(this.indent, this.diagnostics[i]); + this.reporter.diagnostic(this.nesting, this.diagnostics[i]); } - this.reporter.diagnostic(this.indent, `tests ${this.subtests.length}`); - this.reporter.diagnostic(this.indent, `pass ${counters.passed}`); - this.reporter.diagnostic(this.indent, `fail ${counters.failed}`); - this.reporter.diagnostic(this.indent, `cancelled ${counters.cancelled}`); - this.reporter.diagnostic(this.indent, `skipped ${counters.skipped}`); - this.reporter.diagnostic(this.indent, `todo ${counters.todo}`); - this.reporter.diagnostic(this.indent, `duration_ms ${this.#duration()}`); + this.reporter.diagnostic(this.nesting, `tests ${this.subtests.length}`); + this.reporter.diagnostic(this.nesting, `pass ${counters.passed}`); + this.reporter.diagnostic(this.nesting, `fail ${counters.failed}`); + this.reporter.diagnostic(this.nesting, `cancelled ${counters.cancelled}`); + this.reporter.diagnostic(this.nesting, `skipped ${counters.skipped}`); + this.reporter.diagnostic(this.nesting, `todo ${counters.todo}`); + this.reporter.diagnostic(this.nesting, `duration_ms ${this.#duration()}`); this.reporter.push(null); } } @@ -681,9 +680,9 @@ class Test extends AsyncResource { report() { if (this.subtests.length > 0) { - this.reporter.plan(this.subtests[0].indent, this.subtests.length); + this.reporter.plan(this.subtests[0].nesting, this.subtests.length); } else { - this.reportSubtest(); + this.reportStarted(); } let directive; const details = { __proto__: null, duration_ms: this.#duration() }; @@ -695,24 +694,24 @@ class Test extends AsyncResource { } if (this.passed) { - this.reporter.ok(this.indent, this.testNumber, this.name, details, directive); + this.reporter.ok(this.nesting, this.testNumber, this.name, details, directive); } else { details.error = this.error; - this.reporter.fail(this.indent, this.testNumber, this.name, details, directive); + this.reporter.fail(this.nesting, this.testNumber, this.name, details, directive); } for (let i = 0; i < this.diagnostics.length; i++) { - this.reporter.diagnostic(this.indent, this.diagnostics[i]); + this.reporter.diagnostic(this.nesting, this.diagnostics[i]); } } - reportSubtest() { + reportStarted() { if (this.#reportedSubtest || this.parent === null) { return; } this.#reportedSubtest = true; - this.parent.reportSubtest(); - this.reporter.subtest(this.indent, this.name); + this.parent.reportStarted(); + this.reporter.start(this.nesting, this.name); } } @@ -817,7 +816,6 @@ class Suite extends Test { module.exports = { ItTest, kCancelledByParent, - kDefaultIndent, kSubtestsFailed, kTestCodeFailure, kUnwrapErrors, diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js new file mode 100644 index 00000000000000..b016d316154807 --- /dev/null +++ b/lib/internal/test_runner/tests_stream.js @@ -0,0 +1,74 @@ +'use strict'; +const { + ArrayPrototypePush, + ArrayPrototypeShift, +} = primordials; +const Readable = require('internal/streams/readable'); + +class TestsStream extends Readable { + #buffer; + #canPush; + + constructor() { + super({ objectMode: true }); + this.#buffer = []; + this.#canPush = true; + } + + _read() { + this.#canPush = true; + + while (this.#buffer.length > 0) { + const obj = ArrayPrototypeShift(this.#buffer); + + if (!this.#tryPush(obj)) { + return; + } + } + } + + fail(nesting, testNumber, name, details, directive) { + this.#emit('test:fail', { __proto__: null, name, nesting, testNumber, details, ...directive }); + } + + ok(nesting, testNumber, name, details, directive) { + this.#emit('test:pass', { __proto__: null, name, nesting, testNumber, details, ...directive }); + } + + plan(nesting, count) { + this.#emit('test:plan', { __proto__: null, nesting, count }); + } + + getSkip(reason = undefined) { + return { __proto__: null, skip: reason ?? true }; + } + + getTodo(reason = undefined) { + return { __proto__: null, todo: reason ?? true }; + } + + start(nesting, name) { + this.#emit('test:start', { __proto__: null, nesting, name }); + } + + diagnostic(nesting, message) { + this.#emit('test:diagnostic', { __proto__: null, nesting, message }); + } + + #emit(type, data) { + this.emit(type, data); + this.#tryPush({ type, data }); + } + + #tryPush(message) { + if (this.#canPush) { + this.#canPush = this.push(message); + } else { + ArrayPrototypePush(this.#buffer, message); + } + + return this.#canPush; + } +} + +module.exports = { TestsStream }; diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index ad040f010250e2..9dba00de25719e 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -1,7 +1,18 @@ 'use strict'; -const { RegExp, RegExpPrototypeExec } = primordials; +const { + ArrayPrototypePush, + ObjectGetOwnPropertyDescriptor, + SafePromiseAllReturnArrayLike, + RegExp, + RegExpPrototypeExec, + SafeMap, +} = primordials; const { basename } = require('path'); +const { createWriteStream } = require('fs'); const { createDeferredPromise } = require('internal/util'); +const { getOptionValue } = require('internal/options'); +const { requireOrImport } = require('internal/modules/utils'); + const { codes: { ERR_INVALID_ARG_VALUE, @@ -9,6 +20,7 @@ const { }, kIsNodeError, } = require('internal/errors'); +const { compose } = require('stream'); const kMultipleCallbackInvocations = 'multipleCallbackInvocations'; const kRegExpPattern = /^\/(.*)\/([a-z]*)$/; @@ -74,10 +86,71 @@ function convertStringToRegExp(str, name) { } } +const kBuiltinDestinations = new SafeMap([ + ['stdout', process.stdout], + ['stderr', process.stderr], +]); + +const kBuiltinReporters = new SafeMap([ + ['spec', 'node:test/reporter/spec'], + ['dot', 'node:test/reporter/dot'], + ['tap', 'node:test/reporter/tap'], +]); + +const kDefaultReporter = 'tap'; +const kDefaultDestination = 'stdout'; + +async function getReportersMap(reporters, destinations) { + return SafePromiseAllReturnArrayLike(reporters, async (name, i) => { + const destination = kBuiltinDestinations.get(destinations[i]) ?? createWriteStream(destinations[i]); + let reporter = await requireOrImport(kBuiltinReporters.get(name) ?? name); + + if (reporter?.default) { + reporter = reporter.default; + } + + if (reporter?.prototype && ObjectGetOwnPropertyDescriptor(reporter.prototype, 'constructor')) { + reporter = new reporter(); + } + + if (!reporter) { + throw new ERR_INVALID_ARG_VALUE('Reporter', name, 'is not a valid reporter'); + } + + return { __proto__: null, reporter, destination }; + }); +} + + +async function setupTestReporters(testsStream) { + const destinations = getOptionValue('--test-reporter-destination'); + const reporters = getOptionValue('--test-reporter'); + + if (reporters.length === 0 && destinations.length === 0) { + ArrayPrototypePush(reporters, kDefaultReporter); + } + + if (reporters.length === 1 && destinations.length === 0) { + ArrayPrototypePush(destinations, kDefaultDestination); + } + + if (destinations.length !== reporters.length) { + throw new ERR_INVALID_ARG_VALUE('--test-reporter', reporters, + 'must match the number of specified \'--test-reporter-destination\''); + } + + const reportersMap = await getReportersMap(reporters, destinations); + for (let i = 0; i < reportersMap.length; i++) { + const { reporter, destination } = reportersMap[i]; + compose(testsStream, reporter).pipe(destination); + } +} + module.exports = { convertStringToRegExp, createDeferredCallback, doesPathMatchFilter, isSupportedFileType, isTestFailureError, + setupTestReporters, }; diff --git a/lib/internal/test_runner/yaml_parser.js b/lib/internal/test_runner/yaml_to_js.js similarity index 100% rename from lib/internal/test_runner/yaml_parser.js rename to lib/internal/test_runner/yaml_to_js.js diff --git a/lib/internal/util/colors.js b/lib/internal/util/colors.js index 5622a88467d038..79021a2bd9825d 100644 --- a/lib/internal/util/colors.js +++ b/lib/internal/util/colors.js @@ -5,6 +5,7 @@ module.exports = { green: '', white: '', red: '', + gray: '', clear: '', hasColors: false, refresh() { @@ -14,6 +15,7 @@ module.exports = { module.exports.green = hasColors ? '\u001b[32m' : ''; module.exports.white = hasColors ? '\u001b[39m' : ''; module.exports.red = hasColors ? '\u001b[31m' : ''; + module.exports.gray = hasColors ? '\u001b[90m' : ''; module.exports.clear = hasColors ? '\u001bc' : ''; module.exports.hasColors = hasColors; } diff --git a/lib/test/reporter/dot.js b/lib/test/reporter/dot.js new file mode 100644 index 00000000000000..7dbba5a957894e --- /dev/null +++ b/lib/test/reporter/dot.js @@ -0,0 +1,17 @@ +'use strict'; + +module.exports = async function* dot(source) { + let count = 0; + for await (const { type } of source) { + if (type === 'test:pass') { + yield '.'; + } + if (type === 'test:fail') { + yield 'X'; + } + if ((type === 'test:fail' || type === 'test:pass') && ++count % 20 === 0) { + yield '\n'; + } + } + yield '\n'; +}; diff --git a/lib/test/reporter/spec.js b/lib/test/reporter/spec.js new file mode 100644 index 00000000000000..c19d5568d1c5ca --- /dev/null +++ b/lib/test/reporter/spec.js @@ -0,0 +1,107 @@ +'use strict'; + +const { + ArrayPrototypeJoin, + ArrayPrototypePop, + ArrayPrototypeShift, + ArrayPrototypeUnshift, + hardenRegExp, + RegExpPrototypeSymbolSplit, + SafeMap, + StringPrototypeRepeat, +} = primordials; +const assert = require('assert'); +const Transform = require('internal/streams/transform'); +const { inspectWithNoCustomRetry } = require('internal/errors'); +const { green, blue, red, white, gray } = require('internal/util/colors'); + + +const inspectOptions = { __proto__: null, colors: true, breakLength: Infinity }; + +const colors = { + '__proto__': null, + 'test:fail': red, + 'test:pass': green, + 'test:diagnostic': blue, +}; +const symbols = { + '__proto__': null, + 'test:fail': '\u2716 ', + 'test:pass': '\u2714 ', + 'test:diagnostic': '\u2139 ', + 'arrow:right': '\u25B6 ', +}; +class SpecReporter extends Transform { + #stack = []; + #reported = []; + #indentMemo = new SafeMap(); + + constructor() { + super({ writableObjectMode: true }); + } + + #indent(nesting) { + let value = this.#indentMemo.get(nesting); + if (value === undefined) { + value = StringPrototypeRepeat(' ', nesting); + this.#indentMemo.set(nesting, value); + } + + return value; + } + #formatError(error, indent) { + if (!error) return ''; + const err = error.code === 'ERR_TEST_FAILURE' ? error.cause : error; + const message = ArrayPrototypeJoin( + RegExpPrototypeSymbolSplit( + hardenRegExp(/\r?\n/), + inspectWithNoCustomRetry(err, inspectOptions), + ), `\n${indent} `); + return `\n${indent} ${message}\n`; + } + #handleEvent({ type, data }) { + const color = colors[type] ?? white; + const symbol = symbols[type] ?? ' '; + + switch (type) { + case 'test:fail': + case 'test:pass': { + const subtest = ArrayPrototypeShift(this.#stack); // This is the matching `test:start` event + if (subtest) { + assert(subtest.type === 'test:start'); + assert(subtest.data.nesting === data.nesting); + assert(subtest.data.name === data.name); + } + let prefix = ''; + while (this.#stack.length) { + // Report all the parent `test:start` events + const parent = ArrayPrototypePop(this.#stack); + assert(parent.type === 'test:start'); + const msg = parent.data; + ArrayPrototypeUnshift(this.#reported, msg); + prefix += `${this.#indent(msg.nesting)}${symbols['arrow:right']}${msg.name}\n`; + } + const indent = this.#indent(data.nesting); + const duration_ms = data.details?.duration_ms ? ` ${gray}(${data.details.duration_ms}ms)${white}` : ''; + const title = `${data.name}${duration_ms}`; + if (this.#reported[0] && this.#reported[0].nesting === data.nesting && this.#reported[0].name === data.name) { + // If this test has had children - it was already reporter, so slightly modify the output + ArrayPrototypeShift(this.#reported); + return `${prefix}${indent}${color}${symbols['arrow:right']}${white}${title}\n\n`; + } + const error = this.#formatError(data.details?.error, indent); + return `${prefix}${indent}${color}${symbol}${title}${error}${white}\n`; + } + case 'test:start': + ArrayPrototypeUnshift(this.#stack, { __proto__: null, data, type }); + break; + case 'test:diagnostic': + return `${color}${this.#indent(data.nesting)}${symbol}${data.message}${white}\n`; + } + } + _transform({ type, data }, encoding, callback) { + callback(null, this.#handleEvent({ type, data })); + } +} + +module.exports = SpecReporter; diff --git a/lib/internal/test_runner/tap_stream.js b/lib/test/reporter/tap.js similarity index 63% rename from lib/internal/test_runner/tap_stream.js rename to lib/test/reporter/tap.js index 052f8284c8d931..fa5d4684fbb9e3 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/test/reporter/tap.js @@ -2,18 +2,17 @@ const { ArrayPrototypeForEach, ArrayPrototypeJoin, - ArrayPrototypeMap, ArrayPrototypePush, - ArrayPrototypeShift, ObjectEntries, + RegExpPrototypeSymbolReplace, + SafeMap, StringPrototypeReplaceAll, - StringPrototypeToUpperCase, StringPrototypeSplit, - RegExpPrototypeSymbolReplace, + StringPrototypeRepeat, } = primordials; const { inspectWithNoCustomRetry } = require('internal/errors'); -const Readable = require('internal/streams/readable'); const { isError, kEmptyObject } = require('internal/util'); +const kDefaultIndent = ' '; // 4 spaces const kFrameStartRegExp = /^ {4}at /; const kLineBreakRegExp = /\n|\r\n/; const kDefaultTAPVersion = 13; @@ -22,112 +21,77 @@ let testModule; // Lazy loaded due to circular dependency. function lazyLoadTest() { testModule ??= require('internal/test_runner/test'); - return testModule; } -class TapStream extends Readable { - #buffer; - #canPush; - - constructor() { - super(); - this.#buffer = []; - this.#canPush = true; - } - - _read() { - this.#canPush = true; - while (this.#buffer.length > 0) { - const line = ArrayPrototypeShift(this.#buffer); - - if (!this.#tryPush(line)) { - return; - } +async function * tapReporter(source) { + yield `TAP version ${kDefaultTAPVersion}\n`; + for await (const { type, data } of source) { + switch (type) { + case 'test:fail': + yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo); + yield reportDetails(data.nesting, data.details); + break; + case 'test:pass': + yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo); + yield reportDetails(data.nesting, data.details); + break; + case 'test:plan': + yield `${indent(data.nesting)}1..${data.count}\n`; + break; + case 'test:start': + yield `${indent(data.nesting)}# Subtest: ${tapEscape(data.name)}\n`; + break; + case 'test:diagnostic': + yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`; + break; } } +} - bail(message) { - this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`); - } - - fail(indent, testNumber, name, details, directive) { - this.emit('test:fail', { __proto__: null, name, testNumber, details, ...directive }); - this.#test(indent, testNumber, 'not ok', name, directive); - this.#details(indent, details); - } - - ok(indent, testNumber, name, details, directive) { - this.emit('test:pass', { __proto__: null, name, testNumber, details, ...directive }); - this.#test(indent, testNumber, 'ok', name, directive); - this.#details(indent, details); - } - - plan(indent, count, explanation) { - const exp = `${explanation ? ` # ${tapEscape(explanation)}` : ''}`; - - this.#tryPush(`${indent}1..${count}${exp}\n`); - } - - getSkip(reason) { - return { __proto__: null, skip: reason }; - } - - getTodo(reason) { - return { __proto__: null, todo: reason }; - } - - subtest(indent, name) { - this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`); - } - - #details(indent, data = kEmptyObject) { - const { error, duration_ms } = data; - let details = `${indent} ---\n`; +function reportTest(nesting, testNumber, status, name, skip, todo) { + let line = `${indent(nesting)}${status} ${testNumber}`; - details += jsToYaml(indent, 'duration_ms', duration_ms); - details += jsToYaml(indent, null, error); - details += `${indent} ...\n`; - this.#tryPush(details); + if (name) { + line += ` ${tapEscape(`- ${name}`)}`; } - diagnostic(indent, message) { - this.emit('test:diagnostic', message); - this.#tryPush(`${indent}# ${tapEscape(message)}\n`); + if (skip !== undefined) { + line += ` # SKIP${typeof skip === 'string' && skip.length ? ` ${tapEscape(skip)}` : ''}`; + } else if (todo !== undefined) { + line += ` # TODO${typeof todo === 'string' && todo.length ? ` ${tapEscape(todo)}` : ''}`; } - version(spec = kDefaultTAPVersion) { - this.#tryPush(`TAP version ${spec}\n`); - } + line += '\n'; - #test(indent, testNumber, status, name, directive = kEmptyObject) { - let line = `${indent}${status} ${testNumber}`; + return line; +} - if (name) { - line += ` ${tapEscape(`- ${name}`)}`; - } - line += ArrayPrototypeJoin(ArrayPrototypeMap(ObjectEntries(directive), ({ 0: key, 1: value }) => ( - ` # ${StringPrototypeToUpperCase(key)}${value ? ` ${tapEscape(value)}` : ''}` - )), ''); +function reportDetails(nesting, data = kEmptyObject) { + const { error, duration_ms } = data; + const _indent = indent(nesting); + let details = `${_indent} ---\n`; - line += '\n'; + details += jsToYaml(_indent, 'duration_ms', duration_ms); + details += jsToYaml(_indent, null, error); + details += `${_indent} ...\n`; + return details; +} - this.#tryPush(line); +const memo = new SafeMap(); +function indent(nesting) { + let value = memo.get(nesting); + if (value === undefined) { + value = StringPrototypeRepeat(kDefaultIndent, nesting); + memo.set(nesting, value); } - #tryPush(message) { - if (this.#canPush) { - this.#canPush = this.push(message); - } else { - ArrayPrototypePush(this.#buffer, message); - } - - return this.#canPush; - } + return value; } + // In certain places, # and \ need to be escaped as \# and \\. function tapEscape(input) { let result = StringPrototypeReplaceAll(input, '\\', '\\\\'); @@ -266,4 +230,4 @@ function isAssertionLike(value) { return value && typeof value === 'object' && 'expected' in value && 'actual' in value; } -module.exports = { TapStream }; +module.exports = tapReporter; diff --git a/src/node_options.cc b/src/node_options.cc index a9b874b519ebcd..604b6006faf2e7 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -558,6 +558,12 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { AddOption("--test-name-pattern", "run tests whose name matches this regular expression", &EnvironmentOptions::test_name_pattern); + AddOption("--test-reporter", + "report test output using the given reporter", + &EnvironmentOptions::test_reporter); + AddOption("--test-reporter-destination", + "report given reporter to the given destination", + &EnvironmentOptions::test_reporter_destination); AddOption("--test-only", "run tests with 'only' option set", &EnvironmentOptions::test_only, diff --git a/src/node_options.h b/src/node_options.h index 2e4da9decf3b4b..052f57bce6fa6d 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -155,6 +155,8 @@ class EnvironmentOptions : public Options { std::string diagnostic_dir; bool test_runner = false; std::vector test_name_pattern; + std::vector test_reporter; + std::vector test_reporter_destination; bool test_only = false; bool test_udp_no_try_send = false; bool throw_deprecation = false; diff --git a/test/fixtures/test-runner/custom_reporters/custom.cjs b/test/fixtures/test-runner/custom_reporters/custom.cjs new file mode 100644 index 00000000000000..a3f653d11bb981 --- /dev/null +++ b/test/fixtures/test-runner/custom_reporters/custom.cjs @@ -0,0 +1,17 @@ +const { Transform } = require('node:stream'); + +const customReporter = new Transform({ + writableObjectMode: true, + transform(event, encoding, callback) { + this.counters ??= {}; + this.counters[event.type] = (this.counters[event.type] ?? 0) + 1; + callback(); + }, + flush(callback) { + this.push('custom.cjs ') + this.push(JSON.stringify(this.counters)); + callback(); + } +}); + +module.exports = customReporter; diff --git a/test/fixtures/test-runner/custom_reporters/custom.js b/test/fixtures/test-runner/custom_reporters/custom.js new file mode 100644 index 00000000000000..62690f115b7ae1 --- /dev/null +++ b/test/fixtures/test-runner/custom_reporters/custom.js @@ -0,0 +1,8 @@ +module.exports = async function * customReporter(source) { + const counters = {}; + for await (const event of source) { + counters[event.type] = (counters[event.type] ?? 0) + 1; + } + yield "custom.js "; + yield JSON.stringify(counters); +}; diff --git a/test/fixtures/test-runner/custom_reporters/custom.mjs b/test/fixtures/test-runner/custom_reporters/custom.mjs new file mode 100644 index 00000000000000..b202d770c6bf19 --- /dev/null +++ b/test/fixtures/test-runner/custom_reporters/custom.mjs @@ -0,0 +1,8 @@ +export default async function * customReporter(source) { + const counters = {}; + for await (const event of source) { + counters[event.type] = (counters[event.type] ?? 0) + 1; + } + yield "custom.mjs "; + yield JSON.stringify(counters); +} diff --git a/test/fixtures/test-runner/reporters.js b/test/fixtures/test-runner/reporters.js new file mode 100644 index 00000000000000..ed7066023d1299 --- /dev/null +++ b/test/fixtures/test-runner/reporters.js @@ -0,0 +1,11 @@ +'use strict'; +const test = require('node:test'); + +test('nested', { concurrency: 4 }, async (t) => { + t.test('ok', () => {}); + t.test('failing', () => { + throw new Error('error'); + }); +}); + +test('top level', () => {}); diff --git a/test/message/test_runner_describe_it.out b/test/message/test_runner_describe_it.out index 199e834d6f65ae..87207aca71fafa 100644 --- a/test/message/test_runner_describe_it.out +++ b/test/message/test_runner_describe_it.out @@ -24,7 +24,6 @@ not ok 3 - sync fail todo # TODO * * * - * ... # Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo @@ -41,7 +40,6 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * * - * ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -73,7 +71,6 @@ not ok 8 - sync throw fail * * * - * ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -100,7 +97,6 @@ not ok 11 - async throw fail * * * - * ... # Subtest: async skip fail not ok 12 - async skip fail @@ -132,7 +128,6 @@ not ok 13 - async assertion fail * * * - * ... # Subtest: resolve pass ok 14 - resolve pass @@ -154,7 +149,6 @@ not ok 15 - reject fail * * * - * ... # Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns @@ -620,7 +614,6 @@ not ok 58 - rejected thenable code: 'ERR_TEST_FAILURE' stack: |- * - * ... # Subtest: invalid subtest fail not ok 59 - invalid subtest fail diff --git a/test/message/test_runner_hooks.out b/test/message/test_runner_hooks.out index 6bb1705967d043..7c82e9ff292ad5 100644 --- a/test/message/test_runner_hooks.out +++ b/test/message/test_runner_hooks.out @@ -64,7 +64,6 @@ not ok 2 - before throws * * * - * ... # Subtest: after throws # Subtest: 1 @@ -93,7 +92,6 @@ not ok 3 - after throws * * * - * ... # Subtest: beforeEach throws # Subtest: 1 @@ -490,7 +488,6 @@ not ok 13 - t.after() is called if test body throws * * * - * ... # - after() called 1..13 diff --git a/test/message/test_runner_output.js b/test/message/test_runner_output.js index f75b11a848e9de..91caca3038e7b8 100644 --- a/test/message/test_runner_output.js +++ b/test/message/test_runner_output.js @@ -113,7 +113,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p1a = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 1000); + }, 100); }); return p1a; @@ -131,7 +131,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p1c = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 2000); + }, 200); }); return p1c; @@ -141,7 +141,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p1c = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 1500); + }, 150); }); return p1c; @@ -150,7 +150,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p0a = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 3000); + }, 300); }); return p0a; @@ -159,7 +159,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { test('top level', { concurrency: 2 }, async (t) => { t.test('+long running', async (t) => { return new Promise((resolve, reject) => { - setTimeout(resolve, 3000).unref(); + setTimeout(resolve, 300).unref(); }); }); @@ -331,12 +331,12 @@ test('subtest sync throw fails', async (t) => { test('timed out async test', { timeout: 5 }, async (t) => { return new Promise((resolve) => { - setTimeout(resolve, 1000); + setTimeout(resolve, 100); }); }); test('timed out callback test', { timeout: 5 }, (t, done) => { - setTimeout(done, 1000); + setTimeout(done, 100); }); diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index 14479c773bbc86..42eae979daf6dd 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -24,7 +24,6 @@ not ok 3 - sync fail todo # TODO * * * - * ... # Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo @@ -41,7 +40,6 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * * - * ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -74,7 +72,6 @@ not ok 8 - sync throw fail * * * - * ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -101,7 +98,6 @@ not ok 11 - async throw fail * * * - * ... # Subtest: async skip fail not ok 12 - async skip fail # SKIP @@ -118,7 +114,6 @@ not ok 12 - async skip fail # SKIP * * * - * ... # Subtest: async assertion fail not ok 13 - async assertion fail @@ -142,7 +137,6 @@ not ok 13 - async assertion fail * * * - * ... # Subtest: resolve pass ok 14 - resolve pass @@ -164,7 +158,6 @@ not ok 15 - reject fail * * * - * ... # Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns diff --git a/test/message/test_runner_output_cli.js b/test/message/test_runner_output_cli.js index 1058d903c5fee4..5645f1afb1f3a2 100644 --- a/test/message/test_runner_output_cli.js +++ b/test/message/test_runner_output_cli.js @@ -3,4 +3,5 @@ require('../common'); const spawn = require('node:child_process').spawn; spawn(process.execPath, - ['--no-warnings', '--test', 'test/message/test_runner_output.js'], { stdio: 'inherit' }); + ['--no-warnings', '--test', '--test-reporter', 'tap', 'test/message/test_runner_output.js'], + { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_cli.out b/test/message/test_runner_output_cli.out index b33d3e0fbf50b1..044610905755ca 100644 --- a/test/message/test_runner_output_cli.out +++ b/test/message/test_runner_output_cli.out @@ -25,7 +25,6 @@ TAP version 13 * * * - * ... # Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo @@ -42,7 +41,6 @@ TAP version 13 * * * - * ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -74,7 +72,6 @@ TAP version 13 * * * - * ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -101,7 +98,6 @@ TAP version 13 * * * - * ... # Subtest: async skip fail not ok 12 - async skip fail # SKIP @@ -118,7 +114,6 @@ TAP version 13 * * * - * ... # Subtest: async assertion fail not ok 13 - async assertion fail @@ -142,7 +137,6 @@ TAP version 13 * * * - * ... # Subtest: resolve pass ok 14 - resolve pass @@ -164,7 +158,6 @@ TAP version 13 * * * - * ... # Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns diff --git a/test/message/test_runner_output_dot_reporter.js b/test/message/test_runner_output_dot_reporter.js new file mode 100644 index 00000000000000..8c36b9ba245425 --- /dev/null +++ b/test/message/test_runner_output_dot_reporter.js @@ -0,0 +1,6 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const spawn = require('node:child_process').spawn; +spawn(process.execPath, + ['--no-warnings', '--test-reporter', 'dot', 'test/message/test_runner_output.js'], { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_dot_reporter.out b/test/message/test_runner_output_dot_reporter.out new file mode 100644 index 00000000000000..823ecfb146b991 --- /dev/null +++ b/test/message/test_runner_output_dot_reporter.out @@ -0,0 +1,4 @@ +..XX...X..XXX.X..... +XXX.....X..X...X.... +.........X...XXX.XX. +.....XXXXXXX...XXXX diff --git a/test/message/test_runner_output_spec_reporter.js b/test/message/test_runner_output_spec_reporter.js new file mode 100644 index 00000000000000..49d8d3f2293da1 --- /dev/null +++ b/test/message/test_runner_output_spec_reporter.js @@ -0,0 +1,10 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const spawn = require('node:child_process').spawn; +const child = spawn(process.execPath, + ['--no-warnings', '--test-reporter', 'spec', 'test/message/test_runner_output.js'], + { stdio: 'pipe' }); +// eslint-disable-next-line no-control-regex +child.stdout.on('data', (d) => process.stdout.write(d.toString().replace(/[^\x00-\x7F]/g, '').replace(/\u001b\[\d+m/g, ''))); +child.stderr.pipe(process.stderr); diff --git a/test/message/test_runner_output_spec_reporter.out b/test/message/test_runner_output_spec_reporter.out new file mode 100644 index 00000000000000..f7e2b7e66d800a --- /dev/null +++ b/test/message/test_runner_output_spec_reporter.out @@ -0,0 +1,280 @@ + sync pass todo (*ms) + sync pass todo with message (*ms) + sync fail todo (*ms) + Error: thrown from sync fail todo + * + * + * + * + * + * + * + + sync fail todo with message (*ms) + Error: thrown from sync fail todo with message + * + * + * + * + * + * + * + + sync skip pass (*ms) + sync skip pass with message (*ms) + sync pass (*ms) + this test should pass + sync throw fail (*ms) + Error: thrown from sync throw fail + * + * + * + * + * + * + * + + async skip pass (*ms) + async pass (*ms) + async throw fail (*ms) + Error: thrown from async throw fail + * + * + * + * + * + * + * + + async skip fail (*ms) + Error: thrown from async throw fail + * + * + * + * + * + * + * + + async assertion fail (*ms) + AssertionError [ERR_ASSERTION]: Expected values to be strictly equal: + + true !== false + + * + * + * + * + * + * + * { + generatedMessage: true, + code: 'ERR_ASSERTION', + actual: true, + expected: false, + operator: 'strictEqual' + } + + resolve pass (*ms) + reject fail (*ms) + Error: rejected from reject fail + * + * + * + * + * + * + * + + unhandled rejection - passes but warns (*ms) + async unhandled rejection - passes but warns (*ms) + immediate throw - passes but warns (*ms) + immediate reject - passes but warns (*ms) + immediate resolve pass (*ms) + subtest sync throw fail + +sync throw fail (*ms) + Error: thrown from subtest sync throw fail + * + * + * + * + * + * + * + * + * + * + + this subtest should make its parent test fail + subtest sync throw fail (*ms) + + sync throw non-error fail (*ms) + Symbol(thrown symbol from sync throw non-error fail) + + level 0a + level 1a (*ms) + level 1b (*ms) + level 1c (*ms) + level 1d (*ms) + level 0a (*ms) + + top level + +long running (*ms) + 'test did not finish before its parent and was cancelled' + + +short running + ++short running (*ms) + +short running (*ms) + + top level (*ms) + + invalid subtest - pass but subtest fails (*ms) + sync skip option (*ms) + sync skip option with message (*ms) + sync skip option is false fail (*ms) + Error: this should be executed + * + * + * + * + * + * + * + + (*ms) + functionOnly (*ms) + (*ms) + test with only a name provided (*ms) + (*ms) + (*ms) + test with a name and options provided (*ms) + functionAndOptions (*ms) + escaped description \ # * + * + (*ms) + escaped skip message (*ms) + escaped todo message (*ms) + escaped diagnostic (*ms) + #diagnostic + callback pass (*ms) + callback fail (*ms) + Error: callback failure + * + * + + sync t is this in test (*ms) + async t is this in test (*ms) + callback t is this in test (*ms) + callback also returns a Promise (*ms) + 'passed a callback but also returned a Promise' + + callback throw (*ms) + Error: thrown from callback throw + * + * + * + * + * + * + * + + callback called twice (*ms) + 'callback invoked multiple times' + + callback called twice in different ticks (*ms) + callback called twice in future tick (*ms) + Error [ERR_TEST_FAILURE]: callback invoked multiple times + * + failureType: 'multipleCallbackInvocations', + cause: 'callback invoked multiple times', + code: 'ERR_TEST_FAILURE' + } + + callback async throw (*ms) + Error: thrown from callback async throw + * + * + + callback async throw after done (*ms) + only is set but not in only mode + running subtest 1 (*ms) + running subtest 2 (*ms) + running subtest 3 (*ms) + running subtest 4 (*ms) + only is set but not in only mode (*ms) + + custom inspect symbol fail (*ms) + customized + + custom inspect symbol that throws fail (*ms) + { foo: 1, [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] } + + subtest sync throw fails + sync throw fails at first (*ms) + Error: thrown from subtest sync throw fails at first + * + * + * + * + * + * + * + * + * + * + + sync throw fails at second (*ms) + Error: thrown from subtest sync throw fails at second + * + * + * + * + * + * + * + * + * + * + + subtest sync throw fails (*ms) + + timed out async test (*ms) + 'test timed out after *ms' + + timed out callback test (*ms) + 'test timed out after *ms' + + large timeout async test is ok (*ms) + large timeout callback test is ok (*ms) + successful thenable (*ms) + rejected thenable (*ms) + 'custom error' + + unfinished test with uncaughtException (*ms) + Error: foo + * + * + * + + unfinished test with unhandledRejection (*ms) + Error: bar + * + * + * + + invalid subtest fail (*ms) + 'test could not be started because its parent finished' + + Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. + Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. + Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event. + Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. + Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. + Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. + tests 65 + pass 27 + fail 21 + cancelled 2 + skipped 10 + todo 5 + duration_ms * diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 73df0ab053b533..73dce999a5c351 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -75,6 +75,7 @@ const expectedModules = new Set([ 'NativeModule internal/mime', 'NativeModule internal/modules/cjs/helpers', 'NativeModule internal/modules/cjs/loader', + 'NativeModule internal/modules/utils', 'NativeModule internal/modules/esm/assert', 'NativeModule internal/modules/esm/create_dynamic_module', 'NativeModule internal/modules/esm/fetch_module', diff --git a/test/parallel/test-runner-exit-code.js b/test/parallel/test-runner-exit-code.js index 1833fa00f7f7ae..1c28c2439050fc 100644 --- a/test/parallel/test-runner-exit-code.js +++ b/test/parallel/test-runner-exit-code.js @@ -20,8 +20,7 @@ async function runAndKill(file) { }); const [code, signal] = await once(child, 'exit'); await finished(child.stdout); - assert.match(stdout, /not ok 1/); - assert.match(stdout, /# cancelled 1\n/); + assert.strictEqual(stdout, 'TAP version 13\n'); assert.strictEqual(signal, null); assert.strictEqual(code, 1); } diff --git a/test/parallel/test-runner-reporters.js b/test/parallel/test-runner-reporters.js new file mode 100644 index 00000000000000..74cae3401e2843 --- /dev/null +++ b/test/parallel/test-runner-reporters.js @@ -0,0 +1,95 @@ +'use strict'; + +require('../common'); +const fixtures = require('../common/fixtures'); +const tmpdir = require('../common/tmpdir'); +const { describe, it } = require('node:test'); +const { spawnSync } = require('node:child_process'); +const assert = require('node:assert'); +const path = require('node:path'); +const fs = require('node:fs'); + +const testFile = fixtures.path('test-runner/reporters.js'); +tmpdir.refresh(); + +let tmpFiles = 0; +describe('node:test reporters', { concurrency: true }, () => { + it('should default to outputing TAP to stdout', async () => { + const child = spawnSync(process.execPath, ['--test', testFile]); + assert.strictEqual(child.stderr.toString(), ''); + assert.match(child.stdout.toString(), /TAP version 13/); + assert.match(child.stdout.toString(), /ok 1 - ok/); + assert.match(child.stdout.toString(), /not ok 2 - failing/); + assert.match(child.stdout.toString(), /ok 2 - top level/); + }); + + it('should default destination to stdout when passing a single reporter', async () => { + const child = spawnSync(process.execPath, ['--test', '--test-reporter', 'dot', testFile]); + assert.strictEqual(child.stderr.toString(), ''); + assert.strictEqual(child.stdout.toString(), '.XX.X\n'); + }); + + it('should throw when passing reporters without a destination', async () => { + const child = spawnSync(process.execPath, ['--test', '--test-reporter', 'dot', '--test-reporter', 'tap', testFile]); + assert.match(child.stderr.toString(), /The argument '--test-reporter' must match the number of specified '--test-reporter-destination'\. Received \[ 'dot', 'tap' \]/); + assert.strictEqual(child.stdout.toString(), ''); + }); + + it('should throw when passing a destination without a reporter', async () => { + const child = spawnSync(process.execPath, ['--test', '--test-reporter-destination', 'tap', testFile]); + assert.match(child.stderr.toString(), /The argument '--test-reporter' must match the number of specified '--test-reporter-destination'\. Received \[\]/); + assert.strictEqual(child.stdout.toString(), ''); + }); + + it('should support stdout as a destination', async () => { + const child = spawnSync(process.execPath, + ['--test', '--test-reporter', 'dot', '--test-reporter-destination', 'stdout', testFile]); + assert.strictEqual(child.stderr.toString(), ''); + assert.strictEqual(child.stdout.toString(), '.XX.X\n'); + }); + + it('should support stderr as a destination', async () => { + const child = spawnSync(process.execPath, + ['--test', '--test-reporter', 'dot', '--test-reporter-destination', 'stderr', testFile]); + assert.strictEqual(child.stderr.toString(), '.XX.X\n'); + assert.strictEqual(child.stdout.toString(), ''); + }); + + it('should support a file as a destination', async () => { + const file = path.join(tmpdir.path, `${tmpFiles++}.out`); + const child = spawnSync(process.execPath, + ['--test', '--test-reporter', 'dot', '--test-reporter-destination', file, testFile]); + assert.strictEqual(child.stderr.toString(), ''); + assert.strictEqual(child.stdout.toString(), ''); + assert.strictEqual(fs.readFileSync(file, 'utf8'), '.XX.X\n'); + }); + + it('should support multiple reporters', async () => { + const file = path.join(tmpdir.path, `${tmpFiles++}.out`); + const file2 = path.join(tmpdir.path, `${tmpFiles++}.out`); + const child = spawnSync(process.execPath, + ['--test', + '--test-reporter', 'dot', '--test-reporter-destination', file, + '--test-reporter', 'spec', '--test-reporter-destination', file2, + '--test-reporter', 'tap', '--test-reporter-destination', 'stdout', + testFile]); + assert.match(child.stdout.toString(), /TAP version 13/); + assert.match(child.stdout.toString(), /# duration_ms/); + assert.strictEqual(fs.readFileSync(file, 'utf8'), '.XX.X\n'); + const file2Contents = fs.readFileSync(file2, 'utf8'); + assert.match(file2Contents, /▶ nested/); + assert.match(file2Contents, /✔ ok/); + assert.match(file2Contents, /✖ failing/); + }); + + ['js', 'cjs', 'mjs'].forEach((ext) => { + it(`should support a '${ext}' file as a custom reporter`, async () => { + const filename = `custom.${ext}`; + const child = spawnSync(process.execPath, + ['--test', '--test-reporter', fixtures.path('test-runner/custom_reporters/', filename), + testFile]); + assert.strictEqual(child.stderr.toString(), ''); + assert.strictEqual(child.stdout.toString(), `${filename} {"test:start":5,"test:pass":2,"test:fail":3,"test:plan":3,"test:diagnostic":7}`); + }); + }); +}); diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs index 6ce7c8f5d97c5d..b6e0eee3b259ad 100644 --- a/test/parallel/test-runner-run.mjs +++ b/test/parallel/test-runner-run.mjs @@ -10,7 +10,6 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { it('should run with no tests', async () => { const stream = run({ files: [] }); - stream.setEncoding('utf8'); stream.on('test:fail', common.mustNotCall()); stream.on('test:pass', common.mustNotCall()); // eslint-disable-next-line no-unused-vars diff --git a/tools/doc/type-parser.mjs b/tools/doc/type-parser.mjs index 0e886a4785cb19..146981fe1464d0 100644 --- a/tools/doc/type-parser.mjs +++ b/tools/doc/type-parser.mjs @@ -208,7 +208,7 @@ const customTypesMap = { 'Timeout': 'timers.html#class-timeout', 'Timer': 'timers.html#timers', - 'TapStream': 'test.html#class-tapstream', + 'TestsStream': 'test.html#class-testsstream', 'tls.SecureContext': 'tls.html#tlscreatesecurecontextoptions', 'tls.Server': 'tls.html#class-tlsserver',