From c2412dfb4f458cfb9567eee3694f2b41f9f6a38c Mon Sep 17 00:00:00 2001 From: Moshe Atlow Date: Fri, 2 Dec 2022 02:22:28 +0200 Subject: [PATCH] test_runner: add reporters --- doc/api/cli.md | 19 + doc/api/test.md | 247 ++++++- lib/internal/main/test_runner.js | 7 +- lib/internal/modules/run_main.js | 26 +- lib/internal/modules/utils.js | 55 ++ lib/internal/test_runner/harness.js | 4 +- lib/internal/test_runner/runner.js | 33 +- lib/internal/test_runner/test.js | 50 +- lib/internal/test_runner/tests_stream.js | 74 ++ lib/internal/test_runner/utils.js | 76 ++- lib/test/reporter/dot.js | 14 + lib/test/reporter/spec.js | 87 +++ .../tap_stream.js => test/reporter/tap.js} | 148 ++-- src/node_options.cc | 6 + src/node_options.h | 2 + .../test-runner/custom_reporters/custom.cjs | 25 + .../test-runner/custom_reporters/custom.js | 21 + .../test-runner/custom_reporters/custom.mjs | 20 + test/message/test_runner_describe_it.out | 7 - test/message/test_runner_hooks.out | 2 - test/message/test_runner_output.js | 14 +- test/message/test_runner_output.out | 7 - .../test_runner_output_dot_reporter.js | 6 + .../test_runner_output_dot_reporter.out | 4 + .../test_runner_output_spec_reporter.js | 6 + .../test_runner_output_spec_reporter.out | 107 +++ .../test_runner_output_tap_reporter.js | 6 + .../test_runner_output_tap_reporter.out | 646 ++++++++++++++++++ test/parallel/test-bootstrap-modules.js | 1 + test/parallel/test-runner-exit-code.js | 3 +- test/parallel/test-runner-reporters.js | 38 ++ test/parallel/test-runner-run.mjs | 1 - tools/doc/type-parser.mjs | 2 +- 33 files changed, 1542 insertions(+), 222 deletions(-) create mode 100644 lib/internal/modules/utils.js create mode 100644 lib/internal/test_runner/tests_stream.js create mode 100644 lib/test/reporter/dot.js create mode 100644 lib/test/reporter/spec.js rename lib/{internal/test_runner/tap_stream.js => test/reporter/tap.js} (60%) create mode 100644 test/fixtures/test-runner/custom_reporters/custom.cjs create mode 100644 test/fixtures/test-runner/custom_reporters/custom.js create mode 100644 test/fixtures/test-runner/custom_reporters/custom.mjs create mode 100644 test/message/test_runner_output_dot_reporter.js create mode 100644 test/message/test_runner_output_dot_reporter.out create mode 100644 test/message/test_runner_output_spec_reporter.js create mode 100644 test/message/test_runner_output_spec_reporter.out create mode 100644 test/message/test_runner_output_tap_reporter.js create mode 100644 test/message/test_runner_output_tap_reporter.out create mode 100644 test/parallel/test-runner-reporters.js diff --git a/doc/api/cli.md b/doc/api/cli.md index c09066a29a1dec..5056f431d3b1f7 100644 --- a/doc/api/cli.md +++ b/doc/api/cli.md @@ -1230,6 +1230,24 @@ A regular expression that configures the test runner to only execute tests whose name matches the provided pattern. See the documentation on [filtering tests by name][] for more details. +### `--test-reporter` + + + +A test reporter to use when running tests. See the documentation on +[test reporters][] for more details. + +### `--test-reporter-destination` + + + +the destination for each used test reporter. See the documentation on +[test reporters][] for more details. + ### `--test-only` -The `node:test` module facilitates the creation of JavaScript tests that -report results in [TAP][] format. To access it: +The `node:test` module facilitates the creation of JavaScript tests. +To access it: ```mjs import test from 'node:test'; @@ -91,9 +91,7 @@ test('callback failing test', (t, done) => { }); ``` -As a test file executes, TAP is written to the standard output of the Node.js -process. This output can be interpreted by any test harness that understands -the TAP format. If any tests fail, the process exit code is set to `1`. +If any tests fail, the process exit code is set to `1`. ## Subtests @@ -122,8 +120,7 @@ test to fail. ## Skipping tests Individual tests can be skipped by passing the `skip` option to the test, or by -calling the test context's `skip()` method. Both of these options support -including a message that is displayed in the TAP output as shown in the +calling the test context's `skip()` method as shown in the following example. ```js @@ -258,7 +255,7 @@ Test name patterns do not change the set of files that the test runner executes. ## Extraneous asynchronous activity -Once a test function finishes executing, the TAP results are output as quickly +Once a test function finishes executing, the results are reported as quickly as possible while maintaining the order of the tests. However, it is possible for the test function to generate asynchronous activity that outlives the test itself. The test runner handles this type of activity, but does not delay the @@ -267,13 +264,13 @@ reporting of test results in order to accommodate it. In the following example, a test completes with two `setImmediate()` operations still outstanding. The first `setImmediate()` attempts to create a new subtest. Because the parent test has already finished and output its -results, the new subtest is immediately marked as failed, and reported in the -top level of the file's TAP output. +results, the new subtest is immediately marked as failed, and reported later +to the {TestsStream}. The second `setImmediate()` creates an `uncaughtException` event. `uncaughtException` and `unhandledRejection` events originating from a completed test are marked as failed by the `test` module and reported as diagnostic -warnings in the top level of the file's TAP output. +warnings emitted at the top level of by the {TestsStream}. ```js test('a test that creates asynchronous activity', (t) => { @@ -454,6 +451,166 @@ test('spies on an object method', (t) => { }); ``` +## Test reporters + + + +The `node:test` module supports passing [`--test-reporter`][] +flags for the test runner to use a specific reporter. + +the default reporter is the `tap` reporter. + +The following built-reporters are supported: + +### `tap` + +The `tap` reporter is the default reporter used by the test runner. It outputs +the test results in the [TAP][] format. + +### `spec` + +The `spec` reporter outputs the test results in a human-readable format. + +### `dot` + +The `dot` reporter outputs the test results in a dot format. + +### Custom reporters + +[`--test-reporter`][] can be used to specify a path to custom reporter. +a custom reporter is a module that exports a value +accepted by [stream.compose][] wich can be +{stream.Writable|Iterable|AsyncIterable|Function}. +the reporter will transform events emitted by {TestsStream} + +Expamle of a custom reporter using {stream.Transform}: + +```mjs +import { Transform } from 'node:stream'; + +const customReporter = new Transform({ + transform(event, encoding, callback) { + switch (event.type) { + case 'test:start': + callback(null, `test ${event.data.name} started`); + break; + case 'test:pass': + callback(null, `test ${event.data.name} passed`); + break; + case 'test:fail': + callback(null, `test ${event.data.name} failed`); + break; + case 'test:plan': + callback(null, 'test plan'); + break; + case 'test:diagnostic': + callback(null, event.data.message); + break; + } + }, +}); + +export default customReporter; +``` + +```cjs +const { Transform } = require('node:stream'); + +const customReporter = new Transform({ + transform(event, encoding, callback) { + switch (event.type) { + case 'test:start': + callback(null, `test ${event.data.name} started`); + break; + case 'test:pass': + callback(null, `test ${event.data.name} passed`); + break; + case 'test:fail': + callback(null, `test ${event.data.name} failed`); + break; + case 'test:plan': + callback(null, 'test plan'); + break; + case 'test:diagnostic': + callback(null, event.data.message); + break; + } + }, +}); + +module.exports = customReporter; +``` + +Expamle of a custom reporter using a Function: + +```mjs +export default async function * customReporter(source) { + for await (const event of source) { + switch (event.type) { + case 'test:start': + yield `test ${event.data.name} started\n`; + break; + case 'test:pass': + yield `test ${event.data.name} passed\n`; + break; + case 'test:fail': + yield `test ${event.data.name} failed\n`; + break; + case 'test:plan': + yield 'test plan'; + break; + case 'test:diagnostic': + yield `${event.data.message}\n`; + break; + } + } +} +``` + +```cjs +module.exports = async function * customReporter(source) { + for await (const event of source) { + switch (event.type) { + case 'test:start': + yield `test ${event.data.name} started\n`; + break; + case 'test:pass': + yield `test ${event.data.name} passed\n`; + break; + case 'test:fail': + yield `test ${event.data.name} failed\n`; + break; + case 'test:plan': + yield 'test plan\n'; + break; + case 'test:diagnostic': + yield `${event.data.message}\n`; + break; + } + } +}; +``` + +### Multiple reporters + +When passing multiple values to the [`--test-reporter`][] flag, +it is required to specify a destination for each reporter +using [`--test-reporter-destination`][]. +for each reporter specified via [`--test-reporter`][], +the corresponding destination will be used according +to the order they were specified. + +destination can be either `stdout`, `stderr` or a file path. + +```bash +node --test-reporter=spec --test-reporter=dot --test-reporter-destination=stdout --test-reporter-destination=file.txt +``` + +when a single reporter is specified, +the destination will be `stdout` by default. + ## `run([options])` -* `message` {string} Message to be displayed as a TAP diagnostic. +* `message` {string} Message to be reported. -This function is used to write TAP diagnostics to the output. Any diagnostic +This function is used to write diagnostics to the output. Any diagnostic information is included at the end of the test's results. This function does not return a value. @@ -1248,10 +1424,10 @@ added: - v16.17.0 --> -* `message` {string} Optional skip message to be displayed in TAP output. +* `message` {string} Optional skip message. This function causes the test's output to indicate the test as skipped. If -`message` is provided, it is included in the TAP output. Calling `skip()` does +`message` is provided, it is included in the output. Calling `skip()` does not terminate execution of the test function. This function does not return a value. @@ -1270,10 +1446,10 @@ added: - v16.17.0 --> -* `message` {string} Optional `TODO` message to be displayed in TAP output. +* `message` {string} Optional `TODO` message. This function adds a `TODO` directive to the test's output. If `message` is -provided, it is included in the TAP output. Calling `todo()` does not terminate +provided, it is included in the output. Calling `todo()` does not terminate execution of the test function. This function does not return a value. ```js @@ -1380,6 +1556,8 @@ added: [TAP]: https://testanything.org/ [`--test-name-pattern`]: cli.md#--test-name-pattern [`--test-only`]: cli.md#--test-only +[`--test-reporter-destination`]: cli.md#--test-reporter-destination +[`--test-reporter`]: cli.md#--test-reporter [`--test`]: cli.md#--test [`MockFunctionContext`]: #class-mockfunctioncontext [`MockTracker.method`]: #mockmethodobject-methodname-implementation-options @@ -1393,4 +1571,5 @@ added: [`test()`]: #testname-options-fn [describe options]: #describename-options-fn [it options]: #testname-options-fn +[stream.compose]: stream.md#streamcomposestreams [test runner execution model]: #test-runner-execution-model diff --git a/lib/internal/main/test_runner.js b/lib/internal/main/test_runner.js index f7165a0288cf9e..658aab03323a24 100644 --- a/lib/internal/main/test_runner.js +++ b/lib/internal/main/test_runner.js @@ -6,6 +6,7 @@ const { const { getOptionValue } = require('internal/options'); const { isUsingInspector } = require('internal/util/inspector'); const { run } = require('internal/test_runner/runner'); +const { setupTestReporters } = require('internal/test_runner/utils'); const { exitCodes: { kGenericUserError } } = internalBinding('errors'); prepareMainThreadExecution(false); @@ -21,8 +22,8 @@ if (isUsingInspector()) { inspectPort = process.debugPort; } -const tapStream = run({ concurrency, inspectPort, watch: getOptionValue('--watch') }); -tapStream.pipe(process.stdout); -tapStream.once('test:fail', () => { +const testsStream = run({ concurrency, inspectPort, watch: getOptionValue('--watch') }); +testsStream.once('test:fail', () => { process.exitCode = kGenericUserError; }); +setupTestReporters(testsStream); diff --git a/lib/internal/modules/run_main.js b/lib/internal/modules/run_main.js index 738c945bc21c21..2c8b17a3393d2a 100644 --- a/lib/internal/modules/run_main.js +++ b/lib/internal/modules/run_main.js @@ -2,12 +2,12 @@ const { ObjectCreate, - StringPrototypeEndsWith, } = primordials; const CJSLoader = require('internal/modules/cjs/loader'); -const { Module, toRealPath, readPackageScope } = CJSLoader; +const { Module, toRealPath } = CJSLoader; const { getOptionValue } = require('internal/options'); const path = require('path'); +const { shouldUseESMLoader } = require('internal/modules/utils'); function resolveMainPath(main) { // Note extension resolution for the main entry point can be deprecated in a @@ -24,28 +24,6 @@ function resolveMainPath(main) { return mainPath; } -function shouldUseESMLoader(mainPath) { - /** - * @type {string[]} userLoaders A list of custom loaders registered by the user - * (or an empty list when none have been registered). - */ - const userLoaders = getOptionValue('--experimental-loader'); - /** - * @type {string[]} userImports A list of preloaded modules registered by the user - * (or an empty list when none have been registered). - */ - const userImports = getOptionValue('--import'); - if (userLoaders.length > 0 || userImports.length > 0) - return true; - // Determine the module format of the main - if (mainPath && StringPrototypeEndsWith(mainPath, '.mjs')) - return true; - if (!mainPath || StringPrototypeEndsWith(mainPath, '.cjs')) - return false; - const pkg = readPackageScope(mainPath); - return pkg && pkg.data.type === 'module'; -} - function runMainESM(mainPath) { const { loadESM } = require('internal/process/esm_loader'); const { pathToFileURL } = require('internal/url'); diff --git a/lib/internal/modules/utils.js b/lib/internal/modules/utils.js new file mode 100644 index 00000000000000..1a52cc227908e9 --- /dev/null +++ b/lib/internal/modules/utils.js @@ -0,0 +1,55 @@ +'use strict'; + +const { + ObjectCreate, + StringPrototypeEndsWith, +} = primordials; +const CJSLoader = require('internal/modules/cjs/loader'); +const { Module, readPackageScope } = CJSLoader; +const { getOptionValue } = require('internal/options'); + + +function shouldUseESMLoader(filePath) { + /** + * @type {string[]} userLoaders A list of custom loaders registered by the user + * (or an empty list when none have been registered). + */ + const userLoaders = getOptionValue('--experimental-loader'); + /** + * @type {string[]} userImports A list of preloaded modules registered by the user + * (or an empty list when none have been registered). + */ + const userImports = getOptionValue('--import'); + if (userLoaders.length > 0 || userImports.length > 0) + return true; + // Determine the module format of the main + if (filePath && StringPrototypeEndsWith(filePath, '.mjs')) + return true; + if (!filePath || StringPrototypeEndsWith(filePath, '.cjs')) + return false; + const pkg = readPackageScope(filePath); + return pkg && pkg.data.type === 'module'; +} + +/** + * @param {string} filePath + * @returns {any} + * requireOrImport imports a module if the file is an ES module, otherwise it requires it. + */ +function requireOrImport(filePath) { + const useESMLoader = shouldUseESMLoader(filePath); + if (useESMLoader) { + const { esmLoader } = require('internal/process/esm_loader'); + const { pathToFileURL } = require('internal/url'); + const { isAbsolute } = require('path'); + const file = isAbsolute(filePath) ? pathToFileURL(filePath).href : filePath; + return esmLoader.import(file, undefined, ObjectCreate(null)); + } + return new Module._load(filePath, null, false); + +} + +module.exports = { + shouldUseESMLoader, + requireOrImport, +}; diff --git a/lib/internal/test_runner/harness.js b/lib/internal/test_runner/harness.js index 0a6be080e8b7f1..33c0bb5ae8c962 100644 --- a/lib/internal/test_runner/harness.js +++ b/lib/internal/test_runner/harness.js @@ -18,6 +18,7 @@ const { exitCodes: { kGenericUserError } } = internalBinding('errors'); const { kEmptyObject } = require('internal/util'); const { getOptionValue } = require('internal/options'); const { kCancelledByParent, Test, ItTest, Suite } = require('internal/test_runner/test'); +const { setupTestReporters } = require('internal/test_runner/utils'); const { bigint: hrtime } = process.hrtime; const isTestRunnerCli = getOptionValue('--test'); @@ -109,7 +110,6 @@ function setup(root) { } root.startTime = hrtime(); - root.reporter.version(); wasRootSetup.add(root); return root; @@ -119,10 +119,10 @@ let globalRoot; function getGlobalRoot() { if (!globalRoot) { globalRoot = createTestTree(); - globalRoot.reporter.pipe(process.stdout); globalRoot.reporter.once('test:fail', () => { process.exitCode = kGenericUserError; }); + setupTestReporters(globalRoot.reporter); } return globalRoot; } diff --git a/lib/internal/test_runner/runner.js b/lib/internal/test_runner/runner.js index 731f94e5b6b8f7..ca6ebf9c3f62b9 100644 --- a/lib/internal/test_runner/runner.js +++ b/lib/internal/test_runner/runner.js @@ -7,6 +7,7 @@ const { ArrayPrototypeJoin, ArrayPrototypePush, ArrayPrototypeSlice, + ArrayPrototypeSome, ArrayPrototypeSort, ObjectAssign, PromisePrototypeThen, @@ -16,6 +17,7 @@ const { SafeMap, SafeSet, StringPrototypeRepeat, + StringPrototypeStartsWith, } = primordials; const { spawn } = require('child_process'); @@ -33,7 +35,7 @@ const { validateArray, validateBoolean } = require('internal/validators'); const { getInspectPort, isUsingInspector, isInspectorMessage } = require('internal/util/inspector'); const { kEmptyObject } = require('internal/util'); const { createTestTree } = require('internal/test_runner/harness'); -const { kDefaultIndent, kSubtestsFailed, Test } = require('internal/test_runner/test'); +const { kSubtestsFailed, Test } = require('internal/test_runner/test'); const { TapParser } = require('internal/test_runner/tap_parser'); const { TokenKind } = require('internal/test_runner/tap_lexer'); @@ -49,6 +51,7 @@ const { } = internalBinding('errors'); const kFilterArgs = ['--test', '--watch']; +const kFilterArgValues = ['--test-reporter']; // TODO(cjihrig): Replace this with recursive readdir once it lands. function processPath(path, testFiles, options) { @@ -112,8 +115,9 @@ function createTestFileList() { return ArrayPrototypeSort(ArrayFrom(testFiles)); } -function filterExecArgv(arg) { - return !ArrayPrototypeIncludes(kFilterArgs, arg); +function filterExecArgv(arg, i, arr) { + return !ArrayPrototypeIncludes(kFilterArgs, arg) && + !ArrayPrototypeSome(kFilterArgValues, (p) => arg === p || (i > 0 && arr[i - 1] === p) || StringPrototypeStartsWith(arg, `${p}=`)); } function getRunArgs({ path, inspectPort }) { @@ -125,12 +129,15 @@ function getRunArgs({ path, inspectPort }) { return argv; } +const kDefaultIndent = ' '; // 4 spaces + class FileTest extends Test { #buffer = []; #handleReportItem({ kind, node, nesting = 0 }) { - const indent = StringPrototypeRepeat(kDefaultIndent, nesting + 1); + nesting += 1; const details = (diagnostic) => { + const indent = StringPrototypeRepeat(kDefaultIndent, nesting); return ( diagnostic && { __proto__: null, @@ -149,11 +156,11 @@ class FileTest extends Test { break; case TokenKind.TAP_PLAN: - this.reporter.plan(indent, node.end - node.start + 1); + this.reporter.plan(nesting, node.end - node.start + 1); break; case TokenKind.TAP_SUBTEST_POINT: - this.reporter.subtest(indent, node.name); + this.reporter.start(nesting, node.name); break; case TokenKind.TAP_TEST_POINT: @@ -172,7 +179,7 @@ class FileTest extends Test { if (pass) { this.reporter.ok( - indent, + nesting, node.id, node.description, details(node.diagnostics), @@ -180,7 +187,7 @@ class FileTest extends Test { ); } else { this.reporter.fail( - indent, + nesting, node.id, node.description, details(node.diagnostics), @@ -190,15 +197,15 @@ class FileTest extends Test { break; case TokenKind.COMMENT: - if (indent === kDefaultIndent) { + if (nesting === 1) { // Ignore file top level diagnostics break; } - this.reporter.diagnostic(indent, node.comment); + this.reporter.diagnostic(nesting, node.comment); break; case TokenKind.UNKNOWN: - this.reporter.diagnostic(indent, node.value); + this.reporter.diagnostic(nesting, node.value); break; } } @@ -207,11 +214,11 @@ class FileTest extends Test { ArrayPrototypePush(this.#buffer, ast); return; } - this.reportSubtest(); + this.reportStarted(); this.#handleReportItem(ast); } report() { - this.reportSubtest(); + this.reportStarted(); ArrayPrototypeForEach(this.#buffer, (ast) => this.#handleReportItem(ast)); super.report(); } diff --git a/lib/internal/test_runner/test.js b/lib/internal/test_runner/test.js index c56c03c0725586..798f29c3706602 100644 --- a/lib/internal/test_runner/test.js +++ b/lib/internal/test_runner/test.js @@ -33,7 +33,7 @@ const { } = require('internal/errors'); const { getOptionValue } = require('internal/options'); const { MockTracker } = require('internal/test_runner/mock'); -const { TapStream } = require('internal/test_runner/tap_stream'); +const { TestsStream } = require('internal/test_runner/tests_stream'); const { convertStringToRegExp, createDeferredCallback, @@ -63,7 +63,6 @@ const kTestCodeFailure = 'testCodeFailure'; const kTestTimeoutFailure = 'testTimeoutFailure'; const kHookFailure = 'hookFailed'; const kDefaultTimeout = null; -const kDefaultIndent = ' '; // 4 spaces const noop = FunctionPrototype; const isTestRunner = getOptionValue('--test'); const testOnlyFlag = !isTestRunner && getOptionValue('--test-only'); @@ -187,18 +186,18 @@ class Test extends AsyncResource { if (parent === null) { this.concurrency = 1; - this.indent = ''; + this.nesting = 0; this.only = testOnlyFlag; - this.reporter = new TapStream(); + this.reporter = new TestsStream(); this.runOnlySubtests = this.only; this.testNumber = 0; this.timeout = kDefaultTimeout; } else { - const indent = parent.parent === null ? parent.indent : - parent.indent + kDefaultIndent; + const nesting = parent.parent === null ? parent.nesting : + parent.nesting + 1; this.concurrency = parent.concurrency; - this.indent = indent; + this.nesting = nesting; this.only = only ?? !parent.runOnlySubtests; this.reporter = parent.reporter; this.runOnlySubtests = !this.only; @@ -331,7 +330,7 @@ class Test extends AsyncResource { } if (i === 1 && this.parent !== null) { - this.reportSubtest(); + this.reportStarted(); } // Report the subtest's results and remove it from the ready map. @@ -623,19 +622,19 @@ class Test extends AsyncResource { this.parent.processPendingSubtests(); } else if (!this.reported) { this.reported = true; - this.reporter.plan(this.indent, this.subtests.length); + this.reporter.plan(this.nesting, this.subtests.length); for (let i = 0; i < this.diagnostics.length; i++) { - this.reporter.diagnostic(this.indent, this.diagnostics[i]); + this.reporter.diagnostic(this.nesting, this.diagnostics[i]); } - this.reporter.diagnostic(this.indent, `tests ${this.subtests.length}`); - this.reporter.diagnostic(this.indent, `pass ${counters.passed}`); - this.reporter.diagnostic(this.indent, `fail ${counters.failed}`); - this.reporter.diagnostic(this.indent, `cancelled ${counters.cancelled}`); - this.reporter.diagnostic(this.indent, `skipped ${counters.skipped}`); - this.reporter.diagnostic(this.indent, `todo ${counters.todo}`); - this.reporter.diagnostic(this.indent, `duration_ms ${this.#duration()}`); + this.reporter.diagnostic(this.nesting, `tests ${this.subtests.length}`); + this.reporter.diagnostic(this.nesting, `pass ${counters.passed}`); + this.reporter.diagnostic(this.nesting, `fail ${counters.failed}`); + this.reporter.diagnostic(this.nesting, `cancelled ${counters.cancelled}`); + this.reporter.diagnostic(this.nesting, `skipped ${counters.skipped}`); + this.reporter.diagnostic(this.nesting, `todo ${counters.todo}`); + this.reporter.diagnostic(this.nesting, `duration_ms ${this.#duration()}`); this.reporter.push(null); } } @@ -671,9 +670,9 @@ class Test extends AsyncResource { report() { if (this.subtests.length > 0) { - this.reporter.plan(this.subtests[0].indent, this.subtests.length); + this.reporter.plan(this.subtests[0].nesting, this.subtests.length); } else { - this.reportSubtest(); + this.reportStarted(); } let directive; const details = { __proto__: null, duration: this.#duration() }; @@ -685,24 +684,24 @@ class Test extends AsyncResource { } if (this.passed) { - this.reporter.ok(this.indent, this.testNumber, this.name, details, directive); + this.reporter.ok(this.nesting, this.testNumber, this.name, details, directive); } else { details.error = this.error; - this.reporter.fail(this.indent, this.testNumber, this.name, details, directive); + this.reporter.fail(this.nesting, this.testNumber, this.name, details, directive); } for (let i = 0; i < this.diagnostics.length; i++) { - this.reporter.diagnostic(this.indent, this.diagnostics[i]); + this.reporter.diagnostic(this.nesting, this.diagnostics[i]); } } - reportSubtest() { + reportStarted() { if (this.#reportedSubtest || this.parent === null) { return; } this.#reportedSubtest = true; - this.parent.reportSubtest(); - this.reporter.subtest(this.indent, this.name); + this.parent.reportStarted(); + this.reporter.start(this.nesting, this.name); } } @@ -807,7 +806,6 @@ class Suite extends Test { module.exports = { ItTest, kCancelledByParent, - kDefaultIndent, kSubtestsFailed, kTestCodeFailure, kUnwrapErrors, diff --git a/lib/internal/test_runner/tests_stream.js b/lib/internal/test_runner/tests_stream.js new file mode 100644 index 00000000000000..bbdd97ab446664 --- /dev/null +++ b/lib/internal/test_runner/tests_stream.js @@ -0,0 +1,74 @@ +'use strict'; +const { + ArrayPrototypePush, + ArrayPrototypeShift, +} = primordials; +const Readable = require('internal/streams/readable'); + +class TestsStream extends Readable { + #buffer; + #canPush; + + constructor() { + super({ objectMode: true }); + this.#buffer = []; + this.#canPush = true; + } + + _read() { + this.#canPush = true; + + while (this.#buffer.length > 0) { + const obj = ArrayPrototypeShift(this.#buffer); + + if (!this.#tryPush(obj)) { + return; + } + } + } + + fail(nesting, testNumber, name, details, directive) { + this.#emit('test:fail', { __proto__: null, name, nesting, testNumber, details, ...directive }); + } + + ok(nesting, testNumber, name, details, directive) { + this.#emit('test:pass', { __proto__: null, name, nesting, testNumber, details, ...directive }); + } + + plan(nesting, count) { + this.#emit('test:plan', { __proto__: null, nesting, count }); + } + + getSkip(reason) { + return { __proto__: null, skip: reason ?? true }; + } + + getTodo(reason) { + return { __proto__: null, todo: reason ?? true }; + } + + start(nesting, name) { + this.#emit('test:start', { __proto__: null, nesting, name }); + } + + diagnostic(nesting, message) { + this.#emit('test:diagnostic', { __proto__: null, nesting, message }); + } + + #emit(type, data) { + this.emit(type, data); + this.#tryPush({ type, data }); + } + + #tryPush(message) { + if (this.#canPush) { + this.#canPush = this.push(message); + } else { + ArrayPrototypePush(this.#buffer, message); + } + + return this.#canPush; + } +} + +module.exports = { TestsStream }; diff --git a/lib/internal/test_runner/utils.js b/lib/internal/test_runner/utils.js index ad040f010250e2..4856abeae5ed09 100644 --- a/lib/internal/test_runner/utils.js +++ b/lib/internal/test_runner/utils.js @@ -1,7 +1,19 @@ 'use strict'; -const { RegExp, RegExpPrototypeExec } = primordials; +const { + ArrayPrototypeMap, + ArrayPrototypePush, + ObjectGetOwnPropertyDescriptor, + SafePromiseAll, + RegExp, + RegExpPrototypeExec, + SafeMap +} = primordials; const { basename } = require('path'); +const { createWriteStream } = require('fs'); const { createDeferredPromise } = require('internal/util'); +const { getOptionValue } = require('internal/options'); +const { requireOrImport } = require('internal/modules/utils'); + const { codes: { ERR_INVALID_ARG_VALUE, @@ -9,6 +21,7 @@ const { }, kIsNodeError, } = require('internal/errors'); +const { compose } = require('stream'); const kMultipleCallbackInvocations = 'multipleCallbackInvocations'; const kRegExpPattern = /^\/(.*)\/([a-z]*)$/; @@ -74,10 +87,71 @@ function convertStringToRegExp(str, name) { } } +const kBuiltinDestinations = new SafeMap([ + ['stdout', process.stdout], + ['stderr', process.stderr], +]); + +const kBuiltinReporters = new SafeMap([ + ['spec', 'node:test/reporter/spec'], + ['dot', 'node:test/reporter/dot'], + ['tap', 'node:test/reporter/tap'], +]); + +const kDefaultReporter = 'tap'; +const kDefaltDestination = 'stdout'; + +async function getReportersMap(reporters, destinations) { + const result = await SafePromiseAll(ArrayPrototypeMap(reporters, async (name, i) => { + const destination = kBuiltinDestinations.get(destinations[i]) ?? createWriteStream(destinations[i]); + let reporter = await requireOrImport(kBuiltinReporters.get(name) ?? name); + + if (reporter && reporter.default) { + reporter = reporter.default; + } + + if (reporter.prototype && ObjectGetOwnPropertyDescriptor(reporter.prototype, 'constructor')) { + reporter = new reporter(); + } + + if (!reporter) { + throw new ERR_INVALID_ARG_VALUE('Reporter', name, 'is not a valid reporter'); + } + + return { __proto__: null, reporter, destination }; + } + )); + return result; +} + + +async function setupTestReporters(testsStream) { + const destinations = getOptionValue('--test-reporter-destination'); + const reporters = getOptionValue('--test-reporter'); + + if (reporters.length === 0 && destinations.length === 0) { + ArrayPrototypePush(reporters, kDefaultReporter); + } + + if (reporters.length === 1 && destinations.length === 0) { + ArrayPrototypePush(destinations, kDefaltDestination); + } + + if (destinations.length !== reporters.length) { + throw new ERR_INVALID_ARG_VALUE('The number of reporters and destinations must match'); + } + + const reportersMap = await getReportersMap(reporters, destinations); + for (const { reporter, destination } of reportersMap) { + compose(testsStream, reporter).pipe(destination); + } +} + module.exports = { convertStringToRegExp, createDeferredCallback, doesPathMatchFilter, isSupportedFileType, isTestFailureError, + setupTestReporters, }; diff --git a/lib/test/reporter/dot.js b/lib/test/reporter/dot.js new file mode 100644 index 00000000000000..f45b44a31045d2 --- /dev/null +++ b/lib/test/reporter/dot.js @@ -0,0 +1,14 @@ +'use strict'; + +module.exports = async function*(source) { + let count = 0; + for await (const { type } of source) { + if (type === 'test:fail' || type === 'test:pass') { + yield '.'; + if (++count % 20 === 0) { + yield '\n'; + } + } + } + yield '\n'; +}; diff --git a/lib/test/reporter/spec.js b/lib/test/reporter/spec.js new file mode 100644 index 00000000000000..c2a50099530699 --- /dev/null +++ b/lib/test/reporter/spec.js @@ -0,0 +1,87 @@ +'use strict'; + +const { + ArrayPrototypePop, + ArrayPrototypeShift, + ArrayPrototypeUnshift, + SafeMap, + StringPrototypeRepeat, +} = primordials; +const assert = require('assert'); +const Transform = require('internal/streams/transform'); +const { green, blue, red, white } = require('internal/util/colors'); + + +const colors = { + '__proto__': null, + 'test:fail': red, + 'test:pass': green, + 'test:diagnostic': blue, +}; +const symbols = { + '__proto__': null, + 'test:fail': '\u2716 ', + 'test:pass': '\u2714 ', + 'test:diagnostic': '\u2139 ', + 'arrow:right': '\u25B6 ', +}; +class SpecReporter extends Transform { + #stack = []; + #reported = []; + #indentMemo = new SafeMap(); + + constructor() { + super({ writableObjectMode: true }); + } + + #indent(nesting) { + let value = this.#indentMemo.get(nesting); + if (value === undefined) { + value = StringPrototypeRepeat(' ', nesting); + this.#indentMemo.set(nesting, value); + } + + return value; + } + #handleEvent({ type, data }) { + const color = colors[type] ?? white; + const symbol = symbols[type] ?? ' '; + + switch (type) { + case 'test:fail': + case 'test:pass': { + const subtest = ArrayPrototypeShift(this.#stack); // This is the matching `test:start` event + if (subtest) { + assert(subtest.type === 'test:start'); + assert(subtest.data.nesting === data.nesting); + assert(subtest.data.name === data.name); + } + let prefix = ''; + while (this.#stack.length) { + // Report all the parent `test:start` events + const parent = ArrayPrototypePop(this.#stack); + assert(parent.type === 'test:start'); + const msg = parent.data; + ArrayPrototypeUnshift(this.#reported, msg); + prefix += `${this.#indent(msg.nesting)}${symbols['arrow:right']}${msg.name}\n`; + } + if (this.#reported[0] && this.#reported[0].nesting === data.nesting && this.#reported[0].name === data.name) { + // If this test has had children - it was already reporter, so slightly modify the output + ArrayPrototypeShift(this.#reported); + return `${prefix}${this.#indent(data.nesting)}${color}${symbols['arrow:right']}${white}${data.name}\n\n`; + } + return `${prefix}${this.#indent(data.nesting)}${color}${symbol}${data.name}${white}\n`; + } + case 'test:start': + ArrayPrototypeUnshift(this.#stack, { __proto__: null, data, type }); + break; + case 'test:diagnostic': + return `${color}${this.#indent(data.nesting)}${symbol}${data.message}${white}\n`; + } + } + _transform({ type, data }, encoding, callback) { + callback(null, this.#handleEvent({ type, data })); + } +} + +module.exports = SpecReporter; diff --git a/lib/internal/test_runner/tap_stream.js b/lib/test/reporter/tap.js similarity index 60% rename from lib/internal/test_runner/tap_stream.js rename to lib/test/reporter/tap.js index 3abd7ed5560664..29bbbb84bcd8a3 100644 --- a/lib/internal/test_runner/tap_stream.js +++ b/lib/test/reporter/tap.js @@ -2,18 +2,17 @@ const { ArrayPrototypeForEach, ArrayPrototypeJoin, - ArrayPrototypeMap, ArrayPrototypePush, - ArrayPrototypeShift, ObjectEntries, + RegExpPrototypeSymbolReplace, + SafeMap, StringPrototypeReplaceAll, - StringPrototypeToUpperCase, StringPrototypeSplit, - RegExpPrototypeSymbolReplace, + StringPrototypeRepeat, } = primordials; const { inspectWithNoCustomRetry } = require('internal/errors'); -const Readable = require('internal/streams/readable'); const { isError, kEmptyObject } = require('internal/util'); +const kDefaultIndent = ' '; // 4 spaces const kFrameStartRegExp = /^ {4}at /; const kLineBreakRegExp = /\n|\r\n/; const kDefaultTAPVersion = 13; @@ -22,113 +21,78 @@ let testModule; // Lazy loaded due to circular dependency. function lazyLoadTest() { testModule ??= require('internal/test_runner/test'); - return testModule; } -class TapStream extends Readable { - #buffer; - #canPush; - - constructor() { - super(); - this.#buffer = []; - this.#canPush = true; - } - - _read() { - this.#canPush = true; - while (this.#buffer.length > 0) { - const line = ArrayPrototypeShift(this.#buffer); - - if (!this.#tryPush(line)) { - return; - } +async function * tapReporter(source) { + yield `TAP version ${kDefaultTAPVersion}\n`; + for await (const { type, data } of source) { + switch (type) { + case 'test:fail': + yield reportTest(data.nesting, data.testNumber, 'not ok', data.name, data.skip, data.todo); + yield reportDetails(data.nesting, data.details); + break; + case 'test:pass': + yield reportTest(data.nesting, data.testNumber, 'ok', data.name, data.skip, data.todo); + yield reportDetails(data.nesting, data.details); + break; + case 'test:plan': + yield `${indent(data.nesting)}1..${data.count}\n`; + break; + case 'test:start': + yield `${indent(data.nesting)}# Subtest: ${tapEscape(data.name)}\n`; + break; + case 'test:diagnostic': + yield `${indent(data.nesting)}# ${tapEscape(data.message)}\n`; + break; } } +} - bail(message) { - this.#tryPush(`Bail out!${message ? ` ${tapEscape(message)}` : ''}\n`); - } - - fail(indent, testNumber, name, details, directive) { - this.emit('test:fail', { __proto__: null, name, testNumber, details, ...directive }); - this.#test(indent, testNumber, 'not ok', name, directive); - this.#details(indent, details); - } - - ok(indent, testNumber, name, details, directive) { - this.emit('test:pass', { __proto__: null, name, testNumber, details, ...directive }); - this.#test(indent, testNumber, 'ok', name, directive); - this.#details(indent, details); - } - - plan(indent, count, explanation) { - const exp = `${explanation ? ` # ${tapEscape(explanation)}` : ''}`; - - this.#tryPush(`${indent}1..${count}${exp}\n`); - } - - getSkip(reason) { - return { __proto__: null, skip: reason }; - } - - getTodo(reason) { - return { __proto__: null, todo: reason }; - } - - subtest(indent, name) { - this.#tryPush(`${indent}# Subtest: ${tapEscape(name)}\n`); - } - - #details(indent, data = kEmptyObject) { - const { error, duration, yaml } = data; - let details = `${indent} ---\n`; +function reportTest(nesting, testNumber, status, name, skip, todo) { + let line = `${indent(nesting)}${status} ${testNumber}`; - details += `${yaml ? yaml : ''}`; - details += jsToYaml(indent, 'duration_ms', duration); - details += jsToYaml(indent, null, error); - details += `${indent} ...\n`; - this.#tryPush(details); + if (name) { + line += ` ${tapEscape(`- ${name}`)}`; } - diagnostic(indent, message) { - this.emit('test:diagnostic', message); - this.#tryPush(`${indent}# ${tapEscape(message)}\n`); + if (skip) { + line += ` # SKIP${typeof skip === 'string' ? ` ${tapEscape(skip)}` : ''}`; + } else if (todo) { + line += ` # TODO${typeof todo === 'string' ? ` ${tapEscape(todo)}` : ''}`; } - version(spec = kDefaultTAPVersion) { - this.#tryPush(`TAP version ${spec}\n`); - } + line += '\n'; - #test(indent, testNumber, status, name, directive = kEmptyObject) { - let line = `${indent}${status} ${testNumber}`; + return line; +} - if (name) { - line += ` ${tapEscape(`- ${name}`)}`; - } - line += ArrayPrototypeJoin(ArrayPrototypeMap(ObjectEntries(directive), ({ 0: key, 1: value }) => ( - ` # ${StringPrototypeToUpperCase(key)}${value ? ` ${tapEscape(value)}` : ''}` - )), ''); +function reportDetails(nesting, data = kEmptyObject) { + const { error, duration, yaml } = data; + const _indent = indent(nesting); + let details = `${_indent} ---\n`; - line += '\n'; + details += `${yaml ? yaml : ''}`; + details += jsToYaml(_indent, 'duration_ms', duration); + details += jsToYaml(_indent, null, error); + details += `${_indent} ...\n`; + return details; +} - this.#tryPush(line); +const memo = new SafeMap(); +function indent(nesting) { + let value = memo.get(nesting); + if (value === undefined) { + value = StringPrototypeRepeat(kDefaultIndent, nesting); + memo.set(nesting, value); } - #tryPush(message) { - if (this.#canPush) { - this.#canPush = this.push(message); - } else { - ArrayPrototypePush(this.#buffer, message); - } - - return this.#canPush; - } + return value; } + // In certain places, # and \ need to be escaped as \# and \\. function tapEscape(input) { return StringPrototypeReplaceAll( @@ -261,4 +225,4 @@ function isAssertionLike(value) { return value && typeof value === 'object' && 'expected' in value && 'actual' in value; } -module.exports = { TapStream }; +module.exports = tapReporter; diff --git a/src/node_options.cc b/src/node_options.cc index a5365987129ac7..3f4dc3f870aa40 100644 --- a/src/node_options.cc +++ b/src/node_options.cc @@ -548,6 +548,12 @@ EnvironmentOptionsParser::EnvironmentOptionsParser() { AddOption("--test-name-pattern", "run tests whose name matches this regular expression", &EnvironmentOptions::test_name_pattern); + AddOption("--test-reporter", + "report test output using the given reporter", + &EnvironmentOptions::test_reporter); + AddOption("--test-reporter-destination", + "report given reporter to the given destination", + &EnvironmentOptions::test_reporter_destination); AddOption("--test-only", "run tests with 'only' option set", &EnvironmentOptions::test_only, diff --git a/src/node_options.h b/src/node_options.h index 02beddccdf01c8..01db91b1db5498 100644 --- a/src/node_options.h +++ b/src/node_options.h @@ -155,6 +155,8 @@ class EnvironmentOptions : public Options { std::string diagnostic_dir; bool test_runner = false; std::vector test_name_pattern; + std::vector test_reporter; + std::vector test_reporter_destination; bool test_only = false; bool test_udp_no_try_send = false; bool throw_deprecation = false; diff --git a/test/fixtures/test-runner/custom_reporters/custom.cjs b/test/fixtures/test-runner/custom_reporters/custom.cjs new file mode 100644 index 00000000000000..90ed96d3e9de81 --- /dev/null +++ b/test/fixtures/test-runner/custom_reporters/custom.cjs @@ -0,0 +1,25 @@ +const { Transform } = require('node:stream'); + +const customReporter = new Transform({ + transform(event, encoding, callback) { + switch (event.type) { + case 'test:start': + callback(null, `test ${event.name} started`); + break; + case 'test:pass': + callback(null, `test ${event.name} passed`); + break; + case 'test:fail': + callback(null, `test ${event.name} failed`); + break; + case 'test:plan': + callback(null, 'test plan'); + break; + case 'test:diagnostic': + yield 'diagnostic'; + break; + } + }, +}); + +module.exports = customReporter; diff --git a/test/fixtures/test-runner/custom_reporters/custom.js b/test/fixtures/test-runner/custom_reporters/custom.js new file mode 100644 index 00000000000000..f730439b6e726b --- /dev/null +++ b/test/fixtures/test-runner/custom_reporters/custom.js @@ -0,0 +1,21 @@ +module.exports = async function * customReporter(source) { + for await (const event of source) { + switch (event.type) { + case 'test:start': + yield `test ${event.data.name} started\n`; + break; + case 'test:pass': + yield `test ${event.data.name} passed\n`; + break; + case 'test:fail': + yield `test ${event.data.name} failed\n`; + break; + case 'test:plan': + yield 'test plan\n'; + break; + case 'test:diagnostic': + yield `${event.data.message}\n`; + break; + } + } +}; diff --git a/test/fixtures/test-runner/custom_reporters/custom.mjs b/test/fixtures/test-runner/custom_reporters/custom.mjs new file mode 100644 index 00000000000000..6c1961086b3279 --- /dev/null +++ b/test/fixtures/test-runner/custom_reporters/custom.mjs @@ -0,0 +1,20 @@ +export default async function * customReporter(source) { + for await (const event of source) { + switch (event.type) { + case 'test:start': + yield `test ${event.name} started`; + break; + case 'test:pass': + yield `test ${event.name} passed`; + break; + case 'test:fail': + yield `test ${event.name} failed`; + break; + case 'test:plan': + yield 'test plan'; + break; + case 'test:diagnostic': + yield 'diagnostic'; + } + } +} diff --git a/test/message/test_runner_describe_it.out b/test/message/test_runner_describe_it.out index 199e834d6f65ae..87207aca71fafa 100644 --- a/test/message/test_runner_describe_it.out +++ b/test/message/test_runner_describe_it.out @@ -24,7 +24,6 @@ not ok 3 - sync fail todo # TODO * * * - * ... # Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo @@ -41,7 +40,6 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * * - * ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -73,7 +71,6 @@ not ok 8 - sync throw fail * * * - * ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -100,7 +97,6 @@ not ok 11 - async throw fail * * * - * ... # Subtest: async skip fail not ok 12 - async skip fail @@ -132,7 +128,6 @@ not ok 13 - async assertion fail * * * - * ... # Subtest: resolve pass ok 14 - resolve pass @@ -154,7 +149,6 @@ not ok 15 - reject fail * * * - * ... # Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns @@ -620,7 +614,6 @@ not ok 58 - rejected thenable code: 'ERR_TEST_FAILURE' stack: |- * - * ... # Subtest: invalid subtest fail not ok 59 - invalid subtest fail diff --git a/test/message/test_runner_hooks.out b/test/message/test_runner_hooks.out index e0b3e91b8c1376..bbe9837f72db3c 100644 --- a/test/message/test_runner_hooks.out +++ b/test/message/test_runner_hooks.out @@ -64,7 +64,6 @@ not ok 2 - before throws * * * - * ... # Subtest: after throws # Subtest: 1 @@ -93,7 +92,6 @@ not ok 3 - after throws * * * - * ... # Subtest: beforeEach throws # Subtest: 1 diff --git a/test/message/test_runner_output.js b/test/message/test_runner_output.js index 2a71cd3e16e143..63a848268ecabf 100644 --- a/test/message/test_runner_output.js +++ b/test/message/test_runner_output.js @@ -113,7 +113,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p1a = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 1000); + }, 100); }); return p1a; @@ -131,7 +131,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p1c = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 2000); + }, 200); }); return p1c; @@ -141,7 +141,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p1c = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 1500); + }, 150); }); return p1c; @@ -150,7 +150,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { const p0a = new Promise((resolve) => { setTimeout(() => { resolve(); - }, 3000); + }, 300); }); return p0a; @@ -159,7 +159,7 @@ test('level 0a', { concurrency: 4 }, async (t) => { test('top level', { concurrency: 2 }, async (t) => { t.test('+long running', async (t) => { return new Promise((resolve, reject) => { - setTimeout(resolve, 3000).unref(); + setTimeout(resolve, 300).unref(); }); }); @@ -331,12 +331,12 @@ test('subtest sync throw fails', async (t) => { test('timed out async test', { timeout: 5 }, async (t) => { return new Promise((resolve) => { - setTimeout(resolve, 1000); + setTimeout(resolve, 100); }); }); test('timed out callback test', { timeout: 5 }, (t, done) => { - setTimeout(done, 1000); + setTimeout(done, 100); }); diff --git a/test/message/test_runner_output.out b/test/message/test_runner_output.out index 96d977b21c5b1a..062355753a5385 100644 --- a/test/message/test_runner_output.out +++ b/test/message/test_runner_output.out @@ -24,7 +24,6 @@ not ok 3 - sync fail todo # TODO * * * - * ... # Subtest: sync fail todo with message not ok 4 - sync fail todo with message # TODO this is a failing todo @@ -41,7 +40,6 @@ not ok 4 - sync fail todo with message # TODO this is a failing todo * * * - * ... # Subtest: sync skip pass ok 5 - sync skip pass # SKIP @@ -74,7 +72,6 @@ not ok 8 - sync throw fail * * * - * ... # Subtest: async skip pass ok 9 - async skip pass # SKIP @@ -101,7 +98,6 @@ not ok 11 - async throw fail * * * - * ... # Subtest: async skip fail not ok 12 - async skip fail # SKIP @@ -118,7 +114,6 @@ not ok 12 - async skip fail # SKIP * * * - * ... # Subtest: async assertion fail not ok 13 - async assertion fail @@ -142,7 +137,6 @@ not ok 13 - async assertion fail * * * - * ... # Subtest: resolve pass ok 14 - resolve pass @@ -164,7 +158,6 @@ not ok 15 - reject fail * * * - * ... # Subtest: unhandled rejection - passes but warns ok 16 - unhandled rejection - passes but warns diff --git a/test/message/test_runner_output_dot_reporter.js b/test/message/test_runner_output_dot_reporter.js new file mode 100644 index 00000000000000..8c36b9ba245425 --- /dev/null +++ b/test/message/test_runner_output_dot_reporter.js @@ -0,0 +1,6 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const spawn = require('node:child_process').spawn; +spawn(process.execPath, + ['--no-warnings', '--test-reporter', 'dot', 'test/message/test_runner_output.js'], { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_dot_reporter.out b/test/message/test_runner_output_dot_reporter.out new file mode 100644 index 00000000000000..125bfd385607d5 --- /dev/null +++ b/test/message/test_runner_output_dot_reporter.out @@ -0,0 +1,4 @@ +.................... +.................... +.................... +................... diff --git a/test/message/test_runner_output_spec_reporter.js b/test/message/test_runner_output_spec_reporter.js new file mode 100644 index 00000000000000..3389942abed393 --- /dev/null +++ b/test/message/test_runner_output_spec_reporter.js @@ -0,0 +1,6 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const spawn = require('node:child_process').spawn; +spawn(process.execPath, + ['--no-warnings', '--test-reporter', 'spec', 'test/message/test_runner_output.js'], { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_spec_reporter.out b/test/message/test_runner_output_spec_reporter.out new file mode 100644 index 00000000000000..a6bbd184fa3170 --- /dev/null +++ b/test/message/test_runner_output_spec_reporter.out @@ -0,0 +1,107 @@ +✔ sync pass todo +✔ sync pass todo with message +✖ sync fail todo +✖ sync fail todo with message +✔ sync skip pass +✔ sync skip pass with message +✔ sync pass +ℹ this test should pass +✖ sync throw fail +✔ async skip pass +✔ async pass +✖ async throw fail +✖ async skip fail +✖ async assertion fail +✔ resolve pass +✖ reject fail +✔ unhandled rejection - passes but warns +✔ async unhandled rejection - passes but warns +✔ immediate throw - passes but warns +✔ immediate reject - passes but warns +✔ immediate resolve pass +▶ subtest sync throw fail + ✖ +sync throw fail + ℹ this subtest should make its parent test fail +▶ subtest sync throw fail + +✖ sync throw non-error fail +▶ level 0a + ✔ level 1a + ✔ level 1b + ✔ level 1c + ✔ level 1d +▶ level 0a + +▶ top level + ✖ +long running + ▶ +short running + ✔ ++short running + ▶ +short running + +▶ top level + +✔ invalid subtest - pass but subtest fails +✔ sync skip option +✔ sync skip option with message +✖ sync skip option is false fail +✔ +✔ functionOnly +✔ +✔ test with only a name provided +✔ +✔ +✔ test with a name and options provided +✔ functionAndOptions +✔ escaped description \ # \#\ +✔ escaped skip message +✔ escaped todo message +✔ escaped diagnostic +ℹ #diagnostic +✔ callback pass +✖ callback fail +✔ sync t is this in test +✔ async t is this in test +✔ callback t is this in test +✖ callback also returns a Promise +✖ callback throw +✖ callback called twice +✔ callback called twice in different ticks +✖ callback called twice in future tick +✖ callback async throw +✔ callback async throw after done +▶ only is set but not in only mode + ✔ running subtest 1 + ✔ running subtest 2 + ✔ running subtest 3 + ✔ running subtest 4 +▶ only is set but not in only mode + +✖ custom inspect symbol fail +✖ custom inspect symbol that throws fail +▶ subtest sync throw fails + ✖ sync throw fails at first + ✖ sync throw fails at second +▶ subtest sync throw fails + +✖ timed out async test +✖ timed out callback test +✔ large timeout async test is ok +✔ large timeout callback test is ok +✔ successful thenable +✖ rejected thenable +✖ unfinished test with uncaughtException +✖ unfinished test with unhandledRejection +✖ invalid subtest fail +ℹ Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +ℹ Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +ℹ Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event. +ℹ Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +ℹ Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. +ℹ Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. +ℹ tests 65 +ℹ pass 27 +ℹ fail 21 +ℹ cancelled 2 +ℹ skipped 10 +ℹ todo 5 +ℹ duration_ms * diff --git a/test/message/test_runner_output_tap_reporter.js b/test/message/test_runner_output_tap_reporter.js new file mode 100644 index 00000000000000..6a37bf04ca5d01 --- /dev/null +++ b/test/message/test_runner_output_tap_reporter.js @@ -0,0 +1,6 @@ +// Flags: --no-warnings +'use strict'; +require('../common'); +const spawn = require('node:child_process').spawn; +spawn(process.execPath, + ['--no-warnings', '--test-reporter', 'tap', 'test/message/test_runner_output.js'], { stdio: 'inherit' }); diff --git a/test/message/test_runner_output_tap_reporter.out b/test/message/test_runner_output_tap_reporter.out new file mode 100644 index 00000000000000..062355753a5385 --- /dev/null +++ b/test/message/test_runner_output_tap_reporter.out @@ -0,0 +1,646 @@ +TAP version 13 +# Subtest: sync pass todo +ok 1 - sync pass todo # TODO + --- + duration_ms: * + ... +# Subtest: sync pass todo with message +ok 2 - sync pass todo with message # TODO this is a passing todo + --- + duration_ms: * + ... +# Subtest: sync fail todo +not ok 3 - sync fail todo # TODO + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: sync fail todo with message +not ok 4 - sync fail todo with message # TODO this is a failing todo + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync fail todo with message' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: sync skip pass +ok 5 - sync skip pass # SKIP + --- + duration_ms: * + ... +# Subtest: sync skip pass with message +ok 6 - sync skip pass with message # SKIP this is skipped + --- + duration_ms: * + ... +# Subtest: sync pass +ok 7 - sync pass + --- + duration_ms: * + ... +# this test should pass +# Subtest: sync throw fail +not ok 8 - sync throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from sync throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: async skip pass +ok 9 - async skip pass # SKIP + --- + duration_ms: * + ... +# Subtest: async pass +ok 10 - async pass + --- + duration_ms: * + ... +# Subtest: async throw fail +not ok 11 - async throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: async skip fail +not ok 12 - async skip fail # SKIP + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from async throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: async assertion fail +not ok 13 - async assertion fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: |- + Expected values to be strictly equal: + + true !== false + + code: 'ERR_ASSERTION' + expected: false + actual: true + operator: 'strictEqual' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: resolve pass +ok 14 - resolve pass + --- + duration_ms: * + ... +# Subtest: reject fail +not ok 15 - reject fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'rejected from reject fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: unhandled rejection - passes but warns +ok 16 - unhandled rejection - passes but warns + --- + duration_ms: * + ... +# Subtest: async unhandled rejection - passes but warns +ok 17 - async unhandled rejection - passes but warns + --- + duration_ms: * + ... +# Subtest: immediate throw - passes but warns +ok 18 - immediate throw - passes but warns + --- + duration_ms: * + ... +# Subtest: immediate reject - passes but warns +ok 19 - immediate reject - passes but warns + --- + duration_ms: * + ... +# Subtest: immediate resolve pass +ok 20 - immediate resolve pass + --- + duration_ms: * + ... +# Subtest: subtest sync throw fail + # Subtest: +sync throw fail + not ok 1 - +sync throw fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fail' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + * + * + ... + # this subtest should make its parent test fail + 1..1 +not ok 21 - subtest sync throw fail + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: sync throw non-error fail +not ok 22 - sync throw non-error fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'Symbol(thrown symbol from sync throw non-error fail)' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: level 0a + # Subtest: level 1a + ok 1 - level 1a + --- + duration_ms: * + ... + # Subtest: level 1b + ok 2 - level 1b + --- + duration_ms: * + ... + # Subtest: level 1c + ok 3 - level 1c + --- + duration_ms: * + ... + # Subtest: level 1d + ok 4 - level 1d + --- + duration_ms: * + ... + 1..4 +ok 23 - level 0a + --- + duration_ms: * + ... +# Subtest: top level + # Subtest: +long running + not ok 1 - +long running + --- + duration_ms: * + failureType: 'cancelledByParent' + error: 'test did not finish before its parent and was cancelled' + code: 'ERR_TEST_FAILURE' + ... + # Subtest: +short running + # Subtest: ++short running + ok 1 - ++short running + --- + duration_ms: * + ... + 1..1 + ok 2 - +short running + --- + duration_ms: * + ... + 1..2 +not ok 24 - top level + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '1 subtest failed' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: invalid subtest - pass but subtest fails +ok 25 - invalid subtest - pass but subtest fails + --- + duration_ms: * + ... +# Subtest: sync skip option +ok 26 - sync skip option # SKIP + --- + duration_ms: * + ... +# Subtest: sync skip option with message +ok 27 - sync skip option with message # SKIP this is skipped + --- + duration_ms: * + ... +# Subtest: sync skip option is false fail +not ok 28 - sync skip option is false fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'this should be executed' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: +ok 29 - + --- + duration_ms: * + ... +# Subtest: functionOnly +ok 30 - functionOnly + --- + duration_ms: * + ... +# Subtest: +ok 31 - + --- + duration_ms: * + ... +# Subtest: test with only a name provided +ok 32 - test with only a name provided + --- + duration_ms: * + ... +# Subtest: +ok 33 - + --- + duration_ms: * + ... +# Subtest: +ok 34 - # SKIP + --- + duration_ms: * + ... +# Subtest: test with a name and options provided +ok 35 - test with a name and options provided # SKIP + --- + duration_ms: * + ... +# Subtest: functionAndOptions +ok 36 - functionAndOptions # SKIP + --- + duration_ms: * + ... +# Subtest: escaped description \\ \# \\\#\\ +ok 37 - escaped description \\ \# \\\#\\ + --- + duration_ms: * + ... +# Subtest: escaped skip message +ok 38 - escaped skip message # SKIP \#skip + --- + duration_ms: * + ... +# Subtest: escaped todo message +ok 39 - escaped todo message # TODO \#todo + --- + duration_ms: * + ... +# Subtest: escaped diagnostic +ok 40 - escaped diagnostic + --- + duration_ms: * + ... +# \#diagnostic +# Subtest: callback pass +ok 41 - callback pass + --- + duration_ms: * + ... +# Subtest: callback fail +not ok 42 - callback fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'callback failure' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... +# Subtest: sync t is this in test +ok 43 - sync t is this in test + --- + duration_ms: * + ... +# Subtest: async t is this in test +ok 44 - async t is this in test + --- + duration_ms: * + ... +# Subtest: callback t is this in test +ok 45 - callback t is this in test + --- + duration_ms: * + ... +# Subtest: callback also returns a Promise +not ok 46 - callback also returns a Promise + --- + duration_ms: * + failureType: 'callbackAndPromisePresent' + error: 'passed a callback but also returned a Promise' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: callback throw +not ok 47 - callback throw + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from callback throw' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + ... +# Subtest: callback called twice +not ok 48 - callback called twice + --- + duration_ms: * + failureType: 'multipleCallbackInvocations' + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... +# Subtest: callback called twice in different ticks +ok 49 - callback called twice in different ticks + --- + duration_ms: * + ... +# Subtest: callback called twice in future tick +not ok 50 - callback called twice in future tick + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'callback invoked multiple times' + code: 'ERR_TEST_FAILURE' + stack: |- + * + ... +# Subtest: callback async throw +not ok 51 - callback async throw + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'thrown from callback async throw' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + ... +# Subtest: callback async throw after done +ok 52 - callback async throw after done + --- + duration_ms: * + ... +# Subtest: only is set but not in only mode + # Subtest: running subtest 1 + ok 1 - running subtest 1 + --- + duration_ms: * + ... + # Subtest: running subtest 2 + ok 2 - running subtest 2 + --- + duration_ms: * + ... + # Subtest: running subtest 3 + ok 3 - running subtest 3 + --- + duration_ms: * + ... + # Subtest: running subtest 4 + ok 4 - running subtest 4 + --- + duration_ms: * + ... + 1..4 +ok 53 - only is set but not in only mode + --- + duration_ms: * + ... +# Subtest: custom inspect symbol fail +not ok 54 - custom inspect symbol fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'customized' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: custom inspect symbol that throws fail +not ok 55 - custom inspect symbol that throws fail + --- + duration_ms: * + failureType: 'testCodeFailure' + error: |- + { + foo: 1, + [Symbol(nodejs.util.inspect.custom)]: [Function: [nodejs.util.inspect.custom]] + } + code: 'ERR_TEST_FAILURE' + ... +# Subtest: subtest sync throw fails + # Subtest: sync throw fails at first + not ok 1 - sync throw fails at first + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fails at first' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + * + * + ... + # Subtest: sync throw fails at second + not ok 2 - sync throw fails at second + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'thrown from subtest sync throw fails at second' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + * + * + * + * + * + * + * + ... + 1..2 +not ok 56 - subtest sync throw fails + --- + duration_ms: * + failureType: 'subtestsFailed' + error: '2 subtests failed' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: timed out async test +not ok 57 - timed out async test + --- + duration_ms: * + failureType: 'testTimeoutFailure' + error: 'test timed out after 5ms' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: timed out callback test +not ok 58 - timed out callback test + --- + duration_ms: * + failureType: 'testTimeoutFailure' + error: 'test timed out after 5ms' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: large timeout async test is ok +ok 59 - large timeout async test is ok + --- + duration_ms: * + ... +# Subtest: large timeout callback test is ok +ok 60 - large timeout callback test is ok + --- + duration_ms: * + ... +# Subtest: successful thenable +ok 61 - successful thenable + --- + duration_ms: * + ... +# Subtest: rejected thenable +not ok 62 - rejected thenable + --- + duration_ms: * + failureType: 'testCodeFailure' + error: 'custom error' + code: 'ERR_TEST_FAILURE' + ... +# Subtest: unfinished test with uncaughtException +not ok 63 - unfinished test with uncaughtException + --- + duration_ms: * + failureType: 'uncaughtException' + error: 'foo' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + ... +# Subtest: unfinished test with unhandledRejection +not ok 64 - unfinished test with unhandledRejection + --- + duration_ms: * + failureType: 'unhandledRejection' + error: 'bar' + code: 'ERR_TEST_FAILURE' + stack: |- + * + * + * + ... +# Subtest: invalid subtest fail +not ok 65 - invalid subtest fail + --- + duration_ms: * + failureType: 'parentAlreadyFinished' + error: 'test could not be started because its parent finished' + code: 'ERR_TEST_FAILURE' + stack: |- + * + ... +1..65 +# Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event. +# Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event. +# Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event. +# Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event. +# tests 65 +# pass 27 +# fail 21 +# cancelled 2 +# skipped 10 +# todo 5 +# duration_ms * diff --git a/test/parallel/test-bootstrap-modules.js b/test/parallel/test-bootstrap-modules.js index 3809c5c68d4f31..2c8c1a023e2115 100644 --- a/test/parallel/test-bootstrap-modules.js +++ b/test/parallel/test-bootstrap-modules.js @@ -53,6 +53,7 @@ const expectedModules = new Set([ 'NativeModule internal/linkedlist', 'NativeModule internal/modules/cjs/helpers', 'NativeModule internal/modules/cjs/loader', + 'NativeModule internal/modules/utils', 'NativeModule internal/modules/esm/assert', 'NativeModule internal/modules/esm/formats', 'NativeModule internal/modules/esm/get_format', diff --git a/test/parallel/test-runner-exit-code.js b/test/parallel/test-runner-exit-code.js index 1833fa00f7f7ae..1c28c2439050fc 100644 --- a/test/parallel/test-runner-exit-code.js +++ b/test/parallel/test-runner-exit-code.js @@ -20,8 +20,7 @@ async function runAndKill(file) { }); const [code, signal] = await once(child, 'exit'); await finished(child.stdout); - assert.match(stdout, /not ok 1/); - assert.match(stdout, /# cancelled 1\n/); + assert.strictEqual(stdout, 'TAP version 13\n'); assert.strictEqual(signal, null); assert.strictEqual(code, 1); } diff --git a/test/parallel/test-runner-reporters.js b/test/parallel/test-runner-reporters.js new file mode 100644 index 00000000000000..4d3678777795f4 --- /dev/null +++ b/test/parallel/test-runner-reporters.js @@ -0,0 +1,38 @@ +'use strict'; +require('../common'); +const { describe, it } = require('node:test'); + +describe('node:test reporters', () => { + it('should default to outputing TAP to stdout', async () => { + }); + + it('should default destination to stdout when passing a single reporter', async () => { + }); + + it('should throw when passing reporters without a destination', async () => { + }); + + it('should throw when passing a destination without a reporter', async () => { + }); + + it('should support stdout as a destination', async () => { + }); + + it('should support stderr as a destination', async () => { + }); + + it('should support a file as a destination', async () => { + }); + + it('should support an js file as a custom reporter', async () => { + }); + + it('should support an mjs file as a custom reporter', async () => { + }); + + it('should support an cjs file as a custom reporter', async () => { + }); + + it('should support multiple reporters', async () => { + }); +}); diff --git a/test/parallel/test-runner-run.mjs b/test/parallel/test-runner-run.mjs index 8f650509f9ee54..2a7f343cbe0312 100644 --- a/test/parallel/test-runner-run.mjs +++ b/test/parallel/test-runner-run.mjs @@ -10,7 +10,6 @@ describe('require(\'node:test\').run', { concurrency: true }, () => { it('should run with no tests', async () => { const stream = run({ files: [] }); - stream.setEncoding('utf8'); stream.on('test:fail', common.mustNotCall()); stream.on('test:pass', common.mustNotCall()); // eslint-disable-next-line no-unused-vars diff --git a/tools/doc/type-parser.mjs b/tools/doc/type-parser.mjs index 64d499d182f484..d4658a9fd552ba 100644 --- a/tools/doc/type-parser.mjs +++ b/tools/doc/type-parser.mjs @@ -207,7 +207,7 @@ const customTypesMap = { 'Timeout': 'timers.html#class-timeout', 'Timer': 'timers.html#timers', - 'TapStream': 'test.html#class-tapstream', + 'TestsStream': 'test.html#class-testsstream', 'tls.SecureContext': 'tls.html#tlscreatesecurecontextoptions', 'tls.Server': 'tls.html#class-tlsserver',