test_runner: count individual tests

With this change, test reports will count `test`, `it`, and `t.test`
rather than `describe`.

Fixes: nodejs#46762

lib/internal/test_runner/runner.js

@@ -23,6 +23,7 @@ const {
   SafeSet,
   StringPrototypeIndexOf,
   StringPrototypeSlice,
+  StringPrototypeSplit,
   StringPrototypeStartsWith,
 } = primordials;
 
@@ -43,11 +44,8 @@ const { getInspectPort, isUsingInspector, isInspectorMessage } = require('intern
 const { kEmptyObject } = require('internal/util');
 const { createTestTree } = require('internal/test_runner/harness');
 const {
-  kAborted,
-  kCancelledByParent,
   kSubtestsFailed,
   kTestCodeFailure,
-  kTestTimeoutFailure,
   Test,
 } = require('internal/test_runner/test');
 const { TapParser } = require('internal/test_runner/tap_parser');
@@ -69,9 +67,6 @@ const kFilterArgs = ['--test', '--experimental-test-coverage', '--watch'];
 const kFilterArgValues = ['--test-reporter', '--test-reporter-destination'];
 const kDiagnosticsFilterArgs = ['tests', 'pass', 'fail', 'cancelled', 'skipped', 'todo', 'duration_ms'];
 
-const kCanceledTests = new SafeSet()
-  .add(kCancelledByParent).add(kAborted).add(kTestTimeoutFailure);
-
 // TODO(cjihrig): Replace this with recursive readdir once it lands.
 function processPath(path, testFiles, options) {
   const stats = statSync(path);
@@ -153,7 +148,7 @@ class FileTest extends Test {
   #counters = { __proto__: null, all: 0, failed: 0, passed: 0, cancelled: 0, skipped: 0, todo: 0, totalFailed: 0 };
   failedSubtests = false;
   #skipReporting() {
-    return this.#counters.all > 0 && (!this.error || this.error.failureType === kSubtestsFailed);
+    return this.testsStarted && (!this.error || this.error.failureType === kSubtestsFailed);
   }
   #checkNestedComment({ comment }) {
     const firstSpaceIndex = StringPrototypeIndexOf(comment, ' ');
@@ -162,6 +157,40 @@ class FileTest extends Test {
     return secondSpaceIndex === -1 &&
           ArrayPrototypeIncludes(kDiagnosticsFilterArgs, StringPrototypeSlice(comment, 0, firstSpaceIndex));
   }
+  #handleReportComment(comment) {
+    const { 0: status, 1: count } = StringPrototypeSplit(comment, ' ');
+    const countSubtest = super.countSubtest;
+    const thisCounters = this.#counters;
+    function addToCounter(counters) {
+      for (let i = 0; i < +count; i++) {
+        FunctionPrototypeCall(countSubtest,
+                              counters,
+                              thisCounters);
+      }
+    }
+    if (+count) {
+      switch (status) {
+        case 'pass':
+          addToCounter({ finished: true, passed: true });
+          break;
+        case 'fail':
+          // this.failedSubtests = true;
+          addToCounter({ finished: true, passed: false });
+          break;
+        case 'cancelled':
+          addToCounter({ finished: true, cancelled: true });
+          break;
+        case 'skipped':
+          addToCounter({ finished: true, skipped: true });
+          break;
+        case 'todo':
+          addToCounter({ finished: true, isTodo: true });
+          break;
+        default:
+          break;
+      }
+    }
+  }
   #handleReportItem({ kind, node, comments, nesting = 0 }) {
     if (comments) {
       ArrayPrototypeForEach(comments, (comment) => this.reporter.diagnostic(nesting, this.name, comment));
@@ -184,7 +213,6 @@ class FileTest extends Test {
         break;
 
       case TokenKind.TAP_TEST_POINT: {
-
         const { todo, skip, pass } = node.status;
 
         let directive;
@@ -198,24 +226,23 @@ class FileTest extends Test {
         }
 
         const diagnostics = YAMLToJs(node.diagnostics);
-        const cancelled = kCanceledTests.has(diagnostics.error?.failureType);
         const testNumber = nesting === 0 ? (Number(node.id) + this.testNumber - 1) : node.id;
         const method = pass ? 'ok' : 'fail';
         this.reporter[method](nesting, this.name, testNumber, node.description, diagnostics, directive);
         if (nesting === 0) {
-          FunctionPrototypeCall(super.countSubtest,
-                                { finished: true, skipped: skip, isTodo: todo, passed: pass, cancelled },
-                                this.#counters);
+          this.testsStarted = true;
           this.failedSubtests ||= !pass;
         }
         break;
 
       }
       case TokenKind.COMMENT:
+        this.#handleReportComment(node.comment);
         if (nesting === 0 && this.#checkNestedComment(node)) {
           // Ignore file top level diagnostics
           break;
         }
+        // this.#handleReportComment(node.comment);
         this.reporter.diagnostic(nesting, this.name, node.comment);
         break;
 

lib/internal/test_runner/test.js

@@ -576,24 +576,32 @@ class Test extends AsyncResource {
   }
 
   countSubtest(counters) {
-    // Check SKIP and TODO tests first, as those should not be counted as
-    // failures.
-    if (this.skipped) {
-      counters.skipped++;
-    } else if (this.isTodo) {
-      counters.todo++;
-    } else if (this.cancelled) {
-      counters.cancelled++;
-    } else if (!this.passed) {
-      counters.failed++;
-    } else {
-      counters.passed++;
+    if (this.subtests) {
+      for (let i = 0; i < this.subtests.length; i++) {
+        const subtest = this.subtests[i];
+        subtest.countSubtest(counters);
+      }
     }
+    if (!this.notCountable) {
+      // Check SKIP and TODO tests first, as those should not be counted as
+      // failures.
+      if (this.skipped) {
+        counters.skipped++;
+      } else if (this.isTodo) {
+        counters.todo++;
+      } else if (this.cancelled) {
+        counters.cancelled++;
+      } else if (!this.passed) {
+        counters.failed++;
+      } else {
+        counters.passed++;
+      }
 
-    if (!this.passed) {
-      counters.totalFailed++;
+      if (!this.passed) {
+        counters.totalFailed++;
+      }
+      counters.all++;
     }
-    counters.all++;
   }
 
   postRun(pendingSubtestsError) {
@@ -748,6 +756,7 @@ class TestHook extends Test {
 class Suite extends Test {
   constructor(options) {
     super(options);
+    this.notCountable = true;
 
     try {
       const { ctx, args } = this.getRunArgs();

test/message/test_runner_abort.out

@@ -259,11 +259,11 @@ not ok 4 - callback abort signal
     *
     *
   ...
-1..4
-# tests 4
-# pass 0
+1..22
+# tests 22
+# pass 8
 # fail 0
-# cancelled 4
+# cancelled 14
 # skipped 0
 # todo 0
 # duration_ms *

test/message/test_runner_abort_suite.out

@@ -93,11 +93,11 @@ not ok 2 - describe abort signal
     *
     *
   ...
-1..2
-# tests 2
-# pass 0
+1..9
+# tests 9
+# pass 4
 # fail 0
-# cancelled 2
+# cancelled 5
 # skipped 0
 # todo 0
 # duration_ms *

test/message/test_runner_describe_it.out

@@ -636,17 +636,17 @@ not ok 60 - invalid subtest fail
   stack: |-
     *
   ...
-1..60
+1..67
 # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
-# tests 60
-# pass 23
-# fail 22
-# cancelled 0
+# tests 67
+# pass 29
+# fail 19
+# cancelled 4
 # skipped 10
 # todo 5
 # duration_ms *

test/message/test_runner_hooks.out

@@ -490,11 +490,11 @@ not ok 13 - t.after() is called if test body throws
     *
   ...
 # - after() called
-1..13
-# tests 13
-# pass 2
-# fail 11
-# cancelled 0
+1..35
+# tests 35
+# pass 14
+# fail 19
+# cancelled 2
 # skipped 0
 # todo 0
 # duration_ms *

test/message/test_runner_no_refs.out

@@ -20,11 +20,11 @@ not ok 1 - does not keep event loop alive
   stack: |-
     *
   ...
-1..1
-# tests 1
+1..2
+# tests 2
 # pass 0
 # fail 0
-# cancelled 1
+# cancelled 2
 # skipped 0
 # todo 0
 # duration_ms *

test/message/test_runner_only_tests.out

@@ -189,11 +189,11 @@ ok 14 - describe only = true, with subtests
   ---
   duration_ms: *
   ...
-1..14
-# tests 14
-# pass 4
+1..34
+# tests 34
+# pass 14
 # fail 0
 # cancelled 0
-# skipped 10
+# skipped 20
 # todo 0
 # duration_ms *

test/message/test_runner_output.out

@@ -634,17 +634,17 @@ not ok 65 - invalid subtest fail
   stack: |-
     *
   ...
-1..65
+1..79
 # Warning: Test "unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "async unhandled rejection - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from async unhandled rejection fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "immediate throw - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from immediate throw fail" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
-# tests 65
-# pass 27
-# fail 21
-# cancelled 2
+# tests 79
+# pass 37
+# fail 24
+# cancelled 3
 # skipped 10
 # todo 5
 # duration_ms *

test/message/test_runner_output_cli.out

@@ -640,11 +640,11 @@ not ok 65 - invalid subtest fail
 # Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
 # Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
 # Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
-1..65
-# tests 65
-# pass 27
-# fail 21
-# cancelled 2
+1..79
+# tests 79
+# pass 37
+# fail 24
+# cancelled 3
 # skipped 10
 # todo 5
 # duration_ms *

test/message/test_runner_output_spec_reporter.out

@@ -274,10 +274,10 @@
  Warning: Test "immediate reject - passes but warns" generated asynchronous activity after the test ended. This activity created the error "Error: rejected from immediate reject fail" and would have caused the test to fail, but instead triggered an unhandledRejection event.
  Warning: Test "callback called twice in different ticks" generated asynchronous activity after the test ended. This activity created the error "Error [ERR_TEST_FAILURE]: callback invoked multiple times" and would have caused the test to fail, but instead triggered an uncaughtException event.
  Warning: Test "callback async throw after done" generated asynchronous activity after the test ended. This activity created the error "Error: thrown from callback async throw after done" and would have caused the test to fail, but instead triggered an uncaughtException event.
- tests 65
- pass 27
- fail 21
- cancelled 2
+ tests 79
+ pass 37
+ fail 24
+ cancelled 3
  skipped 10
  todo 5
  duration_ms *

test/message/test_runner_test_name_pattern.out

@@ -99,9 +99,9 @@ ok 13 - top level describe enabled
   ...
 1..13
 # tests 13
-# pass 4
+# pass 6
 # fail 0
 # cancelled 0
-# skipped 9
+# skipped 7
 # todo 0
 # duration_ms *

test/message/test_runner_test_name_pattern_with_only.out

@@ -30,11 +30,11 @@ ok 4 - not only and does not match pattern # SKIP 'only' option not set
   ---
   duration_ms: *
   ...
-1..4
-# tests 4
-# pass 1
+1..6
+# tests 6
+# pass 2
 # fail 0
 # cancelled 0
-# skipped 3
+# skipped 4
 # todo 0
 # duration_ms *

test/parallel/test-runner-cli.js

@@ -155,9 +155,9 @@ const testFixtures = fixtures.path('test-runner');
   assert.match(stdout, /# Subtest: level 0b/);
   assert.match(stdout, /not ok 4 - level 0b/);
   assert.match(stdout, / {2}error: 'level 0b error'/);
-  assert.match(stdout, /# tests 4/);
-  assert.match(stdout, /# pass 2/);
-  assert.match(stdout, /# fail 2/);
+  assert.match(stdout, /# tests 8/);
+  assert.match(stdout, /# pass 4/);
+  assert.match(stdout, /# fail 3/);
 }
 
 {