feat: support snapshot tests

Closes #41

package.json

@@ -175,7 +175,39 @@
           }
         }
       }
-    ]
+    ],
+    "commands": [
+      {
+        "title": "Create Snapshot File",
+        "command": "nodejs-testing.rerunWithSnapshot"
+      },
+      {
+        "title": "Update Snapshot File",
+        "command": "nodejs-testing.rerunWithSnapshot2"
+      }
+    ],
+    "menus": {
+      "commandPalette": [
+        {
+          "command": "nodejs-testing.rerunWithSnapshot",
+          "when": "false"
+        },
+        {
+          "command": "nodejs-testing.rerunWithSnapshot2",
+          "when": "false"
+        }
+      ],
+      "testing/message/content": [
+        {
+          "command": "nodejs-testing.rerunWithSnapshot",
+          "when": "testMessage == isNodejsSnapshotMissing && !testResultOutdated"
+        },
+        {
+          "command": "nodejs-testing.rerunWithSnapshot2",
+          "when": "testMessage == isNodejsSnapshotOutdated && !testResultOutdated"
+        }
+      ]
+    }
   },
   "repository": {
     "type": "git",
@@ -215,7 +247,7 @@
     "prettier": "^3.2.5",
     "sinon": "^17.0.1",
     "tsx": "^4.7.3",
-    "typescript": "^5.4.2",
+    "typescript": "^5.7.2",
     "vitest": "^2.1.1"
   },
   "prettier": {

src/constants.ts

@@ -0,0 +1 @@
+export const nodeSnapshotImpl = "node:internal/test_runner/snapshot";

src/extension.ts

@@ -82,13 +82,23 @@ export async function activate(context: vscode.ExtensionContext) {
     );
   };
 
+  function updateSnapshots() {
+    TestRunner.regenerateSnapshotsOnNextRun = true;
+    return vscode.commands.executeCommand("testing.reRunFailTests");
+  }
+
   context.subscriptions.push(
     vscode.workspace.onDidChangeWorkspaceFolders(syncWorkspaceFolders),
     vscode.workspace.onDidChangeTextDocument((e) => syncTextDocument(e.document)),
     vscode.commands.registerCommand("nodejs-testing.get-controllers-for-test", () => {
       refreshFolders();
       return ctrls;
     }),
+    vscode.commands.registerCommand("nodejs-testing.pre-rerun-with-snapshot-for-test", () => {
+      TestRunner.regenerateSnapshotsOnNextRun = true;
+    }),
+    vscode.commands.registerCommand("nodejs-testing.rerunWithSnapshot", updateSnapshots),
+    vscode.commands.registerCommand("nodejs-testing.rerunWithSnapshot2", updateSnapshots),
     includePattern.onChange(refreshFolders),
     excludePatterns.onChange(refreshFolders),
     extensions.onChange(refreshFolders),

src/node-version.ts

@@ -0,0 +1,52 @@
+export const enum Capability {
+  ExperimentalSnapshots = 1 << 0,
+}
+
+export class NodeVersion {
+  private readonly capabilities = 0;
+
+  public static process() {
+    const version = process.versions.node.split(".").map(Number);
+    return new NodeVersion(new Semver(version[0], version[1], version[2]));
+  }
+
+  constructor(public readonly semver: Semver) {
+    // todo@connor4312: currently still experimental in 23, if it's extended
+    // to stable 24 then then should be updated to lt(new Semver(25, 0, 0))
+    if (semver.gte(new Semver(22, 3, 0)) && semver.lt(new Semver(24, 0, 0))) {
+      this.capabilities |= Capability.ExperimentalSnapshots;
+    }
+  }
+
+  public has(capability: Capability) {
+    return (this.capabilities & capability) !== 0;
+  }
+}
+
+class Semver {
+  constructor(
+    public readonly major: number,
+    public readonly minor: number,
+    public readonly patch: number,
+  ) {}
+
+  public compare(other: Semver) {
+    return this.major - other.major || this.minor - other.minor || this.patch - other.patch;
+  }
+
+  public gt(other: Semver) {
+    return this.compare(other) > 0;
+  }
+
+  public gte(other: Semver) {
+    return this.compare(other) >= 0;
+  }
+
+  public lt(other: Semver) {
+    return this.compare(other) < 0;
+  }
+
+  public lte(other: Semver) {
+    return this.compare(other) <= 0;
+  }
+}

src/runner-loader.ts

@@ -8,6 +8,7 @@ import {
   type PrettyFormatOptions,
 } from "pretty-format";
 import { parse } from "stacktrace-parser";
+import { nodeSnapshotImpl } from "./constants";
 import type { JsonFromReporter } from "./runner-protocol";
 
 // Default options borrowed from jest-diff:
@@ -83,6 +84,25 @@ for (const channel of ["stderr", "stdout"] as const) {
   });
 }
 
+function isSnapshotMissingError(err: any): err is Error & {
+  snapshot: string;
+  filename: string;
+} {
+  // https://github.com/nodejs/node/blob/0547dcfc005ae7d9b60d31a7edc90f5a180f907a/lib/internal/test_runner/snapshot.js#L70-L75
+  return (
+    err &&
+    typeof err === "object" &&
+    err.code === "ERR_INVALID_STATE" &&
+    err.cause?.code === "ENOENT"
+  );
+}
+
+function formatSnapValue(s: string) {
+  // snapshot's serializer adds a newline at the start and end of the string,
+  // so remove that for better diffing
+  return typeof s === "string" && s.startsWith("\n") && s.endsWith("\n") ? s.slice(1, -1) : s;
+}
+
 module.exports = async function* reporter(source: AsyncGenerator<TestEvent>) {
   for await (const evt of source) {
     if (evt.type === "test:fail") {
@@ -92,9 +112,19 @@ module.exports = async function* reporter(source: AsyncGenerator<TestEvent>) {
         (err.cause as any)._stack = err.cause.stack ? parse(err.cause.stack) : undefined;
       }
 
+      if (isSnapshotMissingError(err.cause)) {
+        (err.cause as any)._isNodeSnapshotError = true;
+      }
+
       if (err.cause?.hasOwnProperty("expected") && err.cause?.hasOwnProperty("actual")) {
-        let actual = prettyFormat(err.cause.actual, FORMAT_OPTIONS);
-        let expected = prettyFormat(err.cause.expected, FORMAT_OPTIONS);
+        // snapshot always compares as strings, so don't do extra formatting
+        const isSnap = err.cause?.stack?.includes(nodeSnapshotImpl);
+        let actual = isSnap
+          ? formatSnapValue(err.cause.actual)
+          : prettyFormat(err.cause.actual, FORMAT_OPTIONS);
+        let expected = isSnap
+          ? formatSnapValue(err.cause.expected)
+          : prettyFormat(err.cause.expected, FORMAT_OPTIONS);
         if (actual === expected) {
           actual = prettyFormat(err.cause.actual, FALLBACK_FORMAT_OPTIONS);
           expected = prettyFormat(err.cause.expected, FALLBACK_FORMAT_OPTIONS);

src/runner-protocol.ts

@@ -51,6 +51,7 @@ export const contract = makeContract({
         expected: s.optionalProp(s.sString()),
         actual: s.optionalProp(s.sString()),
         error: s.optionalProp(s.sString()),
+        isSnapshotMissing: s.optionalProp(s.sBoolean()),
         stack: s.optionalProp(s.sArrayOf(stackFrame)),
       }),
     }),
@@ -83,6 +84,7 @@ export const contract = makeContract({
         verbose: s.sBoolean(),
         concurrency: s.sNumber(),
         coverageDir: s.optionalProp(s.sString()),
+        regenerateSnapshots: s.sBoolean(),
         files: s.sArrayOf(
           s.sObject({
             // VS Code URI of the file to associate the test with. For sourcemaps,

src/runner-worker.ts

@@ -11,6 +11,7 @@ import { StackFrame } from "stacktrace-parser";
 import { pathToFileURL } from "url";
 import { WebSocket } from "ws";
 import { ExtensionConfig } from "./extension-config";
+import { Capability, NodeVersion } from "./node-version";
 import { escapeRegex } from "./regex";
 import { ITestRunFile, JsonFromReporter, contract } from "./runner-protocol";
 
@@ -46,11 +47,14 @@ const start: (typeof contract)["TClientHandler"]["start"] = async ({
   verbose,
   extraEnv,
   coverageDir,
+  regenerateSnapshots,
 }) => {
   const todo: Promise<void>[] = [];
   for (let i = 0; i < concurrency && i < files.length; i++) {
     const prefix = colors[i % colors.length](`worker${i + 1}> `);
-    todo.push(doWork(prefix, files, extensions, verbose, extraEnv, coverageDir));
+    todo.push(
+      doWork(prefix, files, extensions, verbose, extraEnv, coverageDir, regenerateSnapshots),
+    );
   }
   await Promise.all(todo);
 
@@ -89,13 +93,16 @@ const getNearestPackageJson = async (dir: string): Promise<string | undefined> =
   return result;
 };
 
+const nodeVersion = NodeVersion.process();
+
 async function doWork(
   prefix: string,
   queue: ITestRunFile[],
   extensions: ExtensionConfig[],
   verbose: boolean,
   extraEnv: Record<string, string>,
   coverageDir: string | undefined,
+  regenerateSnapshots: boolean,
 ) {
   while (queue.length) {
     const next = queue.pop()!;
@@ -113,6 +120,13 @@ async function doWork(
         if (parameters) args.push(...parameters);
       }
 
+      if (nodeVersion.has(Capability.ExperimentalSnapshots)) {
+        args.push("--experimental-test-snapshots");
+      }
+      if (regenerateSnapshots) {
+        args.push("--test-update-snapshots");
+      }
+
       args.push("--test-reporter", pathToFileURL(join(__dirname, "runner-loader.js")).toString());
       for (const include of next.include || []) {
         args.push("--test-name-pattern", `^${escapeRegex(include)}$`);
@@ -197,6 +211,7 @@ async function doWork(
               expected?: any;
               _stack?: StackFrame[];
               _message?: string;
+              _isNodeSnapshotError?: true;
             } = cause && typeof cause === "object" ? (cause as any) : {};
             const message =
               causeObj._message ||
@@ -207,6 +222,7 @@ async function doWork(
               duration: json.data.details.duration_ms,
               error: message,
               stack: causeObj._stack,
+              isSnapshotMissing: !!causeObj._isNodeSnapshotError,
               expected: typeof causeObj.expected === "string" ? causeObj.expected : undefined,
               actual: typeof causeObj.actual === "string" ? causeObj.actual : undefined,
             });

src/runner.ts

@@ -11,6 +11,7 @@ import { isAbsolute, join } from "path";
 import split from "split2";
 import * as vscode from "vscode";
 import { ConfigValue } from "./configValue";
+import { nodeSnapshotImpl } from "./constants";
 import { applyC8Coverage } from "./coverage";
 import { DisposableStore } from "./disposable";
 import { ExtensionConfig } from "./extension-config";
@@ -38,6 +39,11 @@ export type RunHandler = (
 ) => Promise<void>;
 
 export class TestRunner implements vscode.Disposable {
+  /**
+   * Set via a command, before tests are re-run, to generate or update snapshots.
+   */
+  public static regenerateSnapshotsOnNextRun = false;
+
   private readonly workerPath: string;
   private readonly disposables = new DisposableStore();
 
@@ -223,11 +229,24 @@ export class TestRunner implements vscode.Disposable {
                   });
                 },
 
-                failed({ id, duration, actual, expected, error, stack }) {
+                failed({ id, duration, actual, expected, error, stack, isSnapshotMissing }) {
                   const test = getTestByPath(id);
                   if (!test) {
                     return;
                   }
+
+                  if (isSnapshotMissing && expected === undefined) {
+                    const message = new vscode.TestMessage(
+                      "Snapshot not found...\n\nPlease click the button to the right to generate them.",
+                    );
+                    message.contextValue = "isNodejsSnapshotMissing";
+                    outputQueue.enqueue(() => {
+                      run.appendOutput(style.failed(test, "Snapshot missing."));
+                      run.failed(test, message);
+                    });
+                    return;
+                  }
+
                   const asText = error || "Test failed";
                   const testMessage =
                     actual !== undefined && expected !== undefined
@@ -247,6 +266,9 @@ export class TestRunner implements vscode.Disposable {
                     if (startOfMessage) {
                       testMessage.message = asText.slice(0, startOfMessage.index - 1);
                     }
+                    if (stack.some((s) => s.file === nodeSnapshotImpl)) {
+                      testMessage.contextValue = "isNodejsSnapshotOutdated";
+                    }
                   }
 
                   const lastFrame = stack?.find((s) => !s.file?.startsWith("node:"));
@@ -277,11 +299,13 @@ export class TestRunner implements vscode.Disposable {
                   files,
                   concurrency,
                   extensions,
+                  regenerateSnapshots: TestRunner.regenerateSnapshotsOnNextRun,
                   verbose: this.verbose.value,
                   extraEnv,
                   coverageDir,
                 });
 
+                TestRunner.regenerateSnapshotsOnNextRun = false;
                 outputQueue.enqueue(() => run.appendOutput(style.done()));
                 await outputQueue.drain();
                 resolve();
@@ -324,7 +348,7 @@ export class TestRunner implements vscode.Disposable {
   ) {
     if (!debug) {
       return new Promise<void>((resolve, reject) => {
-        const stderr: Buffer[] = [];
+        const stderr: Uint8Array[] = [];
         const cp = spawn(this.nodejsPath.value, [
           ...resolvedNodejsParameters,
           this.workerPath,