eng: add support for snapshot tests (#190444)

* eng: add support for snapshot tests

This adds Jest-like support for snapshot testing.
Developers can do something like:

```js
await assertSnapshot(myComplexObject)
```

The first time this is run, the snapshot expectation file is written
to a `__snapshots__` directory beside the test file. Subsequent runs
will compare the object to the snapshot, and fail if it doesn't match.

You can see an example of this in the test for snapshots themselves!

After a successful run, any unused snapshots are cleaned up. On a failed
run, a gitignored `.actual` snapshot file is created beside the
snapshot for easy processing and inspection.

Shortly I will do some integration with the selfhost test extension to
allow developers to easily update snapshots from the vscode UI.

For #189680

cc @ulugbekna @hediet

* fix async stacktraces getting clobbered

* random fixes

* comment out leak detector, for now

* add option to snapshot file extension

.gitignore

@@ -18,3 +18,4 @@ vscode.db
 /cli/target
 /cli/openssl
 product.overrides.json
+*.snap.actual

build/hygiene.js

@@ -150,10 +150,12 @@ function hygiene(some, linting = true) {
 	}
 
 	const productJsonFilter = filter('product.json', { restore: true });
+	const snapshotFilter = filter(['**', '!**/*.snap', '!**/*.snap.actual']);
 	const unicodeFilterStream = filter(unicodeFilter, { restore: true });
 
 	const result = input
 		.pipe(filter((f) => !f.stat.isDirectory()))
+		.pipe(snapshotFilter)
 		.pipe(productJsonFilter)
 		.pipe(process.env['BUILD_SOURCEVERSION'] ? es.through() : productJson)
 		.pipe(productJsonFilter.restore)

src/vs/base/common/iterator.ts

@@ -47,7 +47,7 @@ export namespace Iterable {
 		return false;
 	}
 
-	export function find<T, R extends T>(iterable: Iterable<T>, predicate: (t: T) => t is R): T | undefined;
+	export function find<T, R extends T>(iterable: Iterable<T>, predicate: (t: T) => t is R): R | undefined;
 	export function find<T>(iterable: Iterable<T>, predicate: (t: T) => boolean): T | undefined;
 	export function find<T>(iterable: Iterable<T>, predicate: (t: T) => boolean): T | undefined {
 		for (const element of iterable) {

src/vs/base/test/common/snapshot.ts

@@ -0,0 +1,185 @@
+/*---------------------------------------------------------------------------------------------
+ *  Copyright (c) Microsoft Corporation. All rights reserved.
+ *  Licensed under the MIT License. See License.txt in the project root for license information.
+ *--------------------------------------------------------------------------------------------*/
+
+import { Lazy } from 'vs/base/common/lazy';
+import { FileAccess } from 'vs/base/common/network';
+import { URI } from 'vs/base/common/uri';
+
+declare const __readFileInTests: (path: string) => Promise<string>;
+declare const __writeFileInTests: (path: string, contents: string) => Promise<void>;
+declare const __readDirInTests: (path: string) => Promise<string[]>;
+declare const __unlinkInTests: (path: string) => Promise<void>;
+declare const __mkdirPInTests: (path: string) => Promise<void>;
+
+// setup on import so assertSnapshot has the current context without explicit passing
+let context: Lazy<SnapshotContext> | undefined;
+const sanitizeName = (name: string) => name.replace(/[^a-z0-9_-]/gi, '_');
+const normalizeCrlf = (str: string) => str.replace(/\r\n/g, '\n');
+
+export interface ISnapshotOptions {
+	/** Name for snapshot file, rather than an incremented number */
+	name?: string;
+	/** Extension name of the snapshot file, defaults to `.snap` */
+	extension?: string;
+}
+
+/**
+ * This is exported only for tests against the snapshotting itself! Use
+ * {@link assertSnapshot} as a consumer!
+ */
+export class SnapshotContext {
+	private nextIndex = 0;
+	private readonly namePrefix: string;
+	private readonly snapshotsDir: URI;
+	private readonly usedNames = new Set();
+
+	constructor(private readonly test: Mocha.Test | undefined) {
+		if (!test) {
+			throw new Error('assertSnapshot can only be used in a test');
+		}
+
+		if (!test.file) {
+			throw new Error('currentTest.file is not set, please open an issue with the test you\'re trying to run');
+		}
+
+		const src = FileAccess.asFileUri('');
+		const parts = test.file.split(/[/\\]/g);
+
+		this.namePrefix = sanitizeName(test.fullTitle()) + '_';
+		this.snapshotsDir = URI.joinPath(src, ...[...parts.slice(0, -1), '__snapshots__']);
+	}
+
+	public async assert(value: any, options?: ISnapshotOptions) {
+		const originalStack = new Error().stack!; // save to make the stack nicer on failure
+		const nameOrIndex = (options?.name ? sanitizeName(options.name) : this.nextIndex++);
+		const fileName = this.namePrefix + nameOrIndex + '.' + (options?.extension || 'snap');
+		this.usedNames.add(fileName);
+
+		const fpath = URI.joinPath(this.snapshotsDir, fileName).fsPath;
+		const actual = formatValue(value);
+		let expected: string;
+		try {
+			expected = await __readFileInTests(fpath);
+		} catch {
+			console.info(`Creating new snapshot in: ${fpath}`);
+			await __mkdirPInTests(this.snapshotsDir.fsPath);
+			await __writeFileInTests(fpath, actual);
+			return;
+		}
+
+		if (normalizeCrlf(expected) !== normalizeCrlf(actual)) {
+			await __writeFileInTests(fpath + '.actual', actual);
+			const err: any = new Error(`Snapshot #${nameOrIndex} does not match expected output`);
+			err.expected = expected;
+			err.actual = actual;
+			err.snapshotPath = fpath;
+			err.stack = (err.stack as string)
+				.split('\n')
+				// remove all frames from the async stack and keep the original caller's frame
+				.slice(0, 1)
+				.concat(originalStack.split('\n').slice(3))
+				.join('\n');
+			throw err;
+		}
+	}
+
+	public async removeOldSnapshots() {
+		const contents = await __readDirInTests(this.snapshotsDir.fsPath);
+		const toDelete = contents.filter(f => f.startsWith(this.namePrefix) && !this.usedNames.has(f));
+		if (toDelete.length) {
+			console.info(`Deleting ${toDelete.length} old snapshots for ${this.test?.fullTitle()}`);
+		}
+
+		await Promise.all(toDelete.map(f => __unlinkInTests(URI.joinPath(this.snapshotsDir, f).fsPath)));
+	}
+}
+
+const debugDescriptionSymbol = Symbol.for('debug.description');
+
+function formatValue(value: unknown, level = 0, seen: unknown[] = []): string {
+	switch (typeof value) {
+		case 'bigint':
+		case 'boolean':
+		case 'number':
+		case 'symbol':
+		case 'undefined':
+			return String(value);
+		case 'string':
+			return level === 0 ? value : JSON.stringify(value);
+		case 'function':
+			return `[Function ${value.name}]`;
+		case 'object': {
+			if (value === null) {
+				return 'null';
+			}
+			if (value instanceof RegExp) {
+				return String(value);
+			}
+			if (seen.includes(value)) {
+				return '[Circular]';
+			}
+			if (debugDescriptionSymbol in value && typeof (value as any)[debugDescriptionSymbol] === 'function') {
+				return (value as any)[debugDescriptionSymbol]();
+			}
+			const oi = '  '.repeat(level);
+			const ci = '  '.repeat(level + 1);
+			if (Array.isArray(value)) {
+				const children = value.map(v => formatValue(v, level + 1, [...seen, value]));
+				const multiline = children.some(c => c.includes('\n')) || children.join(', ').length > 80;
+				return multiline ? `[\n${ci}${children.join(`,\n${ci}`)}\n${oi}]` : `[ ${children.join(', ')} ]`;
+			}
+
+			let entries;
+			let prefix = '';
+			if (value instanceof Map) {
+				prefix = 'Map ';
+				entries = [...value.entries()];
+			} else if (value instanceof Set) {
+				prefix = 'Set ';
+				entries = [...value.entries()];
+			} else {
+				entries = Object.entries(value);
+			}
+
+			const lines = entries.map(([k, v]) => `${k}: ${formatValue(v, level + 1, [...seen, value])}`);
+			return prefix + (lines.length > 1
+				? `{\n${ci}${lines.join(`,\n${ci}`)}\n${oi}}`
+				: `{ ${lines.join(',\n')} }`);
+		}
+		default:
+			throw new Error(`Unknown type ${value}`);
+	}
+}
+
+setup(function () {
+	const currentTest = this.currentTest;
+	context = new Lazy(() => new SnapshotContext(currentTest));
+});
+teardown(async function () {
+	if (this.currentTest?.state === 'passed') {
+		await context?.rawValue?.removeOldSnapshots();
+	}
+	context = undefined;
+});
+
+/**
+ * Implements a snapshot testing utility. ⚠️ This is async! ⚠️
+ *
+ * The first time a snapshot test is run, it'll record the value it's called
+ * with as the expected value. Subsequent runs will fail if the value differs,
+ * but the snapshot can be regenerated by hand or using the Selfhost Test
+ * Provider Extension which'll offer to update it.
+ *
+ * The snapshot will be associated with the currently running test and stored
+ * in a `__snapshots__` directory next to the test file, which is expected to
+ * be the first `.test.js` file in the callstack.
+ */
+export function assertSnapshot(value: any, options?: ISnapshotOptions): Promise<void> {
+	if (!context) {
+		throw new Error('assertSnapshot can only be used in a test');
+	}
+
+	return context.value.assert(value, options);
+}

src/vs/base/test/node/__snapshots__/snapshot_cleans_up_old_snapshots_0.snap

@@ -0,0 +1,8 @@
+hello_world__0.snap:
+  { cool: true }
+hello_world__1.snap:
+  { nifty: true }
+hello_world__fourthTest.snap:
+  { customName: 2 }
+hello_world__thirdTest.txt:
+  { customName: 1 }

src/vs/base/test/node/__snapshots__/snapshot_cleans_up_old_snapshots_1.snap

@@ -0,0 +1,4 @@
+hello_world__0.snap:
+  { cool: true }
+hello_world__thirdTest.snap:
+  { customName: 1 }

src/vs/base/test/node/__snapshots__/snapshot_creates_a_snapshot_0.snap

@@ -0,0 +1,2 @@
+hello_world__0.snap:
+  { cool: true }

src/vs/base/test/node/__snapshots__/snapshot_formats_object_nicely_0.snap

@@ -0,0 +1,35 @@
+[
+  1,
+  true,
+  undefined,
+  null,
+  123,
+  Symbol(heyo),
+  "hello",
+  { hello: "world" },
+  { a: [Circular] },
+  Map {
+    hello: 1,
+    goodbye: 2
+  },
+  Set {
+    1: 1,
+    2: 2,
+    3: 3
+  },
+  [Function helloWorld],
+  /hello/g,
+  [
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string",
+    "long stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong stringlong string"
+  ],
+  Range [1 -> 5]
+]