Support durableObjectsPersist option (#517)

* Support `durableObjectsPersist` option

Uses cloudflare/workerd#302 to enable Durable Object persistence.

- `durableObjectsPersist: undefined | false | "memory:"` stores
  "in-memory"
- `durableObjectsPersist: true` stores under `.mf` on the file-system
- `durableObjectsPersist: "(file://)?<path>"` stores under `<path>`
  on the file-system

Note Miniflare 2 persisted in-memory data between options reloads. In
Miniflare 3, `workerd` restarts on every reload, so if we used
`workerd`'s in-memory storage, we'd lose data every time options
changed. To maintain Miniflare 2's behaviour, "in-memory" storage
isn't actually in-memory. Instead, we write to a temporary directory
and clean this up on `dispose()`/exit. 🙈

Also fixes a bug if multiple services bound to the same Durable
Object class. This would result in duplicate entries in
`durableObjectClassNames`. This has been changed from a `Map` of
`string[]`s to a `Map` of `Set<string>`s to enforce the uniqueness
constraint.

Closes #2403
Closes #2458

Internal ticket: DEVX-219

* fixup! Support `durableObjectsPersist` option

* fixup! Support `durableObjectsPersist` option

* fixup! Support `durableObjectsPersist` option

packages/miniflare/package.json

@@ -39,7 +39,7 @@
 		"source-map-support": "0.5.21",
 		"stoppable": "^1.1.0",
 		"undici": "^5.13.0",
-		"workerd": "^1.20221111.5",
+		"workerd": "^1.20230221.0",
 		"ws": "^8.11.0",
 		"youch": "^3.2.2",
 		"zod": "^3.18.0"

packages/miniflare/src/index.ts

@@ -1,6 +1,10 @@
 import assert from "assert";
+import crypto from "crypto";
+import fs from "fs";
 import http from "http";
 import net from "net";
+import os from "os";
+import path from "path";
 import { Duplex } from "stream";
 import type {
 	IncomingRequestCfProperties,
@@ -22,6 +26,7 @@ import {
 	fetch,
 } from "./http";
 import {
+	DurableObjectClassNames,
 	GatewayConstructor,
 	GatewayFactory,
 	HEADER_CF_BLOB,
@@ -124,8 +129,8 @@ function validateOptions(
 // in other services.
 function getDurableObjectClassNames(
 	allWorkerOpts: PluginWorkerOptions[]
-): Map<string, string[]> {
-	const serviceClassNames = new Map<string, string[]>();
+): DurableObjectClassNames {
+	const serviceClassNames: DurableObjectClassNames = new Map();
 	for (const workerOpts of allWorkerOpts) {
 		const workerServiceName = getUserServiceName(workerOpts.core.name);
 		for (const designator of Object.values(
@@ -136,9 +141,10 @@ function getDurableObjectClassNames(
 				normaliseDurableObject(designator);
 			let classNames = serviceClassNames.get(serviceName);
 			if (classNames === undefined) {
-				serviceClassNames.set(serviceName, (classNames = []));
+				classNames = new Set();
+				serviceClassNames.set(serviceName, classNames);
 			}
-			classNames.push(className);
+			classNames.add(className);
 		}
 	}
 	return serviceClassNames;
@@ -201,6 +207,12 @@ export class Miniflare {
 	#removeRuntimeExitHook?: () => void;
 	#runtimeEntryURL?: URL;
 
+	// Path to temporary directory for use as scratch space/"in-memory" Durable
+	// Object storage. Note this may not exist, it's up to the consumers to
+	// create this if needed. Deleted on `dispose()`.
+	readonly #tmpPath: string;
+	readonly #removeTmpPathExitHook: () => void;
+
 	// Mutual exclusion lock for runtime operations (i.e. initialisation and
 	// updating config). This essentially puts initialisation and future updates
 	// in a queue, ensuring they're performed in calling order.
@@ -272,6 +284,17 @@ export class Miniflare {
 			}
 		});
 
+		// Build path for temporary directory. We don't actually want to create this
+		// unless it's needed (i.e. we have Durable Objects enabled). This means we
+		// can't use `fs.mkdtemp()`, as that always creates the directory.
+		this.#tmpPath = path.join(
+			os.tmpdir(),
+			`miniflare-${crypto.randomBytes(16).toString("hex")}`
+		);
+		this.#removeTmpPathExitHook = exitHook(() => {
+			fs.rmSync(this.#tmpPath, { force: true, recursive: true });
+		});
+
 		this.#disposeController = new AbortController();
 		this.#runtimeMutex = new Mutex();
 		this.#initPromise = this.#runtimeMutex.runWith(() => this.#init());
@@ -646,6 +669,7 @@ export class Miniflare {
 					durableObjectClassNames,
 					additionalModules,
 					loopbackPort,
+					tmpPath: this.#tmpPath,
 				});
 				if (pluginServices !== undefined) {
 					for (const service of pluginServices) {
@@ -759,10 +783,15 @@ export class Miniflare {
 			await this.#initPromise;
 			await this.#lastUpdatePromise;
 		} finally {
-			// Cleanup as much as possible even if `#init()` threw
+			// Remove exit hooks, we're cleaning up what they would've cleaned up now
+			this.#removeTmpPathExitHook();
 			this.#removeRuntimeExitHook?.();
+
+			// Cleanup as much as possible even if `#init()` threw
 			await this.#runtime?.dispose();
 			await this.#stopLoopbackServer();
+			// `rm -rf ${#tmpPath}`, this won't throw if `#tmpPath` doesn't exist
+			await fs.promises.rm(this.#tmpPath, { force: true, recursive: true });
 		}
 	}
 }

packages/miniflare/src/plugins/core/index.ts

@@ -7,11 +7,11 @@ import {
 	Service,
 	Worker_Binding,
 	Worker_Module,
-	kVoid,
 	supportedCompatibilityDate,
 } from "../../runtime";
 import { Awaitable, JsonSchema, Log, MiniflareCoreError } from "../../shared";
 import { getCacheServiceName } from "../cache";
+import { DURABLE_OBJECTS_STORAGE_SERVICE_NAME } from "../do";
 import {
 	BINDING_SERVICE_LOOPBACK,
 	CloudflareFetchSchema,
@@ -383,7 +383,9 @@ export const CORE_PLUGIN: Plugin<
 			}
 
 			const name = getUserServiceName(options.name);
-			const classNames = durableObjectClassNames.get(name) ?? [];
+			const classNames = Array.from(
+				durableObjectClassNames.get(name) ?? new Set<string>()
+			);
 			const compatibilityDate = validateCompatibilityDate(
 				log,
 				options.compatibilityDate ?? FALLBACK_COMPATIBILITY_DATE
@@ -398,9 +400,15 @@ export const CORE_PLUGIN: Plugin<
 					bindings: workerBindings,
 					durableObjectNamespaces: classNames.map((className) => ({
 						className,
-						uniqueKey: className,
+						// This `uniqueKey` will (among other things) be used as part of the
+						// path when persisting to the file-system. `-` is invalid in
+						// JavaScript class names, but safe on filesystems (incl. Windows).
+						uniqueKey: `${options.name ?? ""}-${className}`,
 					})),
-					durableObjectStorage: { inMemory: kVoid },
+					durableObjectStorage:
+						classNames.length === 0
+							? undefined
+							: { localDisk: DURABLE_OBJECTS_STORAGE_SERVICE_NAME },
 					cacheApiOutbound: { name: getCacheServiceName(workerIndex) },
 				},
 			});

packages/miniflare/src/plugins/do/index.ts

@@ -1,14 +1,20 @@
+import fs from "fs/promises";
+import path from "path";
+import { fileURLToPath } from "url";
 import { z } from "zod";
 import { Worker_Binding } from "../../runtime";
-import { MiniflareError } from "../../shared";
+import { MiniflareCoreError } from "../../shared";
 import { getUserServiceName } from "../core";
-import { PersistenceSchema, Plugin } from "../shared";
+import {
+	DEFAULT_PERSIST_ROOT,
+	Persistence,
+	PersistenceSchema,
+	Plugin,
+	maybeParseURL,
+} from "../shared";
 import { DurableObjectsStorageGateway } from "./gateway";
 import { DurableObjectsStorageRouter } from "./router";
 
-export type DurableObjectsErrorCode = "ERR_PERSIST_UNSUPPORTED"; // Durable Object persistence is not yet supported
-export class DurableObjectsError extends MiniflareError<DurableObjectsErrorCode> {}
-
 export const DurableObjectsOptionsSchema = z.object({
 	durableObjects: z
 		.record(
@@ -41,6 +47,49 @@ export function normaliseDurableObject(
 }
 
 export const DURABLE_OBJECTS_PLUGIN_NAME = "do";
+
+export const DURABLE_OBJECTS_STORAGE_SERVICE_NAME = `${DURABLE_OBJECTS_PLUGIN_NAME}:storage`;
+function normaliseDurableObjectStoragePath(
+	tmpPath: string,
+	persist: Persistence
+): string {
+	// If persistence is disabled, use "memory" storage. Note we're still
+	// returning a path on the file-system here. Miniflare 2's in-memory storage
+	// persisted between options reloads. However, we restart the `workerd`
+	// process on each reload which would destroy any in-memory data. We'd like to
+	// keep Miniflare 2's behaviour, so persist to a temporary path which we
+	// destroy on `dispose()`.
+	const memoryishPath = path.join(tmpPath, DURABLE_OBJECTS_PLUGIN_NAME);
+	if (persist === undefined || persist === false) {
+		return memoryishPath;
+	}
+
+	// Try parse `persist` as a URL
+	const url = maybeParseURL(persist);
+	if (url !== undefined) {
+		if (url.protocol === "memory:") {
+			return memoryishPath;
+		} else if (url.protocol === "file:") {
+			// Note we're ignoring `PARAM_FILE_UNSANITISE` here, file names should
+			// be Durable Object IDs which are just hex strings.
+			return fileURLToPath(url);
+		}
+		// Omitting `sqlite:` and `remote:`. `sqlite:` expects all data to be stored
+		// in a single SQLite database, which isn't possible here. We could
+		// `path.dirname()` the SQLite database path and use that, but the path
+		// might be ":memory:" which we definitely can't support.
+		throw new MiniflareCoreError(
+			"ERR_PERSIST_UNSUPPORTED",
+			`Unsupported "${url.protocol}" persistence protocol for Durable Object storage: ${url.href}`
+		);
+	}
+
+	// Otherwise, fallback to file storage
+	return persist === true
+		? path.join(DEFAULT_PERSIST_ROOT, DURABLE_OBJECTS_PLUGIN_NAME)
+		: persist;
+}
+
 export const DURABLE_OBJECTS_PLUGIN: Plugin<
 	typeof DurableObjectsOptionsSchema,
 	typeof DurableObjectsSharedOptionsSchema,
@@ -61,19 +110,35 @@ export const DURABLE_OBJECTS_PLUGIN: Plugin<
 			}
 		);
 	},
-	getServices({ options, sharedOptions }) {
-		if (
-			// If we have Durable Object bindings...
-			Object.keys(options.durableObjects ?? {}).length > 0 &&
-			// ...and persistence is enabled...
-			sharedOptions.durableObjectsPersist
-		) {
-			// ...throw, as Durable-Durable Objects are not yet supported
-			throw new DurableObjectsError(
-				"ERR_PERSIST_UNSUPPORTED",
-				"Persisted Durable Objects are not yet supported"
-			);
+	async getServices({ sharedOptions, tmpPath, durableObjectClassNames }) {
+		// Check if we even have any Durable Object bindings, if we don't, we can
+		// skip creating the storage directory
+		let hasDurableObjects = false;
+		for (const classNames of durableObjectClassNames.values()) {
+			if (classNames.size > 0) {
+				hasDurableObjects = true;
+				break;
+			}
 		}
+		if (!hasDurableObjects) return;
+
+		const storagePath = normaliseDurableObjectStoragePath(
+			tmpPath,
+			sharedOptions.durableObjectsPersist
+		);
+		// `workerd` requires the `disk.path` to exist. Setting `recursive: true`
+		// is like `mkdir -p`: it won't fail if the directory already exists, and it
+		// will create all non-existing parents.
+		await fs.mkdir(storagePath, { recursive: true });
+		return [
+			{
+				// Note this service will be de-duped by name if multiple Workers create
+				// it. Each Worker will have the same `sharedOptions` though, so this
+				// isn't a problem.
+				name: DURABLE_OBJECTS_STORAGE_SERVICE_NAME,
+				disk: { path: storagePath, writable: true },
+			},
+		];
 	},
 };
 

packages/miniflare/src/plugins/shared/gateway.ts

@@ -45,11 +45,11 @@ export interface RemoteStorageConstructor {
 	): RemoteStorage;
 }
 
-const DEFAULT_PERSIST_ROOT = ".mf";
+export const DEFAULT_PERSIST_ROOT = ".mf";
 
 export const PARAM_FILE_UNSANITISE = "unsanitise";
 
-function maybeParseURL(url: Persistence): URL | undefined {
+export function maybeParseURL(url: Persistence): URL | undefined {
 	try {
 		if (typeof url === "string") return new URL(url);
 	} catch {}
@@ -101,7 +101,7 @@ export class GatewayFactory<Gateway> {
 			} else if (url.protocol === "sqlite:") {
 				return new SqliteStorage(url.pathname, sanitisedNamespace, this.clock);
 			}
-			// TODO: support Redis/SQLite storages?
+			// TODO: support Redis storage?
 			if (url.protocol === "remote:") {
 				const { cloudflareFetch, remoteStorageClass } = this;
 				if (cloudflareFetch === undefined) {

packages/miniflare/src/plugins/shared/index.ts

@@ -4,7 +4,7 @@ import { Awaitable, Log, OptionalZodTypeOf } from "../../shared";
 import { GatewayConstructor, RemoteStorageConstructor } from "./gateway";
 import { RouterConstructor } from "./router";
 
-export type DurableObjectClassNames = Map<string, string[]>;
+export type DurableObjectClassNames = Map<string, Set<string>>;
 
 export interface PluginServicesOptions<
 	Options extends z.ZodType,
@@ -19,6 +19,7 @@ export interface PluginServicesOptions<
 	durableObjectClassNames: DurableObjectClassNames;
 	additionalModules: Worker_Module[];
 	loopbackPort: number;
+	tmpPath: string;
 }
 
 export interface PluginBase<

packages/miniflare/src/runtime/config/workerd.capnp

@@ -493,8 +493,16 @@ struct Worker {
     #
     # This mode is intended for local testing purposes.
 
-    # TODO(someday): Support storage to a local directory.
-    # TODO(someday): Support storage to a database.
+    localDisk @12 :Text;
+    # ** EXPERIMENTAL; SUBJECT TO BACKWARDS-INCOMPATIBLE CHANGE **
+    #
+    # Durable Object data will be stored in a directory on local disk. This field is the name of
+    # a service, which must be a DiskDirectory service. For each Durable Object class, a
+    # subdirectory will be created using `uniqueKey` as the name. Within the directory, one or
+    # more files are created for each object, with names `<id>.<ext>`, where `.<ext>` may be any of
+    # a number of different extensions depending on the storage mode. (Currently, the main storage
+    # is a file with the extension `.sqlite`, and in certain situations extra files with the
+    # extensions `.sqlite-wal`, and `.sqlite-shm` may also be present.)
   }
 
   # TODO(someday): Support distributing objects across a cluster. At present, objects are always

packages/miniflare/src/runtime/config/workerd.capnp.d.ts

@@ -524,10 +524,12 @@ export declare class Worker_DurableObjectNamespace extends __S {
 export declare enum Worker_DurableObjectStorage_Which {
 	NONE = 0,
 	IN_MEMORY = 1,
+	LOCAL_DISK = 2,
 }
 export declare class Worker_DurableObjectStorage extends __S {
 	static readonly NONE = Worker_DurableObjectStorage_Which.NONE;
 	static readonly IN_MEMORY = Worker_DurableObjectStorage_Which.IN_MEMORY;
+	static readonly LOCAL_DISK = Worker_DurableObjectStorage_Which.LOCAL_DISK;
 	static readonly _capnp: {
 		displayName: string;
 		id: string;
@@ -537,6 +539,9 @@ export declare class Worker_DurableObjectStorage extends __S {
 	setNone(): void;
 	isInMemory(): boolean;
 	setInMemory(): void;
+	getLocalDisk(): string;
+	isLocalDisk(): boolean;
+	setLocalDisk(value: string): void;
 	toString(): string;
 	which(): Worker_DurableObjectStorage_Which;
 }