feat(miniflare): Implement KV/Assets plugin and Workers Assets simula…

…tor (#6403)

This commit adds a new KV/Assets plugin and an Assets Worker
in Miniflare, that are meant to emulate Workers with Assets in
local development.

.changeset/eight-terms-beg.md

@@ -0,0 +1,8 @@
+---
+"miniflare": minor
+"@cloudflare/workers-shared": minor
+---
+
+feat: Extend KV plugin behaviour to support Workers assets
+
+This commit extends Miniflare's KV plugin's behaviour to support Workers assets, and therefore enables the emulation of Workers with assets in local development.

fixtures/workers-with-assets/public/about/index.html

@@ -0,0 +1 @@
+<p>Learn more about Workers with Assets soon!</p>

fixtures/workers-with-assets/public/index.html

@@ -1 +1 @@
-<h1>Hello Workers + Assets World!</h1>
+<h1>Hello Workers + Assets World 🚀!</h1>

fixtures/workers-with-assets/tests/index.test.ts

@@ -17,10 +17,22 @@ describe("[Workers + Assets] `wrangler dev`", () => {
 		await stop?.();
 	});
 
-	it("renders ", async ({ expect }) => {
-		const response = await fetch(`http://${ip}:${port}/`);
-		const text = await response.text();
+	it("should respond with static asset content", async ({ expect }) => {
+		let response = await fetch(`http://${ip}:${port}/index.html`);
+		let text = await response.text();
 		expect(response.status).toBe(200);
-		expect(text).toContain(`Hello from Asset Server Worker 🚀`);
+		expect(text).toContain(`<h1>Hello Workers + Assets World 🚀!</h1>`);
+
+		response = await fetch(`http://${ip}:${port}/about/index.html`);
+		text = await response.text();
+		expect(response.status).toBe(200);
+		expect(text).toContain(`<p>Learn more about Workers with Assets soon!</p>`);
+	});
+
+	it("should not resolve '/' to '/index.html' ", async ({ expect }) => {
+		let response = await fetch(`http://${ip}:${port}/`);
+		let text = await response.text();
+		expect(response.status).toBe(404);
+		expect(text).toContain("Not Found");
 	});
 });

packages/miniflare/README.md

@@ -528,6 +528,16 @@ parameter in module format Workers.
   If set, only files with paths _not_ matching these glob patterns will be
   served.
 
+  - `assetsPath?: string`
+
+  Path to serve Workers assets from.
+
+  - `assetsKVBindingName?: string`
+    Name of the binding to the KV namespace that the assets are in. If `assetsPath` is set, this binding will be injected into this Worker.
+
+  - `assetsManifestBindingName?: string`
+    Name of the binding to an `ArrayBuffer` containing the binary-encoded assets manifest. If `assetsPath` is set, this binding will be injected into this Worker.
+
 #### R2
 
 - `r2Buckets?: Record<string, string> | string[]`

packages/miniflare/src/plugins/kv/assets.ts

@@ -0,0 +1,99 @@
+import { KVOptionsSchema } from "miniflare";
+import SCRIPT_KV_ASSETS from "worker:kv/assets";
+import { z } from "zod";
+import { Service, Worker_Binding } from "../../runtime";
+import { getAssetsBindingsNames, SharedBindings } from "../../workers";
+import { kProxyNodeBinding } from "../shared";
+import { KV_PLUGIN_NAME } from "./constants";
+
+export interface AssetsOptions {
+	assetsPath: string;
+	assetsKVBindingName?: string;
+	assetsManifestBindingName?: string;
+}
+
+export function isWorkersWithAssets(
+	options: z.infer<typeof KVOptionsSchema>
+): options is AssetsOptions {
+	return options.assetsPath !== undefined;
+}
+
+const SERVICE_NAMESPACE_ASSET = `${KV_PLUGIN_NAME}:asset`;
+
+export function buildAssetsManifest(): Uint8Array {
+	const buffer = new ArrayBuffer(20);
+	const assetManifest = new Uint8Array(buffer); // [0, 0, 0, ..., 0, 0]
+	// this will signal to Asset Server Worker that its running in a
+	// local dev "context"
+	assetManifest.set([1], 0); // [1, 0, 0, ..., 0, 0]
+
+	return assetManifest;
+}
+
+export async function getAssetsBindings(
+	options: AssetsOptions
+): Promise<Worker_Binding[]> {
+	const assetsBindings = getAssetsBindingsNames(
+		options?.assetsKVBindingName,
+		options?.assetsManifestBindingName
+	);
+
+	const assetsManifest = buildAssetsManifest();
+
+	return [
+		{
+			// this is the binding to the KV namespace that the assets are in.
+			name: assetsBindings.ASSETS_KV_NAMESPACE,
+			kvNamespace: { name: SERVICE_NAMESPACE_ASSET },
+		},
+		{
+			// this is the binding to an ArrayBuffer containing the binary-encoded
+			// assets manifest.
+			name: assetsBindings.ASSETS_MANIFEST,
+			data: assetsManifest,
+		},
+	];
+}
+
+export async function getAssetsNodeBindings(
+	options: AssetsOptions
+): Promise<Record<string, unknown>> {
+	const assetsManifest = buildAssetsManifest();
+	const assetsBindings = getAssetsBindingsNames(
+		options?.assetsKVBindingName,
+		options?.assetsManifestBindingName
+	);
+
+	return {
+		[assetsBindings.ASSETS_KV_NAMESPACE]: kProxyNodeBinding,
+		[assetsBindings.ASSETS_MANIFEST]: assetsManifest,
+	};
+}
+
+export function getAssetsServices(options: AssetsOptions): Service[] {
+	const storageServiceName = `${SERVICE_NAMESPACE_ASSET}:storage`;
+	const storageService: Service = {
+		name: storageServiceName,
+		disk: { path: options.assetsPath, writable: true },
+	};
+	const namespaceService: Service = {
+		name: SERVICE_NAMESPACE_ASSET,
+		worker: {
+			compatibilityDate: "2023-07-24",
+			compatibilityFlags: ["nodejs_compat"],
+			modules: [
+				{
+					name: "assets.worker.js",
+					esModule: SCRIPT_KV_ASSETS(),
+				},
+			],
+			bindings: [
+				{
+					name: SharedBindings.MAYBE_SERVICE_BLOBS,
+					service: { name: storageServiceName },
+				},
+			],
+		},
+	};
+	return [storageService, namespaceService];
+}

packages/miniflare/src/plugins/kv/index.ts

@@ -20,6 +20,12 @@ import {
 	Plugin,
 	SERVICE_LOOPBACK,
 } from "../shared";
+import {
+	getAssetsBindings,
+	getAssetsNodeBindings,
+	getAssetsServices,
+	isWorkersWithAssets,
+} from "./assets";
 import { KV_PLUGIN_NAME } from "./constants";
 import {
 	getSitesBindings,
@@ -31,6 +37,11 @@ import {
 export const KVOptionsSchema = z.object({
 	kvNamespaces: z.union([z.record(z.string()), z.string().array()]).optional(),
 
+	// Workers + Assets
+	assetsPath: PathSchema.optional(),
+	assetsKVBindingName: z.string().optional(),
+	assetsManifestBindingName: z.string().optional(),
+
 	// Workers Sites
 	sitePath: PathSchema.optional(),
 	siteInclude: z.string().array().optional(),
@@ -71,18 +82,29 @@ export const KV_PLUGIN: Plugin<
 			bindings.push(...(await getSitesBindings(options)));
 		}
 
+		if (isWorkersWithAssets(options)) {
+			bindings.push(...(await getAssetsBindings(options)));
+		}
+
 		return bindings;
 	},
+
 	async getNodeBindings(options) {
 		const namespaces = namespaceKeys(options.kvNamespaces);
 		const bindings = Object.fromEntries(
 			namespaces.map((name) => [name, kProxyNodeBinding])
 		);
+
 		if (isWorkersSitesEnabled(options)) {
 			Object.assign(bindings, await getSitesNodeBindings(options));
 		}
+
+		if (isWorkersWithAssets(options)) {
+			Object.assign(bindings, await getAssetsNodeBindings(options));
+		}
 		return bindings;
 	},
+
 	async getServices({
 		options,
 		sharedOptions,
@@ -151,8 +173,13 @@ export const KV_PLUGIN: Plugin<
 			services.push(...getSitesServices(options));
 		}
 
+		if (isWorkersWithAssets(options)) {
+			services.push(...getAssetsServices(options));
+		}
+
 		return services;
 	},
+
 	getPersistPath({ kvPersist }, tmpPath) {
 		return getPersistPath(KV_PLUGIN_NAME, tmpPath, kvPersist);
 	},

packages/miniflare/src/workers/kv/assets.worker.ts

@@ -0,0 +1,33 @@
+import { SharedBindings } from "miniflare:shared";
+import { KVParams } from "./constants";
+
+interface Env {
+	[SharedBindings.MAYBE_SERVICE_BLOBS]: Fetcher;
+}
+
+export default <ExportedHandler<Env>>{
+	async fetch(request, env) {
+		// Only permit reads
+		if (request.method !== "GET") {
+			const message = `Cannot ${request.method.toLowerCase()}() with Workers Assets namespace`;
+			return new Response(message, { status: 405, statusText: message });
+		}
+
+		// Decode key
+		const url = new URL(request.url);
+		let key = url.pathname.substring(1); // Strip leading "/"
+
+		if (url.searchParams.get(KVParams.URL_ENCODED)?.toLowerCase() === "true") {
+			key = decodeURIComponent(key);
+		}
+
+		const blobsService = env[SharedBindings.MAYBE_SERVICE_BLOBS];
+		if (key === "" || key === "/") {
+			return new Response("Not Found", {
+				status: 404,
+			});
+		} else {
+			return blobsService.fetch(new URL(key, "http://placeholder"));
+		}
+	},
+};

packages/miniflare/src/workers/kv/constants.ts

@@ -45,6 +45,7 @@ export function decodeSitesKey(key: string): string {
 		? decodeURIComponent(key.substring(SITES_NO_CACHE_PREFIX.length))
 		: key;
 }
+
 export function isSitesRequest(request: { url: string }) {
 	const url = new URL(request.url);
 	return url.pathname.startsWith(`/${SITES_NO_CACHE_PREFIX}`);
@@ -116,3 +117,17 @@ export function testSiteRegExps(
 	if (regExps.exclude !== undefined) return !testRegExps(regExps.exclude, key);
 	return true;
 }
+
+export function getAssetsBindingsNames(
+	// __STATIC_CONTENT and __STATIC_CONTENT_MANIFEST binding names are
+	// reserved for Workers Sites. Since we want to allow both sites and
+	// assets to work side by side, we cannot use the same binding name
+	// for assets. Therefore deferring to a different default naming here.
+	assetsKVBindingName = "__STATIC_ASSETS_CONTENT",
+	assetsManifestBindingName = "__STATIC_ASSETS_CONTENT_MANIFEST"
+) {
+	return {
+		ASSETS_KV_NAMESPACE: assetsKVBindingName,
+		ASSETS_MANIFEST: assetsManifestBindingName,
+	} as const;
+}

packages/miniflare/test/fixtures/assets/worker-with-custom-assets-bindings.ts

@@ -0,0 +1,14 @@
+interface Env {
+	//  custom kv binding name
+	CUSTOM_ASSETS_NAMESPACE: KVNamespace;
+}
+
+export default {
+	async fetch(request: Request, env: Env) {
+		const url = new URL(request.url);
+		const { pathname } = url;
+
+		const content = await env.CUSTOM_ASSETS_NAMESPACE.get(pathname);
+		return new Response(content);
+	},
+};

packages/miniflare/test/fixtures/assets/worker-with-default-assets-bindings.ts

@@ -0,0 +1,14 @@
+interface Env {
+	//  this is the default kv binding name
+	__STATIC_ASSETS_CONTENT: KVNamespace;
+}
+
+export default {
+	async fetch(request: Request, env: Env) {
+		const url = new URL(request.url);
+		const { pathname } = url;
+
+		const content = await env.__STATIC_ASSETS_CONTENT.get(pathname);
+		return new Response(content);
+	},
+};