add next-dev internal-package

README.md

@@ -13,6 +13,10 @@ You can see the packages contents (with their documentation) in their respective
 - [`@cloudflare/next-on-pages`](https://github.com/cloudflare/next-on-pages/tree/main/packages/next-on-pages#cloudflarenext-on-pages)
 - [`eslint-plugin-next-on-pages`](https://github.com/cloudflare/next-on-pages/tree/main/packages/eslint-plugin-next-on-pages#eslint-plugin-next-on-pages)
 
+Additionally there is also the `next-dev` submodule which is implemented as a separate package in this repository but included as a submodule of the main `@cloudflare/next-on-pages` package, you can see the submodule's content here:
+
+- [`@cloudflare/next-on-pages/next-dev`](https://github.com/cloudflare/next-on-pages/tree/main/internal-packages/next-dev)
+
 ## Contributing
 
 If you want to contribute to this project (both to the main package and the eslint one) please refer to the [Contributing document](./docs/contributing.md).

internal-packages/docs-scraper/package.json

@@ -1,5 +1,5 @@
 {
-	"name": "docs-scraper",
+	"name": "@cloudflare/next-on-pages-docs-scraper",
 	"private": true,
 	"scripts": {
 		"lint": "eslint scripts",

internal-packages/next-dev/.eslintrc.js

@@ -0,0 +1,7 @@
+module.exports = {
+	root: true,
+	extends: ['@cloudflare/eslint-config-next-on-pages'],
+	rules: {
+		'no-console': 'off',
+	},
+};

internal-packages/next-dev/README.md

@@ -0,0 +1,95 @@
+# Next-on-pages Next-Dev
+
+The `next-dev` submodule of the `@cloudflare/next-on-pages` package implements a utility that allows you to run the [standard Next.js development server](https://nextjs.org/docs/app/api-reference/next-cli#development) (with hot-code reloading, error reporting, HMR and everything it has to offer) with also adding local Cloudflare bindings simulations (implemented via [Miniflare](https://github.com/cloudflare/miniflare)).
+
+IMPORTANT: As mentioned above the module allows you to run the standard Next.js dev server as is and it only makes sure that Cloudflare bindings are accessible, it does not generate a worker nor faithfully represent the final application that will be deployed to Cloudflare Pages, so please use this only as a development tool and make sure to properly test your application with `wrangler pages dev` before actually deploying it to.
+
+## How to use the module
+
+The module is part of the `@cloudflare/next-on-pages` package so it does not need installation, it exports the `setupDevBindings` function which you need to import and call in your `next.config.js` file to declare what bindings your application is using and need to be made available in the development server.
+
+After having added the `setupDevBindings` call to the `next.config.js` you can simply run `next dev` and inside your edge routes you will be able to access your bindings via `process.env` in the exact same way as you would in your production code.
+
+### Example
+
+Let's see an example of how to use the utility, in a Next.js application built in TypeScript using the App router.
+
+Firstly we need to update the `next.config.js` file:
+
+```js
+// file: next.config.js
+
+/** @type {import('next').NextConfig} */
+const nextConfig = {};
+
+module.exports = nextConfig;
+
+// we only need to use the utility during development so we can check NODE_ENV
+// (note: this check is recommended but completely optional)
+if (process.env.NODE_ENV === 'development') {
+	// we import the utility from the next-dev submodule
+	const { setupDevBindings } = require('@cloudflare/next-on-pages/next-dev');
+
+	// we call the utility with the bindings we want to have access to
+	setupDevBindings({
+		kvNamespaces: ['MY_KV_1', 'MY_KV_2'],
+		r2Buckets: ['MY_R2'],
+		durableObjects: {
+			MY_DO: {
+				scriptName: 'do-worker',
+				className: 'DurableObjectClass',
+			},
+		},
+		// ...
+	});
+}
+```
+
+Next (optional but highly recommended) we create a [TypeScript declaration file](https://www.typescriptlang.org/docs/handbook/2/type-declarations.html) so that we can make sure that TypeScript is aware of the bindings added to `process.env`:
+
+```ts
+// file: env.d.ts
+
+declare global {
+	namespace NodeJS {
+		interface ProcessEnv {
+			[key: string]: string | undefined;
+			MY_KV_1: KVNamespace;
+			MY_KV_2: KVNamespace;
+			MY_R2: R2Bucket;
+			MY_DO: DurableObjectNamespace;
+		}
+	}
+}
+
+export {};
+```
+
+> **Note**
+> The binding types used in the above file come from `@cloudflare/workers-types`, in order to use them make sure that you've installed the package as a dev dependency and you've added it to your `tsconfig.js` file under `compilerOptions.types`.
+
+Then we can simply use any of our bindings inside our next application, for example in the following API route:
+
+```ts
+export const runtime = 'edge';
+
+export async function GET(request: NextRequest) {
+	const myKv = process.env.MY_KV;
+
+	const valueA = await myKv.get('key-a');
+
+	return new Response(`The value of key-a in MY_KV is: ${valueA}`);
+}
+```
+
+## Recommended Workflow
+
+When developing a next-on-pages application, this is the development workflow that we recommend:
+
+- **Develop using the standard Next.js dev server**\
+  In order to have a very fast and polished dev experience the standard dev server provided by Next.js is the best available option. So use it to quickly make changes and iterate over them, while still having access to your Cloudflare bindings thanks to the
+  `next-dev` submodule.
+- **Build and preview your worker locally**\
+  In order to make sure that your application is being built in a manner that is fully compatible with Cloudflare Pages, before deploying it, or whenever you're comfortable checking the correctness of the application during your development process, build your worker by using `@cloudflare/next-on-pages` and preview it locally via `wrangler pages dev .vercel/output/static`, this is the only way to locally make sure that every is working as you expect it to.
+- **Deploy your app and iterate**\
+  Once you've previewed your application locally then you can deploy it to Cloudflare Pages (both via direct uploads or git integration) and iterate over the process to make new changes.

internal-packages/next-dev/package.json

@@ -0,0 +1,35 @@
+{
+	"name": "@cloudflare/next-on-pages-next-dev",
+	"version": "0.0.1",
+	"main": "dist/index.cjs",
+	"scripts": {
+		"lint": "eslint src",
+		"types-check": "tsc --noEmit",
+		"build:js": "esbuild --bundle --format=cjs ./src/index.ts --external:miniflare --outfile=./dist/index.cjs --platform=node",
+		"build:types": "tsc --emitDeclarationOnly --declaration --outDir ./dist",
+		"build:js:watch": "npm run build:js -- --watch=forever",
+		"build:types:watch": "npm run build:types -- --watch",
+		"build": "npm run build:js && npm run build:types",
+		"build:watch": "npm run build:js:watch & npm run build:types:watch",
+		"test": "npx vitest --config vitest.config.ts"
+	},
+	"files": [
+		"dist",
+		"dev-init.cjs",
+		"dev-init.d.ts",
+		"devBindingsOptions.ts"
+	],
+	"dependencies": {
+		"miniflare": "^3.20231002.0",
+		"node-fetch": "^3.3.2"
+	},
+	"devDependencies": {
+		"@cloudflare/workers-types": "4.20231002.0",
+		"@tsconfig/strictest": "^2.0.0",
+		"esbuild": "^0.15.3",
+		"eslint": "^8.35.0",
+		"tsconfig": "*",
+		"typescript": "^5.0.4",
+		"vitest": "^0.32.2"
+	}
+}

internal-packages/next-dev/src/durableObjects.ts

@@ -0,0 +1,218 @@
+import type { DevBindingsOptions } from './index';
+import type { WorkerOptions } from 'miniflare';
+import type { WorkerDefinition, WorkerRegistry } from './wrangler';
+import {
+	EXTERNAL_DURABLE_OBJECTS_WORKER_NAME,
+	EXTERNAL_DURABLE_OBJECTS_WORKER_SCRIPT,
+	getIdentifier,
+	getRegisteredWorkers,
+} from './wrangler';
+import { warnAboutExternalBindingsNotFound } from './utils';
+
+/**
+ * Gets information regarding DurableObject bindings that can be passed to miniflare to access external (locally exposed in the local registry) Durable Object bindings.
+ *
+ * @param durableObjects
+ * @returns the durableObjects and WorkersOptions objects to use or undefined if connecting to the registry and/or creating the options has failed
+ */
+export async function getDOBindingInfo(
+	durableObjects: DevBindingsOptions['durableObjects'],
+): Promise<
+	| {
+			workerOptions: WorkerOptions;
+			durableObjects: DevBindingsOptions['durableObjects'];
+	  }
+	| undefined
+> {
+	const requestedDurableObjectsNames = new Set(
+		Object.keys(durableObjects ?? {}),
+	);
+
+	if (requestedDurableObjectsNames.size === 0) {
+		return;
+	}
+
+	let registeredWorkers: WorkerRegistry | undefined;
+
+	try {
+		registeredWorkers = await getRegisteredWorkers();
+	} catch {
+		/* */
+	}
+
+	if (!registeredWorkers) {
+		warnAboutLocalDurableObjectsNotFound(requestedDurableObjectsNames);
+		return;
+	}
+
+	const registeredWorkersWithDOs: RegisteredWorkersWithDOs =
+		getRegisteredWorkersWithDOs(
+			registeredWorkers,
+			requestedDurableObjectsNames,
+		);
+
+	const [foundDurableObjects, missingDurableObjects] = [
+		...requestedDurableObjectsNames.keys(),
+	].reduce(
+		([foundDOs, missingDOs], durableObjectName) => {
+			let found = false;
+			for (const [, worker] of registeredWorkersWithDOs.entries()) {
+				found = !!worker.durableObjects.find(
+					durableObject => durableObject.name === durableObjectName,
+				);
+				if (found) break;
+			}
+			if (found) {
+				foundDOs.add(durableObjectName);
+			} else {
+				missingDOs.add(durableObjectName);
+			}
+			return [foundDOs, missingDOs];
+		},
+		[new Set(), new Set()] as [Set<string>, Set<string>],
+	);
+
+	if (missingDurableObjects.size) {
+		warnAboutLocalDurableObjectsNotFound(missingDurableObjects);
+	}
+
+	const externalDOs = collectExternalDurableObjects(
+		registeredWorkersWithDOs,
+		foundDurableObjects,
+	);
+
+	const script = generateDurableObjectProxyWorkerScript(
+		externalDOs,
+		registeredWorkersWithDOs,
+	);
+
+	// the following is a very simplified version of wrangler's code but tweaked/simplified for our use case
+	// https://github.com/cloudflare/workers-sdk/blob/3077016/packages/wrangler/src/dev/miniflare.ts#L240-L288
+	const externalDurableObjectWorker: WorkerOptions = {
+		name: EXTERNAL_DURABLE_OBJECTS_WORKER_NAME,
+		routes: [`*/${EXTERNAL_DURABLE_OBJECTS_WORKER_NAME}`],
+		unsafeEphemeralDurableObjects: true,
+		modules: true,
+		script,
+	};
+
+	const durableObjectsToUse = externalDOs.reduce(
+		(all, externalDO) => {
+			return {
+				...all,
+				[externalDO.durableObjectName]: externalDO,
+			};
+		},
+		{} as DevBindingsOptions['durableObjects'],
+	);
+
+	return {
+		workerOptions: externalDurableObjectWorker,
+		durableObjects: durableObjectsToUse,
+	};
+}
+
+function getRegisteredWorkersWithDOs(
+	registeredWorkers: WorkerRegistry,
+	durableObjectsNames: Set<string>,
+) {
+	const registeredWorkersWithDOs: Map<string, WorkerRegistry[string]> =
+		new Map();
+
+	for (const workerName of Object.keys(registeredWorkers ?? {})) {
+		// eslint-disable-next-line @typescript-eslint/no-non-null-assertion
+		const worker = registeredWorkers![workerName]!;
+		const containsRequestedDO = !!worker.durableObjects.find(({ name }) =>
+			durableObjectsNames.has(name),
+		);
+		if (containsRequestedDO) {
+			registeredWorkersWithDOs.set(workerName, worker);
+		}
+	}
+	return registeredWorkersWithDOs;
+}
+
+/**
+ * Collects information about durable objects exposed locally by the local registry that we can use to in
+ * miniflare to give access such bindings.
+ *
+ * NOTE: This function contains logic taken from wrangler but customized and updated to our (simpler) use case
+ * see: https://github.com/cloudflare/workers-sdk/blob/3077016/packages/wrangler/src/dev/miniflare.ts#L312-L330
+ *
+ * @param registeredWorkersWithDOs a map containing the registered workers containing Durable Objects we need to proxy to
+ * @param foundDurableObjects durable objects found in the local registry
+ * @returns array of objects containing durable object information (that we can use to generate the local DO proxy worker)
+ */
+function collectExternalDurableObjects(
+	registeredWorkersWithDOs: RegisteredWorkersWithDOs,
+	foundDurableObjects: Set<string>,
+): ExternalDurableObject[] {
+	return [...registeredWorkersWithDOs.entries()]
+		.flatMap(([workerName, worker]) => {
+			const dos = worker.durableObjects;
+			return dos.map(({ name, className }) => {
+				if (!foundDurableObjects.has(name)) return;
+
+				return {
+					workerName,
+					durableObjectName: name,
+					className: getIdentifier(`${workerName}_${className}`),
+					scriptName: EXTERNAL_DURABLE_OBJECTS_WORKER_NAME,
+					unsafeUniqueKey: `${workerName}-${className}`,
+				};
+			});
+		})
+		.filter(Boolean) as ExternalDurableObject[];
+}
+
+/**
+ * Generates the script for a worker that can be used to proxy durable object requests to the appropriate
+ * external (locally exposed in the local registry) bindings.
+ *
+ * NOTE: This function contains logic taken from wrangler but customized and updated to our (simpler) use case
+ * see: https://github.com/cloudflare/workers-sdk/blob/3077016/packages/wrangler/src/dev/miniflare.ts#L259-L284
+ *
+ * @param externalDOs
+ * @param registeredWorkersWithDOs a map containing the registered workers containing Durable Objects we need to proxy to
+ * @returns the worker script
+ */
+function generateDurableObjectProxyWorkerScript(
+	externalDOs: {
+		workerName: string;
+		className: string;
+		scriptName: string;
+		unsafeUniqueKey: string;
+	}[],
+	registeredWorkersWithDOs: RegisteredWorkersWithDOs,
+) {
+	return (
+		EXTERNAL_DURABLE_OBJECTS_WORKER_SCRIPT +
+		externalDOs
+			.map(({ workerName, className }) => {
+				const classNameJson = JSON.stringify(className);
+				const target = registeredWorkersWithDOs.get(workerName);
+				if (!target || !target.host || !target.port) return;
+				const proxyUrl = `http://${target.host}:${target.port}/${EXTERNAL_DURABLE_OBJECTS_WORKER_NAME}`;
+				const proxyUrlJson = JSON.stringify(proxyUrl);
+				return `export const ${className} = createClass({ className: ${classNameJson}, proxyUrl: ${proxyUrlJson} });`;
+			})
+			.filter(Boolean)
+			.join('\n')
+	);
+}
+
+type RegisteredWorkersWithDOs = Map<string, WorkerDefinition>;
+
+type ExternalDurableObject = {
+	workerName: string;
+	durableObjectName: string;
+	className: string;
+	scriptName: string;
+	unsafeUniqueKey: string;
+};
+
+function warnAboutLocalDurableObjectsNotFound(
+	durableObjectsNotFound: Set<string>,
+): void {
+	warnAboutExternalBindingsNotFound(durableObjectsNotFound, 'Durable Objects');
+}