Isolate the Toolpad router from Toolpad server

packages/create-toolpad-app/src/index.ts

@@ -11,7 +11,7 @@ import { execaCommand } from 'execa';
 import { satisfies } from 'semver';
 import { readJsonFile } from '@mui/toolpad-utils/fs';
 import invariant from 'invariant';
-import { bashResolvePath } from '@mui/toolpad-utils/path';
+import { bashResolvePath } from '@mui/toolpad-utils/cli';
 import { PackageJson } from './packageType';
 
 type PackageManager = 'npm' | 'pnpm' | 'yarn';

packages/toolpad-app/cli/appBuilder.ts

@@ -11,7 +11,7 @@ async function main() {
 
  const projectDir = process.env.TOOLPAD_PROJECT_DIR;
 
- const project = await initProject({ dir: projectDir });
+ const project = await initProject({ dev: false, dir: projectDir });
 
  await project.build();
 

packages/toolpad-app/cli/server.ts

@@ -3,7 +3,6 @@ import { IncomingMessage, createServer } from 'http';
 import * as fs from 'fs/promises';
 import { Worker, MessageChannel } from 'worker_threads';
 import express from 'express';
-import invariant from 'invariant';
 import getPort from 'get-port';
 import { createProxyMiddleware } from 'http-proxy-middleware';
 import { mapValues } from '@mui/toolpad-utils/collections';
@@ -125,31 +124,33 @@ interface AppHandler {
  dispose?: () => Promise<void>;
 }
 
-export interface ServerConfig {
- dev?: boolean;
+export interface ToolpadHandlerConfig {
+ dev: boolean;
  dir: string;
- port: number;
  externalUrl: string;
  toolpadDevMode?: boolean;
 }
 
-async function startServer({ dev, port, toolpadDevMode, externalUrl, dir }: ServerConfig) {
+async function createToolpadHandler({
+ dev,
+ toolpadDevMode,
+ externalUrl,
+ dir,
+}: ToolpadHandlerConfig): Promise<AppHandler> {
  const gitSha1 = process.env.GIT_SHA1 || null;
  const circleBuildNum = process.env.CIRCLE_BUILD_NUM || null;
 
- const project = await initProject({ dev, dir, externalUrl });
+ const wsPort = await getPort();
+
+ const project = await initProject({ dev, dir, externalUrl, wsPort });
  await project.start();
 
- const runtimeConfig: RuntimeConfig = {
- projectDir: project.getRoot(),
- externalUrl,
- };
+ const runtimeConfig: RuntimeConfig = project.getRuntimeConfig();
 
- const app = express();
- const httpServer = createServer(app);
+ const router = express.Router();
 
  // See https://nextjs.org/docs/advanced-features/security-headers
- app.use((req, res, expressNext) => {
+ router.use((req, res, expressNext) => {
  // Force the browser to trust the Content-Type header
  // https://stackoverflow.com/questions/18337630/what-is-x-content-type-options-nosniff
  res.setHeader('X-Content-Type-Options', 'nosniff');
@@ -158,12 +159,12 @@ async function startServer({ dev, port, toolpadDevMode, externalUrl, dir }: Serv
  expressNext();
  });
 
- app.get('/', (req, res) => {
+ router.get('/', (req, res) => {
  const redirectUrl = dev ? '/_toolpad' : '/prod';
  res.redirect(302, redirectUrl);
  });
 
- app.get('/health-check', (req, res) => {
+ router.get('/health-check', (req, res) => {
  const memoryUsage = process.memoryUsage();
  res.json({
  gitSha1,
@@ -174,7 +175,7 @@ async function startServer({ dev, port, toolpadDevMode, externalUrl, dir }: Serv
  });
 
  const publicPath = path.resolve(__dirname, '../../public');
- app.use(express.static(publicPath, { index: false }));
+ router.use(express.static(publicPath, { index: false }));
 
  let appHandler: AppHandler | undefined;
 
@@ -184,16 +185,16 @@ async function startServer({ dev, port, toolpadDevMode, externalUrl, dir }: Serv
  runtimeConfig,
  base: previewBase,
  });
- app.use(previewBase, appHandler.handler);
+ router.use(previewBase, appHandler.handler);
  } else {
  appHandler = await createProdHandler(project);
- app.use('/prod', appHandler.handler);
+ router.use('/prod', appHandler.handler);
  }
 
  if (dev) {
  const rpcServer = createRpcServer(project);
- app.use('/api/rpc', createRpcHandler(rpcServer));
- app.use('/api/dataSources', project.dataManager.createDataSourcesHandler());
+ router.use('/api/rpc', createRpcHandler(rpcServer));
+ router.use('/api/dataSources', project.dataManager.createDataSourcesHandler());
 
  const transformIndexHtml = (html: string) => {
  const serializedConfig = serializeJavascript(runtimeConfig, { isJSON: true });
@@ -224,9 +225,9 @@ async function startServer({ dev, port, toolpadDevMode, externalUrl, dir }: Serv
  ],
  });
 
- app.use(editorBasename, viteApp.middlewares);
+ router.use(editorBasename, viteApp.middlewares);
  } else {
- app.use(
+ router.use(
  editorBasename,
  express.static(path.resolve(__dirname, '../../dist/editor'), { index: false }),
  asyncHandler(async (req, res) => {
@@ -239,38 +240,72 @@ async function startServer({ dev, port, toolpadDevMode, externalUrl, dir }: Serv
  }
  }
 
- const runningServer = await listen(httpServer, port);
+ if (dev) {
+ const wsServer = new WebSocketServer({ port: wsPort });
 
- const wsServer = new WebSocketServer({ noServer: true });
+ project.events.on('*', (event, payload) => {
+ wsServer.clients.forEach((client) => {
+ if (client.readyState === WebSocket.OPEN) {
+ client.send(JSON.stringify({ kind: 'projectEvent', event, payload }));
+ }
+ });
+ });
 
- project.events.on('*', (event, payload) => {
- wsServer.clients.forEach((client) => {
- if (client.readyState === WebSocket.OPEN) {
- client.send(JSON.stringify({ kind: 'projectEvent', event, payload }));
- }
+ // eslint-disable-next-line @typescript-eslint/no-unused-vars
+ wsServer.on('connection', (ws: WebSocket, _request: IncomingMessage) => {
+ ws.on('error', console.error);
  });
- });
 
- // eslint-disable-next-line @typescript-eslint/no-unused-vars
- wsServer.on('connection', (ws: WebSocket, _request: IncomingMessage) => {
- ws.on('error', console.error);
- });
+ // TODO(Jan): allow passing a server instance to the handler and attach websocket server to it
+ // httpServer.on('upgrade', (request, socket, head) => {
+ // invariant(request.url, 'request must have a url');
+ // const { pathname } = new URL(request.url, 'http://x');
+ //
+ // if (pathname === '/toolpad-ws') {
+ // wsServer.handleUpgrade(request, socket, head, (ws) => {
+ // wsServer.emit('connection', ws, request);
+ // });
+ // }
+ // });
+ }
 
- httpServer.on('upgrade', (request, socket, head) => {
- invariant(request.url, 'request must have a url');
- const { pathname } = new URL(request.url, 'http://x');
+ return {
+ handler: router,
+ dispose: async () => {
+ await Promise.allSettled([project.dispose(), appHandler?.dispose?.()]);
+ },
+ };
+}
 
- if (pathname === '/toolpad-ws') {
- wsServer.handleUpgrade(request, socket, head, (ws) => {
- wsServer.emit('connection', ws, request);
- });
- }
+export interface ToolpadServerConfig extends Omit<ToolpadHandlerConfig, 'server'> {
+ port: number;
+}
+
+async function startToolpadServer({
+ dev,
+ port,
+ toolpadDevMode,
+ externalUrl,
+ dir,
+}: ToolpadServerConfig) {
+ const app = express();
+ const httpServer = createServer(app);
+
+ const toolpadHandler = await createToolpadHandler({
+ dev,
+ toolpadDevMode,
+ externalUrl,
+ dir,
  });
 
+ app.use(toolpadHandler.handler);
+
+ const runningServer = await listen(httpServer, port);
+
  return {
  port: runningServer.port,
  async dispose() {
- await Promise.allSettled([project.dispose(), runningServer.close(), appHandler?.dispose?.()]);
+ await Promise.allSettled([runningServer.close(), toolpadHandler?.dispose?.()]);
  },
  };
 }
@@ -308,7 +343,7 @@ export async function runApp({ cmd, dir, port, toolpadDevMode = false }: RunAppO
 
  const externalUrl = process.env.TOOLPAD_EXTERNAL_URL || `http://localhost:${port}`;
 
- const server = await startServer({
+ const server = await startToolpadServer({
  dev,
  dir,
  port,

packages/toolpad-app/src/config.ts

@@ -37,7 +37,8 @@ export type BuildEnvVars = Record<
 // Do not add secrets
 export interface RuntimeConfig {
  externalUrl: string;
- projectDir?: string;
+ projectDir: string;
+ wsPort: number;
 }
 
 declare global {

packages/toolpad-app/src/projectEvents.ts

@@ -2,13 +2,14 @@ import * as React from 'react';
 import { Emitter } from '@mui/toolpad-utils/events';
 import useEventCallback from '@mui/utils/useEventCallback';
 import { ProjectEvents } from './types';
+import config from './config';
 
 let ws: WebSocket | null = null;
 const projectEvents = new Emitter<ProjectEvents>();
 
 if (typeof window !== 'undefined') {
  const wsProtocol = window.location.protocol === 'https:' ? 'wss:' : 'ws:';
- ws = new WebSocket(`${wsProtocol}//${window.location.host}/toolpad-ws`);
+ ws = new WebSocket(`${wsProtocol}//${window.location.hostname}:${config.wsPort}/toolpad-ws`);
 
  ws.addEventListener('error', (err) => console.error(err));
 

packages/toolpad-app/src/server/DataManager.ts

@@ -39,16 +39,22 @@ interface IToolpadProject {
 export default class DataManager {
  private project: IToolpadProject;
 
- private dataSources: Map<string, ServerDataSource<any, any, any> | undefined>;
+ private dataSources: Map<string, ServerDataSource<any, any, any> | undefined> | undefined;
 
  constructor(project: IToolpadProject) {
  this.project = project;
- this.dataSources = new Map(
- Object.entries(serverDataSources).map(([key, value]) => [
- key,
- typeof value === 'function' ? value(project) : value,
- ]),
- );
+ }
+
+ getDataSources(): Map<string, ServerDataSource<any, any, any> | undefined> {
+ if (!this.dataSources) {
+ this.dataSources = new Map(
+ Object.entries(serverDataSources).map(([key, value]) => [
+ key,
+ typeof value === 'function' ? value(this.project) : value,
+ ]),
+ );
+ }
+ return this.dataSources;
  }
 
  async getConnectionParams<P = unknown>(connectionId: string | null): Promise<P | null> {
@@ -80,8 +86,9 @@ export default class DataManager {
  dataNode: appDom.QueryNode<Q>,
  params: Q,
  ): Promise<ExecFetchResult<any>> {
+ const dataSources = this.getDataSources();
  const dataSource: ServerDataSource<P, Q, any> | undefined = dataNode.attributes.dataSource
- ? this.dataSources.get(dataNode.attributes.dataSource)
+ ? dataSources.get(dataNode.attributes.dataSource)
  : undefined;
  if (!dataSource) {
  throw new Error(
@@ -137,7 +144,8 @@ export default class DataManager {
  connectionId: NodeId | null,
  query: Q,
  ): Promise<any> {
- const dataSource: ServerDataSource<P, Q, any> | undefined = this.dataSources.get(dataSourceId);
+ const dataSources = this.getDataSources();
+ const dataSource: ServerDataSource<P, Q, any> | undefined = dataSources.get(dataSourceId);
 
  if (!dataSource) {
  throw new Error(`Unknown dataSource "${dataSourceId}"`);
@@ -155,7 +163,8 @@ export default class DataManager {
  method: keyof PQS,
  args: any[],
  ): Promise<any> {
- const dataSource = this.dataSources.get(dataSourceId) as
+ const dataSources = this.getDataSources();
+ const dataSource = dataSources.get(dataSourceId) as
  | ServerDataSource<P, Q, any, PQS>
  | undefined;
 
@@ -202,11 +211,12 @@ export default class DataManager {
  }
 
  createDataSourcesHandler(): Router {
+ const dataSources = this.getDataSources();
  const router = express.Router();
 
  const handlerMap = new Map<String, Function | null | undefined>();
  Object.keys(serverDataSources).forEach((dataSourceId) => {
- const handler = this.dataSources.get(dataSourceId)?.createHandler?.();
+ const handler = dataSources.get(dataSourceId)?.createHandler?.();
  if (handler) {
  invariant(
  typeof handler === 'function',

packages/toolpad-app/src/server/localMode.ts

@@ -993,7 +993,6 @@ class ToolpadProject {
  this.root = root;
  this.options = {
  dev: false,
- externalUrl: 'http://localhost:3000',
  ...options,
  };
 
@@ -1199,9 +1198,16 @@ class ToolpadProject {
  }
 
  getRuntimeConfig(): RuntimeConfig {
+ // When these fail, you are likely trying to retrieve this information during the
+ // toolpad build. It's fundamentally wrong to use this information as it strictly holds
+ // information about the running toolpad instance.
+ invariant(this.options.externalUrl, 'External URL is not set');
+ invariant(this.options.wsPort, 'Websocket port is not set');
+
  return {
  externalUrl: this.options.externalUrl,
  projectDir: this.getRoot(),
+ wsPort: this.options.wsPort,
  };
  }
 }
@@ -1213,21 +1219,19 @@ declare global {
  var __toolpadProject: ToolpadProject | undefined;
 }
 
-export interface InitProjectOptions {
- dev?: boolean;
+export interface InitProjectOptions extends ToolpadProjectOptions {
  dir: string;
- externalUrl?: string;
 }
 
-export async function initProject({ dev, dir, externalUrl }: InitProjectOptions) {
+export async function initProject({ dev, dir, externalUrl, wsPort }: InitProjectOptions) {
  // eslint-disable-next-line no-underscore-dangle
  invariant(!global.__toolpadProject, 'A project is already running');
 
  await migrateLegacyProject(dir);
 
  await initToolpadFolder(dir);
 
- const project = new ToolpadProject(dir, { dev, externalUrl });
+ const project = new ToolpadProject(dir, { dev, externalUrl, wsPort });
  // eslint-disable-next-line no-underscore-dangle
  globalThis.__toolpadProject = project;
 

packages/toolpad-app/src/types.ts

@@ -211,5 +211,6 @@ export type ProjectEvents = {
 
 export interface ToolpadProjectOptions {
  dev: boolean;
- externalUrl: string;
+ externalUrl?: string;
+ wsPort?: number;
 }