diff --git a/packages/toolpad-app/next.config.mjs b/packages/toolpad-app/next.config.mjs index 9a9dbe7bf84..231ec477a82 100644 --- a/packages/toolpad-app/next.config.mjs +++ b/packages/toolpad-app/next.config.mjs @@ -2,6 +2,7 @@ import { createRequire } from 'module'; import * as path from 'path'; const require = createRequire(import.meta.url); +const pkgJson = require('./package.json'); /** * @param {string} input @@ -23,7 +24,8 @@ function parseBuidEnvVars(env) { return { TOOLPAD_TARGET: target, - TOOLPAD_DEMO: env.TOOLPAD_DEMO, + TOOLPAD_DEMO: env.TOOLPAD_DEMO || '', + TOOLPAD_VERSION: pkgJson.version, }; } diff --git a/packages/toolpad-app/package.json b/packages/toolpad-app/package.json index 7178e3e9120..186fac87ca4 100644 --- a/packages/toolpad-app/package.json +++ b/packages/toolpad-app/package.json @@ -32,6 +32,8 @@ "@emotion/react": "^11.9.3", "@emotion/server": "^11.4.0", "@emotion/styled": "^11.9.3", + "@googleapis/drive": "^2.3.0", + "@googleapis/sheets": "^2.0.1", "@mui/icons-material": "^5.8.4", "@mui/lab": "^5.0.0-alpha.88", "@mui/material": "^5.8.6", @@ -54,16 +56,19 @@ "cuid": "^2.1.8", "es-module-shims": "^1.5.4", "esbuild": "^0.14.48", + "fastestsmallesttextencoderdecoder": "^1.0.22", "find-up": "^6.2.0", "fractional-indexing": "^3.0.1", - "@googleapis/drive": "^2.3.0", - "@googleapis/sheets": "^2.0.1", + "headers-polyfill": "^3.0.10", + "isolated-vm": "^4.4.1", "json-to-ts": "^1.7.0", "json5": "^2.2.1", "lodash-es": "^4.17.21", "next": "12.2.0", "node-fetch": "^3.2.6", + "node-fetch-har": "^1.0.1", "path-to-regexp": "^6.2.0", + "perf-cascade": "^2.11.0", "prettier": "^2.6.2", "pretty-bytes": "^6.0.0", "prisma": "^4.0.0", @@ -79,15 +84,18 @@ "react-router-dom": "^6.3.0", "react-split-pane": "^0.1.92", "serialize-javascript": "^6.0.0", + "ses": "^0.15.17", "sucrase": "^3.23.0", "superjson": "^1.8.1", "ts-node": "^10.8.2", - "typescript": "^4.7.4" + "typescript": "^4.7.4", + "whatwg-url": "^11.0.0" }, "devDependencies": { "@types/babel__code-frame": "^7.0.3", "@types/crypto-js": "^4.1.1", "@types/glob": "^7.2.0", + "@types/har-format": "^1.2.8", "@types/lodash-es": "^4.17.6", "@types/node-fetch": "^2.6.2", "@types/react": "^18.0.14", diff --git a/packages/toolpad-app/src/components/AppEditor/BindingEditor.tsx b/packages/toolpad-app/src/components/AppEditor/BindingEditor.tsx index bb288ac147a..9bc657d9ebb 100644 --- a/packages/toolpad-app/src/components/AppEditor/BindingEditor.tsx +++ b/packages/toolpad-app/src/components/AppEditor/BindingEditor.tsx @@ -124,7 +124,15 @@ export function JsBindingEditor({ value, onChange }: JsBindingEditorProps) { - + Make the "{label}" property dynamic with a JavaScript expression. This property expects a type: {propType?.type || 'any'}. diff --git a/packages/toolpad-app/src/components/AppEditor/PageEditor/QueryEditor.tsx b/packages/toolpad-app/src/components/AppEditor/PageEditor/QueryEditor.tsx index be2150a3591..6502ecaeb06 100644 --- a/packages/toolpad-app/src/components/AppEditor/PageEditor/QueryEditor.tsx +++ b/packages/toolpad-app/src/components/AppEditor/PageEditor/QueryEditor.tsx @@ -329,6 +329,7 @@ function QueryNodeEditorDialog({ + {dataSourceId && dataSource ? ( diff --git a/packages/toolpad-app/src/components/Console.tsx b/packages/toolpad-app/src/components/Console.tsx new file mode 100644 index 00000000000..ae5d44993b2 --- /dev/null +++ b/packages/toolpad-app/src/components/Console.tsx @@ -0,0 +1,141 @@ +import { darken, IconButton, lighten, styled, SxProps } from '@mui/material'; +import clsx from 'clsx'; +import * as React from 'react'; +import DoDisturbIcon from '@mui/icons-material/DoDisturb'; +import Inspector, { InspectorProps } from 'react-inspector'; +import inspectorTheme from '../inspectorTheme'; +import { interleave } from '../utils/react'; + +export interface LogEntry { + timestamp: number; + level: string; + args: any[]; +} + +const classes = { + header: 'Toolpad_ConsoleHeader', + logEntriesContainer: 'Toolpad_ConsoleLogEntriesContainer', + logEntries: 'Toolpad_ConsoleLogEntries', + logEntry: 'Toolpad_ConsoleLogEntry', + logEntryText: 'Toolpad_ConsoleLogEntryTExt', +}; + +const ConsoleRoot = styled('div')(({ theme }) => { + const getColor = (color: string) => { + const modify = theme.palette.mode === 'light' ? darken : lighten; + return modify(color, 0.6); + }; + + const getBackgroundColor = (color: string) => { + const modify = theme.palette.mode === 'light' ? lighten : darken; + return modify(color, 0.9); + }; + + return { + width: '100%', + height: '100%', + display: 'flex', + flexDirection: 'column', + alignItems: 'stretch', + + [`& .${classes.header}`]: { + padding: theme.spacing('2px', 1), + borderBottom: `1px solid ${theme.palette.divider}`, + }, + + [`& .${classes.logEntriesContainer}`]: { + flex: 1, + + // This container has only a single item, but the column-reverse has the effect that it + // keeps the scroll position at the bottom when the content grows + display: 'flex', + flexDirection: 'column-reverse', + overflow: 'auto', + + fontSize: 12, + lineHeight: 1.2, + fontFamily: 'Consolas, Menlo, Monaco, "Andale Mono", "Ubuntu Mono", monospace', + }, + + [`& .${classes.logEntry}`]: { + '&:first-of-type': { + borderTop: `1px solid ${theme.palette.divider}`, + }, + borderBottom: `1px solid ${theme.palette.divider}`, + paddingLeft: theme.spacing(1), + paddingRight: theme.spacing(1), + paddingTop: 3, + paddingBottom: 1, + }, + + [`& .${classes.logEntry}.error`]: { + color: getColor(theme.palette.error.light), + background: getBackgroundColor(theme.palette.error.light), + }, + + [`& .${classes.logEntry}.warn`]: { + color: getColor(theme.palette.warning.light), + background: getBackgroundColor(theme.palette.warning.light), + }, + + [`& .${classes.logEntry}.info`]: { + color: getColor(theme.palette.info.light), + background: getBackgroundColor(theme.palette.info.light), + }, + + [`& .${classes.logEntryText} > *`]: { + display: 'inline-block', + verticalAlign: 'top', + }, + }; +}); + +function ConsoleInpector(props: InspectorProps) { + return ; +} + +interface ConsoleEntryProps { + entry: LogEntry; +} + +function ConsoleEntry({ entry }: ConsoleEntryProps) { + return ( +
+
+ {interleave( + entry.args.map((arg, i) => + typeof arg === 'string' ? arg : , + ), + ' ', + )} +
+
+ ); +} + +interface ConsoleProps { + sx?: SxProps; + value?: LogEntry[]; + onChange?: (logEntries: LogEntry[]) => void; +} + +export default function Console({ value = [], onChange, sx }: ConsoleProps) { + return ( + +
+ {onChange ? ( + onChange([])}> + + + ) : null} +
+
+
+ {value.map((entry, i) => ( + + ))} +
+
+ + ); +} diff --git a/packages/toolpad-app/src/components/HarViewer.tsx b/packages/toolpad-app/src/components/HarViewer.tsx new file mode 100644 index 00000000000..ceea43ad57e --- /dev/null +++ b/packages/toolpad-app/src/components/HarViewer.tsx @@ -0,0 +1,52 @@ +import { fromHar } from 'perf-cascade'; +import * as React from 'react'; +import { Har } from 'har-format'; +import { styled, SxProps } from '@mui/material'; +import 'perf-cascade/dist/perf-cascade.css'; + +const HarViewerRoot = styled('div')({}); + +function fixLinks(elm: Element) { + elm.querySelectorAll('a').forEach((link) => link.setAttribute('target', '_blank')); +} + +export interface HarViewerProps { + har?: Har; + sx?: SxProps; +} + +export default function HarViewer({ har, sx }: HarViewerProps) { + const rootRef = React.useRef(null); + + React.useEffect(() => { + const root = rootRef.current; + if (har && root) { + const svg = fromHar(har); + fixLinks(svg); + + const observer = new MutationObserver((entries) => { + for (const entry of entries) { + for (const node of entry.addedNodes) { + if (node instanceof Element) { + fixLinks(node); + } + } + } + }); + + observer.observe(svg, { + subtree: true, + childList: true, + }); + + root.append(svg); + return () => { + observer.disconnect(); + svg.remove(); + }; + } + return () => {}; + }, [har]); + + return ; +} diff --git a/packages/toolpad-app/src/components/JsonView.tsx b/packages/toolpad-app/src/components/JsonView.tsx index 156296af5b7..6140e03b30b 100644 --- a/packages/toolpad-app/src/components/JsonView.tsx +++ b/packages/toolpad-app/src/components/JsonView.tsx @@ -1,6 +1,7 @@ import * as React from 'react'; import { SxProps, styled } from '@mui/material'; import { ObjectInspector, ObjectInspectorProps, ObjectValue, ObjectLabel } from 'react-inspector'; +import inspectorTheme from '../inspectorTheme'; const nodeRenderer: ObjectInspectorProps['nodeRenderer'] = ({ depth, @@ -17,6 +18,10 @@ const nodeRenderer: ObjectInspectorProps['nodeRenderer'] = ({ const JsonViewRoot = styled('div')({ whiteSpace: 'nowrap', + + fontSize: 12, + lineHeight: 1.2, + fontFamily: 'Consolas, Menlo, Monaco, "Andale Mono", "Ubuntu Mono", monospace', }); export interface JsonViewProps { @@ -34,6 +39,7 @@ export default function JsonView({ src, sx }: JsonViewProps) { expandLevel={1} expandPaths={expandPaths} data={src} + theme={inspectorTheme} /> ); diff --git a/packages/toolpad-app/src/components/MonacoEditor.tsx b/packages/toolpad-app/src/components/MonacoEditor.tsx index 669d7684bb1..ba0bdb81e9b 100644 --- a/packages/toolpad-app/src/components/MonacoEditor.tsx +++ b/packages/toolpad-app/src/components/MonacoEditor.tsx @@ -8,6 +8,7 @@ import * as monaco from 'monaco-editor'; import { styled, SxProps } from '@mui/material'; import clsx from 'clsx'; import cuid from 'cuid'; +import monacoEditorTheme from '../monacoEditorTheme'; function getExtension(language: string): string { switch (language) { @@ -189,6 +190,7 @@ export default React.forwardRef(function accessibilitySupport: 'off', tabSize: 2, automaticLayout: true, + theme: monacoEditorTheme, ...extraOptions, }); diff --git a/packages/toolpad-app/src/components/SplitPane.tsx b/packages/toolpad-app/src/components/SplitPane.tsx index 955f985cb7d..8a963e434a8 100644 --- a/packages/toolpad-app/src/components/SplitPane.tsx +++ b/packages/toolpad-app/src/components/SplitPane.tsx @@ -38,7 +38,13 @@ const WrappedSplitPane = React.forwardRef< // Some sensible defaults minSize={20} maxSize={-20} - paneStyle={{ display: 'block', ...props.paneStyle }} + paneStyle={{ + display: 'block', + // Prevent the content from stretching the Panel out + minWidth: 0, + minHeight: 0, + ...props.paneStyle, + }} {...props} /> ); diff --git a/packages/toolpad-app/src/config.ts b/packages/toolpad-app/src/config.ts index a416834b9a9..38d97d1215d 100644 --- a/packages/toolpad-app/src/config.ts +++ b/packages/toolpad-app/src/config.ts @@ -22,7 +22,7 @@ import { RUNTIME_CONFIG_WINDOW_PROPERTY } from './constants'; */ // These are inlined at build time -export type BuildEnvVars = Partial>; +export type BuildEnvVars = Record<'TOOLPAD_TARGET' | 'TOOLPAD_DEMO' | 'TOOLPAD_VERSION', string>; // These are set at runtime and passed to the browser. // Do not add secrets diff --git a/packages/toolpad-app/src/inspectorTheme.ts b/packages/toolpad-app/src/inspectorTheme.ts new file mode 100644 index 00000000000..c0d6e39e67b --- /dev/null +++ b/packages/toolpad-app/src/inspectorTheme.ts @@ -0,0 +1,15 @@ +import { chromeDark, chromeLight, InspectorTheme } from 'react-inspector'; + +import theme from './theme'; + +const inspectorTheme: InspectorTheme = { + ...(theme.palette.mode === 'dark' ? chromeDark : chromeLight), + + BASE_BACKGROUND_COLOR: 'inherit', + TREENODE_FONT_FAMILY: 'inherit', + TREENODE_FONT_SIZE: 'inherit', + ARROW_FONT_SIZE: 'inherit', + TREENODE_LINE_HEIGHT: 'inherit', +}; + +export default inspectorTheme; diff --git a/packages/toolpad-app/src/monacoEditorTheme.ts b/packages/toolpad-app/src/monacoEditorTheme.ts new file mode 100644 index 00000000000..400d2cbe3ab --- /dev/null +++ b/packages/toolpad-app/src/monacoEditorTheme.ts @@ -0,0 +1,5 @@ +import theme from './theme'; + +const monacoEditorTheme: string = theme.palette.mode === 'dark' ? 'vs-dark' : 'vs'; + +export default monacoEditorTheme; diff --git a/packages/toolpad-app/src/server/evalExpression.ts b/packages/toolpad-app/src/server/evalExpression.ts index 58553b0a8a8..6f9f25391d4 100644 --- a/packages/toolpad-app/src/server/evalExpression.ts +++ b/packages/toolpad-app/src/server/evalExpression.ts @@ -9,7 +9,7 @@ export type Serializable = | { [key: string]: Serializable } | ((...args: Serializable[]) => Serializable); -function newJson(ctx: QuickJSContext, json: Serializable): QuickJSHandle { +export function newJson(ctx: QuickJSContext, json: Serializable): QuickJSHandle { switch (typeof json) { case 'string': return ctx.newString(json); diff --git a/packages/toolpad-app/src/theme.ts b/packages/toolpad-app/src/theme.ts index bbb07a9cb8f..78fc09839f1 100644 --- a/packages/toolpad-app/src/theme.ts +++ b/packages/toolpad-app/src/theme.ts @@ -1,6 +1,7 @@ import { createTheme } from '@mui/material/styles'; import { red } from '@mui/material/colors'; import type {} from '@mui/x-data-grid-pro/themeAugmentation'; +import type {} from '@mui/lab/themeAugmentation'; // Create a theme instance. const theme = createTheme({ @@ -65,6 +66,21 @@ const theme = createTheme({ fontSize: 'small', }, }, + MuiTabs: { + styleOverrides: { + root: { + minHeight: 0, + }, + }, + }, + MuiTab: { + styleOverrides: { + root: { + padding: 8, + minHeight: 0, + }, + }, + }, }, palette: { // mode: 'dark', diff --git a/packages/toolpad-app/src/toolpadDataSources/client.tsx b/packages/toolpad-app/src/toolpadDataSources/client.tsx index 1c40f0ca180..50c4a054dd0 100644 --- a/packages/toolpad-app/src/toolpadDataSources/client.tsx +++ b/packages/toolpad-app/src/toolpadDataSources/client.tsx @@ -1,4 +1,5 @@ import movies from './movies/client'; +import functionSrc from './function/client'; // import postgres from './postgres/client'; import rest from './rest/client'; import { ClientDataSource } from '../types'; @@ -12,6 +13,7 @@ const clientDataSources: ClientDataSources = process.env.TOOLPAD_DEMO } : { // postgres, + function: functionSrc, rest, googleSheets, }; diff --git a/packages/toolpad-app/src/toolpadDataSources/function/client.tsx b/packages/toolpad-app/src/toolpadDataSources/function/client.tsx new file mode 100644 index 00000000000..e2e760b8ad3 --- /dev/null +++ b/packages/toolpad-app/src/toolpadDataSources/function/client.tsx @@ -0,0 +1,265 @@ +import * as React from 'react'; +import { Box, Button, Skeleton, Stack, styled, Tab, Toolbar, Typography } from '@mui/material'; +import { BindableAttrValue, BindableAttrValues, LiveBinding } from '@mui/toolpad-core'; + +import { LoadingButton, TabContext, TabList, TabPanel } from '@mui/lab'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import { Controller, useForm } from 'react-hook-form'; +import { ClientDataSource, ConnectionEditorProps, QueryEditorProps } from '../../types'; +import { + FunctionConnectionParams, + FunctionPrivateQuery, + FunctionQuery, + FunctionResult, +} from './types'; +import lazyComponent from '../../utils/lazyComponent'; +import ParametersEditor from '../../components/AppEditor/PageEditor/ParametersEditor'; +import SplitPane from '../../components/SplitPane'; +import { useConnectionContext, usePrivateQuery } from '../context'; +import client from '../../api'; +import JsonView from '../../components/JsonView'; +import ErrorAlert from '../../components/AppEditor/PageEditor/ErrorAlert'; +import Console, { LogEntry } from '../../components/Console'; +import MapEntriesEditor from '../../components/MapEntriesEditor'; +import { Maybe } from '../../utils/types'; +import { isSaveDisabled } from '../../utils/forms'; + +const HarViewer = lazyComponent(() => import('../../components/HarViewer'), {}); + +const DebuggerTabPanel = styled(TabPanel)({ padding: 0, flex: 1, minHeight: 0 }); + +const EVENT_INTERFACE_IDENTIFIER = 'ToolpadFunctionEvent'; + +const TypescriptEditor = lazyComponent(() => import('../../components/TypescriptEditor'), { + noSsr: true, + fallback: , +}); + +function withDefaults(value: Maybe): FunctionConnectionParams { + return { + secrets: [], + ...value, + }; +} + +function ConnectionParamsInput({ + value, + onChange, +}: ConnectionEditorProps) { + const { handleSubmit, formState, reset, control } = useForm({ + defaultValues: withDefaults(value), + reValidateMode: 'onChange', + mode: 'all', + }); + React.useEffect(() => reset(withDefaults(value)), [reset, value]); + + const doSubmit = handleSubmit((connectionParams) => onChange(connectionParams)); + + return ( + + Secrets: + { + return ( + + ); + }} + /> + + + + + + + ); +} + +const DEFAULT_MODULE = `export default async function ({ params }: ${EVENT_INTERFACE_IDENTIFIER}) { + console.info('Executing function with params:', params); + const url = new URL('https://gist.githubusercontent.com/saniyusuf/406b843afdfb9c6a86e25753fe2761f4/raw/523c324c7fcc36efab8224f9ebb7556c09b69a14/Film.JSON'); + url.searchParams.set('timestamp', String(Date.now())); + const response = await fetch(String(url)); + if (!response.ok) { + throw new Error(\`HTTP \${response.status}: \${response.statusText}\`); + } + return response.json(); +}`; + +function QueryEditor({ + globalScope, + liveParams, + value, + onChange, +}: QueryEditorProps) { + const [params, setParams] = React.useState<[string, BindableAttrValue][]>( + Object.entries(value.params || ({} as BindableAttrValue>)), + ); + + React.useEffect( + () => setParams(Object.entries(value.params || ({} as BindableAttrValue>))), + [value.params], + ); + + const handleParamsChange = React.useCallback( + (newParams: [string, BindableAttrValue][]) => { + setParams(newParams); + const paramsObj: BindableAttrValues = Object.fromEntries(newParams); + onChange({ ...value, params: paramsObj }); + }, + [onChange, value], + ); + + const paramsEditorLiveValue: [string, LiveBinding][] = params.map(([key]) => [ + key, + liveParams[key], + ]); + + const { appId, connectionId } = useConnectionContext(); + const [preview, setPreview] = React.useState(null); + const [previewLogs, setPreviewLogs] = React.useState([]); + + const cancelRunPreview = React.useRef<(() => void) | null>(null); + const runPreview = React.useCallback(() => { + let canceled = false; + + cancelRunPreview.current?.(); + cancelRunPreview.current = () => { + canceled = true; + }; + + const currentParams = Object.fromEntries( + paramsEditorLiveValue.map(([key, binding]) => [key, binding.value]), + ); + + client.query + .dataSourceFetchPrivate(appId, connectionId, { + kind: 'debugExec', + query: value.query, + params: currentParams, + } as FunctionPrivateQuery) + .then((result) => { + if (!canceled) { + setPreview(result); + setPreviewLogs((existing) => [...existing, ...result.logs]); + } + }) + .finally(() => { + cancelRunPreview.current = null; + }); + }, [appId, connectionId, paramsEditorLiveValue, value.query]); + + const { data: secretsKeys = [] } = usePrivateQuery({ + kind: 'secretsKeys', + }); + + const extraLibs = React.useMemo(() => { + const paramsKeys = params.map(([key]) => key); + const paramsMembers = paramsKeys.map((key) => `${key}: string`).join('\n'); + const secretsMembers = secretsKeys.map((key) => `${key}: string`).join('\n'); + + const content = ` + interface ${EVENT_INTERFACE_IDENTIFIER} { + params: { + ${paramsMembers} + } + secrets: { + ${secretsMembers} + } + } + `; + + return [{ content, filePath: 'file:///node_modules/@mui/toolpad/index.d.ts' }]; + }, [params, secretsKeys]); + + const [debuggerTab, setDebuggerTab] = React.useState('console'); + const handleDebuggerTabChange = (event: React.SyntheticEvent, newValue: string) => { + setDebuggerTab(newValue); + }; + + return ( + + + + + + } onClick={() => runPreview()}> + Preview + + + + onChange({ ...value, query: { module: newValue } })} + extraLibs={extraLibs} + /> + + + + + Parameters + + + + + + + {preview?.error ? ( + + ) : ( + + )} + + + + + + + + + + + + + + + + + + + + {/* + + */} + + + + ); +} + +function getInitialQueryValue(): FunctionQuery { + return { module: DEFAULT_MODULE }; +} + +const dataSource: ClientDataSource = { + displayName: 'Function', + ConnectionParamsInput, + isConnectionValid: () => true, + QueryEditor, + getInitialQueryValue, +}; + +export default dataSource; diff --git a/packages/toolpad-app/src/toolpadDataSources/function/quickjs.ts b/packages/toolpad-app/src/toolpadDataSources/function/quickjs.ts new file mode 100644 index 00000000000..8a7703d7eef --- /dev/null +++ b/packages/toolpad-app/src/toolpadDataSources/function/quickjs.ts @@ -0,0 +1,88 @@ +import { getQuickJS } from 'quickjs-emscripten'; +import { ApiResult } from '../../types'; +import { FunctionQuery, FunctionConnectionParams } from './types'; +import { Maybe } from '../../utils/types'; + +import { newJson } from '../../server/evalExpression'; + +export default async function execQuickjs( + connection: Maybe, + functionQuery: FunctionQuery, +): Promise> { + const QuickJS = await getQuickJS(); + + const runtime = QuickJS.newRuntime(); + + runtime.setModuleLoader((moduleName) => { + if (moduleName === 'toolpad:serverless-module') { + return functionQuery.module; + } + throw new Error(`Unrecognized module "${moduleName}"`); + }); + const context = runtime.newContext(); + + const fetchHandle = context.newFunction('fetch', (urlHandle) => { + const url = context.getString(urlHandle); + urlHandle.dispose(); + + const promise = context.newPromise(); + + fetch(url).then( + async (res) => { + const resHandle = context.newObject(); + + const jsonFnHandle = context.newFunction('json', () => { + const jsonPromise = context.newPromise(); + res.json().then( + (jsonValue) => { + const jsonHandle = newJson(context, jsonValue); + jsonPromise.resolve(jsonHandle); + jsonHandle.dispose(); + }, + (err) => jsonPromise.reject(context.newError(err.message)), + ); + jsonPromise.settled.then(context.runtime.executePendingJobs); + return jsonPromise.handle; + }); + context.setProp(resHandle, 'json', jsonFnHandle); + jsonFnHandle.dispose(); + + promise.resolve(resHandle); + resHandle.dispose(); + }, + (err) => { + const errHandle = context.newError(err?.message || 'Unknown error'); + promise.reject(errHandle); + errHandle.dispose(); + }, + ); + + promise.settled.then(context.runtime.executePendingJobs); + return promise.handle; + }); + fetchHandle.consume((handle) => context.setProp(context.global, 'fetch', handle)); + + const ok = context.evalCode(` +import fn from 'toolpad:serverless-module'; +globalThis.result = (async () => fn())(); + `); + context.unwrapResult(ok).dispose(); + + const promiseHandle = context.getProp(context.global, 'result'); + + setTimeout(() => { + runtime.executePendingJobs(); + }, 1000); + + const resolvedResult = await context.resolvePromise(promiseHandle); + promiseHandle.dispose(); + + const resolvedHandle = context.unwrapResult(resolvedResult); + const apiResult = context.dump(resolvedHandle); + resolvedHandle.dispose(); + + context.dispose(); + runtime.dispose(); + + return apiResult; +} diff --git a/packages/toolpad-app/src/toolpadDataSources/function/server.ts b/packages/toolpad-app/src/toolpadDataSources/function/server.ts new file mode 100644 index 00000000000..53d5c3539a7 --- /dev/null +++ b/packages/toolpad-app/src/toolpadDataSources/function/server.ts @@ -0,0 +1,258 @@ +import ivm from 'isolated-vm'; +import * as esbuild from 'esbuild'; +import type * as harFormat from 'har-format'; +import { withHar, createHarLog } from 'node-fetch-har'; +import { ServerDataSource } from '../../types'; +import { + FunctionQuery, + FunctionConnectionParams, + FunctionResult, + FunctionPrivateQuery, +} from './types'; +import { Maybe } from '../../utils/types'; +import { LogEntry } from '../../components/Console'; + +async function createPolyfillModule() { + const { outputFiles } = await esbuild.build({ + entryPoints: [], + bundle: true, + format: 'esm', + write: false, + stdin: { + resolveDir: __dirname, + contents: ` + import 'fastestsmallesttextencoderdecoder'; + import { Headers } from 'headers-polyfill'; + import { URL, URLSearchParams } from 'whatwg-url'; + + global.Headers = Headers; + global.URL = URL; + global.URLSearchParams = URLSearchParams; + `, + }, + }); + const code = outputFiles?.[0].text || ''; + return code; +} + +let cachedPolyfills: Promise | undefined; +async function getPolyfills() { + if (!cachedPolyfills) { + cachedPolyfills = createPolyfillModule(); + } + return cachedPolyfills; +} + +const isolate = new ivm.Isolate({ memoryLimit: 128 }); + +async function execBase( + connection: Maybe, + functionQuery: FunctionQuery, + params: Record, +): Promise { + const context = isolate.createContextSync(); + const jail = context.global; + jail.setSync('global', jail.derefInto()); + + const logs: LogEntry[] = []; + const har: harFormat.Har = createHarLog(); + const instrumentedFetch: typeof fetch = withHar(fetch, { har }); + + await context.evalClosure( + ` + function consoleMethod (level) { + return (...args) => { + $0.apply(null, [level, JSON.stringify(args)], { arguments: { copy: true }}); + } + } + + global.console = { + log: consoleMethod('log'), + debug: consoleMethod('debug'), + info: consoleMethod('info'), + warn: consoleMethod('warn'), + error: consoleMethod('error'), + } + `, + [ + (level: string, serializedArgs: string) => { + logs.push({ + timestamp: Date.now(), + level, + args: JSON.parse(serializedArgs), + }); + }, + ], + { arguments: { reference: true } }, + ); + + let nextTimeoutId = 1; + const timeouts = new Map(); + await context.evalClosure( + ` + global.setTimeout = (cb, ms) => { + return $0.applySync(null, [cb, ms], { arguments: { reference: true }, result: { copy: true }}); + } + + global.clearTimeout = (timeout) => { + return $1.applyIgnored(null, [timeout], { arguments: { copy: true }}); + } + `, + [ + (cb: ivm.Reference, ms: ivm.Reference) => { + const id = nextTimeoutId; + nextTimeoutId += 1; + + const timeout = setTimeout(() => { + timeouts.delete(id); + cb.applyIgnored(null, []); + }, ms.copySync()); + + timeouts.set(id, timeout); + + return id; + }, + (id: number) => { + const timeout = timeouts.get(id); + timeouts.delete(id); + clearTimeout(timeout); + }, + ], + { arguments: { reference: true } }, + ); + + const polyfills = await getPolyfills(); + await context.eval(polyfills); + + await context.evalClosure( + ` + const INTERNALS = Symbol('Fetch internals'); + + global.Response = class Response { + constructor() {} + get ok () { + return this[INTERNALS].getSync('ok'); + } + get status () { + return this[INTERNALS].getSync('status'); + } + get statusText () { + return this[INTERNALS].getSync('statusText'); + } + get headers () { + return new Headers(this[INTERNALS].getSync('headers').copy()) + } + json (...args) { + return this[INTERNALS].getSync('json').apply(null, args, { + arguments: { copy: true }, + result: { copy: true, promise: true } + }); + } + text (...args) { + return this[INTERNALS].getSync('text').apply(null, args, { + arguments: { copy: true }, + result: { copy: true, promise: true } + }); + } + } + + global.fetch = async (...args) => { + const response = new Response(); + response[INTERNALS] = await $0.apply(null, args, { arguments: { copy: true }, result: { promise: true }}); + return response; + } + `, + [ + new ivm.Reference((...args: Parameters) => { + const req = new Request(...args); + + return instrumentedFetch(req).then( + (res) => { + const resHeadersInit = Array.from(res.headers.entries()); + + return { + ok: res.ok, + status: res.status, + statusText: res.statusText, + headers: new ivm.ExternalCopy(resHeadersInit), + json: new ivm.Reference(() => res.json()), + text: new ivm.Reference(() => res.text()), + }; + }, + (err) => { + logs.push({ + timestamp: Date.now(), + level: 'error', + args: [{ name: err.name, message: err.message, stack: err.stack }], + }); + + throw err; + }, + ); + }), + ], + { timeout: 30000 }, + ); + + let data; + let error: Error | undefined; + + try { + const { code: userModuleJs } = await esbuild.transform(functionQuery.module, { + loader: 'ts', + }); + + const userModule = await isolate.compileModule(userModuleJs); + + await userModule.instantiate(context, (specifier) => { + throw new Error(`Not found "${specifier}"`); + }); + + userModule.evaluateSync(); + + const secrets = Object.fromEntries(connection?.secrets ?? []); + + const defaultExport = await userModule.namespace.get('default', { reference: true }); + data = await defaultExport.apply(null, [{ params, secrets }], { + arguments: { copy: true }, + result: { copy: true, promise: true }, + }); + } catch (userError) { + error = userError instanceof Error ? userError : new Error(String(userError)); + } + + return { data, logs, error, har }; +} + +async function execPrivate( + connection: Maybe, + query: FunctionPrivateQuery, +) { + switch (query.kind) { + case 'secretsKeys': + return (connection?.secrets ?? []).map(([key]) => key); + case 'debugExec': + return execBase(connection, query.query, query.params); + default: + throw new Error(`Unknown private query "${(query as FunctionPrivateQuery).kind}"`); + } +} + +async function exec( + connection: Maybe, + functionQuery: FunctionQuery, + params: Record, +) { + const { data, error } = await execBase(connection, functionQuery, params); + if (error) { + throw error; + } + return { data }; +} + +const dataSource: ServerDataSource = { + exec, + execPrivate, +}; + +export default dataSource; diff --git a/packages/toolpad-app/src/toolpadDataSources/function/types.ts b/packages/toolpad-app/src/toolpadDataSources/function/types.ts new file mode 100644 index 00000000000..17fe326aeae --- /dev/null +++ b/packages/toolpad-app/src/toolpadDataSources/function/types.ts @@ -0,0 +1,30 @@ +import { Har } from 'har-format'; +import { LogEntry } from '../../components/Console'; + +export interface FunctionConnectionParams { + secrets?: [string, string][]; +} + +export interface FunctionQuery { + readonly module: string; +} + +export interface PrivateQuery { + input: I; + result: R; +} + +export interface PrivateQueries { + debug: PrivateQuery; +} + +export interface FunctionResult { + data: any; + error?: Error; + logs: LogEntry[]; + har: Har; +} + +export type FunctionPrivateQuery = + | { kind: 'debugExec'; query: FunctionQuery; params: Record } + | { kind: 'secretsKeys' }; diff --git a/packages/toolpad-app/src/toolpadDataSources/server.ts b/packages/toolpad-app/src/toolpadDataSources/server.ts index 261912bf674..538c460333c 100644 --- a/packages/toolpad-app/src/toolpadDataSources/server.ts +++ b/packages/toolpad-app/src/toolpadDataSources/server.ts @@ -1,5 +1,6 @@ import { ServerDataSource } from '../types'; import movies from './movies/server'; +import functionSrc from './function/server'; // import postgres from './postgres/server'; import rest from './rest/server'; import googleSheets from './googleSheets/server'; @@ -12,6 +13,7 @@ const serverDataSources: ServerDataSources = process.env.TOOLPAD_DEMO } : { // postgres, + function: functionSrc, rest, googleSheets, }; diff --git a/packages/toolpad-app/src/utils/react.tsx b/packages/toolpad-app/src/utils/react.tsx index 03e4420d864..0b6495623ce 100644 --- a/packages/toolpad-app/src/utils/react.tsx +++ b/packages/toolpad-app/src/utils/react.tsx @@ -47,3 +47,16 @@ export function suspendPromise(promise: Promise): () => T { } }; } + +export function interleave(items: React.ReactNode[], separator: React.ReactNode): React.ReactNode { + const result: React.ReactNode[] = []; + + for (let i = 0; i < items.length; i += 1) { + if (i > 0) { + result.push(separator); + } + result.push(items[i]); + } + + return result; +} diff --git a/packages/toolpad-app/typings/node-fetch-har.d.ts b/packages/toolpad-app/typings/node-fetch-har.d.ts new file mode 100644 index 00000000000..e700aa153e4 --- /dev/null +++ b/packages/toolpad-app/typings/node-fetch-har.d.ts @@ -0,0 +1,10 @@ +declare module 'node-fetch-har' { + import { Har } from 'har-format'; + + export interface WithHarOptions { + har?: Har; + } + + export function withHar(fetch: typeof fetch, options?: WithHarOptions): typeof fetch; + export function createHarLog(): Har; +} diff --git a/yarn.lock b/yarn.lock index 6bb786628af..368eae7bdec 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2281,6 +2281,11 @@ dependencies: "@types/node" "*" +"@types/har-format@^1.2.8": + version "1.2.8" + resolved "https://registry.yarnpkg.com/@types/har-format/-/har-format-1.2.8.tgz#e6908b76d4c88be3db642846bb8b455f0bfb1c4e" + integrity sha512-OP6L9VuZNdskgNN3zFQQ54ceYD8OLq5IbqO4VK91ORLfOm7WdT/CiT/pHEBSQEqCInJ2y3O6iCm/zGtPElpgJQ== + "@types/istanbul-lib-coverage@*", "@types/istanbul-lib-coverage@^2.0.0", "@types/istanbul-lib-coverage@^2.0.1": version "2.0.4" resolved "https://registry.yarnpkg.com/@types/istanbul-lib-coverage/-/istanbul-lib-coverage-2.0.4.tgz#8467d4b3c087805d63580480890791277ce35c44" @@ -3812,6 +3817,11 @@ convert-source-map@^1.4.0, convert-source-map@^1.5.0, convert-source-map@^1.6.0, dependencies: safe-buffer "~5.1.1" +cookie@^0.4.0: + version "0.4.2" + resolved "https://registry.yarnpkg.com/cookie/-/cookie-0.4.2.tgz#0e41f24de5ecf317947c82fc789e06a884824432" + integrity sha512-aSWTXFzaKWkvHO1Ny/s+ePFpvKsPnjc551iI41v3ny/ow6tBG5Vd+FuqGNhh1LxOmVzOlGUriIlOaokOvhaStA== + copy-anything@^3.0.2: version "3.0.2" resolved "https://registry.yarnpkg.com/copy-anything/-/copy-anything-3.0.2.tgz#7189171ff5e1893b2287e8bf574b8cd448ed50b1" @@ -4943,6 +4953,11 @@ fast-text-encoding@^1.0.0: resolved "https://registry.yarnpkg.com/fast-text-encoding/-/fast-text-encoding-1.0.4.tgz#bf1898ad800282a4e53c0ea9690704dd26e4298e" integrity sha512-x6lDDm/tBAzX9kmsPcZsNbvDs3Zey3+scsxaZElS8xWLgUMAg/oFLeewfUz0mu1CblHhhsu15jGkraldkFh8KQ== +fastestsmallesttextencoderdecoder@^1.0.22: + version "1.0.22" + resolved "https://registry.yarnpkg.com/fastestsmallesttextencoderdecoder/-/fastestsmallesttextencoderdecoder-1.0.22.tgz#59b47e7b965f45258629cc6c127bf783281c5e93" + integrity sha512-Pb8d48e+oIuY4MaM64Cd7OW1gt4nxCHs7/ddPPZ/Ic3sg8yVGM7O9wDvZ7us6ScaUupzM+pfBolwtYhN1IxBIw== + fastq@^1.6.0: version "1.13.0" resolved "https://registry.yarnpkg.com/fastq/-/fastq-1.13.0.tgz#616760f88a7526bdfc596b7cab8c18938c36b98c" @@ -5470,6 +5485,11 @@ hash.js@^1.0.3: inherits "^2.0.3" minimalistic-assert "^1.0.1" +headers-polyfill@^3.0.10: + version "3.0.10" + resolved "https://registry.yarnpkg.com/headers-polyfill/-/headers-polyfill-3.0.10.tgz#51a72c0d9c32594fd23854a564c3d6c80b46b065" + integrity sha512-lOhQU7iG3AMcjmb8NIWCa+KwfJw5bY44BoWPtrj5A4iDbSD3ylGf5QcYr0ZyQnhkKQ2GgWNLdF2rfrXtXlF3nQ== + history@^5.2.0: version "5.3.0" resolved "https://registry.yarnpkg.com/history/-/history-5.3.0.tgz#1548abaa245ba47992f063a0783db91ef201c73b" @@ -5973,6 +5993,11 @@ isobject@^3.0.1: resolved "https://registry.yarnpkg.com/isobject/-/isobject-3.0.1.tgz#4e431e92b11a9731636aa1f9c8d1ccbcfdab78df" integrity sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg== +isolated-vm@^4.4.1: + version "4.4.1" + resolved "https://registry.yarnpkg.com/isolated-vm/-/isolated-vm-4.4.1.tgz#651ce31e435e769b2a356bb03db2553a478acebe" + integrity sha512-5aDwxQGm78vHS+qJeUli2ILroG7OS/k3D/Mc0kcT9vyujiL4bV7PYYix1mAvuBm3v44nz2qcfAOqgAbhuACc/w== + istanbul-lib-coverage@^3.0.0, istanbul-lib-coverage@^3.2.0: version "3.2.0" resolved "https://registry.yarnpkg.com/istanbul-lib-coverage/-/istanbul-lib-coverage-3.2.0.tgz#189e7909d0a39fa5a3dfad5b03f71947770191d3" @@ -7135,6 +7160,11 @@ nano-time@1.0.0: dependencies: big-integer "^1.6.16" +nanoid@^2.0.3: + version "2.1.11" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-2.1.11.tgz#ec24b8a758d591561531b4176a01e3ab4f0f0280" + integrity sha512-s/snB+WGm6uwi0WjsZdaVcuf3KJXlfGl2LcxgwkEwJF0D/BWzVWAZW/XY4bFaiR7s0Jk3FPvlnepg1H1b1UwlA== + nanoid@^3.1.30: version "3.3.4" resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" @@ -7186,6 +7216,15 @@ node-domexception@^1.0.0: resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5" integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ== +node-fetch-har@^1.0.1: + version "1.0.1" + resolved "https://registry.yarnpkg.com/node-fetch-har/-/node-fetch-har-1.0.1.tgz#bc665ec72c86619c91152ad3c9d4c53cae98cec3" + integrity sha512-XpTlblmwxdmVHtg6tDB5kXvEziXl8SlNI1Sc9DaKeWJDqreZhHrhijgWdxW84xh2zmYCiiAU9oLriCLu1oyTbg== + dependencies: + cookie "^0.4.0" + nanoid "^2.0.3" + set-cookie-parser "^2.3.5" + node-fetch@^2.6.1, node-fetch@^2.6.7: version "2.6.7" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad" @@ -7816,6 +7855,13 @@ path-type@^4.0.0: resolved "https://registry.yarnpkg.com/path-type/-/path-type-4.0.0.tgz#84ed01c0a7ba380afe09d90a8c180dcd9d03043b" integrity sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw== +perf-cascade@^2.11.0: + version "2.11.0" + resolved "https://registry.yarnpkg.com/perf-cascade/-/perf-cascade-2.11.0.tgz#8cae08dd71ec94e46c71d80d3eaaeb06d2c24ab2" + integrity sha512-z36x0CGUGispOcvWo5ITUcZBNdnI65SEKHInqg2ZJGhOlQH05F6cy5/DfhLQhUl3TfPSkatG7hQIwIUYMnK+Sg== + dependencies: + "@types/har-format" "^1.2.8" + picocolors@^1.0.0: version "1.0.0" resolved "https://registry.yarnpkg.com/picocolors/-/picocolors-1.0.0.tgz#cb5bdc74ff3f51892236eaf79d68bc44564ab81c" @@ -8525,11 +8571,21 @@ serialize-javascript@^6.0.0: dependencies: randombytes "^2.1.0" +ses@^0.15.17: + version "0.15.17" + resolved "https://registry.yarnpkg.com/ses/-/ses-0.15.17.tgz#84e20cd08fb294989c6499942d220bd6604e1884" + integrity sha512-T8XKsR5LGV57ilyqE4InFJKWVniEEFGai0CjuVJPju9LvnjYPXvZ7V8jP7sGtJ500ApRVgNBCu+5ZcSUhiuXig== + set-blocking@^2.0.0: version "2.0.0" resolved "https://registry.yarnpkg.com/set-blocking/-/set-blocking-2.0.0.tgz#045f9782d011ae9a6803ddd382b24392b3d890f7" integrity sha512-KiKBS8AnWGEyLzofFfmvKwpdPzqiy16LvQfK3yv/fVH7Bj13/wl3JSR1J+rfgRE9q7xUJK4qvgS8raSOeLUehw== +set-cookie-parser@^2.3.5: + version "2.5.0" + resolved "https://registry.yarnpkg.com/set-cookie-parser/-/set-cookie-parser-2.5.0.tgz#96b59525e1362c94335c3c761100bb6e8f2da4b0" + integrity sha512-cHMAtSXilfyBePduZEBVPTCftTQWz6ehWJD5YNUg4mqvRosrrjKbo4WS8JkB0/RxonMoohHm7cOGH60mDkRQ9w== + shallow-clone@^3.0.0: version "3.0.1" resolved "https://registry.yarnpkg.com/shallow-clone/-/shallow-clone-3.0.1.tgz#8f2981ad92531f55035b01fb230769a40e02efa3"