From 1aec53e14ab953e265bb7496a5af2408d809d553 Mon Sep 17 00:00:00 2001 From: Pooya Parsa Date: Fri, 29 Dec 2023 01:27:53 +0100 Subject: [PATCH] feat(wasm): migrate to unjs/unwasm (#2037) --- package.json | 3 +- pnpm-lock.yaml | 84 +++++++++++ src/presets/vercel.ts | 2 +- src/rollup/config.ts | 15 +- src/rollup/plugins/wasm.ts | 137 ------------------ src/types/nitro.ts | 28 ++-- .../node_modules/@fixture/wasm/package.json | 17 --- .../node_modules/@fixture/wasm/sum.asc.ts | 6 - .../node_modules/@fixture/wasm/sum.wasm | Bin 93 -> 0 bytes .../node_modules/@fixture/wasm/sum.wasm.d.ts | 1 - test/fixture/routes/wasm/dynamic-import.ts | 9 ++ .../wasm/{dynamic.ts => static-import.ts} | 5 +- test/fixture/routes/wasm/static.ts | 6 - test/presets/cloudflare-module.test.ts | 1 + test/presets/vercel-edge.test.ts | 2 +- test/tests.ts | 19 ++- 16 files changed, 127 insertions(+), 208 deletions(-) delete mode 100644 src/rollup/plugins/wasm.ts delete mode 100644 test/fixture/node_modules/@fixture/wasm/package.json delete mode 100644 test/fixture/node_modules/@fixture/wasm/sum.asc.ts delete mode 100644 test/fixture/node_modules/@fixture/wasm/sum.wasm delete mode 100644 test/fixture/node_modules/@fixture/wasm/sum.wasm.d.ts create mode 100644 test/fixture/routes/wasm/dynamic-import.ts rename test/fixture/routes/wasm/{dynamic.ts => static-import.ts} (59%) delete mode 100644 test/fixture/routes/wasm/static.ts diff --git a/package.json b/package.json index b57e61f52e..0f4947e236 100644 --- a/package.json +++ b/package.json @@ -120,7 +120,8 @@ "unctx": "^2.3.1", "unenv": "^1.8.0", "unimport": "^3.7.0", - "unstorage": "^1.10.1" + "unstorage": "^1.10.1", + "unwasm": "^0.3.2" }, "devDependencies": { "@azure/functions": "^3.5.1", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index a4077a7a3a..55bc9eae01 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -198,6 +198,9 @@ importers: unstorage: specifier: ^1.10.1 version: 1.10.1 + unwasm: + specifier: ^0.3.2 + version: 0.3.2 devDependencies: '@azure/functions': specifier: ^3.5.1 @@ -2207,6 +2210,68 @@ packages: pretty-format: 29.7.0 dev: true + /@webassemblyjs/ast@1.11.6: + resolution: {integrity: sha512-IN1xI7PwOvLPgjcf180gC1bqn3q/QaOCwYUahIOhbYUu8KA/3tw2RT/T0Gidi1l7Hhj5D/INhJxiICObqpMu4Q==} + dependencies: + '@webassemblyjs/helper-numbers': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + dev: false + + /@webassemblyjs/floating-point-hex-parser@1.11.6: + resolution: {integrity: sha512-ejAj9hfRJ2XMsNHk/v6Fu2dGS+i4UaXBXGemOfQ/JfQ6mdQg/WXtwleQRLLS4OvfDhv8rYnVwH27YJLMyYsxhw==} + dev: false + + /@webassemblyjs/helper-api-error@1.11.6: + resolution: {integrity: sha512-o0YkoP4pVu4rN8aTJgAyj9hC2Sv5UlkzCHhxqWj8butaLvnpdc2jOwh4ewE6CX0txSfLn/UYaV/pheS2Txg//Q==} + dev: false + + /@webassemblyjs/helper-numbers@1.11.6: + resolution: {integrity: sha512-vUIhZ8LZoIWHBohiEObxVm6hwP034jwmc9kuq5GdHZH0wiLVLIPcMCdpJzG4C11cHoQ25TFIQj9kaVADVX7N3g==} + dependencies: + '@webassemblyjs/floating-point-hex-parser': 1.11.6 + '@webassemblyjs/helper-api-error': 1.11.6 + '@xtuc/long': 4.2.2 + dev: false + + /@webassemblyjs/helper-wasm-bytecode@1.11.6: + resolution: {integrity: sha512-sFFHKwcmBprO9e7Icf0+gddyWYDViL8bpPjJJl0WHxCdETktXdmtWLGVzoHbqUcY4Be1LkNfwTmXOJUFZYSJdA==} + dev: false + + /@webassemblyjs/ieee754@1.11.6: + resolution: {integrity: sha512-LM4p2csPNvbij6U1f19v6WR56QZ8JcHg3QIJTlSwzFcmx6WSORicYj6I63f9yU1kEUtrpG+kjkiIAkevHpDXrg==} + dependencies: + '@xtuc/ieee754': 1.2.0 + dev: false + + /@webassemblyjs/leb128@1.11.6: + resolution: {integrity: sha512-m7a0FhE67DQXgouf1tbN5XQcdWoNgaAuoULHIfGFIEVKA6tu/edls6XnIlkmS6FrXAquJRPni3ZZKjw6FSPjPQ==} + dependencies: + '@xtuc/long': 4.2.2 + dev: false + + /@webassemblyjs/utf8@1.11.6: + resolution: {integrity: sha512-vtXf2wTQ3+up9Zsg8sa2yWiQpzSsMyXj0qViVP6xKGCUT8p8YJ6HqI7l5eCnWx1T/FYdsv07HQs2wTFbbof/RA==} + dev: false + + /@webassemblyjs/wasm-parser@1.11.6: + resolution: {integrity: sha512-6ZwPeGzMJM3Dqp3hCsLgESxBGtT/OeCvCZ4TA1JUPYgmhAx38tTPR9JaKy0S5H3evQpO/h2uWs2j6Yc/fjkpTQ==} + dependencies: + '@webassemblyjs/ast': 1.11.6 + '@webassemblyjs/helper-api-error': 1.11.6 + '@webassemblyjs/helper-wasm-bytecode': 1.11.6 + '@webassemblyjs/ieee754': 1.11.6 + '@webassemblyjs/leb128': 1.11.6 + '@webassemblyjs/utf8': 1.11.6 + dev: false + + /@xtuc/ieee754@1.2.0: + resolution: {integrity: sha512-DX8nKgqcGwsc0eJSqYt5lwP4DH5FlHnmuWWBRy7X0NcaGR0ZtuyeESgMwTYVEtxmsNGY+qit4QYT/MIYTOTPeA==} + dev: false + + /@xtuc/long@4.2.2: + resolution: {integrity: sha512-NuHqBY1PB/D8xU6s/thBgOAiAP7HOYDQ32+BFZILJ8ivkUkAHQnWfn6WhL79Owj1qmUnoN/YPhktdIoucipkAQ==} + dev: false + /abbrev@1.1.1: resolution: {integrity: sha512-nne9/IiQ/hzIhY6pdDnbBtz7DjPTKrY00P/zvPSm5pOFkl6xuGrGnXn/VtTNNfNtAfZ9/1RtehkszU9qcTii0Q==} dev: false @@ -7332,6 +7397,15 @@ packages: webpack-virtual-modules: 0.6.1 dev: false + /unplugin@1.6.0: + resolution: {integrity: sha512-BfJEpWBu3aE/AyHx8VaNE/WgouoQxgH9baAiH82JjX8cqVyi3uJQstqwD5J+SZxIK326SZIhsSZlALXVBCknTQ==} + dependencies: + acorn: 8.11.2 + chokidar: 3.5.3 + webpack-sources: 3.2.3 + webpack-virtual-modules: 0.6.1 + dev: false + /unstorage@1.10.1: resolution: {integrity: sha512-rWQvLRfZNBpF+x8D3/gda5nUCQL2PgXy2jNG4U7/Rc9BGEv9+CAJd0YyGCROUBKs9v49Hg8huw3aih5Bf5TAVw==} peerDependencies: @@ -7417,6 +7491,16 @@ packages: - supports-color dev: true + /unwasm@0.3.2: + resolution: {integrity: sha512-GGgW8Fo+16UBklAQx+n3Vze2E8xr8jkOPXzXVh1Zdfkre9yrgFacfChQoK5fdZt5XtpS+4I/xmLL0ujiOIkU5w==} + dependencies: + '@webassemblyjs/wasm-parser': 1.11.6 + magic-string: 0.30.5 + mlly: 1.4.2 + pathe: 1.1.1 + unplugin: 1.6.0 + dev: false + /update-browserslist-db@1.0.13(browserslist@4.22.2): resolution: {integrity: sha512-xebP81SNcPuNpPP3uzeW1NYXxI3rxyJzF3pD6sH4jE7o/IX+WtSpwnVU+qIsDPyk0d3hmFQ7mjqc6AtV604hbg==} hasBin: true diff --git a/src/presets/vercel.ts b/src/presets/vercel.ts index 10e231684e..9c91b88edd 100644 --- a/src/presets/vercel.ts +++ b/src/presets/vercel.ts @@ -105,7 +105,7 @@ export const vercelEdge = defineNitroPreset({ }, wasm: { lazy: true, - esmImport: true, + esmImport: false, }, hooks: { "rollup:before": (nitro: Nitro) => { diff --git a/src/rollup/config.ts b/src/rollup/config.ts index e3ffbf9e5e..5bafda9527 100644 --- a/src/rollup/config.ts +++ b/src/rollup/config.ts @@ -1,12 +1,7 @@ import { pathToFileURL } from "node:url"; import { createRequire, builtinModules } from "node:module"; -import { dirname, join, normalize, relative, resolve } from "pathe"; -import type { - InputOptions, - OutputOptions, - Plugin, - PreRenderedChunk, -} from "rollup"; +import { join, normalize, resolve } from "pathe"; +import type { InputOptions, OutputOptions, Plugin } from "rollup"; import { defu } from "defu"; // import terser from "@rollup/plugin-terser"; // TODO: Investigate jiti issue import commonjs from "@rollup/plugin-commonjs"; @@ -18,9 +13,10 @@ import { isWindows } from "std-env"; import { visualizer } from "rollup-plugin-visualizer"; import * as unenv from "unenv"; import type { Preset } from "unenv"; -import { sanitizeFilePath, resolvePath, parseNodeModulePath } from "mlly"; +import { sanitizeFilePath, resolvePath } from "mlly"; import unimportPlugin from "unimport/unplugin"; import { hash } from "ohash"; +import { rollup as unwasm } from "unwasm/plugin"; import type { Nitro, NitroStaticBuildFlags } from "../types"; import { resolveAliases } from "../utils"; import { runtimeDir } from "../dirs"; @@ -28,7 +24,6 @@ import nitroPkg from "../../package.json"; import { nitroRuntimeDependencies } from "../deps"; import { replace } from "./plugins/replace"; import { virtual } from "./plugins/virtual"; -import { wasm } from "./plugins/wasm"; import { dynamicRequire } from "./plugins/dynamic-require"; import { NodeExternalsOptions, externals } from "./plugins/externals"; import { externals as legacyExternals } from "./plugins/externals-legacy"; @@ -174,7 +169,7 @@ export const getRollupConfig = (nitro: Nitro): RollupConfig => { // WASM support if (nitro.options.experimental.wasm) { - rollupConfig.plugins.push(wasm(nitro.options.wasm || {})); + rollupConfig.plugins.push(unwasm(nitro.options.wasm || {})); } // Build-time environment variables diff --git a/src/rollup/plugins/wasm.ts b/src/rollup/plugins/wasm.ts deleted file mode 100644 index 18aeee48c4..0000000000 --- a/src/rollup/plugins/wasm.ts +++ /dev/null @@ -1,137 +0,0 @@ -import { createHash } from "node:crypto"; -import { promises as fs, existsSync } from "node:fs"; -import { basename } from "pathe"; -import type { Plugin } from "rollup"; -import MagicString from "magic-string"; -import { WasmOptions } from "../../types"; - -const WASM_EXTERNAL_ID = "\0nitro:wasm:external:"; -const WASM_HELPERS_ID = "\0nitro:wasm:helpers"; - -export function wasm(opts: WasmOptions): Plugin { - type WasmAsset = { - name: string; - source: Buffer; - }; - - const assets: Record = Object.create(null); - - return { - name: "nitro:wasm", - async resolveId(id, importer) { - if (id.startsWith(WASM_EXTERNAL_ID)) { - return { - id, - external: true, - }; - } - if (id.endsWith(".wasm")) { - const r = await this.resolve(id, importer, { skipSelf: true }); - if (r?.id && r?.id !== id) { - return { - id: r.id.startsWith("file://") ? r.id.slice(7) : r.id, - external: false, - moduleSideEffects: false, - syntheticNamedExports: false, - }; - } - } - }, - async load(id) { - if (!id.endsWith(".wasm") || !existsSync(id)) { - return null; - } - const source = await fs.readFile(id); - const name = `wasm/${basename(id, ".wasm")}-${sha1(source)}.wasm`; - assets[id] = { name, source }; - // TODO: Can we parse wasm to extract exports and avoid syntheticNamedExports? - return `export default "WASM";`; // dummy - }, - transform(_code, id) { - if (!id.endsWith(".wasm")) { - return null; - } - const asset = assets[id]; - if (!asset) { - return null; - } - let _dataStr: string; - if (opts.esmImport) { - _dataStr = `await import("${WASM_EXTERNAL_ID}${id}").then(r => r?.default || r)`; - } else { - const base64Str = asset.source.toString("base64"); - _dataStr = `(()=>{const d=atob("${base64Str}");const s=d.length;const b=new Uint8Array(s);for(let i=0;i r?.exports||r?.instance?.exports || r);`; - if (opts.lazy) { - _str = `(()=>{const e=async()=>{return ${_str}};let _p;const p=()=>{if(!_p)_p=e();return _p;};return {then:cb=>p().then(cb),catch:cb=>p().catch(cb)}})()`; - } - return { - code: `export default ${_str};`, - map: { mappings: "" }, - syntheticNamedExports: true, - }; - }, - generateBundle() { - if (opts.esmImport) { - for (const asset of Object.values(assets)) { - this.emitFile({ - type: "asset", - source: asset.source, - fileName: asset.name, - }); - } - } - }, - renderChunk(code, chunk) { - if ( - !chunk.moduleIds.some((id) => id.endsWith(".wasm")) || - !code.includes(WASM_EXTERNAL_ID) - ) { - return; - } - const s = new MagicString(code); - const resolveImport = (id) => { - if (typeof id !== "string") { - return null; - } - const asset = assets[id]; - if (!asset) { - return null; - } - const nestedLevel = chunk.fileName.split("/").length - 1; - const relativeId = - (nestedLevel ? "../".repeat(nestedLevel) : "./") + asset.name; - return { - relativeId, - asset, - }; - }; - const ReplaceRE = new RegExp(`${WASM_EXTERNAL_ID}([^"']+)`, "g"); - for (const match of code.matchAll(ReplaceRE)) { - const resolved = resolveImport(match[1]); - if (!resolved) { - console.warn( - `Failed to resolve WASM import: ${JSON.stringify(match[1])}` - ); - continue; - } - s.overwrite( - match.index, - match.index + match[0].length, - resolved.relativeId - ); - } - if (s.hasChanged()) { - return { - code: s.toString(), - map: s.generateMap({ includeContent: true }), - }; - } - }, - }; -} - -function sha1(source: Buffer) { - return createHash("sha1").update(source).digest("hex").slice(0, 16); -} diff --git a/src/types/nitro.ts b/src/types/nitro.ts index 7c90ead853..ec976aef0b 100644 --- a/src/types/nitro.ts +++ b/src/types/nitro.ts @@ -11,6 +11,7 @@ import type { Storage, BuiltinDriverName } from "unstorage"; import type { ProxyServerOptions } from "httpxy"; import type { ProxyOptions, RouterMethod } from "h3"; import type { ResolvedConfig, ConfigWatcher } from "c12"; +import type { UnwasmPluginOptions } from "unwasm/plugin"; import type { TSConfig } from "pkg-types"; import type { NodeExternalsOptions } from "../rollup/plugins/externals"; import type { RollupConfig } from "../rollup/config"; @@ -188,23 +189,6 @@ export interface NitroRouteRules proxy?: { to: string } & ProxyOptions; } -export interface WasmOptions { - /** - * Direct import the wasm file instead of bundling, required in Cloudflare Workers - * - * @default false - */ - esmImport?: boolean; - - /** - * Import `.wasm` files using a lazily evaluated promise for compatibility - */ - lazy?: boolean; - - /** @deprecated */ - rollup?: unknown; -} - export interface NitroFrameworkInfo { // eslint-disable-next-line @typescript-eslint/ban-types name?: "nitro" | (string & {}); @@ -265,8 +249,12 @@ export interface NitroOptions extends PresetOptions { renderer?: string; serveStatic: boolean | "node" | "deno" | "inline"; noPublicDir: boolean; - /** @experimental Requires `experimental.wasm` to be effective */ - wasm?: WasmOptions; + /** + * @experimental Requires `experimental.wasm` to work + * + * @see https://github.com/unjs/unwasm + */ + wasm?: UnwasmPluginOptions; experimental?: { legacyExternals?: boolean; openAPI?: boolean; @@ -280,6 +268,8 @@ export interface NitroOptions extends PresetOptions { asyncContext?: boolean; /** * Enable Experimental WebAssembly Support + * + * @see https://github.com/unjs/unwasm */ wasm?: boolean; /** diff --git a/test/fixture/node_modules/@fixture/wasm/package.json b/test/fixture/node_modules/@fixture/wasm/package.json deleted file mode 100644 index 4bffc45752..0000000000 --- a/test/fixture/node_modules/@fixture/wasm/package.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "name": "@fixture/wasm", - "version": "1.0.0", - "type": "module", - "exports": { - "./sum.wasm": { - "types": "./sum.wasm.d.ts", - "default": "./sum.wasm" - } - }, - "scripts": { - "build:sum": "npx --yes --package=assemblyscript -c 'asc ./sum.asc.ts -o sum.wasm'", - "buld": "npm run build:sum", - "test:sum": "NODE_OPTIONS='--experimental-wasm-modules' node -e 'import(`./sum.wasm`).then(mod => console.log(mod.sum(2,3)))'", - "test": "npm run build:sum && npm run test:sum" - } -} diff --git a/test/fixture/node_modules/@fixture/wasm/sum.asc.ts b/test/fixture/node_modules/@fixture/wasm/sum.asc.ts deleted file mode 100644 index c6e2cee20c..0000000000 --- a/test/fixture/node_modules/@fixture/wasm/sum.asc.ts +++ /dev/null @@ -1,6 +0,0 @@ -// @ts-nocheck -// https://www.assemblyscript.org - -export function sum(a: i32, b: i32): i32 { - return a + b; -} diff --git a/test/fixture/node_modules/@fixture/wasm/sum.wasm b/test/fixture/node_modules/@fixture/wasm/sum.wasm deleted file mode 100644 index c28e34e925aa141710512b165936f9a05e3f6033..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 93 zcmWm3F%CdL6h+bZ{x>t2g2Do9Wp+ZLkxU3tsJ_y&jq2XyU_LVeFmMtxnhH@l?j)TS m@q-#t9gXRIB$odZdeQUBh4YfP;}$fYNQ`J())+Bz?cxWEXAPtP diff --git a/test/fixture/node_modules/@fixture/wasm/sum.wasm.d.ts b/test/fixture/node_modules/@fixture/wasm/sum.wasm.d.ts deleted file mode 100644 index 3f529c1298..0000000000 --- a/test/fixture/node_modules/@fixture/wasm/sum.wasm.d.ts +++ /dev/null @@ -1 +0,0 @@ -export declare function sum(a: number, b: number): number; diff --git a/test/fixture/routes/wasm/dynamic-import.ts b/test/fixture/routes/wasm/dynamic-import.ts new file mode 100644 index 0000000000..ffb9cb8a5c --- /dev/null +++ b/test/fixture/routes/wasm/dynamic-import.ts @@ -0,0 +1,9 @@ +export default defineLazyEventHandler(async () => { + // @ts-ignore + const { sum } = await import("unwasm/examples/sum.wasm").then((r) => + r.default() + ); + return eventHandler(() => { + return `2+3=${sum(2, 3)}`; + }); +}); diff --git a/test/fixture/routes/wasm/dynamic.ts b/test/fixture/routes/wasm/static-import.ts similarity index 59% rename from test/fixture/routes/wasm/dynamic.ts rename to test/fixture/routes/wasm/static-import.ts index 09bc112cd7..201c687349 100644 --- a/test/fixture/routes/wasm/dynamic.ts +++ b/test/fixture/routes/wasm/static-import.ts @@ -1,5 +1,8 @@ +// @ts-ignore +import init, { sum } from "unwasm/examples/sum.wasm"; + export default defineLazyEventHandler(async () => { - const { sum } = await import("@fixture/wasm/sum.wasm"); + await init(); return eventHandler(() => { return `2+3=${sum(2, 3)}`; }); diff --git a/test/fixture/routes/wasm/static.ts b/test/fixture/routes/wasm/static.ts deleted file mode 100644 index 918c7ce86f..0000000000 --- a/test/fixture/routes/wasm/static.ts +++ /dev/null @@ -1,6 +0,0 @@ -import _mod from "@fixture/wasm/sum.wasm"; - -export default eventHandler(async () => { - const { sum } = await _mod; - return `2+3=${sum(2, 3)}`; -}); diff --git a/test/presets/cloudflare-module.test.ts b/test/presets/cloudflare-module.test.ts index ba45918004..7ffb35b0d4 100644 --- a/test/presets/cloudflare-module.test.ts +++ b/test/presets/cloudflare-module.test.ts @@ -13,6 +13,7 @@ describe("nitro:preset:cloudflare-module", async () => { const mf = new Miniflare({ modules: true, scriptPath: resolve(ctx.outDir, "server/index.mjs"), + modulesRules: [{ type: "CompiledWasm", include: ["**/*.wasm"] }], sitePath: resolve(ctx.outDir, "public"), compatibilityFlags: ["streams_enable_constructors"], globals: { __env__: {} }, diff --git a/test/presets/vercel-edge.test.ts b/test/presets/vercel-edge.test.ts index 3601f5de36..44d3b2e792 100644 --- a/test/presets/vercel-edge.test.ts +++ b/test/presets/vercel-edge.test.ts @@ -16,7 +16,7 @@ describeIf(!isWindows, "nitro:preset:vercel-edge", async () => { }); runtime.evaluate( initialCode.replace( - "export{handleEvent as default}", + /export ?{ ?handleEvent as default ?}/, "globalThis.handleEvent = handleEvent" ) ); diff --git a/test/tests.ts b/test/tests.ts index 9414753ae4..45a4fa07fc 100644 --- a/test/tests.ts +++ b/test/tests.ts @@ -631,13 +631,16 @@ export function testNitro( }); describe("wasm", () => { - it.skipIf(ctx.isWorker || ctx.preset === "deno-server")( - "dynamic import wasm", - async () => { - expect((await callHandler({ url: "/wasm/dynamic" })).data).toBe( - "2+3=5" - ); - } - ); + it("dynamic import wasm", async () => { + expect((await callHandler({ url: "/wasm/dynamic-import" })).data).toBe( + "2+3=5" + ); + }); + + it("static import wasm", async () => { + expect((await callHandler({ url: "/wasm/static-import" })).data).toBe( + "2+3=5" + ); + }); }); }