From 9fee6e5f1b02a4dff48b355df775b02ad277267f Mon Sep 17 00:00:00 2001 From: hrmny Date: Wed, 26 Jul 2023 18:07:33 +0200 Subject: [PATCH] feat: wasm text format and import support --- Cargo.lock | 153 ++++++----- crates/turbopack-ecmascript/src/chunk/item.rs | 2 +- .../src/references/esm/mod.rs | 2 +- .../turbopack/wasm/complex/input/README.md | 2 + .../turbopack/wasm/complex/input/index.js | 20 ++ .../wasm/complex/input/magic-number.js | 7 + .../turbopack/wasm/complex/input/magic.js | 2 + .../turbopack/wasm/complex/input/magic.wat | 15 ++ .../turbopack/wasm/complex/input/memory.js | 7 + .../turbopack/wasm/simple/input/index.js | 2 +- crates/turbopack-wasm/Cargo.toml | 5 +- crates/turbopack-wasm/src/lib.rs | 239 +++++++++++++----- crates/turbopack-wasm/src/loader.rs | 59 +++++ crates/turbopack-wasm/src/url.rs | 161 ++++++++++++ crates/turbopack/src/module_options/mod.rs | 3 +- 15 files changed, 552 insertions(+), 127 deletions(-) create mode 100644 crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/README.md create mode 100644 crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/index.js create mode 100644 crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic-number.js create mode 100644 crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.js create mode 100644 crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.wat create mode 100644 crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/memory.js create mode 100644 crates/turbopack-wasm/src/loader.rs create mode 100644 crates/turbopack-wasm/src/url.rs diff --git a/Cargo.lock b/Cargo.lock index 60a6845a5359b9..f2b937d61d2354 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -960,7 +960,7 @@ checksum = "a6358dedf60f4d9b8db43ad187391afe959746101346fe51bb978126bec61dfb" dependencies = [ "clap 3.2.23", "heck 0.4.1", - "indexmap", + "indexmap 1.9.3", "log", "proc-macro2", "quote", @@ -1159,7 +1159,7 @@ dependencies = [ "atty", "bitflags 1.3.2", "clap_lex 0.2.4", - "indexmap", + "indexmap 1.9.3", "strsim 0.10.0", "termcolor", "textwrap 0.16.0", @@ -1561,7 +1561,7 @@ dependencies = [ "cranelift-entity", "fxhash", "hashbrown 0.12.3", - "indexmap", + "indexmap 1.9.3", "log", "smallvec", ] @@ -2315,6 +2315,12 @@ dependencies = [ "termcolor", ] +[[package]] +name = "equivalent" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5443807d6dff69373d433ab9ef5378ad8df50ca6298caf15de6e52e24aaf54d5" + [[package]] name = "erased-serde" version = "0.3.25" @@ -2784,7 +2790,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "22030e2c5a68ec659fde1e949a745124b48e6fa8b045b7ed5bd1fe4ccc5c4e5d" dependencies = [ "fallible-iterator", - "indexmap", + "indexmap 1.9.3", "stable_deref_trait", ] @@ -2907,7 +2913,7 @@ dependencies = [ "futures-sink", "futures-util", "http", - "indexmap", + "indexmap 1.9.3", "slab", "tokio", "tokio-util", @@ -2961,6 +2967,12 @@ dependencies = [ "ahash 0.8.3", ] +[[package]] +name = "hashbrown" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c6201b9ff9fd90a5a3bac2e56a830d0caa509576f0e503818ee82c181b3437a" + [[package]] name = "hdrhistogram" version = "7.5.2" @@ -3323,6 +3335,16 @@ dependencies = [ "serde", ] +[[package]] +name = "indexmap" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d5477fe2230a79769d8dc68e0eabf5437907c0457a5614a9e8dddb67f65eb65d" +dependencies = [ + "equivalent", + "hashbrown 0.14.0", +] + [[package]] name = "indicatif" version = "0.17.3" @@ -4918,7 +4940,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4dd7d28ee937e54fe3080c91faa1c3a46c06de6252988a7f4592ba2310ef22a4" dependencies = [ "fixedbitset", - "indexmap", + "indexmap 1.9.3", ] [[package]] @@ -5245,7 +5267,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5ca9c6be70d989d21a136eb86c2d83e4b328447fac4a88dace2143c179c86267" dependencies = [ "autocfg", - "indexmap", + "indexmap 1.9.3", ] [[package]] @@ -5776,7 +5798,7 @@ dependencies = [ "bitvec", "bytecheck", "hashbrown 0.12.3", - "indexmap", + "indexmap 1.9.3", "ptr_meta", "rend", "rkyv_derive", @@ -6158,7 +6180,7 @@ version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ - "indexmap", + "indexmap 1.9.3", "itoa", "ryu", "serde", @@ -6233,7 +6255,7 @@ dependencies = [ "base64 0.13.1", "chrono", "hex", - "indexmap", + "indexmap 1.9.3", "serde", "serde_json", "serde_with_macros", @@ -6258,7 +6280,7 @@ version = "0.8.26" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "578a7433b776b56a35785ed5ce9a7e777ac0598aac5a6dd1b4b18a307c7fc71b" dependencies = [ - "indexmap", + "indexmap 1.9.3", "ryu", "serde", "yaml-rust", @@ -6270,7 +6292,7 @@ version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9d684e3ec7de3bf5466b32bd75303ac16f0736426e5a4e0d6e489559ce1249c" dependencies = [ - "indexmap", + "indexmap 1.9.3", "itoa", "ryu", "serde", @@ -6780,7 +6802,7 @@ dependencies = [ "base64 0.13.1", "dashmap", "either", - "indexmap", + "indexmap 1.9.3", "jsonc-parser", "lru", "napi", @@ -6858,7 +6880,7 @@ dependencies = [ "anyhow", "crc", "dashmap", - "indexmap", + "indexmap 1.9.3", "is-macro", "once_cell", "parking_lot", @@ -6935,7 +6957,7 @@ version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ba1c7a40d38f9dd4e9a046975d3faf95af42937b34b2b963be4d8f01239584b" dependencies = [ - "indexmap", + "indexmap 1.9.3", "serde", "serde_json", "swc_config_macro", @@ -7249,7 +7271,7 @@ checksum = "ad225946cd5070c474941a0cf23d12dbe151143ed2df70ddde91813bf605fa01" dependencies = [ "ahash 0.8.3", "arrayvec 0.7.2", - "indexmap", + "indexmap 1.9.3", "num-bigint", "num_cpus", "once_cell", @@ -7306,7 +7328,7 @@ dependencies = [ "ahash 0.8.3", "anyhow", "dashmap", - "indexmap", + "indexmap 1.9.3", "once_cell", "preset_env_base", "semver 1.0.17", @@ -7381,7 +7403,7 @@ checksum = "cae4d6e3250f61aa71ed1c172cfeb5eee042146417ef17c6b78887fc113bf35d" dependencies = [ "better_scoped_tls", "bitflags 2.3.3", - "indexmap", + "indexmap 1.9.3", "once_cell", "phf", "rayon", @@ -7419,7 +7441,7 @@ checksum = "13fef52d7e0279565d23ccdac8f75e87706792e11570b920a76e8932fa73bf43" dependencies = [ "ahash 0.8.3", "arrayvec 0.7.2", - "indexmap", + "indexmap 1.9.3", "is-macro", "num-bigint", "rayon", @@ -7461,7 +7483,7 @@ dependencies = [ "ahash 0.8.3", "anyhow", "bitflags 2.3.3", - "indexmap", + "indexmap 1.9.3", "is-macro", "path-clean 0.1.0", "pathdiff", @@ -7487,7 +7509,7 @@ checksum = "a219faa289f11a359a07daa2d80225f5126eb1988402214393f2feb24293ed89" dependencies = [ "ahash 0.8.3", "dashmap", - "indexmap", + "indexmap 1.9.3", "once_cell", "petgraph", "rayon", @@ -7534,7 +7556,7 @@ dependencies = [ "ahash 0.8.3", "base64 0.13.1", "dashmap", - "indexmap", + "indexmap 1.9.3", "once_cell", "rayon", "serde", @@ -7600,7 +7622,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "45cc2476ee15d5d4d928d1eacb74de62b3cdfadcbf07998b4f46dbde70b32d87" dependencies = [ "ahash 0.8.3", - "indexmap", + "indexmap 1.9.3", "rustc-hash", "swc_atoms", "swc_common", @@ -7617,7 +7639,7 @@ version = "0.120.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "93562e5b67676f5a60df97725722cc846a48f3cc5ce35a4f7e6c53e064abf76c" dependencies = [ - "indexmap", + "indexmap 1.9.3", "num_cpus", "once_cell", "rayon", @@ -7693,7 +7715,7 @@ version = "0.19.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "11cc84ef676e0901c5a7a01394b98f5219beee0e22f746fbe2c90ee998ceda15" dependencies = [ - "indexmap", + "indexmap 1.9.3", "petgraph", "rustc-hash", "swc_common", @@ -8450,7 +8472,7 @@ version = "0.19.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "239410c8609e8125456927e6707163a3b1fdb40561e4b803bc041f466ccfdc13" dependencies = [ - "indexmap", + "indexmap 1.9.3", "serde", "serde_spanned", "toml_datetime", @@ -8524,7 +8546,7 @@ checksum = "b8fa9be0de6cf49e536ce1851f987bd21a43b771b09473c3549a6c853db37c1c" dependencies = [ "futures-core", "futures-util", - "indexmap", + "indexmap 1.9.3", "pin-project", "pin-project-lite", "rand 0.8.5", @@ -8765,7 +8787,7 @@ dependencies = [ "erased-serde", "event-listener", "futures", - "indexmap", + "indexmap 1.9.3", "mopa", "nohash-hasher", "once_cell", @@ -8817,7 +8839,7 @@ version = "0.1.0" dependencies = [ "anyhow", "dotenvs", - "indexmap", + "indexmap 1.9.3", "serde", "turbo-tasks", "turbo-tasks-build", @@ -8830,7 +8852,7 @@ version = "0.1.0" dependencies = [ "anyhow", "httpmock", - "indexmap", + "indexmap 1.9.3", "lazy_static", "reqwest", "serde", @@ -8858,7 +8880,7 @@ dependencies = [ "futures", "futures-retry", "include_dir", - "indexmap", + "indexmap 1.9.3", "jsonc-parser", "mime", "notify 4.0.17", @@ -8991,7 +9013,7 @@ dependencies = [ "criterion", "difference", "futures", - "indexmap", + "indexmap 1.9.3", "lazy_static", "regex", "rstest", @@ -9107,7 +9129,7 @@ name = "turbopack-build" version = "0.1.0" dependencies = [ "anyhow", - "indexmap", + "indexmap 1.9.3", "indoc", "serde", "serde_json", @@ -9192,7 +9214,7 @@ dependencies = [ "anyhow", "clap 4.1.11", "futures", - "indexmap", + "indexmap 1.9.3", "intervaltree", "itertools", "owo-colors", @@ -9211,7 +9233,7 @@ dependencies = [ "auto-hash-map", "browserslist-rs", "futures", - "indexmap", + "indexmap 1.9.3", "lazy_static", "patricia_tree", "qstring", @@ -9249,7 +9271,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "indexmap", + "indexmap 1.9.3", "indoc", "once_cell", "regex", @@ -9269,7 +9291,7 @@ name = "turbopack-dev" version = "0.1.0" dependencies = [ "anyhow", - "indexmap", + "indexmap 1.9.3", "indoc", "serde", "serde_json", @@ -9297,7 +9319,7 @@ dependencies = [ "futures", "hyper", "hyper-tungstenite", - "indexmap", + "indexmap 1.9.3", "mime", "mime_guess", "once_cell", @@ -9331,7 +9353,7 @@ dependencies = [ "async-trait", "criterion", "futures", - "indexmap", + "indexmap 1.9.3", "indoc", "lazy_static", "num-bigint", @@ -9376,7 +9398,7 @@ version = "0.1.0" dependencies = [ "anyhow", "async-trait", - "indexmap", + "indexmap 1.9.3", "modularize_imports", "serde", "serde_json", @@ -9413,7 +9435,7 @@ name = "turbopack-env" version = "0.1.0" dependencies = [ "anyhow", - "indexmap", + "indexmap 1.9.3", "serde", "turbo-tasks", "turbo-tasks-build", @@ -9430,7 +9452,7 @@ dependencies = [ "anyhow", "base64 0.21.0", "image", - "indexmap", + "indexmap 1.9.3", "mime", "once_cell", "regex", @@ -9480,7 +9502,7 @@ dependencies = [ "const_format", "futures", "futures-retry", - "indexmap", + "indexmap 1.9.3", "mime", "once_cell", "owo-colors", @@ -9581,7 +9603,8 @@ name = "turbopack-wasm" version = "0.1.0" dependencies = [ "anyhow", - "indexmap", + "indexmap 1.9.3", + "indoc", "serde", "turbo-tasks", "turbo-tasks-build", @@ -9590,6 +9613,8 @@ dependencies = [ "turbopack-core", "turbopack-css", "turbopack-ecmascript", + "wasmparser 0.110.0", + "wat", ] [[package]] @@ -9845,8 +9870,8 @@ version = "1.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" dependencies = [ - "cfg-if 0.1.10", - "rand 0.4.6", + "cfg-if 1.0.0", + "rand 0.8.5", "static_assertions", ] @@ -10152,7 +10177,7 @@ dependencies = [ "fs_extra", "futures", "getrandom", - "indexmap", + "indexmap 1.9.3", "lazy_static", "libc", "pin-project-lite", @@ -10439,9 +10464,9 @@ checksum = "0046fef7e28c3804e5e38bfa31ea2a0f73905319b677e57ebe37e49358989b5d" [[package]] name = "wasm-encoder" -version = "0.25.0" +version = "0.31.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4eff853c4f09eec94d76af527eddad4e9de13b11d6286a1ef7134bc30135a2b7" +checksum = "41763f20eafed1399fff1afb466496d3a959f58241436cfdc17e3f5ca954de16" dependencies = [ "leb128", ] @@ -10455,7 +10480,7 @@ dependencies = [ "bytes", "cfg-if 1.0.0", "derivative", - "indexmap", + "indexmap 1.9.3", "js-sys", "more-asserts", "rustc-demangle", @@ -10551,7 +10576,7 @@ dependencies = [ "bytecheck", "enum-iterator 0.7.0", "enumset", - "indexmap", + "indexmap 1.9.3", "more-asserts", "rkyv", "serde", @@ -10573,7 +10598,7 @@ dependencies = [ "derivative", "enum-iterator 0.7.0", "fnv", - "indexmap", + "indexmap 1.9.3", "lazy_static", "libc", "mach", @@ -10677,15 +10702,25 @@ version = "0.95.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2ea896273ea99b15132414be1da01ab0d8836415083298ecaffbe308eaac87a" dependencies = [ - "indexmap", + "indexmap 1.9.3", "url", ] +[[package]] +name = "wasmparser" +version = "0.110.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1dfcdb72d96f01e6c85b6bf20102e7423bdbaad5c337301bab2bbf253d26413c" +dependencies = [ + "indexmap 2.0.0", + "semver 1.0.17", +] + [[package]] name = "wast" -version = "56.0.0" +version = "62.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b54185c051d7bbe23757d50fe575880a2426a2f06d2e9f6a10fd9a4a42920c0" +checksum = "b8ae06f09dbe377b889fbd620ff8fa21e1d49d1d9d364983c0cdbf9870cb9f1f" dependencies = [ "leb128", "memchr", @@ -10695,9 +10730,9 @@ dependencies = [ [[package]] name = "wat" -version = "1.0.62" +version = "1.0.69" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56681922808216ab86d96bb750f70d500b5a7800e41564290fd46bb773581299" +checksum = "842e15861d203fb4a96d314b0751cdeaf0f6f8b35e8d81d2953af2af5e44e637" dependencies = [ "wast", ] @@ -10757,7 +10792,7 @@ dependencies = [ "base64 0.21.0", "byteorder", "bytes", - "indexmap", + "indexmap 1.9.3", "leb128", "lexical-sort", "once_cell", @@ -11123,7 +11158,7 @@ dependencies = [ "cargo-lock", "chrono", "clap 4.1.11", - "indexmap", + "indexmap 1.9.3", "inquire", "num-format", "owo-colors", diff --git a/crates/turbopack-ecmascript/src/chunk/item.rs b/crates/turbopack-ecmascript/src/chunk/item.rs index 7dddde2267ac5b..744e5e9e1528ec 100644 --- a/crates/turbopack-ecmascript/src/chunk/item.rs +++ b/crates/turbopack-ecmascript/src/chunk/item.rs @@ -24,7 +24,7 @@ use crate::{ }; #[turbo_tasks::value(shared)] -#[derive(Default)] +#[derive(Default, Clone)] pub struct EcmascriptChunkItemContent { pub inner_code: Rope, pub source_map: Option>, diff --git a/crates/turbopack-ecmascript/src/references/esm/mod.rs b/crates/turbopack-ecmascript/src/references/esm/mod.rs index ee3e24ab6cddd6..5f0643e72ddca1 100644 --- a/crates/turbopack-ecmascript/src/references/esm/mod.rs +++ b/crates/turbopack-ecmascript/src/references/esm/mod.rs @@ -11,7 +11,7 @@ pub use self::{ base::EsmAssetReference, binding::EsmBinding, dynamic::EsmAsyncAssetReference, - export::EsmExports, + export::{EsmExport, EsmExports}, meta::{ImportMetaBinding, ImportMetaRef}, module_item::EsmModuleItem, url::UrlAssetReference, diff --git a/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/README.md b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/README.md new file mode 100644 index 00000000000000..bcbae787a266de --- /dev/null +++ b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/README.md @@ -0,0 +1,2 @@ +Adapted from webpack +https://github.com/webpack/webpack/blob/6be4065ade1e252c1d8dcba4af0f43e32af1bdc1/examples/wasm-complex/README.md diff --git a/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/index.js b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/index.js new file mode 100644 index 00000000000000..2084b200a5dc47 --- /dev/null +++ b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/index.js @@ -0,0 +1,20 @@ +describe("complex wasm", () => { + it("should be possible to use imported memory", async () => { + // magic.js is an async module, so we import it inside this function to make sure the entrypoint isn't async. + const { get, set } = await import("./magic.js"); + + set(42); + expect(get()).toEqual(42); + set(123); + expect(get()).toEqual(123); + }); + + it("should be possible to use imported functions", async () => { + // magic.js is an async module, so we import it inside this function to make sure the entrypoint isn't async. + const { getNumber } = await import("./magic.js"); + + // random numbers + expect(getNumber()).toBeGreaterThanOrEqual(0); + expect(getNumber()).toBeGreaterThanOrEqual(0); + }); +}) diff --git a/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic-number.js b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic-number.js new file mode 100644 index 00000000000000..0c2eaa5b8c557d --- /dev/null +++ b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic-number.js @@ -0,0 +1,7 @@ +export function getNumber() { + return 42; +} + +export function getRandomNumber() { + return Math.floor(Math.random() * 256); +} diff --git a/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.js b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.js new file mode 100644 index 00000000000000..233b3b85d353a7 --- /dev/null +++ b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.js @@ -0,0 +1,2 @@ +// reexporting +export * from "./magic.wat"; diff --git a/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.wat b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.wat new file mode 100644 index 00000000000000..9032993cac3fad --- /dev/null +++ b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/magic.wat @@ -0,0 +1,15 @@ +(module + (type $t0 (func (result i32))) + (type $t1 (func (param i32))) + (import "./memory.js" "memory" (memory 1)) + (import "./magic-number.js" "getRandomNumber" (func $getRandomNumber (type $t0))) + (func $get (export "get") (type $t0) (result i32) + (i32.load + (i32.const 0))) + (func $set (export "set") (type $t1) (param $p i32) + (i32.store + (i32.const 0) + (get_local $p))) + (func $getNumber (export "getNumber") (type $t0) (result i32) + (call $getRandomNumber)) +) diff --git a/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/memory.js b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/memory.js new file mode 100644 index 00000000000000..3524636e56a5ce --- /dev/null +++ b/crates/turbopack-tests/tests/execution/turbopack/wasm/complex/input/memory.js @@ -0,0 +1,7 @@ +async function getMemoryFromParentInWorker() { + await new Promise(r => setTimeout(r, 200)); + // fake + return new WebAssembly.Memory({ initial: 1 }); +} + +export const memory = await getMemoryFromParentInWorker(); diff --git a/crates/turbopack-tests/tests/execution/turbopack/wasm/simple/input/index.js b/crates/turbopack-tests/tests/execution/turbopack/wasm/simple/input/index.js index 4c6fa60983328c..dba1ba5bd66de4 100644 --- a/crates/turbopack-tests/tests/execution/turbopack/wasm/simple/input/index.js +++ b/crates/turbopack-tests/tests/execution/turbopack/wasm/simple/input/index.js @@ -1,5 +1,5 @@ it("should handle wasm imports", async () => { - // math.js is an async module, so we import it in here + // magic.js is an async module, so we import it inside this function to make sure the entrypoint isn't async. const { add, factorial, diff --git a/crates/turbopack-wasm/Cargo.toml b/crates/turbopack-wasm/Cargo.toml index b87c674fd6f52f..a1646ca52704a0 100644 --- a/crates/turbopack-wasm/Cargo.toml +++ b/crates/turbopack-wasm/Cargo.toml @@ -12,15 +12,16 @@ bench = false [dependencies] anyhow = { workspace = true } indexmap = { workspace = true } - +indoc = { workspace = true } turbo-tasks = { workspace = true } turbo-tasks-fs = { workspace = true } turbo-tasks-hash = { workspace = true } turbopack-core = { workspace = true } turbopack-css = { workspace = true } turbopack-ecmascript = { workspace = true } - serde = { workspace = true } +wat = "1.0.69" +wasmparser = "0.110.0" [build-dependencies] turbo-tasks-build = { workspace = true } diff --git a/crates/turbopack-wasm/src/lib.rs b/crates/turbopack-wasm/src/lib.rs index f219287105c15c..06a9bafe1e6c2f 100644 --- a/crates/turbopack-wasm/src/lib.rs +++ b/crates/turbopack-wasm/src/lib.rs @@ -9,9 +9,15 @@ #![feature(arbitrary_self_types)] #![feature(async_fn_in_trait)] +mod loader; +pub mod url; + +use std::collections::BTreeMap; + use anyhow::{bail, Result}; -use indexmap::IndexSet; -use turbo_tasks::{Value, ValueToString, Vc}; +use indexmap::indexmap; +use turbo_tasks::{Value, Vc}; +use turbo_tasks_fs::{File, FileContent, FileSystemPath}; use turbopack_core::{ asset::{Asset, AssetContent}, chunk::{ @@ -19,10 +25,13 @@ use turbopack_core::{ }, context::AssetContext, ident::AssetIdent, - module::Module, + module::{Module, OptionModule}, output::OutputAsset, - reference::{ModuleReferences, SingleOutputAssetReference}, + reference::ModuleReferences, + reference_type::ReferenceType, + resolve::{origin::ResolveOrigin, parse::Request}, source::Source, + virtual_source::VirtualSource, }; use turbopack_ecmascript::{ chunk::{ @@ -30,15 +39,43 @@ use turbopack_ecmascript::{ EcmascriptChunkItemOptions, EcmascriptChunkPlaceable, EcmascriptChunkingContext, EcmascriptExports, }, - references::async_module::{AsyncModule, OptionAsyncModule}, - utils::StringifyJs, + references::async_module::OptionAsyncModule, + EcmascriptModuleAsset, }; +use wasmparser::{Chunk as WasmChunk, Parser, Payload}; + +use crate::{loader::loader_source, url::WebAssemblyUrlModuleAsset}; #[turbo_tasks::function] fn modifier() -> Vc { Vc::cell("wasm".to_string()) } +#[turbo_tasks::function] +async fn binary_source(source: Vc>) -> Result>> { + let ext = source.ident().path().extension().await?; + if *ext == "wasm" { + return Ok(source); + } + + let content = source.content().file_content().await?; + + let file_content: Vc = if let FileContent::Content(file) = &*content { + let bytes = file.content().to_bytes()?; + + let parsed = wat::parse_bytes(&*bytes)?; + + File::from(&*parsed).into() + } else { + FileContent::NotFound.cell() + }; + + Ok(Vc::upcast(VirtualSource::new( + source.ident().path().append("_.wasm".to_string()), + AssetContent::file(file_content), + ))) +} + #[turbo_tasks::value] #[derive(Clone)] pub struct WebAssemblyModuleAsset { @@ -54,23 +91,110 @@ impl WebAssemblyModuleAsset { } #[turbo_tasks::function] - async fn wasm_asset( - self: Vc, - context: Vc>, - ) -> Result> { - Ok(WebAssemblyAsset::cell(WebAssemblyAsset { + fn wasm_asset(&self, context: Vc>) -> Vc { + WebAssemblyAsset { context, - source: self.await?.source, - })) + source: binary_source(self.source), + } + .cell() + } + + #[turbo_tasks::function] + async fn loader(self: Vc) -> Result> { + let this = self.await?; + + let module = this.context.process( + loader_source(binary_source(this.source).ident(), self.analyze()), + Value::new(ReferenceType::Internal(Vc::cell(indexmap! { + "WASM_PATH".to_string() => Vc::upcast(WebAssemblyUrlModuleAsset::new(this.source, this.context)), + }))), + ); + + let Some(esm_asset) = + Vc::try_resolve_downcast_type::(module).await? + else { + bail!("WASM loader was not processed into an EcmascriptModuleAsset"); + }; + + Ok(esm_asset) + } + + #[turbo_tasks::function] + async fn analyze(&self) -> Result> { + let content = binary_source(self.source).content().file_content().await?; + + let mut analysis = WebAssemblyAnalysis::default(); + + let FileContent::Content(file) = &*content else { + return Ok(analysis.cell()); + }; + + let mut bytes = &*file.content().to_bytes()?; + + let mut parser = Parser::new(0); + loop { + let payload = match parser.parse(&bytes, true)? { + WasmChunk::Parsed { consumed, payload } => { + bytes = &bytes[consumed..]; + payload + } + // this state isn't possible with `eof = true` + WasmChunk::NeedMoreData(_) => unreachable!(), + }; + + match payload { + Payload::ImportSection(s) => { + for import in s { + let import = import?; + + analysis + .imports + .entry(import.module.to_string()) + .or_default() + .push(import.name.to_string()); + } + } + Payload::ExportSection(s) => { + for export in s { + let export = export?; + + analysis.exports.push(export.name.to_string()); + } + } + + // skip over code sections + Payload::CodeSectionStart { size, .. } => { + parser.skip_section(); + bytes = &bytes[size as usize..]; + } + + Payload::End(_) => break, + _ => {} + } + } + + Ok(analysis.cell()) } } +#[turbo_tasks::value] +#[derive(Default)] +struct WebAssemblyAnalysis { + imports: BTreeMap>, + exports: Vec, +} + #[turbo_tasks::value_impl] impl Module for WebAssemblyModuleAsset { #[turbo_tasks::function] fn ident(&self) -> Vc { self.source.ident().with_modifier(modifier()) } + + #[turbo_tasks::function] + async fn references(self: Vc) -> Vc { + self.loader().references() + } } #[turbo_tasks::value_impl] @@ -104,28 +228,41 @@ impl EcmascriptChunkPlaceable for WebAssemblyModuleAsset { self: Vc, context: Vc>, ) -> Vc> { - Vc::upcast(ModuleChunkItem::cell(ModuleChunkItem { - module: self, - context, - wasm_asset: self.wasm_asset(Vc::upcast(context)), - })) + Vc::upcast( + ModuleChunkItem { + module: self, + context, + } + .cell(), + ) } #[turbo_tasks::function] - fn get_exports(&self) -> Vc { - EcmascriptExports::DynamicNamespace.into() + fn get_exports(self: Vc) -> Vc { + self.loader().get_exports() } #[turbo_tasks::function] fn get_async_module(self: Vc) -> Vc { - Vc::cell(Some( - AsyncModule { - placeable: Vc::upcast(self), - references: IndexSet::new(), - has_top_level_await: true, - } - .cell(), - )) + self.loader().get_async_module() + } +} + +#[turbo_tasks::value_impl] +impl ResolveOrigin for WebAssemblyModuleAsset { + #[turbo_tasks::function] + fn origin_path(&self) -> Vc { + self.source.ident().path() + } + + #[turbo_tasks::function] + fn context(&self) -> Vc> { + self.context + } + + #[turbo_tasks::function] + fn get_inner_asset(self: Vc, request: Vc) -> Vc { + self.loader().get_inner_asset(request) } } @@ -159,7 +296,6 @@ impl Asset for WebAssemblyAsset { struct ModuleChunkItem { module: Vc, context: Vc>, - wasm_asset: Vc, } #[turbo_tasks::value_impl] @@ -171,13 +307,9 @@ impl ChunkItem for ModuleChunkItem { #[turbo_tasks::function] async fn references(&self) -> Result> { - Ok(Vc::cell(vec![Vc::upcast(SingleOutputAssetReference::new( - Vc::upcast(self.wasm_asset), - Vc::cell(format!( - "wasm(loader) {}", - self.wasm_asset.ident().to_string().await? - )), - ))])) + let loader = self.module.loader().as_chunk_item(self.context); + + Ok(loader.references()) } } @@ -198,34 +330,19 @@ impl EcmascriptChunkItem for ModuleChunkItem { &self, availability_info: Value, ) -> Result> { - let path = self.wasm_asset.ident().path().await?; - let output_root = self.context.output_root().await?; + let loader_asset = self.module.loader(); - let Some(path) = output_root.get_path_to(&path) else { - bail!("WASM asset ident is not relative to output root"); - }; - - let code = format!( - "__turbopack_dynamic__(await __turbopack_wasm__({path}));", - path = StringifyJs(path) - ) - .into(); - - let options = EcmascriptChunkItemOptions { - async_module: self - .module - .get_async_module() - .module_options(availability_info) - .await? - .clone_value(), - wasm: true, - ..Default::default() - }; + let chunk_item_content = loader_asset + .as_chunk_item(self.context) + .content_with_availability_info(availability_info) + .await?; Ok(EcmascriptChunkItemContent { - inner_code: code, - options, - ..Default::default() + options: EcmascriptChunkItemOptions { + wasm: true, + ..chunk_item_content.options.clone() + }, + ..chunk_item_content.clone_value() } .into()) } diff --git a/crates/turbopack-wasm/src/loader.rs b/crates/turbopack-wasm/src/loader.rs new file mode 100644 index 00000000000000..199d3543c1422e --- /dev/null +++ b/crates/turbopack-wasm/src/loader.rs @@ -0,0 +1,59 @@ +use std::fmt::Write; + +use anyhow::Result; +use indoc::writedoc; +use turbo_tasks::Vc; +use turbo_tasks_fs::File; +use turbopack_core::{ + asset::AssetContent, ident::AssetIdent, source::Source, virtual_source::VirtualSource, +}; +use turbopack_ecmascript::utils::StringifyJs; + +use crate::WebAssemblyAnalysis; + +#[turbo_tasks::function] +pub(crate) async fn loader_source( + wasm_ident: Vc, + analysis: Vc, +) -> Result>> { + let analysis = analysis.await?; + + let mut code = String::new(); + + let mut imports_obj = "{".to_string(); + for (path, items) in &analysis.imports { + writeln!( + code, + "import {{ {} }} from {};", + items.join(", "), + StringifyJs(path) + )?; + + writeln!(imports_obj, "\n {}: {{", StringifyJs(path))?; + for item in items { + writeln!(imports_obj, " {}: {},", StringifyJs(item), item)?; + } + writeln!(imports_obj, " }},")?; + } + writeln!(imports_obj, "}}")?; + + writeln!(code, "import wasmPath from \"WASM_PATH\";")?; + + writeln!(code)?; + + writedoc!( + code, + r#" + const {{ {exports} }} = await __turbopack_wasm__(wasmPath, {imports}); + + export {{ {exports} }}; + "#, + imports = imports_obj, + exports = analysis.exports.join(", "), + )?; + + Ok(Vc::upcast(VirtualSource::new( + wasm_ident.path().append("_.loader.mjs".to_string()), + AssetContent::file(File::from(code).into()), + ))) +} diff --git a/crates/turbopack-wasm/src/url.rs b/crates/turbopack-wasm/src/url.rs new file mode 100644 index 00000000000000..7d101e1ad7e354 --- /dev/null +++ b/crates/turbopack-wasm/src/url.rs @@ -0,0 +1,161 @@ +use anyhow::{bail, Result}; +use turbo_tasks::{Value, ValueToString, Vc}; +use turbopack_core::{ + asset::{Asset, AssetContent}, + chunk::{ + availability_info::AvailabilityInfo, Chunk, ChunkItem, ChunkableModule, ChunkingContext, + }, + context::AssetContext, + ident::AssetIdent, + module::Module, + output::OutputAsset, + reference::{ModuleReferences, SingleOutputAssetReference}, + source::Source, +}; +use turbopack_ecmascript::{ + chunk::{ + EcmascriptChunk, EcmascriptChunkItem, EcmascriptChunkItemContent, EcmascriptChunkPlaceable, + EcmascriptChunkingContext, EcmascriptExports, + }, + utils::StringifyJs, +}; + +use crate::{binary_source, WebAssemblyAsset}; + +#[turbo_tasks::function] +fn modifier() -> Vc { + Vc::cell("wasm url".to_string()) +} + +#[turbo_tasks::value] +#[derive(Clone)] +pub struct WebAssemblyUrlModuleAsset { + pub source: Vc>, + pub context: Vc>, +} + +#[turbo_tasks::value_impl] +impl WebAssemblyUrlModuleAsset { + #[turbo_tasks::function] + pub fn new(source: Vc>, context: Vc>) -> Vc { + Self::cell(WebAssemblyUrlModuleAsset { source, context }) + } + + #[turbo_tasks::function] + fn wasm_asset(&self, context: Vc>) -> Vc { + WebAssemblyAsset { + context, + source: binary_source(self.source), + } + .cell() + } +} + +#[turbo_tasks::value_impl] +impl Module for WebAssemblyUrlModuleAsset { + #[turbo_tasks::function] + fn ident(&self) -> Vc { + self.source.ident().with_modifier(modifier()) + } +} + +#[turbo_tasks::value_impl] +impl Asset for WebAssemblyUrlModuleAsset { + #[turbo_tasks::function] + fn content(&self) -> Vc { + self.source.content() + } +} + +#[turbo_tasks::value_impl] +impl ChunkableModule for WebAssemblyUrlModuleAsset { + #[turbo_tasks::function] + fn as_chunk( + self: Vc, + context: Vc>, + availability_info: Value, + ) -> Vc> { + Vc::upcast(EcmascriptChunk::new( + context, + Vc::upcast(self), + availability_info, + )) + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkPlaceable for WebAssemblyUrlModuleAsset { + #[turbo_tasks::function] + fn as_chunk_item( + self: Vc, + context: Vc>, + ) -> Vc> { + Vc::upcast( + UrlModuleChunkItem { + module: self, + context, + wasm_asset: self.wasm_asset(Vc::upcast(context)), + } + .cell(), + ) + } + + #[turbo_tasks::function] + fn get_exports(self: Vc) -> Vc { + EcmascriptExports::Value.cell() + } +} + +#[turbo_tasks::value] +struct UrlModuleChunkItem { + module: Vc, + context: Vc>, + wasm_asset: Vc, +} + +#[turbo_tasks::value_impl] +impl ChunkItem for UrlModuleChunkItem { + #[turbo_tasks::function] + fn asset_ident(&self) -> Vc { + self.module.ident() + } + + #[turbo_tasks::function] + async fn references(&self) -> Result> { + Ok(Vc::cell(vec![Vc::upcast(SingleOutputAssetReference::new( + Vc::upcast(self.wasm_asset), + Vc::cell(format!( + "wasm(url) {}", + self.wasm_asset.ident().to_string().await? + )), + ))])) + } +} + +#[turbo_tasks::value_impl] +impl EcmascriptChunkItem for UrlModuleChunkItem { + #[turbo_tasks::function] + fn chunking_context(&self) -> Vc> { + self.context + } + + #[turbo_tasks::function] + async fn content(&self) -> Result> { + let path = self.wasm_asset.ident().path().await?; + let output_root = self.context.output_root().await?; + + let Some(path) = output_root.get_path_to(&path) else { + bail!("WASM asset ident is not relative to output root"); + }; + + Ok(EcmascriptChunkItemContent { + inner_code: format!( + "__turbopack_export_value__({path});", + path = StringifyJs(path) + ) + .into(), + ..Default::default() + } + .into()) + } +} diff --git a/crates/turbopack/src/module_options/mod.rs b/crates/turbopack/src/module_options/mod.rs index 5d96c7ab59d457..5bb651e48ca02e 100644 --- a/crates/turbopack/src/module_options/mod.rs +++ b/crates/turbopack/src/module_options/mod.rs @@ -342,8 +342,7 @@ impl ModuleOptions { ModuleRule::new( ModuleRuleCondition::any(vec![ ModuleRuleCondition::ResourcePathEndsWith(".wasm".to_string()), - // TODO(WEB-1316): add support for `wat` files - // ModuleRuleCondition::ResourcePathEndsWith(".wat".to_string()), + ModuleRuleCondition::ResourcePathEndsWith(".wat".to_string()), ]), vec![ModuleRuleEffect::ModuleType(ModuleType::WebAssembly)], ),