diff --git a/Cargo.lock b/Cargo.lock index a6e1af0b9..33a96b130 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -286,6 +286,7 @@ dependencies = [ "deno_unsync", "fastrand", "futures", + "indexmap", "libc", "log", "memoffset", @@ -303,6 +304,7 @@ dependencies = [ "unicycle", "url", "v8", + "wasm_dep_analyzer", ] [[package]] @@ -2004,6 +2006,15 @@ version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm_dep_analyzer" +version = "0.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fda582fdf067ec11fbbf893797c9b17742d994bc6f5cdd05863f111b84c23cf" +dependencies = [ + "thiserror", +] + [[package]] name = "which" version = "5.0.0" diff --git a/core/Cargo.toml b/core/Cargo.toml index dcd0bb1f6..ad25ef9bf 100644 --- a/core/Cargo.toml +++ b/core/Cargo.toml @@ -35,6 +35,7 @@ deno_core_icudata = { workspace = true, optional = true } deno_ops.workspace = true deno_unsync.workspace = true futures.workspace = true +indexmap = "2.1.0" libc.workspace = true log.workspace = true memoffset.workspace = true @@ -49,6 +50,7 @@ static_assertions.workspace = true tokio.workspace = true url.workspace = true v8.workspace = true +wasm_dep_analyzer = "0.0.1" [dev-dependencies] bencher.workspace = true diff --git a/core/modules/loaders.rs b/core/modules/loaders.rs index 64a23feb0..e68e26618 100644 --- a/core/modules/loaders.rs +++ b/core/modules/loaders.rs @@ -294,6 +294,8 @@ impl ModuleLoader for FsModuleLoader { // can decide what to do with it. if ext == "json" { ModuleType::Json + } else if ext == "wasm" { + ModuleType::Wasm } else { match &requested_module_type { RequestedModuleType::Other(ty) => ModuleType::Other(ty.clone()), diff --git a/core/modules/map.rs b/core/modules/map.rs index 3f40cff4c..9204d1bd8 100644 --- a/core/modules/map.rs +++ b/core/modules/map.rs @@ -23,8 +23,10 @@ use crate::runtime::JsRealm; use crate::ExtensionFileSource; use crate::FastString; use crate::JsRuntime; +use crate::ModuleCodeBytes; use crate::ModuleLoadResponse; use crate::ModuleSource; +use crate::ModuleSourceCode; use crate::ModuleSpecifier; use anyhow::bail; use anyhow::Context as _; @@ -36,9 +38,11 @@ use futures::task::noop_waker_ref; use futures::task::AtomicWaker; use futures::Future; use futures::StreamExt; +use indexmap::IndexMap; use log::debug; use v8::Function; use v8::PromiseState; +use wasm_dep_analyzer::WasmDeps; use std::cell::Cell; use std::cell::RefCell; @@ -298,7 +302,6 @@ impl ModuleMap { ); return Ok(module_id); } - let module_id = match module_type { ModuleType::JavaScript => { let code = @@ -314,9 +317,12 @@ impl ModuleMap { )? } ModuleType::Wasm => { - return Err(ModuleError::Other(generic_error( - "Importing Wasm modules is currently not supported.", - ))); + let ModuleSourceCode::Bytes(code) = code else { + return Err(ModuleError::Other(generic_error( + "Source code for Wasm module must be provided as bytes", + ))); + }; + self.new_wasm_module(scope, module_url_found, code, dynamic)? } ModuleType::Json => { let code = @@ -598,6 +604,53 @@ impl ModuleMap { Ok(id) } + pub(crate) fn new_wasm_module( + &self, + scope: &mut v8::HandleScope, + name: ModuleName, + source: ModuleCodeBytes, + is_dynamic_import: bool, + ) -> Result { + let bytes = source.as_bytes(); + let wasm_module_analysis = WasmDeps::parse(bytes).map_err(|e| { + let err = Error::from(e); + ModuleError::Other(err) + })?; + + let Some(wasm_module) = v8::WasmModuleObject::compile(scope, bytes) else { + return Err(ModuleError::Other(generic_error(format!( + "Failed to compile WASM module '{}'", + name.as_str() + )))); + }; + let wasm_module_value: v8::Local = wasm_module.into(); + + let js_wasm_module_source = + render_js_wasm_module(name.as_str(), wasm_module_analysis); + + let synthetic_module_type = + ModuleType::Other("$$deno-core-internal-wasm-module".into()); + + let (name1, name2) = name.into_cheap_copy(); + let value = v8::Local::new(scope, wasm_module_value); + let exports = vec![(FastString::StaticAscii("default"), value)]; + let _synthetic_mod_id = self.new_synthetic_module( + scope, + name1, + synthetic_module_type, + exports, + )?; + + self.new_module_from_js_source( + scope, + false, + ModuleType::Wasm, + name2, + js_wasm_module_source.into(), + is_dynamic_import, + ) + } + pub(crate) fn new_json_module( &self, scope: &mut v8::HandleScope, @@ -1617,3 +1670,187 @@ pub fn module_origin<'a>( true, ) } + +fn render_js_wasm_module(specifier: &str, wasm_deps: WasmDeps) -> String { + // NOTE(bartlomieju): it's unlikely the generated file will have more lines, + // but it's better to overallocate than to have to mem copy. + let mut src = Vec::with_capacity(512); + + fn aggregate_wasm_module_imports( + imports: &[wasm_dep_analyzer::Import], + ) -> IndexMap> { + let mut imports_map = IndexMap::default(); + + for import in imports.iter().filter(|i| { + matches!(i.import_type, wasm_dep_analyzer::ImportType::Function(..)) + }) { + let entry = imports_map + .entry(import.module.to_string()) + .or_insert(vec![]); + entry.push(import.name.to_string()); + } + + imports_map + } + + src.push(format!( + r#"import wasmMod from "{}" with {{ type: "$$deno-core-internal-wasm-module" }};"#, + specifier, + )); + + // TODO(bartlomieju): handle imports collisions? + if !wasm_deps.imports.is_empty() { + let aggregated_imports = aggregate_wasm_module_imports(&wasm_deps.imports); + + for (key, value) in aggregated_imports.iter() { + src.push(format!( + r#"import {{ {} }} from "{}";"#, + value.join(", "), + key + )); + } + + src.push("const importsObject = {".to_string()); + + for (key, value) in aggregated_imports.iter() { + src.push(format!(" \"{}\": {{", key).to_string()); + + for el in value { + src.push(format!(" {},", el)); + } + + src.push(" },".to_string()); + } + + src.push("};".to_string()); + + src.push( + "const modInstance = await import.meta.wasmInstantiate(wasmMod, importsObject);".to_string(), + ) + } else { + src.push( + "const modInstance = await import.meta.wasmInstantiate(wasmMod);" + .to_string(), + ) + } + + if !wasm_deps.exports.is_empty() { + for export_desc in wasm_deps.exports.iter().filter(|e| { + matches!(e.export_type, wasm_dep_analyzer::ExportType::Function) + }) { + if export_desc.name == "default" { + src.push(format!( + "export default modInstance.exports.{};", + export_desc.name + )); + } else { + src.push(format!( + "export const {} = modInstance.exports.{};", + export_desc.name, export_desc.name + )); + } + } + } + + src.join("\n") +} + +#[test] +fn test_render_js_wasm_module() { + let deps = WasmDeps { + imports: vec![], + exports: vec![], + }; + let rendered = render_js_wasm_module("./foo.wasm", deps); + pretty_assertions::assert_eq!( + rendered, + r#"import wasmMod from "./foo.wasm" with { type: "$$deno-core-internal-wasm-module" }; +const modInstance = await import.meta.wasmInstantiate(wasmMod);"#, + ); + + let deps = WasmDeps { + imports: vec![ + wasm_dep_analyzer::Import { + name: "foo", + module: "./import.js", + import_type: wasm_dep_analyzer::ImportType::Tag( + wasm_dep_analyzer::TagType { + kind: 1, + type_index: 1, + }, + ), + }, + wasm_dep_analyzer::Import { + name: "bar", + module: "./import.js", + import_type: wasm_dep_analyzer::ImportType::Function(1), + }, + wasm_dep_analyzer::Import { + name: "fizz", + module: "./import.js", + import_type: wasm_dep_analyzer::ImportType::Function(2), + }, + wasm_dep_analyzer::Import { + name: "buzz", + module: "./buzz.js", + import_type: wasm_dep_analyzer::ImportType::Function(3), + }, + ], + exports: vec![ + wasm_dep_analyzer::Export { + name: "export1", + index: 0, + export_type: wasm_dep_analyzer::ExportType::Function, + }, + wasm_dep_analyzer::Export { + name: "export2", + index: 1, + export_type: wasm_dep_analyzer::ExportType::Table, + }, + wasm_dep_analyzer::Export { + name: "export3", + index: 2, + export_type: wasm_dep_analyzer::ExportType::Memory, + }, + wasm_dep_analyzer::Export { + name: "export4", + index: 3, + export_type: wasm_dep_analyzer::ExportType::Global, + }, + wasm_dep_analyzer::Export { + name: "export5", + index: 4, + export_type: wasm_dep_analyzer::ExportType::Tag, + }, + wasm_dep_analyzer::Export { + name: "export6", + index: 5, + export_type: wasm_dep_analyzer::ExportType::Unknown, + }, + wasm_dep_analyzer::Export { + name: "default", + index: 6, + export_type: wasm_dep_analyzer::ExportType::Function, + }, + ], + }; + let rendered = render_js_wasm_module("./foo.wasm", deps); + pretty_assertions::assert_eq!( + rendered, + r#"import wasmMod from "./foo.wasm" with { type: "$$deno-core-internal-wasm-module" }; +import { bar, fizz } from "./import.js"; +import { buzz } from "./buzz.js"; +const importsObject = { + "./import.js": { + bar, + fizz, + }, + "./buzz.js": { + buzz, + }, +}; +const modInstance = await import.meta.wasmInstantiate(wasmMod, importsObject); +export const export1 = modInstance.exports.export1; +export default modInstance.exports.default;"#, + ); +} diff --git a/core/modules/tests.rs b/core/modules/tests.rs index 9429afcfe..a6ed2793c 100644 --- a/core/modules/tests.rs +++ b/core/modules/tests.rs @@ -67,6 +67,7 @@ if (b() != 'b') throw Error(); if (c() != 'c') throw Error(); if (!import.meta.main) throw Error(); if (import.meta.url != 'file:///a.js') throw Error(); +if (import.meta.wasmInstantiate !== undefined) throw Error(); "#; const B_SRC: &str = r#" diff --git a/testing/Cargo.toml b/testing/Cargo.toml index 62b5a99b6..a94e0d694 100644 --- a/testing/Cargo.toml +++ b/testing/Cargo.toml @@ -3,10 +3,10 @@ [package] name = "deno_core_testing" version = "0.0.0" -publish = false authors.workspace = true edition.workspace = true license.workspace = true +publish = false readme = "README.md" repository.workspace = true @@ -14,12 +14,12 @@ repository.workspace = true path = "./lib.rs" [dev-dependencies] +anyhow.workspace = true +deno_ast.workspace = true deno_core.workspace = true deno_core.features = ["unsafe_use_unprotected_platform"] +futures.workspace = true pretty_assertions.workspace = true prettyplease.workspace = true testing_macros.workspace = true -anyhow.workspace = true -deno_ast.workspace = true -futures.workspace = true tokio.workspace = true diff --git a/testing/checkin/runner/mod.rs b/testing/checkin/runner/mod.rs index 03fc2d1df..81730f805 100644 --- a/testing/checkin/runner/mod.rs +++ b/testing/checkin/runner/mod.rs @@ -241,7 +241,7 @@ fn get_test_url(test_dir: &Path, test: &str) -> Result { } } let Some(path) = path else { - bail!("Test file not found"); + bail!("Test file not found: {}.[ts.js.nocompile]", test); }; let path = path.canonicalize()?.to_owned(); let url = Url::from_file_path(path).unwrap().to_string(); diff --git a/testing/checkin/runner/ts_module_loader.rs b/testing/checkin/runner/ts_module_loader.rs index 5d87299ad..1d70d16a9 100644 --- a/testing/checkin/runner/ts_module_loader.rs +++ b/testing/checkin/runner/ts_module_loader.rs @@ -85,6 +85,7 @@ impl ModuleLoader for TypescriptModuleLoader { | MediaType::Dcts | MediaType::Tsx => (ModuleType::JavaScript, true), MediaType::Json => (ModuleType::Json, false), + MediaType::Wasm => (ModuleType::Wasm, false), _ => { if path.extension().unwrap_or_default() == "nocompile" { (ModuleType::JavaScript, false) @@ -94,8 +95,9 @@ impl ModuleLoader for TypescriptModuleLoader { } }; - let code = std::fs::read_to_string(&path)?; let code = if should_transpile { + let code = std::fs::read_to_string(&path)?; + let parsed = deno_ast::parse_module(ParseParams { specifier: module_specifier.to_string(), text_info: SourceTextInfo::from_string(code), @@ -115,15 +117,12 @@ impl ModuleLoader for TypescriptModuleLoader { .0 .borrow_mut() .insert(module_specifier.to_string(), source_map.into_bytes()); - res.text + ModuleSourceCode::String(res.text.into()) } else { - code + let code = std::fs::read(&path)?; + ModuleSourceCode::Bytes(code.into_boxed_slice().into()) }; - Ok(ModuleSource::new( - module_type, - ModuleSourceCode::String(code.into()), - module_specifier, - )) + Ok(ModuleSource::new(module_type, code, module_specifier)) } ModuleLoadResponse::Sync(load(source_maps, module_specifier)) diff --git a/testing/integration/wasm_imports/add.wasm b/testing/integration/wasm_imports/add.wasm new file mode 100644 index 000000000..9d9a4f32e Binary files /dev/null and b/testing/integration/wasm_imports/add.wasm differ diff --git a/testing/integration/wasm_imports/add.wat b/testing/integration/wasm_imports/add.wat new file mode 100644 index 000000000..5d1835dff --- /dev/null +++ b/testing/integration/wasm_imports/add.wat @@ -0,0 +1,8 @@ +(module + (func $add (import "./import_from_wasm.mjs" "add") (param i32) (param i32) (result i32)) + (func (export "exported_add") (result i32) + i32.const 21 + i32.const 21 + call $add + ) +) \ No newline at end of file diff --git a/testing/integration/wasm_imports/import_from_wasm.mjs b/testing/integration/wasm_imports/import_from_wasm.mjs new file mode 100644 index 000000000..f83ee9b4d --- /dev/null +++ b/testing/integration/wasm_imports/import_from_wasm.mjs @@ -0,0 +1,8 @@ +import { sleep } from "./lib.mjs"; +export { add } from "./lib.mjs"; + +console.log("import_inner.js before"); + +await sleep(100); + +console.log("import_inner.js after"); diff --git a/testing/integration/wasm_imports/lib.mjs b/testing/integration/wasm_imports/lib.mjs new file mode 100644 index 000000000..8ff33bc7a --- /dev/null +++ b/testing/integration/wasm_imports/lib.mjs @@ -0,0 +1,21 @@ +console.log("lib.js before"); + +export function sleep(timeout) { + return new Promise((resolve) => { + Deno.core.queueTimer( + Deno.core.getTimerDepth() + 1, + false, + timeout, + resolve, + ); + }); +} +await sleep(100); + +console.log("lib.js after"); + +const abc = 1 + 2; +export function add(a, b) { + console.log(`abc: ${abc}`); + return a + b; +} diff --git a/testing/integration/wasm_imports/wasm_imports.js b/testing/integration/wasm_imports/wasm_imports.js new file mode 100644 index 000000000..660a4b547 --- /dev/null +++ b/testing/integration/wasm_imports/wasm_imports.js @@ -0,0 +1,7 @@ +// Copyright 2018-2023 the Deno authors. All rights reserved. MIT license. +import { exported_add } from "./add.wasm"; + +// To regenerate WASM file use: +// npx -p wabt wat2wasm ./testing/integration/wasm_imports/add.wat -o ./testing/integration/wasm_imports/add.wasm + +console.log(`exported_add: ${exported_add(4, 5)}`); diff --git a/testing/integration/wasm_imports/wasm_imports.out b/testing/integration/wasm_imports/wasm_imports.out new file mode 100644 index 000000000..10e7f6b2c --- /dev/null +++ b/testing/integration/wasm_imports/wasm_imports.out @@ -0,0 +1,6 @@ +lib.js before +lib.js after +import_inner.js before +import_inner.js after +abc: 3 +exported_add: 42 diff --git a/testing/lib.rs b/testing/lib.rs index f0c95f4d6..b66fd6c61 100644 --- a/testing/lib.rs +++ b/testing/lib.rs @@ -60,6 +60,7 @@ integration_test!( timer_ref, timer_ref_and_cancel, timer_many, + wasm_imports, worker_spawn, worker_terminate, worker_terminate_op,