diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 94edff7..189d746 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,16 @@ jobs: run: | ./install-wasi-sdk.sh go install github.com/extism/cli/extism@latest + cd /tmp + # get just wasm-merge and wasm-opt + curl -L https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz > binaryen.tar.gz + tar xvzf binaryen.tar.gz + sudo cp binaryen-version_116/bin/wasm-merge /usr/local/bin + sudo cp binaryen-version_116/bin/wasm-opt /usr/local/bin - name: Test + env: + QUICKJS_WASM_SYS_WASI_SDK_PATH: "${{ github.workspace }}/wasi-sdk" run: | make make test diff --git a/Cargo.toml b/Cargo.toml index 281434e..fceafcc 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -6,10 +6,10 @@ members = [ ] [workspace.package] -version = "1.0.0-rc2" +version = "1.0.0-rc3" edition = "2021" authors = ["The Extism Authors"] license = "BSD-Clause-3" [workspace.dependencies] -anyhow = "1.0.68" +anyhow = "^1.0.68" diff --git a/Makefile b/Makefile index 9f339e6..3051d01 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ install: cargo install --path crates/cli cli: core - cd crates/cli && QUICKJS_WASM_SYS_WASI_SDK_PATH="$(CURDIR)/wasi-sdk/" cargo build --release && cd - + cd crates/cli && cargo build --release && cd - core: cd crates/core \ @@ -16,7 +16,7 @@ core: && npm install \ && npm run build \ && cd ../.. \ - && QUICKJS_WASM_SYS_WASI_SDK_PATH="$(CURDIR)/wasi-sdk/" cargo build --release --target=wasm32-wasi \ + && cargo build --release --target=wasm32-wasi \ && cd - fmt: fmt-core fmt-cli @@ -46,5 +46,5 @@ test: compile-examples @extism call examples/bundled.wasm greet --wasi --input="Benjamin" compile-examples: - ./target/release/extism-js examples/simple_js/script.js -o examples/simple_js.wasm + ./target/release/extism-js examples/simple_js/script.js -i examples/simple_js/script.d.ts -o examples/simple_js.wasm cd examples/bundled && npm install && npm run build && cd ../.. diff --git a/README.md b/README.md index 6dc831c..8348f17 100644 --- a/README.md +++ b/README.md @@ -23,15 +23,19 @@ curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh sh install.sh ``` +> *Note*: [Binaryen](https://github.com/WebAssembly/binaryen), specifcally the wasm-merge tool +> is required as a dependency. We will try to package this up eventually but for now it must be reachable +> on your machine. You can install on mac with `brew install binaryen` or see their [releases page](https://github.com/WebAssembly/binaryen/releases). + Then run command with no args to see the help: ``` extism-js error: The following required arguments were not provided: - + USAGE: - extism-js -o + extism-js -i -o For more information try --help ``` @@ -62,10 +66,21 @@ Some things to note about this code: 2. Currently, you must use [CJS Module syntax](https://nodejs.org/api/modules.html#modules-commonjs-modules) when not using a bundler. So the `export` keyword is not directly supported. See the [Using with a Bundler](#using-with-a-bundler) section for more. 3. In this PDK we code directly to the ABI. We get input from the using using `Host.input*` functions and we return data back with the `Host.output*` functions. + +We must also describe the Wasm interface for our plug-in. We do this with a typescript module DTS file. +Here is our `plugin.d.ts` file: + +```typescript +declare module 'main' { + // Extism exports take no params and return an I32 + export function greet(): I32; +} +``` + Let's compile this to Wasm now using the `extism-js` tool: ```bash -extism-js plugin.js -o plugin.wasm +extism-js plugin.js -i plugin.d.ts -o plugin.wasm ``` We can now test `plugin.wasm` using the [Extism CLI](https://github.com/extism/cli)'s `run` @@ -99,7 +114,7 @@ module.exports = { greet } Now compile and run: ```bash -extism-js plugin.js -o plugin.wasm +extism-js plugin.js -i plugin.d.ts -o plugin.wasm extism call plugin.wasm greet --input="Benjamin" --wasi # => Error: Uncaught Error: Sorry, we don't greet Benjamins! # => at greet (script.js:4) @@ -265,7 +280,7 @@ Add a `build` script to your `package.json`: // ... "scripts": { // ... - "build": "node esbuild.js && extism-js dist/index.js -o dist/plugin.wasm" + "build": "node esbuild.js && extism-js dist/index.js -i src/index.d.ts -o dist/plugin.wasm" }, // ... } @@ -324,7 +339,7 @@ make To test the built compiler (ensure you have Extism installed): ```bash -./target/release/extism-js bundle.js -o out.wasm +./target/release/extism-js bundle.js -i bundle.d.ts -o out.wasm extism call out.wasm count_vowels --wasi --input='Hello World Test!' # => "{\"count\":4}" ``` diff --git a/crates/cli/Cargo.toml b/crates/cli/Cargo.toml index 343be91..d51141b 100644 --- a/crates/cli/Cargo.toml +++ b/crates/cli/Cargo.toml @@ -12,11 +12,14 @@ path = "src/main.rs" [dependencies] anyhow = { workspace = true } -wasm-encoder = "0.20.0" -wasmparser = "0.96.0" -parity-wasm = { version = "^0.45.0", features = ["bulk", "sign_ext"] } wizer = "^3.0.0" structopt = "0.3" binaryen = "0.12.0" -quick-js = "0.4.1" - +swc_atoms = "0.6.5" +swc_common = "0.33.10" +swc_ecma_ast = "0.110.11" +swc_ecma_parser = "0.141.29" +wasm-encoder = "0.38.1" +wasmparser = "0.118.1" +log = "0.4.20" +tempfile = "3.8.1" diff --git a/crates/cli/src/main.rs b/crates/cli/src/main.rs index 544c9fc..db70819 100644 --- a/crates/cli/src/main.rs +++ b/crates/cli/src/main.rs @@ -1,15 +1,18 @@ mod opt; mod options; +mod shim; use crate::options::Options; use anyhow::{bail, Result}; -use quick_js::Context; +use shim::create_shims; use std::env; +use std::fs::remove_dir_all; use std::io::{Read, Write}; -use std::path::Path; +use std::path::PathBuf; use std::process::Stdio; use std::{fs, process::Command}; use structopt::StructOpt; +use tempfile::TempDir; fn main() -> Result<()> { let opts = Options::from_args(); @@ -26,129 +29,51 @@ fn main() -> Result<()> { return Ok(()); } - let mut input_file = fs::File::open(&opts.input)?; + let mut input_file = fs::File::open(&opts.input_js)?; let mut contents: Vec = vec![]; input_file.read_to_end(&mut contents)?; - let self_cmd = env::args().next().unwrap(); + let self_cmd = env::args().next().expect("Expected a command argument"); + let tmp_dir = TempDir::new()?; + let core_path = tmp_dir.path().join("core.wasm"); + let export_shim_path = tmp_dir.path().join("export-shim.wasm"); { env::set_var("EXTISM_WIZEN", "1"); let mut command = Command::new(self_cmd) - .arg(&opts.input) + .arg(&opts.input_js) .arg("-o") - .arg(&opts.output) + .arg(&core_path) .stdin(Stdio::piped()) .spawn()?; - command.stdin.take().unwrap().write_all(&contents)?; + command + .stdin + .take() + .expect("Expected to get writeable stdin") + .write_all(&contents)?; let status = command.wait()?; if !status.success() { bail!("Couldn't create wasm from input"); } } - add_extism_shim_exports(&opts.output, contents)?; - - Ok(()) -} - -fn add_extism_shim_exports>(file: P, contents: Vec) -> Result<()> { - use parity_wasm::elements::*; - - let code = String::from_utf8(contents)?; - - let context = Context::new().unwrap(); - let _ = context.eval("module = {exports: {}}").unwrap(); - let _ = context.eval(&code).unwrap(); - - let global_functions = context - .eval_as::>("Object.keys(module.exports)") - .unwrap(); - - let mut exported_functions: Vec = global_functions - .into_iter() - .filter(|name| name != "module") - .collect(); - exported_functions.sort(); - - let mut module = parity_wasm::deserialize_file(&file)?; - - let invoke_func_idx = if let Some(Internal::Function(idx)) = module - .export_section() - .unwrap() - .entries() - .iter() - .find_map(|e| { - if e.field() == "__invoke" { - Some(e.internal()) - } else { - None - } - }) { - idx - } else { - bail!("Could not find __invoke function") - }; - - let wrapper_type_idx = module - .type_section() - .unwrap() - .types() - .iter() - .enumerate() - .find_map(|(idx, t)| { - let Type::Function(ft) = t; - // we are looking for the function (type () (result i32)) - // it takes no params and returns an i32. this is the extism call interface - if ft.params() == vec![] && ft.results() == vec![ValueType::I32] { - Some(idx) - } else { - None - } - }); - - // TODO create the type if it doesn't exist - let wrapper_type_idx = wrapper_type_idx.unwrap(); - - let mut function_bodies = vec![]; - - for (func_id, _export_name) in exported_functions.iter().enumerate() { - function_bodies.push(FuncBody::new( - vec![], - Instructions::new(vec![ - Instruction::I32Const(func_id as i32), - Instruction::Call(*invoke_func_idx), - Instruction::End, - ]), - )); - } - - for (idx, f) in function_bodies.into_iter().enumerate() { - // put the code body in the code section - let bodies = module.code_section_mut().unwrap().bodies_mut(); - bodies.push(f); - - // put the function type in the function section table - let func = Func::new(wrapper_type_idx as u32); - module - .function_section_mut() - .unwrap() - .entries_mut() - .push(func); - - //get the index of the function we just made - let max_func_index = module.functions_space() - 1; - - // put the function in the exports table - let export_section = module.export_section_mut().unwrap(); - let entry = ExportEntry::new( - exported_functions.get(idx).unwrap().to_string(), - Internal::Function(max_func_index as u32), - ); - export_section.entries_mut().push(entry); + let interface_path = PathBuf::from(&opts.interface_file); + create_shims(&interface_path, &export_shim_path)?; + + let mut command = Command::new("wasm-merge") + .arg(&core_path) + .arg("coremod") + .arg(&export_shim_path) + .arg("codemod") + .arg("-o") + .arg(&opts.output) + .spawn()?; + let status = command.wait()?; + if !status.success() { + bail!("Couldn't run wasm-merge"); } - parity_wasm::serialize_to_file(&file, module)?; + remove_dir_all(tmp_dir)?; Ok(()) } diff --git a/crates/cli/src/options.rs b/crates/cli/src/options.rs index b772b0a..15b8532 100644 --- a/crates/cli/src/options.rs +++ b/crates/cli/src/options.rs @@ -5,7 +5,10 @@ use structopt::StructOpt; #[structopt(name = "extism-js", about = "Extism JavaScript PDK Plugin Compiler")] pub struct Options { #[structopt(parse(from_os_str))] - pub input: PathBuf, + pub input_js: PathBuf, + + #[structopt(short = "i", parse(from_os_str), default_value = "index.d.ts")] + pub interface_file: PathBuf, #[structopt(short = "o", parse(from_os_str), default_value = "index.wasm")] pub output: PathBuf, diff --git a/crates/cli/src/shim.rs b/crates/cli/src/shim.rs new file mode 100644 index 0000000..e9398ff --- /dev/null +++ b/crates/cli/src/shim.rs @@ -0,0 +1,228 @@ +extern crate swc_common; +extern crate swc_ecma_parser; +use anyhow::{bail, Context, Result}; +use std::fs::File; +use std::io::prelude::*; +use std::path::PathBuf; + +use swc_common::sync::Lrc; +use swc_common::SourceMap; +use swc_ecma_ast::ModuleItem; +use swc_ecma_ast::{Decl, Module, ModuleDecl, Stmt, TsModuleDecl}; +use swc_ecma_parser::{lexer::Lexer, Parser, StringInput, Syntax}; + +use wasm_encoder::{ + CodeSection, EntityType, ExportKind, ExportSection, Function, FunctionSection, Instruction, + TypeSection, ValType, +}; +use wasm_encoder::{ImportSection, Module as WasmModule}; + +#[derive(Debug, Clone)] +struct Param { + pub name: String, + pub ptype: String, +} + +#[derive(Debug, Clone)] +struct Signature { + pub name: String, + pub params: Vec, + pub results: Vec, +} + +#[derive(Debug, Clone)] +struct Interface { + pub name: String, + pub functions: Vec, +} + +fn parse_module_decl(tsmod: &Box) -> Result { + let mut signatures = Vec::new(); + + for block in &tsmod.body { + if let Some(block) = block.as_ts_module_block() { + for decl in &block.body { + if let ModuleItem::ModuleDecl(ModuleDecl::ExportDecl(e)) = decl { + if let Some(fndecl) = e.decl.as_fn_decl() { + let name = fndecl.ident.sym.as_str().to_string(); + let params = vec![]; // TODO ignoring params for now + let return_type = &fndecl + .function + .clone() + .return_type + .context("Missing return type")? + .clone(); + let return_type = &return_type + .type_ann + .as_ts_type_ref() + .context("Illegal return type")? + .type_name + .as_ident() + .context("Illegal return type")? + .sym; + let results = vec![Param { + name: "result".to_string(), + ptype: return_type.to_string(), + }]; + + if !params.is_empty() { + bail!("An Extism export should take no params and return I32") + } + if results.len() != 1 { + bail!("An Extism export should return an I32") + } + let return_type = &results.get(0).unwrap().ptype; + if return_type != "I32" { + bail!("An Extism export should return an I32 not {}", return_type) + } + + let signature = Signature { + name, + params, + results, + }; + + signatures.push(signature); + } + } else { + bail!("Don't know what to do with non export on main module"); + } + } + } + } + + Ok(Interface { + name: "main".to_string(), + functions: signatures, + }) +} + +fn parse_module(module: Module) -> Result> { + let mut interfaces = Vec::new(); + for statement in &module.body { + if let ModuleItem::Stmt(Stmt::Decl(Decl::TsModule(submod))) = statement { + let name = if let Some(name) = submod.id.as_str() { + Some(name.value.as_str()) + } else { + None + }; + + if let Some("main") = name { + interfaces.push(parse_module_decl(submod)?); + } else { + bail!("Could not parse module with name {:#?}", name); + } + } + } + + Ok(interfaces) +} + +/// Generates the wasm shim for the exports +fn generate_export_wasm_shim(exports: &Interface, export_path: &PathBuf) -> Result<()> { + let mut wasm_mod = WasmModule::new(); + + // Note: the order in which you set the sections + // with `wasm_mod.section()` is important + + // Encode the type section. + let mut types = TypeSection::new(); + // __invoke's type + let params = vec![ValType::I32]; + let results = vec![ValType::I32]; + types.function(params, results); + // Extism Export type + let params = vec![]; + let results = vec![ValType::I32]; + types.function(params, results); + wasm_mod.section(&types); + + //Encode the import section + let mut import_sec = ImportSection::new(); + import_sec.import("coremod", "__invoke", EntityType::Function(0)); + wasm_mod.section(&import_sec); + + // Encode the function section. + let mut functions = FunctionSection::new(); + + // we will have 1 thunk function per export + let type_index = 1; // these are exports () -> i32 + for _ in exports.functions.iter() { + functions.function(type_index); + } + wasm_mod.section(&functions); + + let mut func_index = 1; + + // Encode the export section. + let mut export_sec = ExportSection::new(); + // we need to sort them alphabetically because that is + // how the runtime maps indexes + let mut export_functions = exports.functions.clone(); + export_functions.sort_by(|a, b| a.name.cmp(&b.name)); + for i in export_functions.iter() { + export_sec.export(i.name.as_str(), ExportKind::Func, func_index); + func_index += 1; + } + wasm_mod.section(&export_sec); + + // Encode the code section. + let mut codes = CodeSection::new(); + let mut export_idx: i32 = 0; + + // create a single thunk per export + for _ in exports.functions.iter() { + let locals = vec![]; + let mut f = Function::new(locals); + // we will essentially call the eval function (__invoke) + f.instruction(&Instruction::I32Const(export_idx)); + f.instruction(&Instruction::Call(0)); + f.instruction(&Instruction::End); + codes.function(&f); + export_idx += 1; + } + wasm_mod.section(&codes); + + // Extract the encoded Wasm bytes for this module. + let wasm_bytes = wasm_mod.finish(); + let mut file = File::create(export_path)?; + file.write_all(wasm_bytes.as_ref())?; + Ok(()) +} + +pub fn create_shims(interface_path: &PathBuf, export_path: &PathBuf) -> Result<()> { + let cm: Lrc = Default::default(); + if !interface_path.exists() { + bail!( + "Could not find interface file {}. Set to a valid d.ts file with the -i flag", + &interface_path.to_str().unwrap() + ); + } + let fm = cm.load_file(&interface_path)?; + let lexer = Lexer::new( + Syntax::Typescript(Default::default()), + Default::default(), + StringInput::from(&*fm), + None, + ); + + let mut parser = Parser::new_from(lexer); + let parse_errs = parser.take_errors(); + if !parse_errs.is_empty() { + for e in parse_errs { + log::warn!("{:#?}", e); + } + bail!("Failed to parse typescript interface file."); + } + + let module = parser.parse_module().expect("failed to parser module"); + let interfaces = parse_module(module)?; + let exports = interfaces + .iter() + .find(|i| i.name == "main") + .context("You need to declare a 'main' module")?; + + generate_export_wasm_shim(exports, export_path)?; + + Ok(()) +} diff --git a/examples/bundled/package-lock.json b/examples/bundled/package-lock.json index b41e559..ef1b4b6 100644 --- a/examples/bundled/package-lock.json +++ b/examples/bundled/package-lock.json @@ -1,11 +1,10 @@ { - "name": "bundled-plugin", + "name": "bundled", "version": "1.0.0", "lockfileVersion": 3, "requires": true, "packages": { "": { - "name": "bundled-plugin", "version": "1.0.0", "license": "BSD-3-Clause", "devDependencies": { diff --git a/examples/bundled/package.json b/examples/bundled/package.json index b1baa92..8d678ed 100644 --- a/examples/bundled/package.json +++ b/examples/bundled/package.json @@ -4,7 +4,7 @@ "description": "", "main": "src/index.ts", "scripts": { - "build": "node esbuild.js && ../../target/release/extism-js dist/index.js -o ../bundled.wasm" + "build": "node esbuild.js && ../../target/release/extism-js dist/index.js -i src/index.d.ts -o ../bundled.wasm" }, "keywords": [], "author": "", diff --git a/examples/bundled/src/index.d.ts b/examples/bundled/src/index.d.ts new file mode 100644 index 0000000..729c147 --- /dev/null +++ b/examples/bundled/src/index.d.ts @@ -0,0 +1,3 @@ +declare module 'main' { + export function greet(): I32; +} diff --git a/examples/bundled/src/index.ts b/examples/bundled/src/index.ts index f0b4642..5beff83 100644 --- a/examples/bundled/src/index.ts +++ b/examples/bundled/src/index.ts @@ -1,3 +1,5 @@ export function greet() { - Host.outputString(`Hello, ${Host.inputString()}`) + let extra = new TextEncoder().encode("aaa") + let decoded = new TextDecoder().decode(extra) + Host.outputString(`Hello, ${Host.inputString()} ${decoded}`) } diff --git a/examples/simple_js/script.d.ts b/examples/simple_js/script.d.ts new file mode 100644 index 0000000..729c147 --- /dev/null +++ b/examples/simple_js/script.d.ts @@ -0,0 +1,3 @@ +declare module 'main' { + export function greet(): I32; +} diff --git a/install.sh b/install.sh index 695e378..69bdd44 100755 --- a/install.sh +++ b/install.sh @@ -15,7 +15,7 @@ case "$ARCH" in esac -export TAG="v1.0.0-rc2" +export TAG="v1.0.0-rc3" curl -L -O "https://github.com/extism/js-pdk/releases/download/$TAG/extism-js-$ARCH-$OS-$TAG.gz" gunzip extism-js*.gz sudo mv extism-js-* /usr/local/bin/extism-js