refactor: allow more flexibility in the types of imports/exports

.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://docs.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "cargo" # See documentation for possible values
+    directory: "/" # Location of package manifests
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
       - uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: 1.73.0
+          toolchain: stable
           target: wasm32-wasi
           default: true
 
@@ -33,6 +33,7 @@ jobs:
           curl -L https://github.com/WebAssembly/binaryen/releases/download/version_116/binaryen-version_116-x86_64-linux.tar.gz > binaryen.tar.gz
           tar xvzf binaryen.tar.gz
           sudo cp binaryen-version_116/bin/wasm-merge /usr/local/bin
+          sudo cp binaryen-version_116/bin/wasm-opt /usr/local/bin
 
       - name: Run Tests
         env:

.github/workflows/publish.yml

@@ -17,7 +17,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: 1.73.0
+          toolchain: stable
           target: wasm32-wasi
           default: true
 
@@ -86,7 +86,7 @@ jobs:
         uses: actions-rs/toolchain@v1
         with:
           profile: default
-          toolchain: 1.73.0
+          toolchain: stable
           target: ${{ matrix.target }}
           default: true
 

Makefile

@@ -44,10 +44,14 @@ clean-wasi-sdk:
 test: compile-examples
 		@extism call examples/simple_js.wasm greet --wasi --input="Benjamin"
 		@extism call examples/bundled.wasm greet --wasi --input="Benjamin"
-		@pip install -r examples/host_funcs/requirements.txt
-		@python examples/host_funcs/host.py examples/host_funcs.wasm
+		@python3 -m venv ./.venv && \
+			. ./.venv/bin/activate && \
+			pip install -r examples/host_funcs/requirements.txt && \
+			python3 examples/host_funcs/host.py examples/host_funcs.wasm && \
+			deactivate
 
-compile-examples:
+compile-examples: cli
 		./target/release/extism-js examples/simple_js/script.js -i examples/simple_js/script.d.ts -o examples/simple_js.wasm
 		cd examples/bundled && npm install && npm run build && cd ../..
 		./target/release/extism-js examples/host_funcs/script.js -i examples/host_funcs/script.d.ts -o examples/host_funcs.wasm
+		./target/release/extism-js examples/exports/script.js -i examples/exports/script.d.ts -o examples/exports.wasm

README.md

@@ -23,8 +23,8 @@ curl -O https://raw.githubusercontent.com/extism/js-pdk/main/install.sh
 sh install.sh
 ```
 
-> *Note*: [Binaryen](https://github.com/WebAssembly/binaryen), specifcally the wasm-merge tool
-> is required as a dependency. We will try to package this up eventually but for now it must be reachable
+> *Note*: [Binaryen](https://github.com/WebAssembly/binaryen), specifcally the `wasm-merge` and `wasm-opt` tools
+> are required as a dependency. We will try to package this up eventually but for now it must be reachable
 > on your machine. You can install on mac with `brew install binaryen` or see their [releases page](https://github.com/WebAssembly/binaryen/releases).
 
 Then run command with no args to see the help:

crates/cli/Cargo.toml

@@ -12,15 +12,13 @@ path = "src/main.rs"
 
 [dependencies]
 anyhow = { workspace = true }
-wizer = "^3.0.0"
+wizer = "4.0.0"
 structopt = "0.3"
-binaryen = "0.12.0"
 swc_atoms = "0.6.5"
 swc_common = "0.33.10"
-swc_ecma_ast = "0.110.11"
-swc_ecma_parser = "0.141.29"
-wasm-encoder = "0.38.1"
-wasmparser = "0.118.1"
+swc_ecma_ast = "0.112"
+swc_ecma_parser = "0.143"
+wagen = "0.1"
 log = "0.4.20"
 tempfile = "3.8.1"
-env_logger = "^0.10"
+env_logger = "0.11"

crates/cli/src/main.rs

@@ -6,35 +6,29 @@ mod ts_parser;
 use crate::options::Options;
 use crate::ts_parser::parse_interface_file;
 use anyhow::{bail, Result};
-use env_logger::{Builder, Target};
 use log::LevelFilter;
 use shims::generate_wasm_shims;
 use std::env;
-use std::io::{Read, Write};
 use std::path::PathBuf;
 use std::process::Stdio;
-use std::{fs, process::Command};
+use std::{fs, io::Write, process::Command};
 use structopt::StructOpt;
 use tempfile::TempDir;
 
+const CORE: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/engine.wasm"));
+
 fn main() -> Result<()> {
-    let mut builder = Builder::new();
+    let mut builder = env_logger::Builder::new();
     builder
         .filter(None, LevelFilter::Info)
-        .target(Target::Stdout)
+        .target(env_logger::Target::Stdout)
         .init();
 
     let opts = Options::from_args();
-    let wizen = env::var("EXTISM_WIZEN");
-
-    if wizen.eq(&Ok("1".into())) {
-        let wasm: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/engine.wasm"));
-        opt::Optimizer::new(wasm)
-            .optimize(true)
+    if opts.core {
+        opt::Optimizer::new(CORE)
+            .wizen(true)
             .write_optimized_wasm(opts.output)?;
-
-        env::remove_var("EXTISM_WIZEN");
-
         return Ok(());
     }
 
@@ -48,43 +42,37 @@ fn main() -> Result<()> {
     }
     let plugin_interface = parse_interface_file(&interface_path)?;
 
-    // Copy in the user's js code from stdin
-    let mut input_file = fs::File::open(&opts.input_js)?;
-    let mut user_code: Vec<u8> = vec![];
-    input_file.read_to_end(&mut user_code)?;
+    // Copy in the user's js code from the configured file
+    let mut user_code = fs::read(&opts.input_js)?;
 
     // If we have imports, we need to inject some state needed for host function support
-    let names = &plugin_interface
-        .imports
-        .functions
-        .iter()
-        .map(|s| format!("'{}'", &s.name))
-        .collect::<Vec<String>>()
-        .join(",");
-    let mut contents = format!("Host.__hostFunctions = [{}].sort();\n", names)
-        .as_bytes()
-        .to_owned();
+    let mut contents = Vec::new();
+    let mut names = Vec::new();
+    let mut sorted_names = Vec::new();
+    for ns in &plugin_interface.imports {
+        sorted_names.extend(ns.functions.iter().map(|s| (&s.name, s.results.len())));
+    }
+    sorted_names.sort_by_key(|x| x.0.as_str());
+
+    for (name, results) in sorted_names {
+        names.push(format!("{{ name: '{}', results: {} }}", &name, results));
+    }
+
+    contents
+        .extend_from_slice(format!("Host.__hostFunctions = [{}];\n", names.join(", ")).as_bytes());
     contents.append(&mut user_code);
 
     // Create a tmp dir to hold all the library objects
     // This can go away once we do all the wasm-merge stuff in process
     let tmp_dir = TempDir::new()?;
     let core_path = tmp_dir.path().join("core.wasm");
-    let export_shim_path = tmp_dir.path().join("export-shim.wasm");
-    let import_shim_path = tmp_dir.path().join("import-shim.wasm");
-    let linked_shim_path = tmp_dir.path().join("linked.wasm");
-
-    // let tmp_dir = "/tmp/derp";
-    // let core_path = PathBuf::from("/tmp/derp/core.wasm");
-    // let export_shim_path = PathBuf::from("/tmp/derp/export-shim.wasm");
-    // let import_shim_path = PathBuf::from("/tmp/derp/import-shim.wasm");
-    // let linked_shim_path = PathBuf::from("/tmp/derp/linked.wasm");
+    let shim_path = tmp_dir.path().join("shim.wasm");
 
     // First wizen the core module
     let self_cmd = env::args().next().expect("Expected a command argument");
     {
-        env::set_var("EXTISM_WIZEN", "1");
         let mut command = Command::new(self_cmd)
+            .arg("-c")
             .arg(&opts.input_js)
             .arg("-o")
             .arg(&core_path)
@@ -101,49 +89,38 @@ fn main() -> Result<()> {
         }
     }
 
-    // Create our shim files given our parsed TS module object
-    // We have a shim file for exports and one optional one for imports
+    // Create our shim file given our parsed TS module object
     generate_wasm_shims(
-        plugin_interface.exports,
-        &export_shim_path,
-        plugin_interface.imports,
-        &import_shim_path,
+        &shim_path,
+        &plugin_interface.exports,
+        &plugin_interface.imports,
     )?;
 
-    let output = Command::new("wasm-merge").arg("--version").output();
-    if let Err(_) = output {
-        bail!("Failed to execute wasm-merge. Please install binaryen and make sure wasm-merge is on your path: https://github.com/WebAssembly/binaryen");
+    let output = Command::new("wasm-merge")
+        .arg("--version")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status();
+    if output.is_err() {
+        bail!("Failed to detect wasm-merge. Please install binaryen and make sure wasm-merge is on your path: https://github.com/WebAssembly/binaryen");
     }
 
-    // Merge the export shim with the core module
-    let mut command = Command::new("wasm-merge")
+    // Merge the shim with the core module
+    let status = Command::new("wasm-merge")
         .arg(&core_path)
-        .arg("coremod")
-        .arg(&export_shim_path)
-        .arg("codemod")
-        .arg("-o")
-        .arg(&linked_shim_path)
-        .spawn()?;
-    let status = command.wait()?;
-    if !status.success() {
-        bail!("wasm-merge failed. Couldn't merge export shim");
-    }
-
-    // Merge the import shim with the core+export (linked) module
-    let mut command = Command::new("wasm-merge")
-        .arg(&linked_shim_path)
-        .arg("coremod")
-        .arg(&import_shim_path)
-        .arg("codemod")
+        .arg("core")
+        .arg(&shim_path)
+        .arg("shim")
         .arg("-o")
         .arg(&opts.output)
         .arg("--enable-reference-types")
         .arg("--enable-bulk-memory")
-        .spawn()?;
-    let status = command.wait()?;
+        .status()?;
     if !status.success() {
-        bail!("wasm-merge failed. Couldn't merge import shim.");
+        bail!("wasm-merge failed. Couldn't merge shim");
     }
 
+    opt::optimize_wasm_file(opts.output)?;
+
     Ok(())
 }

crates/cli/src/opt.rs

@@ -1,9 +1,12 @@
-use anyhow::{bail, Error, Result};
-use binaryen::{CodegenConfig, Module};
-use std::path::Path;
+use anyhow::{Error, Result};
+use std::{
+    path::Path,
+    process::{Command, Stdio},
+};
 use wizer::Wizer;
 
 pub(crate) struct Optimizer<'a> {
+    wizen: bool,
     optimize: bool,
     wasm: &'a [u8],
 }
@@ -13,40 +16,56 @@ impl<'a> Optimizer<'a> {
         Self {
             wasm,
             optimize: false,
+            wizen: false,
         }
     }
 
+    #[allow(unused)]
     pub fn optimize(self, optimize: bool) -> Self {
         Self { optimize, ..self }
     }
 
+    pub fn wizen(self, wizen: bool) -> Self {
+        Self { wizen, ..self }
+    }
+
     pub fn write_optimized_wasm(self, dest: impl AsRef<Path>) -> Result<(), Error> {
-        let mut wasm = Wizer::new()
-            .allow_wasi(true)?
-            .inherit_stdio(true)
-            .wasm_bulk_memory(true)
-            .run(self.wasm)?;
+        if self.wizen {
+            let wasm = Wizer::new()
+                .allow_wasi(true)?
+                .inherit_stdio(true)
+                .wasm_bulk_memory(true)
+                .run(self.wasm)?;
+            std::fs::write(&dest, wasm)?;
+        } else {
+            std::fs::write(&dest, self.wasm)?;
+        }
 
         if self.optimize {
-            let codegen_cfg = CodegenConfig {
-                optimization_level: 3, // Aggressively optimize for speed.
-                shrink_level: 0,       // Don't optimize for size at the expense of performance.
-                debug_info: false,
-            };
-
-            if let Ok(mut module) = Module::read(&wasm) {
-                module.optimize(&codegen_cfg);
-                module
-                    .run_optimization_passes(vec!["strip"], &codegen_cfg)
-                    .unwrap();
-                wasm = module.write();
-            } else {
-                bail!("Unable to read wasm binary for wasm-opt optimizations");
-            }
+            optimize_wasm_file(dest)?;
         }
 
-        std::fs::write(dest.as_ref(), wasm)?;
-
         Ok(())
     }
 }
+
+pub(crate) fn optimize_wasm_file(dest: impl AsRef<Path>) -> Result<(), Error> {
+    let output = Command::new("wasm-opt")
+        .arg("--version")
+        .stdout(Stdio::null())
+        .stderr(Stdio::null())
+        .status();
+    if output.is_err() {
+        anyhow::bail!("Failed to detect wasm-opt. Please install binaryen and make sure wasm-opt is on your path: https://github.com/WebAssembly/binaryen");
+    }
+    Command::new("wasm-opt")
+        .arg("--enable-reference-types")
+        .arg("--enable-bulk-memory")
+        .arg("--strip")
+        .arg("-O3")
+        .arg(dest.as_ref())
+        .arg("-o")
+        .arg(dest.as_ref())
+        .status()?;
+    Ok(())
+}

crates/cli/src/options.rs

@@ -12,4 +12,7 @@ pub struct Options {
 
     #[structopt(short = "o", parse(from_os_str), default_value = "index.wasm")]
     pub output: PathBuf,
+
+    #[structopt(short = "c")]
+    pub core: bool,
 }