diff --git a/Cargo.lock b/Cargo.lock index 5f78cc0..e3ebe3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1940,6 +1940,7 @@ dependencies = [ "wasmtime", "wasmtime-wasi", "wit-component", + "wit-parser 0.212.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bea6b80..664c5b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ wasm-compose = "0.215.0" wasm-metadata = "0.212.0" wasm-opt = { version = "0.116.1", optional = true } wit-component = "0.212.0" +wit-parser = "0.212.0" [build-dependencies] anyhow = "1" diff --git a/README.md b/README.md index c3b38b0..49d0a3c 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,11 @@ fn main() { ("host.txt", FsEntry::RuntimeFile("/runtime/host/path.txt")) ]))); + // compose the virtualization with a component + virt.compose("path/to/component.wasm"); + // filter enabled subsystems by imports on the composed component + virt.filter_imports().unwrap(); + let virt_component_bytes = virt.finish().unwrap(); fs::write("virt.component.wasm", virt_component_bytes).unwrap(); } @@ -195,6 +200,8 @@ By default, when using the `wasi-virt` CLI command, all virtualizations are enab Selective subsystem virtualization can be performed directly with the WASI Virt library as above (which does not virtualize all subsystems by default). This allows virtualizing just a single subsystem like `env`, where it is possible to virtualize only that subsystem and skip other virtualizations and end up creating a smaller virtualization component. +When directly composing another component, individual subsystems are disabled if the composed component does not import that subsystem. This behavior reduces imports in the output component that are never actually used. + There is an important caveat to this: _as soon as any subsystem uses IO, all subsystems using IO need to be virtualized in order to fully subclass streams and polls in the virtualization layer_. In future this caveat requirement should weaken as these features lower into the core ABI in subsequent WASI versions. `wasm-tools compose` will error in these scenarios, and better [error messaging](https://github.com/bytecodealliance/wasm-tools/issues/1147) may be provided in future when performing invalid compositions like the above. Missing subsystems can be usually detected from the composition warnings list. diff --git a/src/bin/wasi-virt.rs b/src/bin/wasi-virt.rs index 256749e..e078966 100644 --- a/src/bin/wasi-virt.rs +++ b/src/bin/wasi-virt.rs @@ -1,8 +1,7 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use clap::{ArgAction, Parser}; -use std::{env, error::Error, fs, path::PathBuf, time::SystemTime}; +use std::{error::Error, fs, path::PathBuf}; use wasi_virt::{StdioCfg, WasiVirt}; -use wasm_compose::composer::ComponentComposer; #[derive(Parser, Debug)] #[command(verbatim_doc_comment, author, version, about, long_about = None)] @@ -104,13 +103,6 @@ where Ok((s[..pos].parse()?, s[pos + 1..].parse()?)) } -fn timestamp() -> u64 { - match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) { - Ok(n) => n.as_secs(), - Err(_) => panic!(), - } -} - fn main() -> Result<()> { let args = Args::parse(); @@ -189,37 +181,14 @@ fn main() -> Result<()> { fs.allow_host_preopens(); } + if let Some(compose) = args.compose { + virt_opts.compose(compose); + virt_opts.filter_imports()?; + } + let virt_component = virt_opts.finish()?; let out_path = PathBuf::from(args.out); - - let out_bytes = if let Some(compose_path) = args.compose { - let compose_path = PathBuf::from(compose_path); - let dir = env::temp_dir(); - let tmp_virt = dir.join(format!("virt{}.wasm", timestamp())); - fs::write(&tmp_virt, virt_component.adapter)?; - - let composed_bytes = ComponentComposer::new( - &compose_path, - &wasm_compose::config::Config { - definitions: vec![tmp_virt.clone()], - ..Default::default() - }, - ) - .compose() - .with_context(|| "Unable to compose virtualized adapter into component.\nMake sure virtualizations are enabled and being used.") - .or_else(|e| { - fs::remove_file(&tmp_virt)?; - Err(e) - })?; - - fs::remove_file(&tmp_virt)?; - - composed_bytes - } else { - virt_component.adapter - }; - if virt_component.virtual_files.len() > 0 { println!("Virtualized files from local filesystem:\n"); for (virtual_path, original_path) in virt_component.virtual_files { @@ -227,7 +196,7 @@ fn main() -> Result<()> { } } - fs::write(&out_path, out_bytes)?; + fs::write(&out_path, virt_component.adapter)?; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index ce83938..3b4df17 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,14 +1,16 @@ -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use serde::Deserialize; -use std::env; +use std::{env, fs, path::PathBuf, time::SystemTime}; use virt_deny::{ deny_clocks_virt, deny_exit_virt, deny_http_virt, deny_random_virt, deny_sockets_virt, }; use virt_env::{create_env_virt, strip_env_virt}; use virt_io::{create_io_virt, VirtStdio}; use walrus_ops::strip_virt; +use wasm_compose::composer::ComponentComposer; use wasm_metadata::Producers; use wit_component::{metadata, ComponentEncoder, DecodedWasm, StringEncoding}; +use wit_parser::WorldItem; mod data; mod stub_preview1; @@ -58,6 +60,8 @@ pub struct WasiVirt { pub random: Option, /// Disable wasm-opt run if desired pub wasm_opt: Option, + /// Path to compose component + pub compose: Option, } pub struct VirtResult { @@ -128,6 +132,67 @@ impl WasiVirt { self.wasm_opt = Some(opt); } + pub fn compose(&mut self, compose: String) { + self.compose = Some(compose); + } + + /// drop capabilities that are not imported by the composed component + pub fn filter_imports(&mut self) -> Result<()> { + match &self.compose { + Some(compose) => { + let imports = { + let module_bytes = fs::read(compose).map_err(anyhow::Error::new)?; + let (resolve, world_id) = match wit_component::decode(&module_bytes)? { + DecodedWasm::WitPackages(..) => { + bail!("expected a component, found a WIT package") + } + DecodedWasm::Component(resolve, world_id) => (resolve, world_id), + }; + + let mut import_ids: Vec = vec![]; + for (_, import) in &resolve.worlds[world_id].imports { + if let WorldItem::Interface { id, .. } = import { + if let Some(id) = resolve.id_of(*id) { + import_ids.push(id); + } + } + } + + import_ids + }; + let matches = |prefix: &str| imports.iter().any(|i| i.starts_with(prefix)); + + if !matches("wasi:cli/environment") { + self.env = None; + } + if !matches("wasi:filesystem/") { + self.fs = None; + } + if !(matches("wasi:cli/std") || matches("wasi:cli/terminal")) { + self.stdio = None; + } + if !matches("wasi:cli/exit") { + self.exit = None; + } + if !matches("wasi:clocks/") { + self.clocks = None; + } + if !matches("wasi:http/") { + self.http = None; + } + if !matches("wasi:sockets/") { + self.sockets = None; + } + if !matches("wasi:random/") { + self.random = None; + } + + Ok(()) + } + None => bail!("filtering imports can only be applied to composed components"), + } + } + pub fn finish(&mut self) -> Result { let mut config = walrus::ModuleConfig::new(); config.generate_name_section(self.debug); @@ -299,10 +364,37 @@ impl WasiVirt { // now adapt the virtualized component let encoder = ComponentEncoder::default().validate(true).module(&bytes)?; - let encoded = encoder.encode()?; + let encoded_bytes = encoder.encode()?; + + let adapter = if let Some(compose_path) = &self.compose { + let compose_path = PathBuf::from(compose_path); + let dir = env::temp_dir(); + let tmp_virt = dir.join(format!("virt{}.wasm", timestamp())); + fs::write(&tmp_virt, encoded_bytes)?; + + let composed_bytes = ComponentComposer::new( + &compose_path, + &wasm_compose::config::Config { + definitions: vec![tmp_virt.clone()], + ..Default::default() + }, + ) + .compose() + .with_context(|| "Unable to compose virtualized adapter into component.\nMake sure virtualizations are enabled and being used.") + .or_else(|e| { + fs::remove_file(&tmp_virt)?; + Err(e) + })?; + + fs::remove_file(&tmp_virt)?; + + composed_bytes + } else { + encoded_bytes + }; Ok(VirtResult { - adapter: encoded, + adapter, virtual_files, }) } @@ -347,3 +439,10 @@ fn apply_wasm_opt(bytes: Vec, debug: bool) -> Result> { Ok(bytes) } } + +fn timestamp() -> u64 { + match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) { + Ok(n) => n.as_secs(), + Err(_) => panic!(), + } +} diff --git a/tests/cases/encapsulate-component.toml b/tests/cases/encapsulate-component.toml new file mode 100644 index 0000000..99c50e4 --- /dev/null +++ b/tests/cases/encapsulate-component.toml @@ -0,0 +1,18 @@ +component = "do-everything" +compose = true + +[virt-opts] +clocks = true +# http will be filtered out because the "do-everything" component doesn't import it +http = true +stdio.stdin = "ignore" +stdio.stdout = "ignore" +stdio.stderr = "ignore" + +[expect.imports] +required = [ + "wasi:clocks/wall-clock", +] +disallowed = [ + "wasi:http/incoming-handler", +] diff --git a/tests/virt.rs b/tests/virt.rs index 1598514..7444b77 100644 --- a/tests/virt.rs +++ b/tests/virt.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use heck::ToSnakeCase; use serde::Deserialize; use std::collections::BTreeMap; @@ -13,7 +13,8 @@ use wasmtime::{ Config, Engine, Store, WasmBacktraceDetails, }; use wasmtime_wasi::{DirPerms, FilePerms, WasiCtx, WasiCtxBuilder, WasiView}; -use wit_component::ComponentEncoder; +use wit_component::{ComponentEncoder, DecodedWasm}; +use wit_parser::WorldItem; wasmtime::component::bindgen!({ world: "virt-test", @@ -49,12 +50,21 @@ struct TestExpectation { file_read: Option, encapsulation: Option, stdout: Option, + imports: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct TestExpectationImports { + required: Option>, + disallowed: Option>, } #[derive(Deserialize, Debug)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] struct TestCase { component: String, + compose: Option, host_env: Option>, host_fs_path: Option, virt_opts: Option, @@ -136,6 +146,17 @@ async fn virt_test() -> Result<()> { virt_opts.wasm_opt = Some(false); } } + if let Some(compose) = test.compose { + if compose { + let compose_path = generated_component_path + .clone() + .into_os_string() + .into_string() + .unwrap(); + virt_opts.compose = Some(compose_path); + virt_opts.filter_imports()?; + } + } let virt_component = virt_opts.finish().with_context(|| { format!( @@ -156,24 +177,32 @@ async fn virt_test() -> Result<()> { } } - // compose the test component with the defined test virtualization - if DEBUG { - println!("- Composing virtualization"); - } - let component_bytes = ComponentComposer::new( - &generated_component_path, - &wasm_compose::config::Config { - definitions: vec![virt_component_path], - ..Default::default() - }, - ) - .compose()?; - - if true { - let mut composed_path = generated_path.join(test_case_name); - composed_path.set_extension("composed.wasm"); - fs::write(composed_path, &component_bytes)?; - } + let mut composed_path = generated_path.join(test_case_name); + composed_path.set_extension("composed.wasm"); + let component_bytes = match test.compose { + Some(true) => { + // adapter is already composed + virt_component.adapter + } + _ => { + // compose the test component with the defined test virtualization + if DEBUG { + println!("- Composing virtualization"); + } + let component_bytes = ComponentComposer::new( + &generated_component_path, + &wasm_compose::config::Config { + definitions: vec![virt_component_path], + ..Default::default() + }, + ) + .compose()?; + + fs::write(&composed_path, &component_bytes)?; + + component_bytes + } + }; // execute the composed virtualized component test function if DEBUG { @@ -267,6 +296,37 @@ async fn virt_test() -> Result<()> { instance.call_test_stdio(&mut store).await?; } + if let Some(expect_imports) = &test.expect.imports { + let component_imports = collect_component_imports(component_bytes)?; + + if let Some(required_imports) = &expect_imports.required { + for required_import in required_imports { + if !component_imports + .iter() + .any(|i| i.starts_with(required_import)) + { + return Err(anyhow!( + "Required import missing {required_import} {:?}", + test_case_path + )); + } + } + } + if let Some(disallowed_imports) = &expect_imports.disallowed { + for disallowed_import in disallowed_imports { + if component_imports + .iter() + .any(|i| i.starts_with(disallowed_import)) + { + return Err(anyhow!( + "Disallowed import {disallowed_import} {:?}", + test_case_path + )); + } + } + } + } + println!("\x1b[1;32m√\x1b[0m {:?}", test_case_path); } Ok(()) @@ -316,3 +376,23 @@ fn has_component_import(bytes: &[u8]) -> Result> { } } } + +fn collect_component_imports(component_bytes: Vec) -> Result> { + let (resolve, world_id) = match wit_component::decode(&component_bytes)? { + DecodedWasm::WitPackages(..) => { + bail!("expected a component, found a WIT package") + } + DecodedWasm::Component(resolve, world_id) => (resolve, world_id), + }; + + let mut import_ids: Vec = vec![]; + for (_, import) in &resolve.worlds[world_id].imports { + if let WorldItem::Interface { id, .. } = import { + if let Some(id) = resolve.id_of(*id) { + import_ids.push(id); + } + } + } + + Ok(import_ids) +}