From 24f00b9cf3352ae17234bc2951d632ced7234885 Mon Sep 17 00:00:00 2001 From: Scott Andrews Date: Mon, 26 Aug 2024 14:18:25 -0400 Subject: [PATCH 1/2] Limit imported interfaces for composed components Components that are directly composed will never use any virtualized interface that it does not already import. Virtualizing additional interfaces limits the compatibility of the resulting component as it places an additional requirement on the host environment to satisfy each import, used or not. We now inspect the composed component to match the interfaces it imports with the virtualized capabilities. If the composed component does not import the respective interface the resulting component will not virtualize that interface. In effect, use of `--allow-all` will behave as if the caller specifically approved the interfaces actually used. The granularity of filtered imports is coarse. Interfaces that support a consumed capability may still appear as imports even if the composed component does not consume them explicitly. Signed-off-by: Scott Andrews --- Cargo.lock | 1 + Cargo.toml | 1 + src/bin/wasi-virt.rs | 3 +- src/lib.rs | 71 +++++++++++++++++++++++- tests/cases/encapsulate-component.toml | 18 +++++++ tests/virt.rs | 75 +++++++++++++++++++++++++- 6 files changed, 164 insertions(+), 5 deletions(-) create mode 100644 tests/cases/encapsulate-component.toml diff --git a/Cargo.lock b/Cargo.lock index 5f78cc0..e3ebe3a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1940,6 +1940,7 @@ dependencies = [ "wasmtime", "wasmtime-wasi", "wit-component", + "wit-parser 0.212.0", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index bea6b80..664c5b4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -35,6 +35,7 @@ wasm-compose = "0.215.0" wasm-metadata = "0.212.0" wasm-opt = { version = "0.116.1", optional = true } wit-component = "0.212.0" +wit-parser = "0.212.0" [build-dependencies] anyhow = "1" diff --git a/src/bin/wasi-virt.rs b/src/bin/wasi-virt.rs index 256749e..78b6c10 100644 --- a/src/bin/wasi-virt.rs +++ b/src/bin/wasi-virt.rs @@ -117,6 +117,7 @@ fn main() -> Result<()> { let mut virt_opts = WasiVirt::default(); virt_opts.debug = args.debug.unwrap_or_default(); + virt_opts.compose = args.compose; // By default, we virtualize all subsystems // This ensures full encapsulation in the default (no argument) case @@ -193,7 +194,7 @@ fn main() -> Result<()> { let out_path = PathBuf::from(args.out); - let out_bytes = if let Some(compose_path) = args.compose { + let out_bytes = if let Some(compose_path) = virt_opts.compose { let compose_path = PathBuf::from(compose_path); let dir = env::temp_dir(); let tmp_virt = dir.join(format!("virt{}.wasm", timestamp())); diff --git a/src/lib.rs b/src/lib.rs index ce83938..5317062 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,6 +1,6 @@ -use anyhow::{Context, Result}; +use anyhow::{bail, Context, Result}; use serde::Deserialize; -use std::env; +use std::{env, fs}; use virt_deny::{ deny_clocks_virt, deny_exit_virt, deny_http_virt, deny_random_virt, deny_sockets_virt, }; @@ -9,6 +9,7 @@ use virt_io::{create_io_virt, VirtStdio}; use walrus_ops::strip_virt; use wasm_metadata::Producers; use wit_component::{metadata, ComponentEncoder, DecodedWasm, StringEncoding}; +use wit_parser::WorldItem; mod data; mod stub_preview1; @@ -58,6 +59,9 @@ pub struct WasiVirt { pub random: Option, /// Disable wasm-opt run if desired pub wasm_opt: Option, + /// Path to compose component + pub compose: Option, + compose_imports: Option>, } pub struct VirtResult { @@ -138,6 +142,38 @@ impl WasiVirt { }?; module.name = Some("wasi_virt".into()); + // drop capabilities that are not imported by the composed component + if self.compose.is_some() { + self.collect_compose_imports()?; + + if !self.contains_compose_import("wasi:cli/environment") { + self.env = None; + } + if !self.contains_compose_import("wasi:filesystem/") { + self.fs = None; + } + if !(self.contains_compose_import("wasi:cli/std") + || self.contains_compose_import("wasi:cli/terminal")) + { + self.stdio = None; + } + if !self.contains_compose_import("wasi:cli/exit") { + self.exit = None; + } + if !self.contains_compose_import("wasi:clocks/") { + self.clocks = None; + } + if !self.contains_compose_import("wasi:http/") { + self.http = None; + } + if !self.contains_compose_import("wasi:sockets/") { + self.sockets = None; + } + if !self.contains_compose_import("wasi:random/") { + self.random = None; + } + } + // only env virtualization is independent of io if let Some(env) = &self.env { create_env_virt(&mut module, env)?; @@ -306,6 +342,37 @@ impl WasiVirt { virtual_files, }) } + + // parse the compose component to collect its imported interfaces + fn collect_compose_imports(&mut self) -> Result<()> { + let module_bytes = fs::read(self.compose.as_ref().unwrap()).map_err(anyhow::Error::new)?; + let (resolve, world_id) = match wit_component::decode(&module_bytes)? { + DecodedWasm::WitPackages(..) => { + bail!("expected a component, found a WIT package") + } + DecodedWasm::Component(resolve, world_id) => (resolve, world_id), + }; + + let mut import_ids: Vec = vec![]; + for (_, import) in &resolve.worlds[world_id].imports { + if let WorldItem::Interface { id, .. } = import { + if let Some(id) = resolve.id_of(*id) { + import_ids.push(id); + } + } + } + + self.compose_imports = Some(import_ids); + + Ok(()) + } + + fn contains_compose_import(&self, prefix: &str) -> bool { + match &self.compose_imports { + Some(imports) => imports.iter().any(|i| i.starts_with(prefix)), + None => false, + } + } } fn apply_wasm_opt(bytes: Vec, debug: bool) -> Result> { diff --git a/tests/cases/encapsulate-component.toml b/tests/cases/encapsulate-component.toml new file mode 100644 index 0000000..99c50e4 --- /dev/null +++ b/tests/cases/encapsulate-component.toml @@ -0,0 +1,18 @@ +component = "do-everything" +compose = true + +[virt-opts] +clocks = true +# http will be filtered out because the "do-everything" component doesn't import it +http = true +stdio.stdin = "ignore" +stdio.stdout = "ignore" +stdio.stderr = "ignore" + +[expect.imports] +required = [ + "wasi:clocks/wall-clock", +] +disallowed = [ + "wasi:http/incoming-handler", +] diff --git a/tests/virt.rs b/tests/virt.rs index 1598514..70bfc03 100644 --- a/tests/virt.rs +++ b/tests/virt.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, Context, Result}; +use anyhow::{anyhow, bail, Context, Result}; use heck::ToSnakeCase; use serde::Deserialize; use std::collections::BTreeMap; @@ -13,7 +13,8 @@ use wasmtime::{ Config, Engine, Store, WasmBacktraceDetails, }; use wasmtime_wasi::{DirPerms, FilePerms, WasiCtx, WasiCtxBuilder, WasiView}; -use wit_component::ComponentEncoder; +use wit_component::{ComponentEncoder, DecodedWasm}; +use wit_parser::WorldItem; wasmtime::component::bindgen!({ world: "virt-test", @@ -49,12 +50,21 @@ struct TestExpectation { file_read: Option, encapsulation: Option, stdout: Option, + imports: Option, +} + +#[derive(Deserialize, Debug)] +#[serde(rename_all = "kebab-case", deny_unknown_fields)] +struct TestExpectationImports { + required: Option>, + disallowed: Option>, } #[derive(Deserialize, Debug)] #[serde(rename_all = "kebab-case", deny_unknown_fields)] struct TestCase { component: String, + compose: Option, host_env: Option>, host_fs_path: Option, virt_opts: Option, @@ -136,6 +146,16 @@ async fn virt_test() -> Result<()> { virt_opts.wasm_opt = Some(false); } } + if let Some(compose) = test.compose { + if compose { + let compose_path = generated_component_path + .clone() + .into_os_string() + .into_string() + .unwrap(); + virt_opts.compose = Some(compose_path); + } + } let virt_component = virt_opts.finish().with_context(|| { format!( @@ -267,6 +287,37 @@ async fn virt_test() -> Result<()> { instance.call_test_stdio(&mut store).await?; } + if let Some(expect_imports) = &test.expect.imports { + let component_imports = collect_component_imports(component_bytes)?; + + if let Some(required_imports) = &expect_imports.required { + for required_import in required_imports { + if !component_imports + .iter() + .any(|i| i.starts_with(required_import)) + { + return Err(anyhow!( + "Required import missing {required_import} {:?}", + test_case_path + )); + } + } + } + if let Some(disallowed_imports) = &expect_imports.disallowed { + for disallowed_import in disallowed_imports { + if component_imports + .iter() + .any(|i| i.starts_with(disallowed_import)) + { + return Err(anyhow!( + "Disallowed import {disallowed_import} {:?}", + test_case_path + )); + } + } + } + } + println!("\x1b[1;32m√\x1b[0m {:?}", test_case_path); } Ok(()) @@ -316,3 +367,23 @@ fn has_component_import(bytes: &[u8]) -> Result> { } } } + +fn collect_component_imports(component_bytes: Vec) -> Result> { + let (resolve, world_id) = match wit_component::decode(&component_bytes)? { + DecodedWasm::WitPackages(..) => { + bail!("expected a component, found a WIT package") + } + DecodedWasm::Component(resolve, world_id) => (resolve, world_id), + }; + + let mut import_ids: Vec = vec![]; + for (_, import) in &resolve.worlds[world_id].imports { + if let WorldItem::Interface { id, .. } = import { + if let Some(id) = resolve.id_of(*id) { + import_ids.push(id); + } + } + } + + Ok(import_ids) +} From 9da26416b2157538e8130a0f13b003b6527e1751 Mon Sep 17 00:00:00 2001 From: Scott Andrews Date: Wed, 28 Aug 2024 14:10:48 -0400 Subject: [PATCH 2/2] Review feedback - move filtering to be a separate method from finalize - perform composition in finalize when compose is requested - update readme Signed-off-by: Scott Andrews --- README.md | 7 ++ src/bin/wasi-virt.rs | 48 +++---------- src/lib.rs | 166 ++++++++++++++++++++++++++----------------- tests/virt.rs | 45 +++++++----- 4 files changed, 141 insertions(+), 125 deletions(-) diff --git a/README.md b/README.md index c3b38b0..49d0a3c 100644 --- a/README.md +++ b/README.md @@ -182,6 +182,11 @@ fn main() { ("host.txt", FsEntry::RuntimeFile("/runtime/host/path.txt")) ]))); + // compose the virtualization with a component + virt.compose("path/to/component.wasm"); + // filter enabled subsystems by imports on the composed component + virt.filter_imports().unwrap(); + let virt_component_bytes = virt.finish().unwrap(); fs::write("virt.component.wasm", virt_component_bytes).unwrap(); } @@ -195,6 +200,8 @@ By default, when using the `wasi-virt` CLI command, all virtualizations are enab Selective subsystem virtualization can be performed directly with the WASI Virt library as above (which does not virtualize all subsystems by default). This allows virtualizing just a single subsystem like `env`, where it is possible to virtualize only that subsystem and skip other virtualizations and end up creating a smaller virtualization component. +When directly composing another component, individual subsystems are disabled if the composed component does not import that subsystem. This behavior reduces imports in the output component that are never actually used. + There is an important caveat to this: _as soon as any subsystem uses IO, all subsystems using IO need to be virtualized in order to fully subclass streams and polls in the virtualization layer_. In future this caveat requirement should weaken as these features lower into the core ABI in subsequent WASI versions. `wasm-tools compose` will error in these scenarios, and better [error messaging](https://github.com/bytecodealliance/wasm-tools/issues/1147) may be provided in future when performing invalid compositions like the above. Missing subsystems can be usually detected from the composition warnings list. diff --git a/src/bin/wasi-virt.rs b/src/bin/wasi-virt.rs index 78b6c10..e078966 100644 --- a/src/bin/wasi-virt.rs +++ b/src/bin/wasi-virt.rs @@ -1,8 +1,7 @@ -use anyhow::{bail, Context, Result}; +use anyhow::{bail, Result}; use clap::{ArgAction, Parser}; -use std::{env, error::Error, fs, path::PathBuf, time::SystemTime}; +use std::{error::Error, fs, path::PathBuf}; use wasi_virt::{StdioCfg, WasiVirt}; -use wasm_compose::composer::ComponentComposer; #[derive(Parser, Debug)] #[command(verbatim_doc_comment, author, version, about, long_about = None)] @@ -104,20 +103,12 @@ where Ok((s[..pos].parse()?, s[pos + 1..].parse()?)) } -fn timestamp() -> u64 { - match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) { - Ok(n) => n.as_secs(), - Err(_) => panic!(), - } -} - fn main() -> Result<()> { let args = Args::parse(); let mut virt_opts = WasiVirt::default(); virt_opts.debug = args.debug.unwrap_or_default(); - virt_opts.compose = args.compose; // By default, we virtualize all subsystems // This ensures full encapsulation in the default (no argument) case @@ -190,37 +181,14 @@ fn main() -> Result<()> { fs.allow_host_preopens(); } + if let Some(compose) = args.compose { + virt_opts.compose(compose); + virt_opts.filter_imports()?; + } + let virt_component = virt_opts.finish()?; let out_path = PathBuf::from(args.out); - - let out_bytes = if let Some(compose_path) = virt_opts.compose { - let compose_path = PathBuf::from(compose_path); - let dir = env::temp_dir(); - let tmp_virt = dir.join(format!("virt{}.wasm", timestamp())); - fs::write(&tmp_virt, virt_component.adapter)?; - - let composed_bytes = ComponentComposer::new( - &compose_path, - &wasm_compose::config::Config { - definitions: vec![tmp_virt.clone()], - ..Default::default() - }, - ) - .compose() - .with_context(|| "Unable to compose virtualized adapter into component.\nMake sure virtualizations are enabled and being used.") - .or_else(|e| { - fs::remove_file(&tmp_virt)?; - Err(e) - })?; - - fs::remove_file(&tmp_virt)?; - - composed_bytes - } else { - virt_component.adapter - }; - if virt_component.virtual_files.len() > 0 { println!("Virtualized files from local filesystem:\n"); for (virtual_path, original_path) in virt_component.virtual_files { @@ -228,7 +196,7 @@ fn main() -> Result<()> { } } - fs::write(&out_path, out_bytes)?; + fs::write(&out_path, virt_component.adapter)?; Ok(()) } diff --git a/src/lib.rs b/src/lib.rs index 5317062..3b4df17 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -1,12 +1,13 @@ use anyhow::{bail, Context, Result}; use serde::Deserialize; -use std::{env, fs}; +use std::{env, fs, path::PathBuf, time::SystemTime}; use virt_deny::{ deny_clocks_virt, deny_exit_virt, deny_http_virt, deny_random_virt, deny_sockets_virt, }; use virt_env::{create_env_virt, strip_env_virt}; use virt_io::{create_io_virt, VirtStdio}; use walrus_ops::strip_virt; +use wasm_compose::composer::ComponentComposer; use wasm_metadata::Producers; use wit_component::{metadata, ComponentEncoder, DecodedWasm, StringEncoding}; use wit_parser::WorldItem; @@ -61,7 +62,6 @@ pub struct WasiVirt { pub wasm_opt: Option, /// Path to compose component pub compose: Option, - compose_imports: Option>, } pub struct VirtResult { @@ -132,6 +132,67 @@ impl WasiVirt { self.wasm_opt = Some(opt); } + pub fn compose(&mut self, compose: String) { + self.compose = Some(compose); + } + + /// drop capabilities that are not imported by the composed component + pub fn filter_imports(&mut self) -> Result<()> { + match &self.compose { + Some(compose) => { + let imports = { + let module_bytes = fs::read(compose).map_err(anyhow::Error::new)?; + let (resolve, world_id) = match wit_component::decode(&module_bytes)? { + DecodedWasm::WitPackages(..) => { + bail!("expected a component, found a WIT package") + } + DecodedWasm::Component(resolve, world_id) => (resolve, world_id), + }; + + let mut import_ids: Vec = vec![]; + for (_, import) in &resolve.worlds[world_id].imports { + if let WorldItem::Interface { id, .. } = import { + if let Some(id) = resolve.id_of(*id) { + import_ids.push(id); + } + } + } + + import_ids + }; + let matches = |prefix: &str| imports.iter().any(|i| i.starts_with(prefix)); + + if !matches("wasi:cli/environment") { + self.env = None; + } + if !matches("wasi:filesystem/") { + self.fs = None; + } + if !(matches("wasi:cli/std") || matches("wasi:cli/terminal")) { + self.stdio = None; + } + if !matches("wasi:cli/exit") { + self.exit = None; + } + if !matches("wasi:clocks/") { + self.clocks = None; + } + if !matches("wasi:http/") { + self.http = None; + } + if !matches("wasi:sockets/") { + self.sockets = None; + } + if !matches("wasi:random/") { + self.random = None; + } + + Ok(()) + } + None => bail!("filtering imports can only be applied to composed components"), + } + } + pub fn finish(&mut self) -> Result { let mut config = walrus::ModuleConfig::new(); config.generate_name_section(self.debug); @@ -142,38 +203,6 @@ impl WasiVirt { }?; module.name = Some("wasi_virt".into()); - // drop capabilities that are not imported by the composed component - if self.compose.is_some() { - self.collect_compose_imports()?; - - if !self.contains_compose_import("wasi:cli/environment") { - self.env = None; - } - if !self.contains_compose_import("wasi:filesystem/") { - self.fs = None; - } - if !(self.contains_compose_import("wasi:cli/std") - || self.contains_compose_import("wasi:cli/terminal")) - { - self.stdio = None; - } - if !self.contains_compose_import("wasi:cli/exit") { - self.exit = None; - } - if !self.contains_compose_import("wasi:clocks/") { - self.clocks = None; - } - if !self.contains_compose_import("wasi:http/") { - self.http = None; - } - if !self.contains_compose_import("wasi:sockets/") { - self.sockets = None; - } - if !self.contains_compose_import("wasi:random/") { - self.random = None; - } - } - // only env virtualization is independent of io if let Some(env) = &self.env { create_env_virt(&mut module, env)?; @@ -335,43 +364,39 @@ impl WasiVirt { // now adapt the virtualized component let encoder = ComponentEncoder::default().validate(true).module(&bytes)?; - let encoded = encoder.encode()?; + let encoded_bytes = encoder.encode()?; + + let adapter = if let Some(compose_path) = &self.compose { + let compose_path = PathBuf::from(compose_path); + let dir = env::temp_dir(); + let tmp_virt = dir.join(format!("virt{}.wasm", timestamp())); + fs::write(&tmp_virt, encoded_bytes)?; + + let composed_bytes = ComponentComposer::new( + &compose_path, + &wasm_compose::config::Config { + definitions: vec![tmp_virt.clone()], + ..Default::default() + }, + ) + .compose() + .with_context(|| "Unable to compose virtualized adapter into component.\nMake sure virtualizations are enabled and being used.") + .or_else(|e| { + fs::remove_file(&tmp_virt)?; + Err(e) + })?; - Ok(VirtResult { - adapter: encoded, - virtual_files, - }) - } + fs::remove_file(&tmp_virt)?; - // parse the compose component to collect its imported interfaces - fn collect_compose_imports(&mut self) -> Result<()> { - let module_bytes = fs::read(self.compose.as_ref().unwrap()).map_err(anyhow::Error::new)?; - let (resolve, world_id) = match wit_component::decode(&module_bytes)? { - DecodedWasm::WitPackages(..) => { - bail!("expected a component, found a WIT package") - } - DecodedWasm::Component(resolve, world_id) => (resolve, world_id), + composed_bytes + } else { + encoded_bytes }; - let mut import_ids: Vec = vec![]; - for (_, import) in &resolve.worlds[world_id].imports { - if let WorldItem::Interface { id, .. } = import { - if let Some(id) = resolve.id_of(*id) { - import_ids.push(id); - } - } - } - - self.compose_imports = Some(import_ids); - - Ok(()) - } - - fn contains_compose_import(&self, prefix: &str) -> bool { - match &self.compose_imports { - Some(imports) => imports.iter().any(|i| i.starts_with(prefix)), - None => false, - } + Ok(VirtResult { + adapter, + virtual_files, + }) } } @@ -414,3 +439,10 @@ fn apply_wasm_opt(bytes: Vec, debug: bool) -> Result> { Ok(bytes) } } + +fn timestamp() -> u64 { + match SystemTime::now().duration_since(SystemTime::UNIX_EPOCH) { + Ok(n) => n.as_secs(), + Err(_) => panic!(), + } +} diff --git a/tests/virt.rs b/tests/virt.rs index 70bfc03..7444b77 100644 --- a/tests/virt.rs +++ b/tests/virt.rs @@ -154,6 +154,7 @@ async fn virt_test() -> Result<()> { .into_string() .unwrap(); virt_opts.compose = Some(compose_path); + virt_opts.filter_imports()?; } } @@ -176,24 +177,32 @@ async fn virt_test() -> Result<()> { } } - // compose the test component with the defined test virtualization - if DEBUG { - println!("- Composing virtualization"); - } - let component_bytes = ComponentComposer::new( - &generated_component_path, - &wasm_compose::config::Config { - definitions: vec![virt_component_path], - ..Default::default() - }, - ) - .compose()?; - - if true { - let mut composed_path = generated_path.join(test_case_name); - composed_path.set_extension("composed.wasm"); - fs::write(composed_path, &component_bytes)?; - } + let mut composed_path = generated_path.join(test_case_name); + composed_path.set_extension("composed.wasm"); + let component_bytes = match test.compose { + Some(true) => { + // adapter is already composed + virt_component.adapter + } + _ => { + // compose the test component with the defined test virtualization + if DEBUG { + println!("- Composing virtualization"); + } + let component_bytes = ComponentComposer::new( + &generated_component_path, + &wasm_compose::config::Config { + definitions: vec![virt_component_path], + ..Default::default() + }, + ) + .compose()?; + + fs::write(&composed_path, &component_bytes)?; + + component_bytes + } + }; // execute the composed virtualized component test function if DEBUG {