[wit-component] add shared-everything linking support

Per https://github.com/WebAssembly/component-model/blob/main/design/mvp/examples/SharedEverythingDynamicLinking.md and https://hackmd.io/IlY4lICRRNy9wQbNLdb2Wg.

This adds a new `component link` subcommand, which resembles `component new` but
accepts an arbitrary number of "dynamic library" modules (as defined by
https://github.com/WebAssembly/tool-conventions/blob/main/DynamicLinking.md),
linking them together to create a component whose type is the union of the
component types found in the input modules (via the usual custom section
representation).  This process can also be invoked programmatically via a new
`Linker` API.

`Linker` analyzes and topologically sorts the input modules, then sythesizes two
additional modules:

- `main`: hosts the component's single memory and function table and exports any
  functions needed to break dependency cycles discovered in the input modules.
  Those functions use `call.indirect` to invoke the real functions, references
  to which are placed in the table by the `init` module.

- `init`: populates the function table as described above, initializes global
  variables per the dynamic linking tool convention, and calls any static
  constructors and/or link-time fixup functions

Finally, `Linker` generates a directed graph of modules and their instantiations
and passes it to `ComponentEncoder` to actually produce the component.

The implementation relies on a few modifications to the existing code in
`ComponentEncoder` and friends.  Specifically, I've expanded the meaning of
"adapter" to include dynamic libraries as well as traditional adapters.  The
former are distiguished from the latter by the presence of a new `LibraryInfo`
struct.  This approach minimized the number of code changes in the interest of
keeping this PR comprehensible, but we will probably want to refactor
`ComponentEncoder` in the future to make the distinction between traditional
adapters and dynamic libraries clearer, possibly generalizing how we analyze and
instantiate modules of various flavors.

I've created a demo which ties everything together, for reference:
https://github.com/dicej/component-linking-demo.  It includes `wasi-sdk` and
`wasi-libc` patches which have not yet been upstreamed; I'm planning to work on
that next.

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

skip wasmtime part of linking test under Wasm

Signed-off-by: Joel Dice <joel.dice@fermyon.com>

Author: dicej

Cargo.toml

@@ -38,9 +38,7 @@ num_cpus = "1.13"
 rand = { version = "0.8.4", features = ["small_rng"] }
 rayon = "1.3"
 serde = { version = "1.0.166", features = ["derive"] }
-wasmtime = { version = "3.0.0", default-features = false, features = [
-  'cranelift',
-] }
+wasmtime = { version = "11.0.1", default-features = false, features = ['cranelift', 'component-model'] }
 url = "2.0.0"
 pretty_assertions = "1.3.0"
 semver = "1.0.0"

crates/fuzz-stats/src/limits.rs

@@ -1,3 +1,4 @@
+use anyhow::Result;
 use wasmtime::*;
 
 #[derive(Clone)]
@@ -22,12 +23,17 @@ impl StoreLimits {
 }
 
 impl ResourceLimiter for StoreLimits {
-    fn memory_growing(&mut self, current: usize, desired: usize, _maximum: Option<usize>) -> bool {
-        self.alloc(desired - current)
+    fn memory_growing(
+        &mut self,
+        current: usize,
+        desired: usize,
+        _maximum: Option<usize>,
+    ) -> Result<bool> {
+        Ok(self.alloc(desired - current))
     }
 
-    fn table_growing(&mut self, current: u32, desired: u32, _maximum: Option<u32>) -> bool {
+    fn table_growing(&mut self, current: u32, desired: u32, _maximum: Option<u32>) -> Result<bool> {
         let delta = (desired - current) as usize * std::mem::size_of::<usize>();
-        self.alloc(delta)
+        Ok(self.alloc(delta))
     }
 }

crates/wit-component/Cargo.toml

@@ -30,5 +30,8 @@ pretty_assertions = "1.3.0"
 env_logger = { workspace = true }
 wat = { workspace = true }
 
+[target.'cfg(not(target_family = "wasm"))'.dev-dependencies]
+wasmtime = { workspace = true }
+
 [features]
 dummy-module = ['dep:wat']

crates/wit-component/src/encoding.rs

@@ -1,11 +1,11 @@
-use super::{ComponentEncoder, RequiredOptions};
+use super::{Adapter, ComponentEncoder, LibraryInfo, RequiredOptions};
 use crate::validation::{
     validate_adapter_module, validate_module, RequiredImports, ValidatedAdapter, ValidatedModule,
     BARE_FUNC_MODULE_NAME, RESOURCE_DROP,
 };
 use anyhow::{Context, Result};
 use indexmap::{IndexMap, IndexSet};
-use std::borrow::Borrow;
+use std::borrow::{Borrow, Cow};
 use std::collections::{HashMap, HashSet};
 use std::hash::Hash;
 use wasmparser::FuncType;
@@ -15,6 +15,12 @@ use wit_parser::{
     WorldKey,
 };
 
+pub struct WorldAdapter<'a> {
+    pub wasm: Cow<'a, [u8]>,
+    pub info: ValidatedAdapter<'a>,
+    pub library_info: Option<&'a LibraryInfo>,
+}
+
 /// Metadata discovered from the state configured in a `ComponentEncoder`.
 ///
 /// This is stored separately from `EncodingState` to be stored as a borrow in
@@ -28,7 +34,7 @@ pub struct ComponentWorld<'a> {
     pub info: ValidatedModule<'a>,
     /// Validation information about adapters populated only for required
     /// adapters. Additionally stores the gc'd wasm for each adapter.
-    pub adapters: IndexMap<&'a str, (ValidatedAdapter<'a>, Vec<u8>)>,
+    pub adapters: IndexMap<&'a str, WorldAdapter<'a>>,
     /// Map of all imports and descriptions of what they're importing.
     pub import_map: IndexMap<Option<String>, ImportedInterface>,
     /// Set of all live types which must be exported either because they're
@@ -96,18 +102,47 @@ impl<'a> ComponentWorld<'a> {
     fn process_adapters(&mut self) -> Result<()> {
         let resolve = &self.encoder.metadata.resolve;
         let world = self.encoder.metadata.world;
-        for (name, (wasm, metadata, required_exports)) in self.encoder.adapters.iter() {
+        for (
+            name,
+            Adapter {
+                wasm,
+                metadata,
+                required_exports,
+                library_info,
+            },
+        ) in self.encoder.adapters.iter()
+        {
             let required_by_import = self.info.adapters_required.get(name.as_str());
             let required =
                 self.required_adapter_exports(resolve, world, required_exports, required_by_import);
-            if required.is_empty() {
+            if required.is_empty() && library_info.is_none() {
                 continue;
             }
-            let wasm = crate::gc::run(wasm, &required, self.info.realloc)
-                .context("failed to reduce input adapter module to its minimal size")?;
-            let info = validate_adapter_module(&wasm, resolve, world, metadata, &required)
-                .context("failed to validate the imports of the minimized adapter module")?;
-            self.adapters.insert(name, (info, wasm));
+            let wasm = if library_info.is_some() {
+                Cow::Borrowed(wasm as &[u8])
+            } else {
+                Cow::Owned(
+                    crate::gc::run(wasm, &required, self.info.realloc)
+                        .context("failed to reduce input adapter module to its minimal size")?,
+                )
+            };
+            let info = validate_adapter_module(
+                &wasm,
+                resolve,
+                world,
+                metadata,
+                &required,
+                library_info.is_some(),
+            )
+            .context("failed to validate the imports of the minimized adapter module")?;
+            self.adapters.insert(
+                name,
+                WorldAdapter {
+                    info,
+                    wasm,
+                    library_info: library_info.as_ref(),
+                },
+            );
         }
         Ok(())
     }
@@ -173,7 +208,7 @@ impl<'a> ComponentWorld<'a> {
         let resolve = &self.encoder.metadata.resolve;
         let world = self.encoder.metadata.world;
         let mut all_required_imports = IndexMap::new();
-        for map in self.adapters.values().map(|(i, _)| &i.required_imports) {
+        for map in self.adapters.values().map(|a| &a.info.required_imports) {
             for (k, v) in map {
                 all_required_imports
                     .entry(k.as_str())
@@ -256,9 +291,9 @@ impl<'a> ComponentWorld<'a> {
         // First use the previously calculated metadata about live imports to
         // determine the set of live types in those imports.
         self.add_live_imports(world, &self.info.required_imports, &mut live);
-        for (adapter_name, (info, _wasm)) in self.adapters.iter() {
+        for (adapter_name, adapter) in self.adapters.iter() {
             log::trace!("processing adapter `{adapter_name}`");
-            self.add_live_imports(world, &info.required_imports, &mut live);
+            self.add_live_imports(world, &adapter.info.required_imports, &mut live);
         }
 
         // Next any imported types used by an export must also be considered

crates/wit-component/src/encoding/world.rs

@@ -10,12 +10,14 @@ use wasm_encoder::CanonicalOption;
 mod decoding;
 mod encoding;
 mod gc;
+mod linking;
 mod printing;
 mod targets;
 mod validation;
 
 pub use decoding::{decode, DecodedWasm};
 pub use encoding::{encode, ComponentEncoder};
+pub use linking::Linker;
 pub use printing::*;
 pub use targets::*;