feat: align wasi-http with component linker

.github/workflows/main.yml

@@ -454,6 +454,12 @@ jobs:
  RUST_BACKTRACE: 1
  if: matrix.target == '' && matrix.os != 'windows-latest' && needs.determine.outputs.test-capi
 
+ # Ensure wit definitions are in sync: both wasmtime-wasi and wasmtime-wasi-http need their own
+ # copy of the wit definitions so publishing works, but we need to ensure they are identical copies.
+ - run: |
+ cp -TRv 'crates/wasi/wit/' 'crates/wasi-http/wit/'
+ git diff --exit-code --diff-filter=AM $GITHUB_SHA -- 'crates/wasi-http/wit/'
+
  # Build and test all features
  - run: ./ci/run-tests.sh --locked
  env:

crates/test-programs/Cargo.toml

@@ -14,14 +14,23 @@ wit-component = { workspace = true }
 heck = { workspace = true }
 
 [dependencies]
+anyhow = { workspace = true }
+http = { version = "0.2.9" }
+http-body = "1.0.0-rc.2"
+http-body-util = "0.1.0-rc.2"
+hyper = { version = "1.0.0-rc.3", features = ["full"] }
 is-terminal = { workspace = true }
+tokio = { workspace = true, features = ["net", "rt-multi-thread", "macros"] }
 
 [dev-dependencies]
 anyhow = { workspace = true }
 tempfile = { workspace = true }
 test-log = { version = "0.2", default-features = false, features = ["trace"] }
 tracing = { workspace = true }
-tracing-subscriber = { version = "0.3.1", default-features = false, features = ['fmt', 'env-filter'] }
+tracing-subscriber = { version = "0.3.1", default-features = false, features = [
+ 'fmt',
+ 'env-filter',
+] }
 lazy_static = "1"
 wasmtime = { workspace = true, features = ['cranelift', 'component-model'] }
 
@@ -30,14 +39,9 @@ wasi-cap-std-sync = { workspace = true }
 wasmtime-wasi = { workspace = true, features = ["tokio"] }
 cap-std = { workspace = true }
 cap-rand = { workspace = true }
-tokio = { version = "1.8.0", features = ["net", "rt-multi-thread", "macros"] }
+tokio = { workspace = true, features = ["rt-multi-thread", "macros"] }
 
 wasmtime-wasi-http = { workspace = true }
-hyper = { version = "1.0.0-rc.3", features = ["full"] }
-http = { version = "0.2.9" }
-http-body = "1.0.0-rc.2"
-http-body-util = "0.1.0-rc.2"
 
 [features]
 test_programs = []
-test_programs_http = [ "wasmtime/component-model" ]

crates/test-programs/build.rs

@@ -59,18 +59,15 @@ fn build_and_generate_tests() {
 
  if BUILD_WASI_HTTP_TESTS {
  modules_rs(&meta, "wasi-http-tests", "bin", &out_dir);
- // FIXME this is broken at the moment because guest bindgen is embedding the proxy world type,
- // so wit-component expects the module to contain the proxy's exports. we need a different
- // world to pass guest bindgen that is just "a command that also can do outbound http"
- //components_rs(&meta, "wasi-http-tests", "bin", &command_adapter, &out_dir);
+ components_rs(&meta, "wasi-http-tests", "bin", &reactor_adapter, &out_dir);
  }
 
  components_rs(&meta, "command-tests", "bin", &command_adapter, &out_dir);
  components_rs(&meta, "reactor-tests", "cdylib", &reactor_adapter, &out_dir);
 }
 
 // Creates an `${out_dir}/${package}_modules.rs` file that exposes a `get_module(&str) -> Module`,
-// and a contains a `use self::{module} as _;` for each module that ensures that the user defines
+// and contains a `use self::{module} as _;` for each module that ensures that the user defines
 // a symbol (ideally a #[test]) corresponding to each module.
 fn modules_rs(meta: &cargo_metadata::Metadata, package: &str, kind: &str, out_dir: &PathBuf) {
  let modules = targets_in_package(meta, package, kind)

crates/test-programs/src/http_server.rs

@@ -0,0 +1,96 @@
+use http_body_util::{combinators::BoxBody, BodyExt, Full};
+use hyper::{body::Bytes, service::service_fn, Request, Response};
+use std::{
+ net::{SocketAddr, TcpListener, TcpStream},
+ sync::OnceLock,
+};
+
+async fn test(
+ mut req: Request<hyper::body::Incoming>,
+) -> http::Result<Response<BoxBody<Bytes, std::convert::Infallible>>> {
+ let method = req.method().to_string();
+ let body = req.body_mut().collect().await.unwrap();
+ let buf = body.to_bytes();
+
+ Response::builder()
+ .status(http::StatusCode::OK)
+ .header("x-wasmtime-test-method", method)
+ .header("x-wasmtime-test-uri", req.uri().to_string())
+ .body(Full::<Bytes>::from(buf).boxed())
+}
+
+async fn serve_http1_connection(stream: TcpStream) -> Result<(), hyper::Error> {
+ let mut builder = hyper::server::conn::http1::Builder::new();
+ let http = builder.keep_alive(false).pipeline_flush(true);
+ stream.set_nonblocking(true).unwrap();
+ let io = tokio::net::TcpStream::from_std(stream).unwrap();
+ http.serve_connection(io, service_fn(test)).await
+}
+
+#[derive(Clone)]
+/// An Executor that uses the tokio runtime.
+pub struct TokioExecutor;
+
+impl<F> hyper::rt::Executor<F> for TokioExecutor
+where
+ F: std::future::Future + Send + 'static,
+ F::Output: Send + 'static,
+{
+ fn execute(&self, fut: F) {
+ tokio::task::spawn(fut);
+ }
+}
+
+async fn serve_http2_connection(stream: TcpStream) -> Result<(), hyper::Error> {
+ let mut builder = hyper::server::conn::http2::Builder::new(TokioExecutor);
+ let http = builder.max_concurrent_streams(20);
+ let io = tokio::net::TcpStream::from_std(stream).unwrap();
+ http.serve_connection(io, service_fn(test)).await
+}
+
+pub async fn setup_http1(
+ future: impl std::future::Future<Output = anyhow::Result<()>>,
+) -> Result<(), anyhow::Error> {
+ static CELL_HTTP1: OnceLock<TcpListener> = OnceLock::new();
+ let listener = CELL_HTTP1.get_or_init(|| {
+ let addr = SocketAddr::from(([127, 0, 0, 1], 3000));
+ TcpListener::bind(addr).unwrap()
+ });
+
+ let thread = tokio::task::spawn(async move {
+ let (stream, _) = listener.accept().unwrap();
+ let conn = serve_http1_connection(stream).await;
+ if let Err(err) = conn {
+ eprintln!("Error serving connection: {:?}", err);
+ }
+ });
+
+ let (future_result, thread_result) = tokio::join!(future, thread);
+ future_result?;
+ thread_result.unwrap();
+
+ Ok(())
+}
+
+pub async fn setup_http2(
+ future: impl std::future::Future<Output = anyhow::Result<()>>,
+) -> anyhow::Result<()> {
+ static CELL_HTTP2: OnceLock<TcpListener> = OnceLock::new();
+ let listener = CELL_HTTP2.get_or_init(|| {
+ let addr = SocketAddr::from(([127, 0, 0, 1], 3001));
+ TcpListener::bind(addr).unwrap()
+ });
+ let thread = tokio::task::spawn(async move {
+ let (stream, _) = listener.accept().unwrap();
+ let conn = serve_http2_connection(stream).await;
+ if let Err(err) = conn {
+ eprintln!("Error serving connection: {:?}", err);
+ }
+ });
+
+ let (future_result, thread_result) = tokio::join!(future, thread);
+ future_result?;
+ thread_result.unwrap();
+
+ Ok(())
+}

crates/test-programs/src/lib.rs

@@ -1,6 +1,9 @@
 ///! This crate exists to build crates for wasm32-wasi in build.rs, and execute
 ///! these wasm programs in harnesses defined under tests/.
 
+#[cfg(all(feature = "test_programs", not(skip_wasi_http_tests)))]
+pub mod http_server;
+
 /// The wasi-tests binaries use these environment variables to determine their
 /// expected behavior.
 /// Used by all of the tests/ which execute the wasi-tests binaries.

crates/test-programs/tests/wasi-http-components.rs

@@ -0,0 +1,131 @@
+#![cfg(all(feature = "test_programs", not(skip_wasi_http_tests)))]
+use wasmtime::{
+ component::{Component, Linker},
+ Config, Engine, Store,
+};
+use wasmtime_wasi::preview2::{
+ command::{add_to_linker, Command},
+ Table, WasiCtx, WasiCtxBuilder, WasiView,
+};
+use wasmtime_wasi_http::{WasiHttpCtx, WasiHttpView};
+
+use test_programs::http_server::{setup_http1, setup_http2};
+
+lazy_static::lazy_static! {
+ static ref ENGINE: Engine = {
+ let mut config = Config::new();
+ config.wasm_backtrace_details(wasmtime::WasmBacktraceDetails::Enable);
+ config.wasm_component_model(true);
+ config.async_support(true);
+ let engine = Engine::new(&config).unwrap();
+ engine
+ };
+}
+// uses ENGINE, creates a fn get_module(&str) -> Module
+include!(concat!(env!("OUT_DIR"), "/wasi_http_tests_components.rs"));
+
+struct Ctx {
+ table: Table,
+ wasi: WasiCtx,
+ http: WasiHttpCtx,
+}
+
+impl WasiView for Ctx {
+ fn table(&self) -> &Table {
+ &self.table
+ }
+ fn table_mut(&mut self) -> &mut Table {
+ &mut self.table
+ }
+ fn ctx(&self) -> &WasiCtx {
+ &self.wasi
+ }
+ fn ctx_mut(&mut self) -> &mut WasiCtx {
+ &mut self.wasi
+ }
+}
+
+impl WasiHttpView for Ctx {
+ fn http_ctx(&self) -> &WasiHttpCtx {
+ &self.http
+ }
+ fn http_ctx_mut(&mut self) -> &mut WasiHttpCtx {
+ &mut self.http
+ }
+}
+
+async fn instantiate_component(
+ component: Component,
+ ctx: Ctx,
+) -> Result<(Store<Ctx>, Command), anyhow::Error> {
+ let mut linker = Linker::new(&ENGINE);
+ add_to_linker(&mut linker)?;
+ wasmtime_wasi_http::add_to_component_linker(&mut linker)?;
+
+ let mut store = Store::new(&ENGINE, ctx);
+
+ let (command, _instance) = Command::instantiate_async(&mut store, &component, &linker).await?;
+ Ok((store, command))
+}
+
+async fn run(name: &str) -> anyhow::Result<()> {
+ let mut table = Table::new();
+ let component = get_component(name);
+
+ // Create our wasi context.
+ let wasi = WasiCtxBuilder::new()
+ .inherit_stdio()
+ .arg(name)
+ .build(&mut table)?;
+ let http = WasiHttpCtx::new();
+
+ let (mut store, command) = instantiate_component(component, Ctx { table, wasi, http }).await?;
+ command
+ .wasi_cli_run()
+ .call_run(&mut store)
+ .await
+ .map_err(|e| anyhow::anyhow!("wasm failed with {e:?}"))?
+ .map_err(|e| anyhow::anyhow!("command returned with failing exit status {e:?}"))
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_get() {
+ setup_http1(run("outbound_request_get")).await.unwrap();
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_post() {
+ setup_http1(run("outbound_request_post")).await.unwrap();
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_put() {
+ setup_http1(run("outbound_request_put")).await.unwrap();
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_invalid_version() {
+ setup_http2(run("outbound_request_invalid_version"))
+ .await
+ .unwrap();
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_unknown_method() {
+ run("outbound_request_unknown_method").await.unwrap();
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_unsupported_scheme() {
+ run("outbound_request_unsupported_scheme").await.unwrap();
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_invalid_port() {
+ run("outbound_request_invalid_port").await.unwrap();
+}
+
+#[test_log::test(tokio::test(flavor = "multi_thread"))]
+async fn outbound_request_invalid_dnsname() {
+ run("outbound_request_invalid_dnsname").await.unwrap();
+}