Use wasi-streams in the wasi-http implementation (bytecodealliance#7056)

* Start refactoring wasi-http

* Checkpoint

* Initial implementation of response future handling

* Lazily initialize response headers and body

* make wasmtime-wasi-http compile

* wasi-http wit: make a way to reject outgoing-request in outgoing-handler

before waiting for the future to resolve

* wasi: sync wit from wasi-http

* outgoing handler impl: report errors to userland

* test-programs: get wasi-http-components kicking over, delete modules and components-sync tests

wasi-http-components-sync will come back once we get done with other
stuff, but its superfulous for now. wasi-http-modules will not be
returning.

* Process headers

* Add HostIncomingBody::new

* Add trailers functions

* Add TODO for body task outline

* Rework incoming-response-consume to return a future-trailers value as well

* Fix the wit

* First cut at the worker loop

* wasi-http: change how we represent bodies/trailers, and annotate own/borrow/child throughout

* Update types_impl.rs for wit changes

* Split body management into its own module

* Checkpoint

* more work on incoming body and future trailers

* Fill out some more functions

* Implement future-trailers-{subscribe,get}

* Implement drop-future-trailers

* Rework fields, but make the borrow checker mad

* Fix borrow error

* wasi-http-tests: fix build

* test-runner: report errors with stdout/stderr properly

* fix two trivial wasi-http tests

the error type here changed from a types::Error to an
outbound_handler::Error

* Remove unnecessary drops

* Convert a `bail!` to a `todo!`

* Remove a TODO that documented the body worker structure

* fill in a bunch more of OutputBody

* Remove the custom FrameFut future in favor of using http_body_util

* Move the outgoing body types to body.rs

* Rework the handling of outgoing bodies

* Fix the `outgoing request get` test

* Avoid deadlocking the post tests

* future_incoming_request_get shouldn't delete the resource

* Fix the invalid_dnsname test

* implement drop-future-incoming-response

* Fix invalid_port and invalid_dnsname tests

* Fix the post test

* Passing a too large string to println! caused the large post test to fail

* Format

* Plumb through `between_bytes_timeout`

* Downgrade hyper

* Revert "Downgrade hyper"

This reverts commit fa0750e.

* Restore old https connection setup

* Sync the wasi and wasi-http http deps

* Fix tests

* Remove the module and component-sync tests, as they are currently not
  supported
* Fix the reference to the large_post test in the components test

* Fix wasi-http integration

* sync implementation of wasi-http

* Slightly more robust error checking

* Ignore the wasi-http cli test

prtest:full

* Consistent ignore attributes between sync and async tests

* Fix doc errors

* code motion: introduce intermediate `HostIncomingBodyBuilder` rather than a tuple

* explain design

* Turn FieldMap into a type synonym

* Tidy up some future state (bytecodealliance#7073)

Co-authored-by: Pat Hickey <phickey@fastly.com>

* body HostInputStream: report runtime errors with StreamRuntimeError

HostInputStream is designed wrong to need that in the first place. We
will fix it in a follow-up as soon as resources land.

---------

Co-authored-by: Pat Hickey <phickey@fastly.com>
Co-authored-by: Alex Crichton <alex@alexcrichton.com>

crates/test-programs/tests/wasi-http-components-sync.rs

@@ -46,12 +46,13 @@ impl WasiView for Ctx {
 }
 
 impl WasiHttpView for Ctx {
- fn http_ctx(&self) -> &WasiHttpCtx {
- &self.http
- }
- fn http_ctx_mut(&mut self) -> &mut WasiHttpCtx {
+ fn ctx(&mut self) -> &mut WasiHttpCtx {
  &mut self.http
  }
+
+ fn table(&mut self) -> &mut Table {
+ &mut self.table
+ }
 }
 
 fn instantiate_component(
@@ -60,7 +61,7 @@ fn instantiate_component(
 ) -> Result<(Store<Ctx>, Command), anyhow::Error> {
  let mut linker = Linker::new(&ENGINE);
  add_to_linker(&mut linker)?;
- wasmtime_wasi_http::proxy::sync::add_to_linker(&mut linker)?;
+ wasmtime_wasi_http::proxy::add_to_linker(&mut linker)?;
 
  let mut store = Store::new(&ENGINE, ctx);
 
@@ -84,7 +85,7 @@ fn run(name: &str) -> anyhow::Result<()> {
  builder.env(var, val);
  }
  let wasi = builder.build(&mut table)?;
- let http = WasiHttpCtx::new();
+ let http = WasiHttpCtx {};
 
  let (mut store, command) = instantiate_component(component, Ctx { table, wasi, http })?;
  command

crates/test-programs/tests/wasi-http-components.rs

@@ -47,10 +47,10 @@ impl WasiView for Ctx {
 }
 
 impl WasiHttpView for Ctx {
- fn http_ctx(&self) -> &WasiHttpCtx {
- &self.http
+ fn table(&mut self) -> &mut Table {
+ &mut self.table
  }
- fn http_ctx_mut(&mut self) -> &mut WasiHttpCtx {
+ fn ctx(&mut self) -> &mut WasiHttpCtx {
  &mut self.http
  }
 }
@@ -85,16 +85,11 @@ async fn run(name: &str) -> anyhow::Result<()> {
  builder.env(var, val);
  }
  let wasi = builder.build(&mut table)?;
- let http = WasiHttpCtx::new();
+ let http = WasiHttpCtx;
 
  let (mut store, command) =
  instantiate_component(component, Ctx { table, wasi, http }).await?;
- command
- .wasi_cli_run()
- .call_run(&mut store)
- .await?
- .map_err(|()| anyhow::anyhow!("run returned a failure"))?;
- Ok(())
+ command.wasi_cli_run().call_run(&mut store).await
  };
  r.map_err(move |trap: anyhow::Error| {
  let stdout = stdout.try_into_inner().expect("single ref to stdout");
@@ -109,7 +104,8 @@ async fn run(name: &str) -> anyhow::Result<()> {
  "error while testing wasi-tests {} with http-components",
  name
  ))
- })?;
+ })?
+ .map_err(|()| anyhow::anyhow!("run returned an error"))?;
  Ok(())
 }
 

crates/test-programs/wasi-http-tests/src/bin/outbound_request_invalid_dnsname.rs

@@ -15,6 +15,6 @@ async fn run() {
  )
  .await;
 
- let error = res.unwrap_err();
- assert_eq!(error.to_string(), "Error::InvalidUrl(\"invalid dnsname\")");
+ let error = res.unwrap_err().to_string();
+ assert!(error.starts_with("Error::InvalidUrl(\"failed to lookup address information:"));
 }

crates/test-programs/wasi-http-tests/src/bin/outbound_request_invalid_version.rs

@@ -17,7 +17,7 @@ async fn run() {
 
  let error = res.unwrap_err().to_string();
  if error.ne("Error::ProtocolError(\"invalid HTTP version parsed\")")
- && error.ne("Error::ProtocolError(\"operation was canceled\")")
+ && !error.starts_with("Error::ProtocolError(\"operation was canceled")
  {
  panic!(
  r#"assertion failed: `(left == right)`

crates/test-programs/wasi-http-tests/src/bin/outbound_request_large_post.rs

@@ -23,7 +23,7 @@ async fn run() {
  .context("localhost:3000 /post large")
  .unwrap();
 
- println!("localhost:3000 /post large: {res:?}");
+ println!("localhost:3000 /post large: {}", res.status);
  assert_eq!(res.status, 200);
  let method = res.header("x-wasmtime-test-method").unwrap();
  assert_eq!(std::str::from_utf8(method).unwrap(), "POST");

crates/test-programs/wasi-http-tests/src/bin/outbound_request_post.rs

@@ -22,5 +22,5 @@ async fn run() {
  assert_eq!(res.status, 200);
  let method = res.header("x-wasmtime-test-method").unwrap();
  assert_eq!(std::str::from_utf8(method).unwrap(), "POST");
- assert_eq!(res.body, b"{\"foo\": \"bar\"}");
+ assert_eq!(res.body, b"{\"foo\": \"bar\"}", "invalid body returned");
 }

crates/test-programs/wasi-http-tests/src/bin/outbound_request_unknown_method.rs

@@ -18,6 +18,6 @@ async fn run() {
  let error = res.unwrap_err();
  assert_eq!(
  error.to_string(),
- "Error::InvalidUrl(\"unknown method OTHER\")"
+ "Error::Invalid(\"unknown method OTHER\")"
  );
 }

crates/test-programs/wasi-http-tests/src/bin/outbound_request_unsupported_scheme.rs

@@ -18,6 +18,6 @@ async fn run() {
  let error = res.unwrap_err();
  assert_eq!(
  error.to_string(),
- "Error::InvalidUrl(\"unsupported scheme WS\")"
+ "Error::Invalid(\"unsupported scheme WS\")"
  );
 }