Support async in lucet-wiggle, and switch lucet-wasi to use tokio (#655)

* vendor wiggle and wasi-common crates from alexcrichton/new-api

The wasmtime new-api branch constains futures to be Send, and wiggle and
wasi-common got considerable changes to fix this. Lucet futures are Send
as well, so using these versions of wiggle and wasi-common solves a
bunch of integration issues.

We expect this branch to take a little bit of time to land because it is
gated behind an RFC, so vendoring these changes allows us to ship the async
wasi-common in Lucet a lot sooner.

* port lucet-wiggle & lucet-wasi to ctx syntax changes from wiggle

* lucet-wiggle: wip adding async

* lucet-wiggle: start fixing

* lucet-wasi: replace test suite with atoms_async; fix async execution

* lucet-wiggle: split out an integration macro just like wasmtime

* lucet-runtime-internals: fix wiggle dep

* lucet-wasi: delete generate subcrate, use lucet_wiggle::lucet_integration

* root cargo stuff

* rustfmt

* benchmarks: wasi benchmark requires tokio now

* lucet-wasi-fuzz: use tokio instead of rayon

* rustfmt

* lucet-wiggle: run post_hook after ctx borrow has been returned.

* wasi-common build.rs: use CARGO_MANIFEST_DIR instead of cwd

* wasmtime-new-api: add README breadcrumbs

Cargo.toml

@@ -11,14 +11,20 @@ members = [
  "lucet-runtime/lucet-runtime-tests",
  "lucet-spectest",
  "lucet-wasi",
- "lucet-wasi/generate",
  "lucet-wasi-fuzz",
  "lucet-wasi-sdk",
  "lucet-wiggle",
  "lucet-wiggle/macro",
  "lucet-wiggle/generate",
  "lucetc",
  "fuzz",
+ "wasmtime-new-api/crates/wiggle",
+ "wasmtime-new-api/crates/wiggle/borrow",
+ "wasmtime-new-api/crates/wiggle/generate",
+ "wasmtime-new-api/crates/wiggle/macro",
+ "wasmtime-new-api/crates/wasi-common",
+ "wasmtime-new-api/crates/wasi-common/cap-std-sync",
+ "wasmtime-new-api/crates/wasi-common/tokio",
 ]
 exclude = ["wasmtime", "sightglass"]
 

benchmarks/lucet-benchmarks/Cargo.toml

@@ -22,6 +22,7 @@ nix = "0.17"
 num_cpus = "1.0"
 rayon = "1.0"
 tempfile = "3.0"
+tokio = { version = "1", features = ["rt-multi-thread"] }
 
 [lib]
 bench = false

benchmarks/lucet-benchmarks/src/seq.rs

@@ -233,7 +233,10 @@ fn run_fib<R: RegionCreate + 'static>(c: &mut Criterion) {
 /// Run a trivial WASI program.
 fn run_hello<R: RegionCreate + 'static>(c: &mut Criterion) {
  fn body(inst: &mut InstanceHandle) {
- inst.run("_start", &[]).unwrap();
+ tokio::runtime::Runtime::new()
+ .expect("create runtime")
+ .block_on(inst.run_async("_start", &[], None))
+ .unwrap();
  }
 
  let workdir = TempDir::new().expect("create working directory");

lucet-runtime/lucet-runtime-internals/Cargo.toml

@@ -12,7 +12,7 @@ edition = "2018"
 [dependencies]
 lucet-module = { path = "../../lucet-module", version = "=0.7.0-dev" }
 lucet-runtime-macros = { path = "../lucet-runtime-macros", version = "=0.7.0-dev" }
-wiggle = { path = "../../wasmtime/crates/wiggle", version = "0.23.0" }
+wiggle = { path = "../../wasmtime-new-api/crates/wiggle", version = "0.26.0" }
 
 anyhow = "1.0"
 bitflags = "1.0"

lucet-wasi-fuzz/Cargo.toml

@@ -23,8 +23,9 @@ num_cpus = "1.10"
 progress = "0.2"
 rand = "0.6"
 regex = "1.1"
-rayon = "1.0"
 structopt = "0.3.3"
+task-group = "0.2"
 tempfile = "3.0"
+tokio = { version = "1", features = ["rt-multi-thread", "macros"] }
 wait-timeout = "0.2"
-wasi-common = { path = "../wasmtime/crates/wasi-common" }
+wasi-common = { path = "../wasmtime-new-api/crates/wasi-common" }

lucet-wasi-fuzz/src/main.rs

@@ -7,7 +7,6 @@ use lucet_wasi::{types::Exitcode, WasiCtx, WasiCtxBuilder};
 use lucet_wasi_sdk::{CompileOpts, Link};
 use lucetc::{Lucetc, LucetcOpts};
 use rand::prelude::random;
-use rayon::prelude::*;
 use regex::Regex;
 use std::io::{Read, Write};
 use std::os::unix::prelude::OpenOptionsExt;
@@ -45,7 +44,8 @@ enum Config {
  TestSeed { seed: Seed },
 }
 
-fn main() {
+#[tokio::main]
+async fn main() {
  eprintln!(
  "⚠️⚠️⚠️ WARNING ⚠️⚠️⚠️
 
@@ -62,10 +62,12 @@ the differences in alignment for `long long`s. See #445 for updates:
  lucet_wasi::export_wasi_funcs();
 
  match Config::from_args() {
- Config::Fuzz { num_tests } => run_many(num_tests),
+ Config::Fuzz { num_tests } => run_many(num_tests).await,
  Config::Creduce { seed } => run_creduce_driver(seed),
- Config::CreduceInteresting { creduce_src } => run_creduce_interestingness(creduce_src),
- Config::TestSeed { seed } => run_one_seed(seed),
+ Config::CreduceInteresting { creduce_src } => {
+ run_creduce_interestingness(creduce_src).await
+ }
+ Config::TestSeed { seed } => run_one_seed(seed).await,
  }
 }
 
@@ -134,10 +136,10 @@ fn run_creduce_driver(seed: Seed) {
  );
 }
 
-fn run_creduce_interestingness<P: AsRef<Path>>(src: P) {
+async fn run_creduce_interestingness<P: AsRef<Path>>(src: P) {
  let tmpdir = TempDir::new().unwrap();
 
- match run_both(&tmpdir, src, None) {
+ match run_both(&tmpdir, src, None).await {
  Ok(TestResult::Passed) => println!("test passed"),
  Ok(TestResult::Ignored) => println!("native build/execution failed"),
  Ok(TestResult::Failed {
@@ -190,8 +192,8 @@ fn run_creduce_interestingness<P: AsRef<Path>>(src: P) {
  exit(1);
 }
 
-fn run_one_seed(seed: Seed) {
- match run_one(Some(seed)) {
+async fn run_one_seed(seed: Seed) {
+ match run_one(Some(seed)).await {
  Ok(TestResult::Passed) => println!("test passed"),
  Ok(TestResult::Ignored) => println!("native build/execution failed"),
  Ok(TestResult::Failed {
@@ -209,44 +211,61 @@ fn run_one_seed(seed: Seed) {
  }
 }
 
-fn run_many(num_tests: usize) {
+async fn run_many(num_tests: usize) {
  let mut progress = progress::Bar::new();
  progress.set_job_title(&format!("Running {} tests", num_tests));
 
  let progress = Arc::new(Mutex::new(progress));
  let num_finished = Arc::new(Mutex::new(0));
 
- let res = (0..num_tests)
- .into_par_iter()
- .try_for_each(|_| match run_one(None) {
- Ok(TestResult::Passed) | Ok(TestResult::Ignored) => {
- let mut num_finished = num_finished.lock().unwrap();
- *num_finished += 1;
- let percentage = (*num_finished as f32 / num_tests as f32) * 100.0;
- progress
- .lock()
- .unwrap()
- .reach_percent(percentage.floor() as i32);
- Ok(())
- }
- Ok(fail) => Err(fail),
- Err(error) => Err(TestResult::Errored { error }),
- });
+ let (task_group, task_manager) = task_group::TaskGroup::new();
+ for _ in 0..num_tests {
+ let progress = progress.clone();
+ let num_finished = num_finished.clone();
+ task_group
+ .spawn("worker", async move {
+ match run_one(None).await {
+ Ok(TestResult::Passed) | Ok(TestResult::Ignored) => {
+ let mut num_finished = num_finished.lock().unwrap();
+ *num_finished += 1;
+ let percentage = (*num_finished as f32 / num_tests as f32) * 100.0;
+ progress
+ .lock()
+ .unwrap()
+ .reach_percent(percentage.floor() as i32);
+ Ok(())
+ }
+ Ok(fail) => Err(fail),
+ Err(error) => Err(TestResult::Errored { error }),
+ }
+ })
+ .await
+ .expect("spawn task");
+ }
+
+ let res = task_manager.await;
 
  progress.lock().unwrap().jobs_done();
 
  match res {
- Err(TestResult::Failed {
- seed,
- expected,
- actual,
+ Err(task_group::RuntimeError::Application {
+ error:
+ TestResult::Failed {
+ seed,
+ expected,
+ actual,
+ },
+ ..
  }) => {
  println!("test failed with seed {}\n", seed.unwrap());
  println!("native: {}", String::from_utf8_lossy(&expected));
  println!("lucet-wasi: {}", String::from_utf8_lossy(&actual));
  exit(1);
  }
- Err(TestResult::Errored { error }) => println!("test errored: {:?}", error),
+ Err(task_group::RuntimeError::Application {
+ error: TestResult::Errored { error },
+ ..
+ }) => println!("test errored: {:?}", error),
  Err(_) => unreachable!(),
  Ok(()) => println!("all tests passed"),
  }
@@ -263,7 +282,7 @@ fn gen_c<P: AsRef<Path>>(gen_c_path: P, seed: Seed) -> Result<(), Error> {
  Ok(())
 }
 
-fn run_both<P: AsRef<Path>>(
+async fn run_both<P: AsRef<Path>>(
  tmpdir: &TempDir,
  src: P,
  seed: Option<Seed>,
@@ -274,7 +293,7 @@ fn run_both<P: AsRef<Path>>(
  return Ok(TestResult::Ignored);
  };
 
- let (exitcode, wasm_stdout) = run_with_stdout(&tmpdir, src.as_ref())?;
+ let (exitcode, wasm_stdout) = run_with_stdout(&tmpdir, src.as_ref()).await?;
 
  assert_eq!(exitcode, 0);
 
@@ -362,18 +381,18 @@ enum TestResult {
  },
 }
 
-fn run_one(seed: Option<Seed>) -> Result<TestResult, Error> {
+async fn run_one(seed: Option<Seed>) -> Result<TestResult, Error> {
  let tmpdir = TempDir::new().unwrap();
 
  let gen_c_path = tmpdir.path().join("gen.c");
 
  let seed = seed.unwrap_or(random::<Seed>());
  gen_c(&gen_c_path, seed)?;
 
- run_both(&tmpdir, &gen_c_path, Some(seed))
+ run_both(&tmpdir, &gen_c_path, Some(seed)).await
 }
 
-fn run_with_stdout<P: AsRef<Path>>(
+async fn run_with_stdout<P: AsRef<Path>>(
  tmpdir: &TempDir,
  path: P,
 ) -> Result<(Exitcode, Vec<u8>), Error> {
@@ -383,7 +402,7 @@ fn run_with_stdout<P: AsRef<Path>>(
  .stdout(Box::new(stdout.clone()))
  .build()?;
 
- let exitcode = run(tmpdir, path, ctx)?;
+ let exitcode = run(tmpdir, path, ctx).await?;
 
  Ok((
  exitcode,
@@ -394,7 +413,7 @@ fn run_with_stdout<P: AsRef<Path>>(
  ))
 }
 
-fn run<P: AsRef<Path>>(tmpdir: &TempDir, path: P, ctx: WasiCtx) -> Result<Exitcode, Error> {
+async fn run<P: AsRef<Path>>(tmpdir: &TempDir, path: P, ctx: WasiCtx) -> Result<Exitcode, Error> {
  let region = MmapRegion::create(1, &Limits::default())?;
  let module = wasi_test(tmpdir, path)?;
 
@@ -403,7 +422,7 @@ fn run<P: AsRef<Path>>(tmpdir: &TempDir, path: P, ctx: WasiCtx) -> Result<Exitco
  .with_embed_ctx(ctx)
  .build()?;
 
- match inst.run("_start", &[]) {
+ match inst.run_async("_start", &[], None).await {
  // normal termination implies 0 exit code
  Ok(_) => Ok(0),
  Err(lucet_runtime::Error::RuntimeTerminated(

lucet-wasi/Cargo.toml

@@ -8,6 +8,7 @@ license = "Apache-2.0 WITH LLVM-exception"
 categories = ["wasm"]
 authors = ["Lucet team <lucet@fastly.com>"]
 edition = "2018"
+build = "build.rs"
 
 [dependencies]
 anyhow = "1"
@@ -17,18 +18,18 @@ human-size = "0.4"
 lucet-runtime = { path = "../lucet-runtime", version = "=0.7.0-dev" }
 lucet-runtime-internals = { path = "../lucet-runtime/lucet-runtime-internals", version = "=0.7.0-dev" }
 lucet-module = { path = "../lucet-module", version = "=0.7.0-dev" }
-lucet-wasi-generate = { path = "./generate", version = "=0.7.0-dev" }
 lucet-wiggle = { path = "../lucet-wiggle", version = "=0.7.0-dev" }
 lucet-wiggle-generate = { path = "../lucet-wiggle/generate", version = "=0.7.0-dev" }
 libc = "0.2.65"
 nix = "0.17"
 rand = "0.6"
-wasi-common = { path = "../wasmtime/crates/wasi-common", version = "0.23.0", default-features = false, features = ["wiggle_metadata"] }
-wasi-cap-std-sync = { path = "../wasmtime/crates/wasi-common/cap-std-sync", version = "0.23.0"}
-wiggle = { path = "../wasmtime/crates/wiggle", version = "0.23.0" }
+wasi-common = { path = "../wasmtime-new-api/crates/wasi-common", version = "0.26.0", default-features = false, features = ["wiggle_metadata"] }
+wasi-tokio = { path = "../wasmtime-new-api/crates/wasi-common/tokio", version = "0.26.0"}
+wiggle = { path = "../wasmtime-new-api/crates/wiggle", version = "0.26.0" }
 tracing = "0.1.19"
 tracing-subscriber = "0.2.0"
 cap-std = "0.13"
+tokio = { version = "1", features = ["rt-multi-thread", "macros"]}
 
 [dev-dependencies]
 lucet-wasi-sdk = { path = "../lucet-wasi-sdk" }

lucet-wasi/build.rs

@@ -0,0 +1,6 @@
+fn main() {
+ // wasi-common's links & build.rs ensure this variable points to the wasi root:
+ let wasi_root = std::env::var("DEP_WASI_COMMON_19_WASI").unwrap();
+ // Make it available as WASI_ROOT:
+ println!("cargo:rustc-env=WASI_ROOT={}", wasi_root);
+}