Merge pull request #2900 from peterhuene/benchmark-instantiation

Implement simple benchmarks for instantiation.

.github/workflows/main.yml

@@ -363,6 +363,19 @@ jobs:
       env:
         RUST_BACKTRACE: 1
 
+  bench:
+    name: Run benchmarks
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+      with:
+        submodules: true
+    - run: rustup target add wasm32-wasi
+    - name: Install Rust
+      run: rustup update stable && rustup default stable
+    - run: cargo bench
+    - run: cargo bench --features uffd
+
   # Verify that cranelift's code generation is deterministic
   meta_determinist_check:
     name: Meta deterministic check

Cargo.toml

@@ -41,7 +41,7 @@ file-per-thread-logger = "0.1.1"
 wat = "1.0.37"
 libc = "0.2.60"
 log = "0.4.8"
-rayon = "1.2.1"
+rayon = "1.5.0"
 humantime = "2.0.0"
 wasmparser = "0.78.1"
 lazy_static = "1.4.0"
@@ -57,6 +57,8 @@ wasmtime-runtime = { path = "crates/runtime" }
 tokio = { version = "1.5.0", features = ["rt", "time", "macros", "rt-multi-thread"] }
 tracing-subscriber = "0.2.16"
 wast = "35.0.0"
+criterion = "0.3.4"
+num_cpus = "1.13.0"
 
 [build-dependencies]
 anyhow = "1.0.19"
@@ -116,3 +118,7 @@ required-features = ["wasmtime-wasi/tokio"]
 
 [profile.dev.package.backtrace]
 debug = false # FIXME(#1813)
+
+[[bench]]
+name = "instantiation"
+harness = false

benches/.gitignore

@@ -0,0 +1 @@
+instantiation/wasi.wasm

benches/instantiation.rs

@@ -0,0 +1,158 @@
+use anyhow::Result;
+use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion};
+use rayon::{prelude::*, ThreadPoolBuilder};
+use std::{path::PathBuf, process::Command};
+use wasmtime::*;
+use wasmtime_wasi::{sync::WasiCtxBuilder, Wasi};
+
+fn instantiate(module: &Module) -> Result<Instance> {
+    let store = Store::new(&module.engine());
+
+    // As we don't actually invoke Wasm code in this benchmark, we still add
+    // the WASI context to the store as it is considered part of getting a
+    // module that depends on WASI "ready to run".
+    Wasi::set_context(&store, WasiCtxBuilder::new().build()?)
+        .map_err(|_| anyhow::anyhow!("wasi set_context failed"))?;
+
+    let linker = Linker::new(&store);
+    let instance = linker.instantiate(module)?;
+
+    Ok(instance)
+}
+
+fn benchmark_name<'a>(strategy: &InstanceAllocationStrategy) -> &'static str {
+    match strategy {
+        InstanceAllocationStrategy::OnDemand => "default",
+        #[cfg(any(not(feature = "uffd"), not(target_os = "linux")))]
+        InstanceAllocationStrategy::Pooling { .. } => "pooling",
+        #[cfg(all(feature = "uffd", target_os = "linux"))]
+        InstanceAllocationStrategy::Pooling { .. } => "uffd",
+    }
+}
+
+fn bench_sequential(c: &mut Criterion, modules: &[&str]) {
+    let mut group = c.benchmark_group("sequential");
+
+    for strategy in &[
+        // Skip the on-demand allocator when uffd is enabled
+        #[cfg(any(not(feature = "uffd"), not(target_os = "linux")))]
+        InstanceAllocationStrategy::OnDemand,
+        InstanceAllocationStrategy::pooling(),
+    ] {
+        for file_name in modules {
+            let mut path = PathBuf::new();
+            path.push("benches");
+            path.push("instantiation");
+            path.push(file_name);
+
+            let mut config = Config::default();
+            Wasi::add_to_config(&mut config);
+            config.allocation_strategy(strategy.clone());
+
+            let engine = Engine::new(&config).expect("failed to create engine");
+            let module = Module::from_file(&engine, &path)
+                .expect(&format!("failed to load benchmark `{}`", path.display()));
+
+            group.bench_function(BenchmarkId::new(benchmark_name(strategy), file_name), |b| {
+                b.iter(|| instantiate(&module).expect("failed to instantiate module"));
+            });
+        }
+    }
+
+    group.finish();
+}
+
+fn bench_parallel(c: &mut Criterion) {
+    const PARALLEL_INSTANCES: usize = 1000;
+
+    let mut group = c.benchmark_group("parallel");
+
+    for strategy in &[
+        // Skip the on-demand allocator when uffd is enabled
+        #[cfg(any(not(feature = "uffd"), not(target_os = "linux")))]
+        InstanceAllocationStrategy::OnDemand,
+        InstanceAllocationStrategy::pooling(),
+    ] {
+        let mut config = Config::default();
+        Wasi::add_to_config(&mut config);
+        config.allocation_strategy(strategy.clone());
+
+        let engine = Engine::new(&config).expect("failed to create engine");
+        let module = Module::from_file(&engine, "benches/instantiation/wasi.wasm")
+            .expect("failed to load WASI example module");
+
+        for threads in 1..=num_cpus::get_physical() {
+            let pool = ThreadPoolBuilder::new()
+                .num_threads(threads)
+                .build()
+                .unwrap();
+
+            group.bench_function(
+                BenchmarkId::new(
+                    benchmark_name(strategy),
+                    format!(
+                        "{} instances with {} thread{}",
+                        PARALLEL_INSTANCES,
+                        threads,
+                        if threads == 1 { "" } else { "s" }
+                    ),
+                ),
+                |b| {
+                    b.iter(|| {
+                        pool.install(|| {
+                            (0..PARALLEL_INSTANCES).into_par_iter().for_each(|_| {
+                                instantiate(&module).expect("failed to instantiate module");
+                            })
+                        })
+                    });
+                },
+            );
+        }
+    }
+
+    group.finish();
+}
+
+fn build_wasi_example() {
+    println!("Building WASI example module...");
+    if !Command::new("cargo")
+        .args(&[
+            "build",
+            "--release",
+            "-p",
+            "example-wasi-wasm",
+            "--target",
+            "wasm32-wasi",
+        ])
+        .spawn()
+        .expect("failed to run cargo to build WASI example")
+        .wait()
+        .expect("failed to wait for cargo to build")
+        .success()
+    {
+        panic!("failed to build WASI example for target `wasm32-wasi`");
+    }
+
+    std::fs::copy(
+        "target/wasm32-wasi/release/wasi.wasm",
+        "benches/instantiation/wasi.wasm",
+    )
+    .expect("failed to copy WASI example module");
+}
+
+fn bench_instantiation(c: &mut Criterion) {
+    build_wasi_example();
+    bench_sequential(
+        c,
+        &[
+            "empty.wat",
+            "small_memory.wat",
+            "data_segments.wat",
+            "wasi.wasm",
+        ],
+    );
+    bench_parallel(c);
+}
+
+criterion_group!(benches, bench_instantiation);
+criterion_main!(benches);

benches/instantiation/data_segments.wat

@@ -0,0 +1,5 @@
+(module
+    (memory 17)
+    (func (export "_start"))
+    (data (i32.const 0) "0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789")
+)

benches/instantiation/empty.wat

@@ -0,0 +1,3 @@
+(module
+    (func (export "_start"))
+)

benches/instantiation/small_memory.wat

@@ -0,0 +1,4 @@
+(module
+    (memory 1)
+    (func (export "_start"))
+)

crates/jit/src/instantiate.rs

@@ -35,7 +35,7 @@ pub enum SetupError {
     #[error("Validation error: {0}")]
     Validate(String),
 
-    /// A wasm translation error occured.
+    /// A wasm translation error occurred.
     #[error("WebAssembly failed to compile")]
     Compile(#[from] CompileError),
 
@@ -44,7 +44,7 @@ pub enum SetupError {
     #[error("Instantiation failed during setup")]
     Instantiate(#[from] InstantiationError),
 
-    /// Debug information generation error occured.
+    /// Debug information generation error occurred.
     #[error("Debug information error")]
     DebugInfo(#[from] anyhow::Error),
 }

crates/runtime/src/instance/allocator/pooling.rs

@@ -1610,7 +1610,7 @@ mod test {
     }
 
     #[test]
-    fn test_pooling_allocator_with_memory_pages_exeeded() {
+    fn test_pooling_allocator_with_memory_pages_exceeded() {
         assert_eq!(
             PoolingInstanceAllocator::new(
                 PoolingAllocationStrategy::Random,
@@ -1631,7 +1631,7 @@ mod test {
     }
 
     #[test]
-    fn test_pooling_allocator_with_reservation_size_exeeded() {
+    fn test_pooling_allocator_with_reservation_size_exceeded() {
         assert_eq!(
             PoolingInstanceAllocator::new(
                 PoolingAllocationStrategy::Random,