Enable jitdump profiling support by default

This the result of some of the investigation I was doing for bytecodealliance#1017. I've
done a number of refactorings here which culminated in a number of
changes that all amount to what I think should result in jitdump support being
enabled by default:

* Pass in a list of finished functions instead of just a range to
  ensure that we're emitting jit dump data for a specific module rather
  than a whole `CodeMemory` which may have other modules.
* Define `ProfilingStrategy` in the `wasmtime` crate to have everything
  locally-defined
* Add support to the C API to enable profiling
* Documentation added for profiling with jitdump to the book
* Split out supported/unsupported files in `jitdump.rs` to avoid having
  lots of `#[cfg]`.
* Make dependencies optional that are only used for `jitdump`.
* Move initialization up-front to `JitDumpAgent::new()` instead of
  deferring it to the first module.
* Pass around `Arc<dyn ProfilingAgent>` instead of
  `Option<Arc<Mutex<Box<dyn ProfilingAgent>>>>`

The `jitdump` Cargo feature is now enabled by default which means that
our published binaries, C API artifacts, and crates will support
profiling at runtime by default. The support I don't think is fully
fleshed out and working but I think it's probably in a good enough spot
we can get users playing around with it!

Cargo.toml

@@ -23,12 +23,11 @@ doc = false
 
 [dependencies]
 # Enable all supported architectures by default.
-wasmtime = { path = "crates/api" }
+wasmtime = { path = "crates/api", default-features = false }
 wasmtime-debug = { path = "crates/debug" }
 wasmtime-environ = { path = "crates/environ" }
 wasmtime-jit = { path = "crates/jit" }
 wasmtime-obj = { path = "crates/obj" }
-wasmtime-profiling = { path = "crates/profiling" }
 wasmtime-wast = { path = "crates/wast" }
 wasmtime-wasi = { path = "crates/wasi" }
 wasi-common = { path = "crates/wasi-common" }
@@ -72,13 +71,14 @@ members = [
 ]
 
 [features]
+default = ["jitdump"]
 lightbeam = [
     "wasmtime-environ/lightbeam",
     "wasmtime-jit/lightbeam",
     "wasmtime-wast/lightbeam",
     "wasmtime/lightbeam",
 ]
-jitdump = ["wasmtime-profiling/jitdump"]
+jitdump = ["wasmtime/jitdump"]
 test_programs = ["test-programs/test_programs"]
 
 [badges]

crates/api/Cargo.toml

@@ -40,13 +40,16 @@ tempfile = "3.1"
 maintenance = { status = "actively-developed" }
 
 [features]
-default = ['wat']
+default = ['wat', 'jitdump']
 
 # Enables experimental support for the lightbeam codegen backend, an alternative
 # to cranelift. Requires Nightly Rust currently, and this is not enabled by
 # default.
 lightbeam = ["wasmtime-jit/lightbeam"]
 
+# Enables support for the `perf` jitdump profiler
+jitdump = ["wasmtime-jit/jitdump"]
+
 [[test]]
 name = "host-segfault"
 harness = false

crates/api/src/lib.rs

@@ -26,7 +26,7 @@ pub use crate::func::*;
 pub use crate::instance::Instance;
 pub use crate::module::Module;
 pub use crate::r#ref::{AnyRef, HostInfo, HostRef};
-pub use crate::runtime::{Config, Engine, OptLevel, Store, Strategy};
+pub use crate::runtime::*;
 pub use crate::trap::Trap;
 pub use crate::types::*;
 pub use crate::values::*;

crates/api/src/module.rs

@@ -367,7 +367,7 @@ impl Module {
             &mut store.compiler_mut(),
             binary,
             store.engine().config().debug_info,
-            store.engine().config().profiler.as_ref(),
+            &*store.engine().config().profiler,
         )?;
 
         Ok(Module {

crates/api/src/runtime.rs

@@ -3,12 +3,12 @@ use std::cell::RefCell;
 use std::fmt;
 use std::path::Path;
 use std::rc::Rc;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use wasmparser::{OperatorValidatorConfig, ValidatingParserConfig};
 use wasmtime_environ::settings::{self, Configurable};
 use wasmtime_environ::CacheConfig;
 use wasmtime_jit::{native, CompilationStrategy, Compiler};
-use wasmtime_profiling::{JitDumpAgent, ProfilingAgent, ProfilingStrategy};
+use wasmtime_profiling::{JitDumpAgent, NullProfilerAgent, ProfilingAgent};
 
 // Runtime Environment
 
@@ -26,7 +26,7 @@ pub struct Config {
     pub(crate) debug_info: bool,
     pub(crate) strategy: CompilationStrategy,
     pub(crate) cache_config: CacheConfig,
-    pub(crate) profiler: Option<Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
+    pub(crate) profiler: Arc<dyn ProfilingAgent>,
 }
 
 impl Config {
@@ -65,7 +65,7 @@ impl Config {
             flags,
             strategy: CompilationStrategy::Auto,
             cache_config: CacheConfig::new_cache_disabled(),
-            profiler: None,
+            profiler: Arc::new(NullProfilerAgent),
         }
     }
 
@@ -225,11 +225,9 @@ impl Config {
     /// Profiler creation calls the type's default initializer where the purpose is
     /// really just to put in place the type used for profiling.
     pub fn profiler(&mut self, profile: ProfilingStrategy) -> Result<&mut Self> {
-        match profile {
-            ProfilingStrategy::JitDumpProfiler => {
-                self.profiler = { Some(Arc::new(Mutex::new(Box::new(JitDumpAgent::default())))) }
-            }
-            _ => self.profiler = { None },
+        self.profiler = match profile {
+            ProfilingStrategy::JitDump => Arc::new(JitDumpAgent::new()?) as Arc<dyn ProfilingAgent>,
+            ProfilingStrategy::None => Arc::new(NullProfilerAgent),
         };
         Ok(self)
     }
@@ -381,6 +379,16 @@ pub enum OptLevel {
     SpeedAndSize,
 }
 
+/// Select which profiling technique to use
+#[derive(Debug, Clone, Copy)]
+pub enum ProfilingStrategy {
+    /// No profiler support
+    None,
+
+    /// Collect profile for jitdump file format, used with `perf` on Linux
+    JitDump,
+}
+
 // Engine
 
 /// An `Engine` which is a global context for compilation and management of wasm

crates/c-api/Cargo.toml

@@ -17,7 +17,12 @@ test = false
 doctest = false
 
 [dependencies]
-wasmtime = { path = "../api" }
+wasmtime = { path = "../api", default-features = false }
 wasi-common = { path = "../wasi-common" }
 wasmtime-wasi = { path = "../wasi" }
 wat = "1.0"
+
+[features]
+default = ['jitdump']
+lightbeam = ["wasmtime/lightbeam"]
+jitdump = ["wasmtime/jitdump"]

crates/c-api/include/wasmtime.h

@@ -25,6 +25,12 @@ enum wasmtime_opt_level_enum { // OptLevel
   WASMTIME_OPT_LEVEL_SPEED_AND_SIZE,
 };
 
+typedef uint8_t wasmtime_profiling_strategy_t;
+enum wasmtime_profiling_strategy_t { // ProfilingStrategy
+  WASMTIME_PROFILING_STRATEGY_NONE,
+  WASMTIME_PROFILING_STRATEGY_JITDUMP,
+};
+
 #define WASMTIME_CONFIG_PROP(name, ty) \
     WASM_API_EXTERN void wasmtime_config_##name##_set(wasm_config_t*, ty);
 
@@ -37,6 +43,7 @@ WASMTIME_CONFIG_PROP(wasm_multi_value, bool)
 WASMTIME_CONFIG_PROP(strategy, wasmtime_strategy_t)
 WASMTIME_CONFIG_PROP(cranelift_debug_verifier, bool)
 WASMTIME_CONFIG_PROP(cranelift_opt_level, wasmtime_opt_level_t)
+WASMTIME_CONFIG_PROP(profiler, wasmtime_profiling_strategy_t)
 
 ///////////////////////////////////////////////////////////////////////////////
 

crates/c-api/src/ext.rs

@@ -3,7 +3,7 @@
 
 use crate::{wasm_byte_vec_t, wasm_config_t, wasm_engine_t};
 use std::str;
-use wasmtime::{OptLevel, Strategy};
+use wasmtime::{OptLevel, Strategy, ProfilingStrategy};
 
 #[repr(u8)]
 #[derive(Clone)]
@@ -21,6 +21,13 @@ pub enum wasmtime_opt_level_t {
     WASMTIME_OPT_LEVEL_SPEED_AND_SIZE,
 }
 
+#[repr(u8)]
+#[derive(Clone)]
+pub enum wasmtime_profiling_strategy_t {
+    WASMTIME_PROFILING_STRATEGY_NONE,
+    WASMTIME_PROFILING_STRATEGY_JITDUMP,
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn wasmtime_config_debug_info_set(c: *mut wasm_config_t, enable: bool) {
     (*c).config.debug_info(enable);
@@ -88,6 +95,18 @@ pub unsafe extern "C" fn wasmtime_config_cranelift_opt_level_set(
     });
 }
 
+#[no_mangle]
+pub unsafe extern "C" fn wasmtime_config_profiler_set(
+    c: *mut wasm_config_t,
+    strategy: wasmtime_profiling_strategy_t,
+) {
+    use wasmtime_profiling_strategy_t::*;
+    drop((*c).config.profiler(match strategy {
+        WASMTIME_PROFILING_STRATEGY_NONE => ProfilingStrategy::None,
+        WASMTIME_PROFILING_STRATEGY_JITDUMP => ProfilingStrategy::JitDump,
+    }));
+}
+
 #[no_mangle]
 pub unsafe extern "C" fn wasmtime_wat2wasm(
     _engine: *mut wasm_engine_t,

crates/jit/Cargo.toml

@@ -34,6 +34,7 @@ winapi = { version = "0.3.7", features = ["winnt", "impl-default"] }
 
 [features]
 lightbeam = ["wasmtime-environ/lightbeam"]
+jitdump = ["wasmtime-profiling/jitdump"]
 
 [badges]
 maintenance = { status = "actively-developed" }

crates/jit/src/code_memory.rs

@@ -5,7 +5,6 @@ use region;
 use std::mem::ManuallyDrop;
 use std::{cmp, mem};
 use wasmtime_environ::{Compilation, CompiledFunction};
-use wasmtime_profiling::ProfilingAgent;
 use wasmtime_runtime::{Mmap, VMFunctionBody};
 
 struct CodeMemoryEntry {
@@ -237,22 +236,4 @@ impl CodeMemory {
 
         Ok(())
     }
-
-    /// Calls the module_load for a given ProfilerAgent. Includes
-    /// all memory address and length for the given module.
-    /// TODO: Properly handle the possibilities of multiple mmapped regions
-    /// which may, amongst other things, influence being more specific about
-    /// the module name.
-    pub fn profiler_module_load(
-        &mut self,
-        profiler: &mut Box<dyn ProfilingAgent + Send>,
-        module_name: &str,
-        dbg_image: Option<&[u8]>,
-    ) -> () {
-        for CodeMemoryEntry { mmap: m, table: _t } in &mut self.entries {
-            if m.len() > 0 {
-                profiler.module_load(module_name, m.as_ptr(), m.len(), dbg_image);
-            }
-        }
-    }
 }

crates/jit/src/compiler.rs

@@ -22,7 +22,6 @@ use wasmtime_environ::{
     FunctionBodyData, Module, ModuleMemoryOffset, ModuleVmctxInfo, Relocation, Relocations, Traps,
     Tunables, VMOffsets,
 };
-use wasmtime_profiling::ProfilingAgent;
 use wasmtime_runtime::{
     InstantiationError, SignatureRegistry, TrapRegistration, TrapRegistry, VMFunctionBody,
     VMSharedSignatureIndex, VMTrampoline,
@@ -246,16 +245,6 @@ impl Compiler {
         self.code_memory.publish();
     }
 
-    pub(crate) fn profiler_module_load(
-        &mut self,
-        profiler: &mut Box<dyn ProfilingAgent + Send>,
-        module_name: &str,
-        dbg_image: Option<&[u8]>,
-    ) -> () {
-        self.code_memory
-            .profiler_module_load(profiler, module_name, dbg_image);
-    }
-
     /// Shared signature registry.
     pub fn signatures(&self) -> &SignatureRegistry {
         &self.signatures

crates/jit/src/instantiate.rs

@@ -10,7 +10,7 @@ use crate::resolver::Resolver;
 use std::collections::HashMap;
 use std::io::Write;
 use std::rc::Rc;
-use std::sync::{Arc, Mutex};
+use std::sync::Arc;
 use thiserror::Error;
 use wasmtime_debug::read_debuginfo;
 use wasmtime_environ::entity::{BoxedSlice, PrimaryMap};
@@ -64,7 +64,7 @@ impl<'data> RawCompiledModule<'data> {
         compiler: &mut Compiler,
         data: &'data [u8],
         debug_info: bool,
-        profiler: Option<&Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
+        profiler: &dyn ProfilingAgent,
     ) -> Result<Self, SetupError> {
         let environ = ModuleEnvironment::new(compiler.frontend_config(), compiler.tunables());
 
@@ -105,19 +105,11 @@ impl<'data> RawCompiledModule<'data> {
         compiler.publish_compiled_code();
 
         // Initialize profiler and load the wasm module
-        match profiler {
-            Some(_) => {
-                let region_name = String::from("wasm_module");
-                let mut profiler = profiler.unwrap().lock().unwrap();
-                match &compilation.dbg_image {
-                    Some(dbg) => {
-                        compiler.profiler_module_load(&mut profiler, &region_name, Some(&dbg))
-                    }
-                    _ => compiler.profiler_module_load(&mut profiler, &region_name, None),
-                };
-            }
-            _ => (),
-        };
+        profiler.module_load(
+            &translation.module,
+            &compilation.finished_functions,
+            compilation.dbg_image.as_deref(),
+        );
 
         let dbg_jit_registration = if let Some(img) = compilation.dbg_image {
             let mut bytes = Vec::new();
@@ -157,7 +149,7 @@ impl CompiledModule {
         compiler: &mut Compiler,
         data: &'data [u8],
         debug_info: bool,
-        profiler: Option<&Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
+        profiler: &dyn ProfilingAgent,
     ) -> Result<Self, SetupError> {
         let raw = RawCompiledModule::<'data>::new(compiler, data, debug_info, profiler)?;
 
@@ -290,7 +282,7 @@ pub unsafe fn instantiate(
     resolver: &mut dyn Resolver,
     debug_info: bool,
     is_bulk_memory: bool,
-    profiler: Option<&Arc<Mutex<Box<dyn ProfilingAgent + Send>>>>,
+    profiler: &dyn ProfilingAgent,
 ) -> Result<InstanceHandle, SetupError> {
     let instance = CompiledModule::new(compiler, data, debug_info, profiler)?.instantiate(
         is_bulk_memory,

crates/profiling/Cargo.toml

@@ -11,17 +11,21 @@ readme = "README.md"
 edition = "2018"
 
 [dependencies]
+anyhow = "1.0"
+cfg-if = "0.1"
+gimli = { version = "0.20.0", optional = true }
+goblin = { version = "0.1.3", optional = true }
+lazy_static = "1.4"
 libc = { version = "0.2.60", default-features = false }
-goblin = "0.1.3"
+object = { version = "0.17.0", optional = true }
+scroll = { version = "0.10.1", optional = true }
 serde = { version = "1.0.99", features = ["derive"] }
-scroll = "0.10.1"
-gimli = "0.20.0"
-object = "0.17.0"
 target-lexicon = "0.10.0"
-lazy_static = "1.4"
+wasmtime-environ = { path = "../environ", version = "0.12.0" }
+wasmtime-runtime = { path = "../runtime", version = "0.12.0" }
 
 [badges]
 maintenance = { status = "actively-developed" }
 
 [features]
-jitdump = []
+jitdump = ['goblin', 'object', 'scroll', 'gimli']