diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index 71bcf215de2a..277d30213984 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -1248,10 +1248,16 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m mut pos: cranelift_codegen::cursor::FuncCursor<'_>, func_index: FuncIndex, ) -> WasmResult { - let vmctx = self.vmctx(&mut pos.func); - let vmctx = pos.ins().global_value(self.pointer_type(), vmctx); - let offset = self.offsets.vmctx_anyfunc(func_index); - Ok(pos.ins().iadd_imm(vmctx, i64::from(offset))) + let func_index = pos.ins().iconst(I32, func_index.as_u32() as i64); + let builtin_index = BuiltinFunctionIndex::ref_func(); + let builtin_sig = self.builtin_function_signatures.ref_func(&mut pos.func); + let (vmctx, builtin_addr) = + self.translate_load_builtin_function_address(&mut pos, builtin_index); + + let call_inst = pos + .ins() + .call_indirect(builtin_sig, builtin_addr, &[vmctx, func_index]); + Ok(pos.func.dfg.first_result(call_inst)) } fn translate_custom_global_get( diff --git a/crates/environ/src/builtin.rs b/crates/environ/src/builtin.rs index e660ffc2d7cd..46491a3be3b9 100644 --- a/crates/environ/src/builtin.rs +++ b/crates/environ/src/builtin.rs @@ -18,6 +18,8 @@ macro_rules! foreach_builtin_function { memory_fill(vmctx, i32, i64, i32, i64) -> (); /// Returns an index for wasm's `memory.init` instruction. memory_init(vmctx, i32, i32, i64, i32, i32) -> (); + /// Returns a value for wasm's `ref.func` instruction. + ref_func(vmctx, i32) -> (pointer); /// Returns an index for wasm's `data.drop` instruction. data_drop(vmctx, i32) -> (); /// Returns an index for Wasm's `table.grow` instruction for `funcref`s. diff --git a/crates/environ/src/vmoffsets.rs b/crates/environ/src/vmoffsets.rs index a2f0b9d558f4..069e5ecb3950 100644 --- a/crates/environ/src/vmoffsets.rs +++ b/crates/environ/src/vmoffsets.rs @@ -85,6 +85,8 @@ pub struct VMOffsets

{ defined_memories: u32, defined_globals: u32, defined_anyfuncs: u32, + anyfuncs_init: u32, + anyfuncs_init_u64_words: u32, builtin_functions: u32, size: u32, } @@ -187,6 +189,8 @@ impl From> for VMOffsets

{ defined_memories: 0, defined_globals: 0, defined_anyfuncs: 0, + anyfuncs_init: 0, + anyfuncs_init_u64_words: 0, builtin_functions: 0, size: 0, }; @@ -275,7 +279,7 @@ impl From> for VMOffsets

{ .unwrap(), ) .unwrap(); - ret.builtin_functions = ret + ret.anyfuncs_init = ret .defined_anyfuncs .checked_add( ret.num_imported_functions @@ -285,6 +289,11 @@ impl From> for VMOffsets

{ .unwrap(), ) .unwrap(); + ret.anyfuncs_init_u64_words = (ret.num_defined_functions + 63) / 64; + ret.builtin_functions = ret + .anyfuncs_init + .checked_add(ret.anyfuncs_init_u64_words.checked_mul(8).unwrap()) + .unwrap(); ret.size = ret .builtin_functions .checked_add( @@ -589,6 +598,18 @@ impl VMOffsets

{ self.defined_globals } + /// The offset of the `anyfuncs_init` bitset. + #[inline] + pub fn vmctx_anyfuncs_init_begin(&self) -> u32 { + self.anyfuncs_init + } + + /// The length of the `anyfuncs_init` bitset in bytes. + #[inline] + pub fn vmctx_anyfuncs_init_len(&self) -> u32 { + self.anyfuncs_init_u64_words * 8 + } + /// The offset of the `anyfuncs` array. #[inline] pub fn vmctx_anyfuncs_begin(&self) -> u32 { diff --git a/crates/jit/src/instantiate.rs b/crates/jit/src/instantiate.rs index cc6a3844d187..9119126b7692 100644 --- a/crates/jit/src/instantiate.rs +++ b/crates/jit/src/instantiate.rs @@ -242,7 +242,7 @@ pub struct CompiledModule { address_map_data: Range, trap_data: Range, module: Arc, - funcs: PrimaryMap, + funcs: Arc>, trampolines: Vec, meta: Metadata, code: Range, @@ -298,7 +298,7 @@ impl CompiledModule { let mut ret = Self { module: Arc::new(info.module), - funcs: info.funcs, + funcs: Arc::new(info.funcs), trampolines: info.trampolines, wasm_data: subslice_range(section(ELF_WASM_DATA)?, code.mmap), address_map_data: code @@ -374,7 +374,7 @@ impl CompiledModule { } /// Returns the `FunctionInfo` map for all defined functions. - pub fn functions(&self) -> &PrimaryMap { + pub fn functions(&self) -> &Arc> { &self.funcs } diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 2c9487d75ec9..8c1fc4078d88 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -54,6 +54,9 @@ pub(crate) struct Instance { /// The `Module` this `Instance` was instantiated from. module: Arc, + /// The instantiation info needed for deferred initialization. + info: Arc, + /// Offsets in the `vmctx` region, precomputed from the `module` above. offsets: VMOffsets, @@ -433,6 +436,48 @@ impl Instance { Layout::from_size_align(size, align).unwrap() } + /// Construct a new VMCallerCheckedAnyfunc for the given function + /// (imported or defined in this module). Used during lazy + /// initialization the first time the VMCallerCheckedAnyfunc in + /// the VMContext is referenced; written into place in a careful + /// atomic way by `get_caller_checked_anyfunc()` below. + fn construct_anyfunc(&self, index: FuncIndex) -> VMCallerCheckedAnyfunc { + let sig = self.module.functions[index]; + let type_index = self.info.shared_signatures.lookup(sig); + + let (func_ptr, vmctx) = if let Some(def_index) = self.module.defined_func_index(index) { + ( + (self.info.image_base + self.info.functions[def_index].start as usize) as *mut _, + self.vmctx_ptr(), + ) + } else { + let import = self.imported_function(index); + (import.body.as_ptr(), import.vmctx) + }; + + VMCallerCheckedAnyfunc { + func_ptr, + type_index, + vmctx, + } + } + + /// Get a word of the bitmap of anyfunc-initialized bits, and the + /// bitmask for the particular bit. + pub(crate) fn get_anyfunc_bitmap_word(&self, index: FuncIndex) -> (&mut u64, u64) { + let word_index = index.as_u32() / 64; + let bit_index = index.as_u32() % 64; + let word = unsafe { + self.vmctx_plus_offset::( + self.offsets.vmctx_anyfuncs_init_begin() + (word_index * 8), + ) + .as_mut() + .unwrap() + }; + let mask = 1u64 << (bit_index as u64); + (word, mask) + } + /// Get a `&VMCallerCheckedAnyfunc` for the given `FuncIndex`. /// /// Returns `None` if the index is the reserved index value. @@ -447,7 +492,61 @@ impl Instance { return None; } - unsafe { Some(&*self.vmctx_plus_offset(self.offsets.vmctx_anyfunc(index))) } + unsafe { + let anyfunc = + self.vmctx_plus_offset::(self.offsets.vmctx_anyfunc(index)); + let (bitmap_word, bitmask) = self.get_anyfunc_bitmap_word(index); + if *bitmap_word & bitmask == 0 { + // N.B.: both the anyfunc write and the bitmap-word + // write are non-atomic, yet this is inside a `&self` + // method. How does that work? In fact it is a benign + // race. In the most common case, an anyfunc is + // queried either as part of table init or as part of + // executing a function, both of which have a `&mut + // Store` so are actually exclusive. But if + // e.g. export-lookups happen in parallel, we might + // actually race. + // + // The race is benign because (i) the write is + // idempotent -- the constructed anyfunc will always + // be exactly the same, once it is constructed, so a + // second write, even partially complete, is fine + // (even if fragmented/split; any fragment written + // will just overwrite the same bytes); and (ii) + // racing to set a bit in the bitmap might miss some + // updates, and falsely clear a bit, but that just + // means that we do another init later, which as per + // (i) is safe. + // + // The important safety property is that before we + // hand out a reference to an anyfunc, we have + // initialized it *at least once*, and this property + // is always true because (i) unrelated bits are never + // *set* in the bitmap spontaneously, even by a race; + // (ii) this function always verifies set bit before + // returning; and (iii) once initialized, an anyfunc + // stays initialized (idempotent writes discussed + // above). + // + // Note that we tried it the other way first, with + // atomics rather than benign races. Using an atomic + // in the anyfunc itself to tell whether it's + // initialized is expensive because we have to zero + // all anyfuncs at instantiation, and zeroing words, + // located sparsely in the vmcontext, is expensive + // compared to zeroing a dense bitmap. Using atomic + // updates to the bitmap is expensive because + // `fetch_or` is expensive; the `lock or` instruction + // on x86-64 was the majority of the profile in an + // instantiation microbenchmark. + *anyfunc = self.construct_anyfunc(index); + let (bitmap_word, bitmask) = self.get_anyfunc_bitmap_word(index); + *bitmap_word |= bitmask; + } + + let anyfunc = anyfunc.as_ref().unwrap(); + Some(anyfunc) + } } unsafe fn anyfunc_base(&self) -> *mut VMCallerCheckedAnyfunc { @@ -513,6 +612,7 @@ impl Instance { ptr::null_mut() } else { debug_assert!(idx.as_u32() < self.offsets.num_defined_functions); + self.get_caller_checked_anyfunc(*idx); // Force lazy init base.add(usize::try_from(idx.as_u32()).unwrap()) } }), diff --git a/crates/runtime/src/instance/allocator.rs b/crates/runtime/src/instance/allocator.rs index 82c1eec31e8e..0f015601fcfb 100644 --- a/crates/runtime/src/instance/allocator.rs +++ b/crates/runtime/src/instance/allocator.rs @@ -13,7 +13,7 @@ use std::alloc; use std::any::Any; use std::convert::TryFrom; use std::marker; -use std::ptr::{self, NonNull}; +use std::ptr; use std::slice; use std::sync::Arc; use thiserror::Error; @@ -31,24 +31,34 @@ pub use self::pooling::{ InstanceLimits, ModuleLimits, PoolingAllocationStrategy, PoolingInstanceAllocator, }; -/// Represents a request for a new runtime instance. -pub struct InstanceAllocationRequest<'a> { - /// The module being instantiated. - pub module: Arc, - +/// Information needed for the instance allocation request and +/// afterward as well (for lazy initialization). Will be held alive by +/// the instance. +#[derive(Default)] +pub struct InstanceAllocationInfo { /// The base address of where JIT functions are located. pub image_base: usize, /// Descriptors about each compiled function, such as the offset from /// `image_base`. - pub functions: &'a PrimaryMap, + pub functions: Arc>, + + /// Translation from `SignatureIndex` to `VMSharedSignatureIndex` + pub shared_signatures: SharedSignatures, +} + +/// Represents a request for a new runtime instance. +pub struct InstanceAllocationRequest<'a> { + /// The info needed for both the allocation request and deferred + /// initialization. + pub info: Arc, + + /// The module being instantiated. + pub module: Arc, /// The imports to use for the instantiation. pub imports: Imports<'a>, - /// Translation from `SignatureIndex` to `VMSharedSignatureIndex` - pub shared_signatures: SharedSignatures<'a>, - /// The host state to associate with the instance. pub host_state: Box, @@ -213,17 +223,18 @@ pub unsafe trait InstanceAllocator: Send + Sync { unsafe fn deallocate_fiber_stack(&self, stack: &wasmtime_fiber::FiberStack); } -pub enum SharedSignatures<'a> { +/// Shared signatures. +pub enum SharedSignatures { /// Used for instantiating user-defined modules - Table(&'a PrimaryMap), + Table(PrimaryMap), /// Used for instance creation that has only a single function Always(VMSharedSignatureIndex), /// Used for instance creation that has no functions None, } -impl SharedSignatures<'_> { - fn lookup(&self, index: SignatureIndex) -> VMSharedSignatureIndex { +impl SharedSignatures { + pub(crate) fn lookup(&self, index: SignatureIndex) -> VMSharedSignatureIndex { match self { SharedSignatures::Table(table) => table[index], SharedSignatures::Always(index) => *index, @@ -232,14 +243,20 @@ impl SharedSignatures<'_> { } } -impl<'a> From for SharedSignatures<'a> { - fn from(val: VMSharedSignatureIndex) -> SharedSignatures<'a> { +impl std::default::Default for SharedSignatures { + fn default() -> Self { + SharedSignatures::None + } +} + +impl From for SharedSignatures { + fn from(val: VMSharedSignatureIndex) -> SharedSignatures { SharedSignatures::Always(val) } } -impl<'a> From> for SharedSignatures<'a> { - fn from(val: Option) -> SharedSignatures<'a> { +impl From> for SharedSignatures { + fn from(val: Option) -> SharedSignatures { match val { Some(idx) => SharedSignatures::Always(idx), None => SharedSignatures::None, @@ -247,9 +264,9 @@ impl<'a> From> for SharedSignatures<'a> { } } -impl<'a> From<&'a PrimaryMap> for SharedSignatures<'a> { - fn from(val: &'a PrimaryMap) -> SharedSignatures<'a> { - SharedSignatures::Table(val) +impl From<&PrimaryMap> for SharedSignatures { + fn from(val: &PrimaryMap) -> SharedSignatures { + SharedSignatures::Table(val.clone()) } } @@ -474,7 +491,7 @@ unsafe fn initialize_vmcontext(instance: &mut Instance, req: InstanceAllocationR let mut ptr = instance.vmctx_plus_offset(instance.offsets.vmctx_signature_ids_begin()); for sig in module.types.values() { *ptr = match sig { - ModuleType::Function(sig) => req.shared_signatures.lookup(*sig), + ModuleType::Function(sig) => req.info.shared_signatures.lookup(*sig), _ => VMSharedSignatureIndex::new(u32::max_value()), }; ptr = ptr.add(1); @@ -512,32 +529,12 @@ unsafe fn initialize_vmcontext(instance: &mut Instance, req: InstanceAllocationR req.imports.globals.len(), ); - // Initialize the functions - let mut base = instance.anyfunc_base(); - for (index, sig) in instance.module.functions.iter() { - let type_index = req.shared_signatures.lookup(*sig); - - let (func_ptr, vmctx) = if let Some(def_index) = instance.module.defined_func_index(index) { - ( - NonNull::new((req.image_base + req.functions[def_index].start as usize) as *mut _) - .unwrap(), - instance.vmctx_ptr(), - ) - } else { - let import = instance.imported_function(index); - (import.body, import.vmctx) - }; - - ptr::write( - base, - VMCallerCheckedAnyfunc { - func_ptr, - type_index, - vmctx, - }, - ); - base = base.add(1); - } + // Zero the anyfunc-initialized bitmap -- they will be lazily initialized as requred + let base = instance.vmctx_plus_offset(instance.offsets.vmctx_anyfuncs_init_begin()); + let bitmap_words = (instance.module.functions.len() + 63) / 64; + let len = bitmap_words * 8; + let slice = std::slice::from_raw_parts_mut(base as *mut u8, len); + slice.fill(0); // Initialize the defined tables let mut ptr = instance.vmctx_plus_offset(instance.offsets.vmctx_tables_begin()); @@ -693,6 +690,7 @@ unsafe impl InstanceAllocator for OnDemandInstanceAllocator { let mut handle = { let instance = Instance { module: req.module.clone(), + info: req.info.clone(), offsets: VMOffsets::new(HostPtr, &req.module), memories, tables, diff --git a/crates/runtime/src/instance/allocator/pooling.rs b/crates/runtime/src/instance/allocator/pooling.rs index 76614137d5e9..710447c91c91 100644 --- a/crates/runtime/src/instance/allocator/pooling.rs +++ b/crates/runtime/src/instance/allocator/pooling.rs @@ -11,6 +11,7 @@ use super::{ initialize_instance, initialize_vmcontext, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, }; +use crate::InstanceAllocationInfo; use crate::{instance::Instance, Memory, Mmap, Table, VMContext}; use anyhow::{anyhow, bail, Context, Result}; use rand::Rng; @@ -357,6 +358,7 @@ impl InstancePool { instance as _, Instance { module: self.empty_module.clone(), + info: Arc::new(InstanceAllocationInfo::default()), offsets: VMOffsets::new(HostPtr, &self.empty_module), memories: PrimaryMap::with_capacity(limits.memories as usize), tables: PrimaryMap::with_capacity(limits.tables as usize), @@ -380,6 +382,7 @@ impl InstancePool { let instance = self.instance(index); instance.module = req.module.clone(); + instance.info = req.info.clone(); instance.offsets = VMOffsets::new(HostPtr, instance.module.as_ref()); instance.host_state = std::mem::replace(&mut req.host_state, Box::new(())); instance.wasm_data = &*req.wasm_data; @@ -1392,7 +1395,7 @@ mod test { let mut handles = Vec::new(); let module = Arc::new(Module::default()); - let functions = &PrimaryMap::new(); + let functions = Arc::new(PrimaryMap::new()); for _ in (0..3).rev() { handles.push( @@ -1401,18 +1404,20 @@ mod test { PoolingAllocationStrategy::NextAvailable, InstanceAllocationRequest { module: module.clone(), - image_base: 0, - functions, imports: Imports { functions: &[], tables: &[], memories: &[], globals: &[], }, - shared_signatures: VMSharedSignatureIndex::default().into(), host_state: Box::new(()), store: StorePtr::empty(), wasm_data: &[], + info: Arc::new(InstanceAllocationInfo { + image_base: 0, + functions: functions.clone(), + shared_signatures: VMSharedSignatureIndex::default().into(), + }), }, ) .expect("allocation should succeed"), @@ -1425,18 +1430,20 @@ mod test { PoolingAllocationStrategy::NextAvailable, InstanceAllocationRequest { module: module.clone(), - functions, - image_base: 0, imports: Imports { functions: &[], tables: &[], memories: &[], globals: &[], }, - shared_signatures: VMSharedSignatureIndex::default().into(), host_state: Box::new(()), store: StorePtr::empty(), wasm_data: &[], + info: Arc::new(InstanceAllocationInfo { + functions: functions.clone(), + image_base: 0, + shared_signatures: VMSharedSignatureIndex::default().into(), + }), }, ) { Err(InstantiationError::Limit(3)) => {} diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index f96e7d8dda27..522bcadb6fdb 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -42,8 +42,8 @@ pub use crate::export::*; pub use crate::externref::*; pub use crate::imports::Imports; pub use crate::instance::{ - InstanceAllocationRequest, InstanceAllocator, InstanceHandle, InstantiationError, LinkError, - OnDemandInstanceAllocator, StorePtr, + InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, + InstantiationError, LinkError, OnDemandInstanceAllocator, SharedSignatures, StorePtr, }; #[cfg(feature = "pooling-allocator")] pub use crate::instance::{ diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 2544e1673987..4ae82a888257 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -64,7 +64,9 @@ use crate::vmcontext::{VMCallerCheckedAnyfunc, VMContext}; use backtrace::Backtrace; use std::mem; use std::ptr::{self, NonNull}; -use wasmtime_environ::{DataIndex, ElemIndex, GlobalIndex, MemoryIndex, TableIndex, TrapCode}; +use wasmtime_environ::{ + DataIndex, ElemIndex, FuncIndex, GlobalIndex, MemoryIndex, TableIndex, TrapCode, +}; const TOINT_32: f32 = 1.0 / f32::EPSILON; const TOINT_64: f64 = 1.0 / f64::EPSILON; @@ -379,6 +381,15 @@ pub unsafe extern "C" fn wasmtime_memory_init( } } +/// Implementation of `ref.func`. +pub unsafe extern "C" fn wasmtime_ref_func(vmctx: *mut VMContext, func_index: u32) -> usize { + let instance = (*vmctx).instance(); + let anyfunc = instance + .get_caller_checked_anyfunc(FuncIndex::from_u32(func_index)) + .unwrap(); + anyfunc as *const VMCallerCheckedAnyfunc as usize +} + /// Implementation of `data.drop`. pub unsafe extern "C" fn wasmtime_data_drop(vmctx: *mut VMContext, data_index: u32) { let data_index = DataIndex::from_u32(data_index); diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index d2832eb6ceef..7823402c0982 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -541,7 +541,7 @@ impl Default for VMSharedSignatureIndex { #[repr(C)] pub struct VMCallerCheckedAnyfunc { /// Function body. - pub func_ptr: NonNull, + pub func_ptr: *mut VMFunctionBody, /// Function signature id. pub type_index: VMSharedSignatureIndex, /// Function `VMContext`. @@ -611,6 +611,7 @@ impl VMBuiltinFunctionsArray { ptrs[BuiltinFunctionIndex::memory_copy().index() as usize] = wasmtime_memory_copy as usize; ptrs[BuiltinFunctionIndex::memory_fill().index() as usize] = wasmtime_memory_fill as usize; ptrs[BuiltinFunctionIndex::memory_init().index() as usize] = wasmtime_memory_init as usize; + ptrs[BuiltinFunctionIndex::ref_func().index() as usize] = wasmtime_ref_func as usize; ptrs[BuiltinFunctionIndex::data_drop().index() as usize] = wasmtime_data_drop as usize; ptrs[BuiltinFunctionIndex::drop_externref().index() as usize] = wasmtime_drop_externref as usize; diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 3f4b2ad496d9..e4de20ff8f9b 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -754,7 +754,7 @@ impl Func { trampoline( (*anyfunc.as_ptr()).vmctx, callee, - (*anyfunc.as_ptr()).func_ptr.as_ptr(), + (*anyfunc.as_ptr()).func_ptr, params_and_returns, ) }) @@ -947,7 +947,7 @@ impl Func { unsafe { let f = self.caller_checked_anyfunc(store); VMFunctionImport { - body: f.as_ref().func_ptr, + body: NonNull::new(f.as_ref().func_ptr).unwrap(), vmctx: f.as_ref().vmctx, } } diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index 4ddced224d1e..620188309236 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -160,12 +160,8 @@ where let result = invoke_wasm_and_catch_traps(store, |callee| { let (anyfunc, ret, params, returned) = &mut captures; let anyfunc = anyfunc.as_ref(); - let result = Params::invoke::( - anyfunc.func_ptr.as_ptr(), - anyfunc.vmctx, - callee, - *params, - ); + let result = + Params::invoke::(anyfunc.func_ptr, anyfunc.vmctx, callee, *params); ptr::write(ret.as_mut_ptr(), result); *returned = true }); diff --git a/crates/wasmtime/src/instance.rs b/crates/wasmtime/src/instance.rs index aec6c1ba06f6..42edc49123a8 100644 --- a/crates/wasmtime/src/instance.rs +++ b/crates/wasmtime/src/instance.rs @@ -701,16 +701,15 @@ impl<'a> Instantiator<'a> { // this instance, so we determine what the ID is and then assert // it's the same later when we do actually insert it. let instance_to_be = store.store_data().next_id::(); + let mut instance_handle = store .engine() .allocator() .allocate(InstanceAllocationRequest { module: compiled_module.module().clone(), - image_base: compiled_module.code().as_ptr() as usize, - functions: compiled_module.functions(), + info: self.cur.module.alloc_info().clone(), imports: self.cur.build(), - shared_signatures: self.cur.module.signatures().as_module_map().into(), host_state: Box::new(Instance(instance_to_be)), store: StorePtr::new(store.traitobj()), wasm_data: compiled_module.wasm_data(), @@ -827,9 +826,7 @@ impl<'a> Instantiator<'a> { mem::transmute::< *const VMFunctionBody, unsafe extern "C" fn(*mut VMContext, *mut VMContext), - >(f.anyfunc.as_ref().func_ptr.as_ptr())( - f.anyfunc.as_ref().vmctx, vmctx - ) + >(f.anyfunc.as_ref().func_ptr)(f.anyfunc.as_ref().vmctx, vmctx) })?; } Ok(()) diff --git a/crates/wasmtime/src/module.rs b/crates/wasmtime/src/module.rs index 04c695f2141e..861a5be99540 100644 --- a/crates/wasmtime/src/module.rs +++ b/crates/wasmtime/src/module.rs @@ -11,6 +11,7 @@ use std::sync::Arc; use wasmparser::{Parser, ValidPayload, Validator}; use wasmtime_environ::{ModuleEnvironment, ModuleIndex, PrimaryMap}; use wasmtime_jit::{CompiledModule, CompiledModuleInfo, MmapVec, TypeTables}; +use wasmtime_runtime::InstanceAllocationInfo; mod registry; mod serialization; @@ -107,6 +108,8 @@ struct ModuleInner { types: Arc, /// Registered shared signature for the module. signatures: Arc, + /// InstanceAllocationInfo shared by all instantiated modules, used for lazy initialization. + alloc_info: Arc, } impl Module { @@ -488,6 +491,17 @@ impl Module { module.into_module(engine) } + fn create_alloc_info( + module: &CompiledModule, + sigs: &SignatureCollection, + ) -> Arc { + Arc::new(InstanceAllocationInfo { + image_base: module.code().as_ptr() as usize, + functions: module.functions().clone(), + shared_signatures: sigs.as_module_map().into(), + }) + } + fn from_parts( engine: &Engine, mut modules: Vec>, @@ -508,6 +522,8 @@ impl Module { let module = modules.remove(main_module); + let alloc_info = Module::create_alloc_info(&module, &signatures); + let module_upvars = module_upvars .iter() .map(|m| { @@ -531,6 +547,7 @@ impl Module { artifact_upvars: modules, module_upvars, signatures, + alloc_info, }), }); @@ -543,6 +560,7 @@ impl Module { module_upvars: &[serialization::SerializedModuleUpvar], signatures: &Arc, ) -> Result { + let alloc_info = Module::create_alloc_info(&artifacts[module_index], signatures); Ok(Module { inner: Arc::new(ModuleInner { engine: engine.clone(), @@ -567,6 +585,7 @@ impl Module { }) .collect::>>()?, signatures: signatures.clone(), + alloc_info, }), }) } @@ -667,6 +686,10 @@ impl Module { module_upvars: &[wasmtime_environ::ModuleUpvar], modules: &PrimaryMap, ) -> Module { + let alloc_info = Module::create_alloc_info( + &self.inner.artifact_upvars[artifact_index], + &self.inner.signatures, + ); Module { inner: Arc::new(ModuleInner { types: self.inner.types.clone(), @@ -686,6 +709,7 @@ impl Module { }) .collect(), signatures: self.inner.signatures.clone(), + alloc_info, }), } } @@ -706,6 +730,10 @@ impl Module { &self.inner.signatures } + pub(crate) fn alloc_info(&self) -> &Arc { + &self.inner.alloc_info + } + /// Looks up the module upvar value at the `index` specified. /// /// Note that this panics if `index` is out of bounds since this should diff --git a/crates/wasmtime/src/module/registry.rs b/crates/wasmtime/src/module/registry.rs index 60bbbfffd783..a1a97e740622 100644 --- a/crates/wasmtime/src/module/registry.rs +++ b/crates/wasmtime/src/module/registry.rs @@ -95,7 +95,7 @@ impl ModuleRegistry { /// Looks up a trampoline from an anyfunc. pub fn lookup_trampoline(&self, anyfunc: &VMCallerCheckedAnyfunc) -> Option { - let module = self.module(anyfunc.func_ptr.as_ptr() as usize)?; + let module = self.module(anyfunc.func_ptr as usize)?; module.signatures.trampoline(anyfunc.type_index) } } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index c6d7914e4742..73be2ef4f313 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -92,9 +92,10 @@ use std::sync::atomic::AtomicU64; use std::sync::Arc; use std::task::{Context, Poll}; use wasmtime_runtime::{ - InstanceAllocationRequest, InstanceAllocator, InstanceHandle, ModuleInfo, - OnDemandInstanceAllocator, SignalHandler, StorePtr, VMCallerCheckedAnyfunc, VMContext, - VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, VMTrampoline, + InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, + ModuleInfo, OnDemandInstanceAllocator, SignalHandler, StorePtr, VMCallerCheckedAnyfunc, + VMContext, VMExternRef, VMExternRefActivationsTable, VMInterrupts, VMSharedSignatureIndex, + VMTrampoline, }; mod context; @@ -404,7 +405,6 @@ impl Store { /// tables created to 10,000. This can be overridden with the /// [`Store::limiter`] configuration method. pub fn new(engine: &Engine, data: T) -> Self { - let functions = &Default::default(); // Wasmtime uses the callee argument to host functions to learn about // the original pointer to the `Store` itself, allowing it to // reconstruct a `StoreContextMut`. When we initially call a `Func`, @@ -413,16 +413,19 @@ impl Store { // part of `Func::call` to guarantee that the `callee: *mut VMContext` // is never null. let default_callee = unsafe { + let info = Arc::new(InstanceAllocationInfo { + image_base: 0, + functions: Default::default(), + shared_signatures: None.into(), + }); OnDemandInstanceAllocator::default() .allocate(InstanceAllocationRequest { host_state: Box::new(()), - image_base: 0, - functions, - shared_signatures: None.into(), imports: Default::default(), module: Arc::new(wasmtime_environ::Module::default()), store: StorePtr::empty(), wasm_data: &[], + info, }) .expect("failed to allocate default callee") }; diff --git a/crates/wasmtime/src/trampoline.rs b/crates/wasmtime/src/trampoline.rs index c1f8038a5ace..b2835bb381da 100644 --- a/crates/wasmtime/src/trampoline.rs +++ b/crates/wasmtime/src/trampoline.rs @@ -18,8 +18,8 @@ use std::any::Any; use std::sync::Arc; use wasmtime_environ::{EntityIndex, GlobalIndex, MemoryIndex, Module, TableIndex}; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstanceAllocator, OnDemandInstanceAllocator, StorePtr, - VMFunctionImport, VMSharedSignatureIndex, + Imports, InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, + OnDemandInstanceAllocator, StorePtr, VMFunctionImport, VMSharedSignatureIndex, }; fn create_handle( @@ -31,7 +31,7 @@ fn create_handle( ) -> Result { let mut imports = Imports::default(); imports.functions = func_imports; - let functions = &Default::default(); + let functions = Default::default(); unsafe { let config = store.engine().config(); @@ -41,13 +41,15 @@ fn create_handle( let handle = OnDemandInstanceAllocator::new(config.mem_creator.clone(), 0).allocate( InstanceAllocationRequest { module: Arc::new(module), - functions, - image_base: 0, imports, - shared_signatures: shared_signature_id.into(), host_state, store: StorePtr::new(store.traitobj()), wasm_data: &[], + info: Arc::new(InstanceAllocationInfo { + functions, + image_base: 0, + shared_signatures: shared_signature_id.into(), + }), }, )?; diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 67d57fc3345d..dc8944fb7288 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -8,7 +8,7 @@ use std::sync::Arc; use wasmtime_environ::{EntityIndex, Module, ModuleType, PrimaryMap, SignatureIndex}; use wasmtime_jit::{CodeMemory, MmapVec, ProfilingAgent}; use wasmtime_runtime::{ - Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, + Imports, InstanceAllocationInfo, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, StorePtr, VMContext, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, }; @@ -158,16 +158,19 @@ pub unsafe fn create_raw_function( .exports .insert(String::new(), EntityIndex::Function(func_id)); + let info = Arc::new(InstanceAllocationInfo { + functions: Arc::new(functions), + image_base: (*func).as_ptr() as usize, + shared_signatures: sig.into(), + }); Ok( OnDemandInstanceAllocator::default().allocate(InstanceAllocationRequest { module: Arc::new(module), - functions: &functions, - image_base: (*func).as_ptr() as usize, imports: Imports::default(), - shared_signatures: sig.into(), host_state, store: StorePtr::empty(), wasm_data: &[], + info, })?, ) } diff --git a/crates/wasmtime/src/trampoline/global.rs b/crates/wasmtime/src/trampoline/global.rs index 3feb599b26ee..5c6a8c9f519a 100644 --- a/crates/wasmtime/src/trampoline/global.rs +++ b/crates/wasmtime/src/trampoline/global.rs @@ -1,3 +1,5 @@ +use std::ptr::NonNull; + use crate::store::{InstanceId, StoreOpaque}; use crate::trampoline::create_handle; use crate::{GlobalType, Mutability, Val}; @@ -51,7 +53,7 @@ pub fn create_global(store: &mut StoreOpaque, gt: &GlobalType, val: Val) -> Resu }); func_imports.push(VMFunctionImport { - body: f.func_ptr, + body: NonNull::new(f.func_ptr).unwrap(), vmctx: f.vmctx, });