diff --git a/lib/runtime-core-tests/tests/imports.rs b/lib/runtime-core-tests/tests/imports.rs index b461ad2b758..42a09d84460 100644 --- a/lib/runtime-core-tests/tests/imports.rs +++ b/lib/runtime-core-tests/tests/imports.rs @@ -1,6 +1,13 @@ +use std::sync::Arc; use wasmer_runtime_core::{ - compile_with, error::RuntimeError, imports, memory::Memory, typed_func::Func, - types::MemoryDescriptor, units::Pages, vm, Instance, + compile_with, + error::RuntimeError, + imports, + memory::Memory, + typed_func::Func, + types::{FuncSig, MemoryDescriptor, Type, Value}, + units::Pages, + vm, Instance, }; use wasmer_runtime_core_tests::{get_compiler, wat2wasm}; @@ -68,6 +75,7 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { (import "env" "memory" (memory 1 1)) (import "env" "callback_fn" (func $callback_fn (type $type))) (import "env" "callback_closure" (func $callback_closure (type $type))) + (import "env" "callback_closure_polymorphic" (func $callback_closure_polymorphic (type $type))) (import "env" "callback_closure_with_env" (func $callback_closure_with_env (type $type))) (import "env" "callback_fn_with_vmctx" (func $callback_fn_with_vmctx (type $type))) (import "env" "callback_closure_with_vmctx" (func $callback_closure_with_vmctx (type $type))) @@ -86,6 +94,10 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { get_local 0 call $callback_closure) + (func (export "function_closure_polymorphic") (type $type) + get_local 0 + call $callback_closure_polymorphic) + (func (export "function_closure_with_env") (type $type) get_local 0 call $callback_closure_with_env) @@ -142,6 +154,16 @@ fn imported_functions_forms(test: &dyn Fn(&Instance)) { Ok(n + 1) }), + "callback_closure_polymorphic" => Func::::new_polymorphic( + Arc::new(FuncSig::new(vec![Type::I32], vec![Type::I32])), + |_, params| -> Vec { + match params[0] { + Value::I32(x) => vec![Value::I32(x + 1)], + _ => unreachable!() + } + } + ), + // Closure with a captured environment (a single variable + an instance of `Memory`). "callback_closure_with_env" => Func::new(move |n: i32| -> Result { let shift_ = shift + memory.view::()[0].get(); @@ -236,6 +258,11 @@ macro_rules! test { test!(test_fn, function_fn, Ok(2)); test!(test_closure, function_closure, Ok(2)); +test!( + test_closure_polymorphic, + function_closure_polymorphic, + Ok(2) +); test!( test_closure_with_env, function_closure_with_env, diff --git a/lib/runtime-core/src/loader.rs b/lib/runtime-core/src/loader.rs index f516643d063..ea1ca0130ac 100644 --- a/lib/runtime-core/src/loader.rs +++ b/lib/runtime-core/src/loader.rs @@ -1,7 +1,9 @@ //! The loader module functions are used to load an instance. use crate::{backend::RunnableModule, module::ModuleInfo, types::Type, types::Value, vm::Ctx}; #[cfg(unix)] -use libc::{mmap, mprotect, munmap, MAP_ANON, MAP_PRIVATE, PROT_EXEC, PROT_READ, PROT_WRITE}; +use libc::{ + mmap, mprotect, munmap, MAP_ANON, MAP_NORESERVE, MAP_PRIVATE, PROT_EXEC, PROT_READ, PROT_WRITE, +}; use std::{ fmt::Debug, ops::{Deref, DerefMut}, @@ -169,7 +171,7 @@ impl CodeMemory { std::ptr::null_mut(), size, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON, + MAP_PRIVATE | MAP_ANON | MAP_NORESERVE, -1, 0, ) @@ -196,6 +198,20 @@ impl CodeMemory { panic!("cannot set code memory to writable"); } } + + /// Makes this code memory both writable and executable. + /// + /// Avoid using this if a combination `make_executable` and `make_writable` can be used. + pub fn make_writable_executable(&self) { + if unsafe { mprotect(self.ptr as _, self.size, PROT_READ | PROT_WRITE | PROT_EXEC) } != 0 { + panic!("cannot set code memory to writable and executable"); + } + } + + /// Returns the backing pointer of this code memory. + pub fn get_backing_ptr(&self) -> *mut u8 { + self.ptr + } } #[cfg(unix)] diff --git a/lib/runtime-core/src/trampoline_x64.rs b/lib/runtime-core/src/trampoline_x64.rs index 3d07484c715..c549498f3a3 100644 --- a/lib/runtime-core/src/trampoline_x64.rs +++ b/lib/runtime-core/src/trampoline_x64.rs @@ -9,6 +9,8 @@ use crate::loader::CodeMemory; use crate::vm::Ctx; use std::fmt; +use std::ptr::NonNull; +use std::sync::atomic::{AtomicUsize, Ordering}; use std::{mem, slice}; lazy_static! { @@ -29,6 +31,50 @@ lazy_static! { mem::transmute(ptr) } }; + + static ref TRAMPOLINES: TrampBuffer = TrampBuffer::new(); +} + +struct TrampBuffer { + buffer: CodeMemory, + len: AtomicUsize, +} + +impl TrampBuffer { + /// Creates a trampoline buffer. + fn new() -> TrampBuffer { + // Pre-allocate 64 MiB of virtual memory for code. + let mem = CodeMemory::new(64 * 1048576); + mem.make_writable_executable(); + TrampBuffer { + buffer: mem, + len: AtomicUsize::new(0), + } + } + + /// Bump allocation. Copies `buf` to the end of this code memory. + /// + /// FIXME: Proper storage recycling. + fn append(&self, buf: &[u8]) -> Option> { + let begin = self.len.fetch_add(buf.len(), Ordering::SeqCst); + let end = begin + buf.len(); + + // Length overflowed. Revert and return None. + if end > self.buffer.len() { + self.len.fetch_sub(buf.len(), Ordering::SeqCst); + return None; + } + + // Now we have unique ownership to `self.buffer[begin..end]`. + let slice = unsafe { + std::slice::from_raw_parts_mut( + self.buffer.get_backing_ptr().offset(begin as _), + buf.len(), + ) + }; + slice.copy_from_slice(buf); + Some(NonNull::new(slice.as_mut_ptr()).unwrap()) + } } /// An opaque type for pointers to a callable memory location. @@ -219,6 +265,11 @@ impl TrampolineBufferBuilder { idx } + /// Appends to the global trampoline buffer. + pub fn append_global(self) -> Option> { + TRAMPOLINES.append(&self.code) + } + /// Consumes the builder and builds the trampoline buffer. pub fn build(self) -> TrampolineBuffer { get_context(); // ensure lazy initialization is completed diff --git a/lib/runtime-core/src/typed_func.rs b/lib/runtime-core/src/typed_func.rs index 513ede7fbc0..84fbe8addd7 100644 --- a/lib/runtime-core/src/typed_func.rs +++ b/lib/runtime-core/src/typed_func.rs @@ -240,6 +240,79 @@ where _phantom: PhantomData, } } + + /// Creates a polymorphic function. + #[allow(unused_variables)] + #[cfg(all(unix, target_arch = "x86_64"))] + pub fn new_polymorphic(signature: Arc, func: F) -> Func<'a, Args, Rets, Host> + where + F: Fn(&mut vm::Ctx, &[crate::types::Value]) -> Vec + 'static, + { + use crate::trampoline_x64::*; + use crate::types::Value; + use std::convert::TryFrom; + + struct PolymorphicContext { + arg_types: Vec, + func: Box Vec>, + } + unsafe extern "C" fn enter_host_polymorphic( + ctx: *const CallContext, + args: *const u64, + ) -> u64 { + let ctx = &*(ctx as *const PolymorphicContext); + let vmctx = &mut *(*args.offset(0) as *mut vm::Ctx); + let args: Vec = ctx + .arg_types + .iter() + .enumerate() + .map(|(i, t)| { + let i = i + 1; // skip vmctx + match *t { + Type::I32 => Value::I32(*args.offset(i as _) as i32), + Type::I64 => Value::I64(*args.offset(i as _) as i64), + Type::F32 => Value::F32(f32::from_bits(*args.offset(i as _) as u32)), + Type::F64 => Value::F64(f64::from_bits(*args.offset(i as _) as u64)), + Type::V128 => { + panic!("enter_host_polymorphic: 128-bit types are not supported") + } + } + }) + .collect(); + let rets = (ctx.func)(vmctx, &args); + if rets.len() == 0 { + 0 + } else if rets.len() == 1 { + u64::try_from(rets[0].to_u128()).expect( + "128-bit return value from polymorphic host functions is not yet supported", + ) + } else { + panic!( + "multiple return values from polymorphic host functions is not yet supported" + ); + } + } + let mut builder = TrampolineBufferBuilder::new(); + let ctx = Box::new(PolymorphicContext { + arg_types: signature.params().to_vec(), + func: Box::new(func), + }); + builder.add_callinfo_trampoline( + enter_host_polymorphic, + Box::into_raw(ctx) as *const _, + (signature.params().len() + 1) as u32, // +vmctx + ); + let ptr = builder + .append_global() + .expect("cannot bump-allocate global trampoline memory"); + Func { + inner: Host(()), + func: ptr.cast::(), + func_env: None, + vmctx: ptr::null_mut(), + _phantom: PhantomData, + } + } } impl<'a, Args, Rets, Inner> Func<'a, Args, Rets, Inner> diff --git a/lib/runtime-core/src/vm.rs b/lib/runtime-core/src/vm.rs index 2a39bdec10f..a25ee24ff6b 100644 --- a/lib/runtime-core/src/vm.rs +++ b/lib/runtime-core/src/vm.rs @@ -545,13 +545,13 @@ impl Ctx { /// `typed_func` module within the `wrap` functions, to wrap imported /// functions. #[repr(transparent)] -pub struct Func(pub(self) *mut c_void); +pub struct Func(*mut c_void); /// Represents a function environment pointer, like a captured /// environment of a closure. It is mostly used in the `typed_func` /// module within the `wrap` functions, to wrap imported functions. #[repr(transparent)] -pub struct FuncEnv(pub(self) *mut c_void); +pub struct FuncEnv(*mut c_void); /// Represents a function context. It is used by imported functions /// only.