From 81e96c047da8de6139bb060fd67f80dcd7f69aee Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 07:24:35 -0700 Subject: [PATCH 01/16] Fully support multiple returns in Wasmtime For quite some time now Wasmtime has "supported" multiple return values, but only in the mose bare bones ways. Up until recently you couldn't get a typed version of functions with multiple return values, and never have you been able to use `Func::wrap` with functions that return multiple values. Even recently where `Func::typed` can call functions that return multiple values it uses a double-indirection by calling a trampoline which calls the real function. The underlying reason for this lack of support is that cranelift's ABI for returning multiple values is not possible to write in Rust. For example if a wasm function returns two `i32` values there is no Rust (or C!) function you can write to correspond to that. This commit, however fixes that. This commit adds two new ABIs to Cranelift: `WasmtimeSystemV` and `WasmtimeFastcall`. The intention is that these Wasmtime-specific ABIs match their corresponding ABI (e.g. `SystemV` or `WindowsFastcall`) for everything *except* how multiple values are returned. For multiple return values we simply define our own version of the ABI which Wasmtime implements, which is that for N return values the first is returned as if the function only returned that and the latter N-1 return values are returned via an out-ptr that's the last parameter to the function. These custom ABIs provides the ability for Wasmtime to bind these in Rust meaning that `Func::wrap` can now wrap functions that return multiple values and `Func::typed` no longer uses trampolines when calling functions that return multiple values. Although there's lots of internal changes there's no actual changes in the API surface area of Wasmtime, just a few more impls of more public traits which means that more types are supported in more places! Another change made with this PR is a consolidation of how the ABI of each function in a wasm module is selected. The native `SystemV` ABI, for example, is more efficient at returning multiple values than the wasmtime version of the ABI (since more things are in more registers). To continue to take advantage of this Wasmtime will now classify some functions in a wasm module with the "fast" ABI. Only functions that are not reachable externally from the module are classified with the fast ABI (e.g. those not exported, used in tables, or used with `ref.func`). This should enable purely internal functions of modules to have a faster calling convention than those which might be exposed to Wasmtime itself. Closes #1178 --- Cargo.lock | 1 + cranelift/codegen/src/context.rs | 8 +- cranelift/codegen/src/isa/aarch64/abi.rs | 54 +-- cranelift/codegen/src/isa/call_conv.rs | 32 +- cranelift/codegen/src/isa/x64/abi.rs | 50 +-- cranelift/codegen/src/isa/x86/abi.rs | 15 +- .../codegen/src/isa/x86/unwind/systemv.rs | 6 +- cranelift/codegen/src/machinst/abi_impl.rs | 13 +- cranelift/wasm/src/environ/dummy.rs | 5 +- cranelift/wasm/src/environ/spec.rs | 6 +- cranelift/wasm/src/sections_translator.rs | 14 +- cranelift/wasm/src/translation_utils.rs | 1 + crates/cranelift/Cargo.toml | 1 + crates/cranelift/src/func_environ.rs | 47 ++- crates/cranelift/src/lib.rs | 98 +++++- crates/environ/src/module.rs | 8 +- crates/environ/src/module_environ.rs | 48 +-- crates/jit/src/lib.rs | 1 + crates/jit/src/native.rs | 5 - crates/jit/src/object.rs | 16 +- crates/jit/src/trampoline.rs | 14 +- crates/wasmtime/src/config.rs | 4 +- crates/wasmtime/src/func.rs | 311 ++++++++++++------ crates/wasmtime/src/func/typed.rs | 199 +++++------ crates/wasmtime/src/sig_registry.rs | 19 +- crates/wasmtime/src/store.rs | 10 +- crates/wasmtime/src/trampoline/func.rs | 25 +- crates/wasmtime/src/types.rs | 22 -- tests/all/func.rs | 159 +++++++++ 29 files changed, 748 insertions(+), 444 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 148263def6a7..e034571ae584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3310,6 +3310,7 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "cranelift-wasm", + "target-lexicon", "wasmparser", "wasmtime-environ", ] diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index b831f9966a31..f668fe596a9d 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -267,13 +267,7 @@ impl Context { isa: &dyn TargetIsa, ) -> CodegenResult> { if let Some(backend) = isa.get_mach_backend() { - use crate::isa::CallConv; - use crate::machinst::UnwindInfoKind; - let unwind_info_kind = match self.func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => UnwindInfoKind::SystemV, - CallConv::WindowsFastcall => UnwindInfoKind::Windows, - _ => UnwindInfoKind::None, - }; + let unwind_info_kind = self.func.signature.call_conv.unwind_info_kind(); let result = self.mach_compile_result.as_ref().unwrap(); return backend.emit_unwind_info(result, unwind_info_kind); } diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 8f3b2e6f1eb1..b2a95e314be9 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -197,18 +197,26 @@ impl ABIMachineSpec for AArch64MachineDeps { next_stack = 16; } - // Note on return values: on the regular non-baldrdash ABI, we may return values in 8 - // registers for V128 and I64 registers independently of the number of register values - // returned in the other class. That is, we can return values in up to 8 integer and 8 - // vector registers at once. - // In Baldrdash, we can only use one register for return value for all the register - // classes. That is, we can't return values in both one integer and one vector register; - // only one return value may be in a register. - - let (max_per_class_reg_vals, mut remaining_reg_vals) = match (args_or_rets, is_baldrdash) { - (ArgsOrRets::Args, _) => (8, 16), // x0-x7 and v0-v7 - (ArgsOrRets::Rets, false) => (8, 16), // x0-x7 and v0-v7 - (ArgsOrRets::Rets, true) => (1, 1), // x0 or v0, but not both + let (max_per_class_reg_vals, mut remaining_reg_vals) = match args_or_rets { + ArgsOrRets::Args => (8, 16), // x0-x7 and v0-v7 + + // Note on return values: on the regular ABI, we may return values + // in 8 registers for V128 and I64 registers independently of the + // number of register values returned in the other class. That is, + // we can return values in up to 8 integer and + // 8 vector registers at once. + // + // In Baldrdash and Wasmtime, we can only use one register for + // return value for all the register classes. That is, we can't + // return values in both one integer and one vector register; only + // one return value may be in a register. + ArgsOrRets::Rets => { + if is_baldrdash || call_conv.extends_wasmtime() { + (1, 1) // x0 or v0, but not both + } else { + (8, 16) // x0-x7 and v0-v7 + } + } }; for i in 0..params.len() { @@ -282,16 +290,18 @@ impl ABIMachineSpec for AArch64MachineDeps { // Compute the stack slot's size. let size = (ty_bits(param.value_type) / 8) as u64; - let size = if call_conv != isa::CallConv::AppleAarch64 { - // Every arg takes a minimum slot of 8 bytes. (16-byte stack - // alignment happens separately after all args.) - std::cmp::max(size, 8) - } else { - // MacOS aarch64 allows stack slots with sizes less than 8 - // bytes. They still need to be properly aligned on their - // natural data alignment, though. - size - }; + let size = + if call_conv == isa::CallConv::AppleAarch64 || call_conv.extends_wasmtime() { + // MacOS aarch64 and Wasmtime allow stack slots with + // sizes less than 8 bytes. They still need to be + // properly aligned on their natural data alignment, + // though. + size + } else { + // Every arg takes a minimum slot of 8 bytes. (16-byte stack + // alignment happens separately after all args.) + std::cmp::max(size, 8) + }; // Align the stack slot. debug_assert!(size.is_power_of_two()); diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs index e764f33c6e37..fb0a2ca2635c 100644 --- a/cranelift/codegen/src/isa/call_conv.rs +++ b/cranelift/codegen/src/isa/call_conv.rs @@ -1,3 +1,4 @@ +use crate::machinst::UnwindInfoKind; use crate::settings::{self, LibcallCallConv}; use core::fmt; use core::str; @@ -29,6 +30,10 @@ pub enum CallConv { Baldrdash2020, /// Specialized convention for the probestack function. Probestack, + /// Wasmtime equivalent of SystemV, except the multi-return ABI is tweaked. + WasmtimeSystemV, + /// Wasmtime equivalent of WindowsFastcall, except the multi-return ABI is tweaked. + WasmtimeFastcall, } impl CallConv { @@ -63,7 +68,7 @@ impl CallConv { /// Is the calling convention extending the Windows Fastcall ABI? pub fn extends_windows_fastcall(self) -> bool { match self { - Self::WindowsFastcall | Self::BaldrdashWindows => true, + Self::WindowsFastcall | Self::BaldrdashWindows | Self::WasmtimeFastcall => true, _ => false, } } @@ -75,6 +80,27 @@ impl CallConv { _ => false, } } + + /// Is the calling convention extending the Wasmtime ABI? + pub fn extends_wasmtime(self) -> bool { + match self { + Self::WasmtimeSystemV | Self::WasmtimeFastcall => true, + _ => false, + } + } + + /// Returns the kind of unwind info used for this calling convention + pub fn unwind_info_kind(self) -> UnwindInfoKind { + match self { + #[cfg(feature = "unwind")] + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => { + UnwindInfoKind::SystemV + } + #[cfg(feature = "unwind")] + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => UnwindInfoKind::Windows, + _ => UnwindInfoKind::None, + } + } } impl fmt::Display for CallConv { @@ -89,6 +115,8 @@ impl fmt::Display for CallConv { Self::BaldrdashWindows => "baldrdash_windows", Self::Baldrdash2020 => "baldrdash_2020", Self::Probestack => "probestack", + Self::WasmtimeSystemV => "wasmtime_system_v", + Self::WasmtimeFastcall => "wasmtime_fastcall", }) } } @@ -106,6 +134,8 @@ impl str::FromStr for CallConv { "baldrdash_windows" => Ok(Self::BaldrdashWindows), "baldrdash_2020" => Ok(Self::Baldrdash2020), "probestack" => Ok(Self::Probestack), + "wasmtime_system_v" => Ok(Self::WasmtimeSystemV), + "wasmtime_fastcall" => Ok(Self::WasmtimeFastcall), _ => Err(()), } } diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index f55478a1c8d3..29a0dbd98250 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -237,10 +237,20 @@ impl ABIMachineSpec for X64ABIMachineSpec { extension: param.extension, }); } else { - // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte - // stack alignment happens separately after all args.) + // Compute size. For the wasmtime ABI it differs from native + // ABIs in how multiple values are returned, so we take a + // leaf out of arm64's book by not rounding everything up to + // 8 bytes. For all ABI arguments, and other ABI returns, + // though, each slot takes a minimum of 8 bytes. + // + // Note that in all cases 16-byte stack alignment happens + // separately after all args. let size = (reg_ty.bits() / 8) as u64; - let size = std::cmp::max(size, 8); + let size = if args_or_rets == ArgsOrRets::Rets && call_conv.extends_wasmtime() { + size + } else { + std::cmp::max(size, 8) + }; // Align. debug_assert!(size.is_power_of_two()); next_stack = align_to(next_stack, size); @@ -824,15 +834,7 @@ impl From for SyntheticAmode { } fn get_intreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Option { - let is_fastcall = match call_conv { - CallConv::Fast - | CallConv::Cold - | CallConv::SystemV - | CallConv::BaldrdashSystemV - | CallConv::Baldrdash2020 => false, - CallConv::WindowsFastcall => true, - _ => panic!("int args only supported for SysV or Fastcall calling convention"), - }; + let is_fastcall = call_conv.extends_windows_fastcall(); // Fastcall counts by absolute argument number; SysV counts by argument of // this (integer) class. @@ -853,15 +855,7 @@ fn get_intreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Optio } fn get_fltreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Option { - let is_fastcall = match call_conv { - CallConv::Fast - | CallConv::Cold - | CallConv::SystemV - | CallConv::BaldrdashSystemV - | CallConv::Baldrdash2020 => false, - CallConv::WindowsFastcall => true, - _ => panic!("float args only supported for SysV or Fastcall calling convention"), - }; + let is_fastcall = call_conv.extends_windows_fastcall(); // Fastcall counts by absolute argument number; SysV counts by argument of // this (floating-point) class. @@ -894,7 +888,10 @@ fn get_intreg_for_retval( 1 => Some(regs::rdx()), _ => None, }, - CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => { + CallConv::BaldrdashSystemV + | CallConv::Baldrdash2020 + | CallConv::WasmtimeSystemV + | CallConv::WasmtimeFastcall => { if intreg_idx == 0 && retval_idx == 0 { Some(regs::rax()) } else { @@ -922,7 +919,10 @@ fn get_fltreg_for_retval( 1 => Some(regs::xmm1()), _ => None, }, - CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => { + CallConv::BaldrdashSystemV + | CallConv::Baldrdash2020 + | CallConv::WasmtimeFastcall + | CallConv::WasmtimeSystemV => { if fltreg_idx == 0 && retval_idx == 0 { Some(regs::xmm0()) } else { @@ -992,12 +992,12 @@ fn get_callee_saves(call_conv: &CallConv, regs: &Set>) -> Vec< CallConv::BaldrdashWindows => { todo!("baldrdash windows"); } - CallConv::Fast | CallConv::Cold | CallConv::SystemV => regs + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => regs .iter() .cloned() .filter(|r| is_callee_save_systemv(r.to_reg())) .collect(), - CallConv::WindowsFastcall => regs + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => regs .iter() .cloned() .filter(|r| is_callee_save_fastcall(r.to_reg())) diff --git a/cranelift/codegen/src/isa/x86/abi.rs b/cranelift/codegen/src/isa/x86/abi.rs index c6df87ae2203..83b44be1b09a 100644 --- a/cranelift/codegen/src/isa/x86/abi.rs +++ b/cranelift/codegen/src/isa/x86/abi.rs @@ -503,10 +503,12 @@ fn callee_saved_regs_used(isa: &dyn TargetIsa, func: &ir::Function) -> RegisterS pub fn prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> { match func.signature.call_conv { // For now, just translate fast and cold as system_v. - CallConv::Fast | CallConv::Cold | CallConv::SystemV => { + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => { system_v_prologue_epilogue(func, isa) } - CallConv::WindowsFastcall => fastcall_prologue_epilogue(func, isa), + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => { + fastcall_prologue_epilogue(func, isa) + } CallConv::BaldrdashSystemV | CallConv::BaldrdashWindows => { baldrdash_prologue_epilogue(func, isa) } @@ -1084,16 +1086,17 @@ pub fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { use crate::isa::unwind::UnwindInfo; + use crate::machinst::UnwindInfoKind; // Assumption: RBP is being used as the frame pointer for both calling conventions // In the future, we should be omitting frame pointer as an optimization, so this will change - Ok(match func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => { + Ok(match func.signature.call_conv.unwind_info_kind() { + UnwindInfoKind::SystemV => { super::unwind::systemv::create_unwind_info(func, isa)?.map(|u| UnwindInfo::SystemV(u)) } - CallConv::WindowsFastcall => { + UnwindInfoKind::Windows => { super::unwind::winx64::create_unwind_info(func, isa)?.map(|u| UnwindInfo::WindowsX64(u)) } - _ => None, + UnwindInfoKind::None => None, }) } diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs index f3e1cbea84a1..26e1ccbda7d0 100644 --- a/cranelift/codegen/src/isa/x86/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -3,7 +3,7 @@ use crate::ir::Function; use crate::isa::{ unwind::systemv::{RegisterMappingError, UnwindInfo}, - CallConv, RegUnit, TargetIsa, + RegUnit, TargetIsa, }; use crate::result::CodegenResult; use gimli::{write::CommonInformationEntry, Encoding, Format, Register, X86_64}; @@ -97,8 +97,8 @@ pub(crate) fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { // Only System V-like calling conventions are supported - match func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => {} + match func.signature.call_conv.unwind_info_kind() { + crate::machinst::UnwindInfoKind::SystemV => {} _ => return Ok(None), } diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 17995e18df90..765712c2cf5c 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -647,7 +647,8 @@ impl ABICalleeImpl { || call_conv == isa::CallConv::Cold || call_conv.extends_baldrdash() || call_conv.extends_windows_fastcall() - || call_conv == isa::CallConv::AppleAarch64, + || call_conv == isa::CallConv::AppleAarch64 + || call_conv == isa::CallConv::WasmtimeSystemV, "Unsupported calling convention: {:?}", call_conv ); @@ -1372,15 +1373,7 @@ impl ABICallee for ABICalleeImpl { } fn unwind_info_kind(&self) -> UnwindInfoKind { - match self.sig.call_conv { - #[cfg(feature = "unwind")] - isa::CallConv::Fast | isa::CallConv::Cold | isa::CallConv::SystemV => { - UnwindInfoKind::SystemV - } - #[cfg(feature = "unwind")] - isa::CallConv::WindowsFastcall => UnwindInfoKind::Windows, - _ => UnwindInfoKind::None, - } + self.sig.call_conv.unwind_info_kind() } } diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index 2bcd004dbe1e..7a56712b7cee 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -660,8 +660,9 @@ impl TargetEnvironment for DummyEnvironment { } impl<'data> ModuleEnvironment<'data> for DummyEnvironment { - fn declare_type_func(&mut self, _wasm: WasmFuncType, sig: ir::Signature) -> WasmResult<()> { - self.info.signatures.push(sig); + fn declare_type_func(&mut self, _wasm: WasmFuncType) -> WasmResult<()> { + // TODO + // self.info.signatures.push(sig); Ok(()) } diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 0bd3c48745f9..3a2389c44d8e 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -702,11 +702,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { } /// Declares a function signature to the environment. - fn declare_type_func( - &mut self, - wasm_func_type: WasmFuncType, - sig: ir::Signature, - ) -> WasmResult<()>; + fn declare_type_func(&mut self, wasm_func_type: WasmFuncType) -> WasmResult<()>; /// Declares a module type signature to the environment. fn declare_type_module( diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index e9f0f784a5b1..c617c9943c89 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -18,7 +18,6 @@ use crate::wasm_unsupported; use core::convert::TryFrom; use core::convert::TryInto; use cranelift_codegen::ir::immediates::V128Imm; -use cranelift_codegen::ir::{self, AbiParam, Signature}; use cranelift_entity::packed_option::ReservedValue; use cranelift_entity::EntityRef; use std::boxed::Box; @@ -110,18 +109,7 @@ pub fn parse_type_section<'a>( for entry in types { match entry? { TypeDef::Func(wasm_func_ty) => { - let mut sig = Signature::new(environ.target_config().default_call_conv); - sig.params.extend(wasm_func_ty.params.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - sig.returns.extend(wasm_func_ty.returns.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - environ.declare_type_func(wasm_func_ty.clone().try_into()?, sig)?; + environ.declare_type_func(wasm_func_ty.clone().try_into()?)?; module_translation_state .wasm_types .push((wasm_func_ty.params, wasm_func_ty.returns)); diff --git a/cranelift/wasm/src/translation_utils.rs b/cranelift/wasm/src/translation_utils.rs index c8493d4b51e5..fffb7b375e27 100644 --- a/cranelift/wasm/src/translation_utils.rs +++ b/cranelift/wasm/src/translation_utils.rs @@ -19,6 +19,7 @@ entity_impl!(FuncIndex); /// Index type of a defined function inside the WebAssembly module. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DefinedFuncIndex(u32); entity_impl!(DefinedFuncIndex); diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index f9b763dccc2d..764177223c67 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -18,3 +18,4 @@ cranelift-codegen = { path = "../../cranelift/codegen", version = "0.72.0" } cranelift-frontend = { path = "../../cranelift/frontend", version = "0.72.0" } cranelift-entity = { path = "../../cranelift/entity", version = "0.72.0" } wasmparser = "0.77.0" +target-lexicon = "0.12" diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index a9481aca93ee..29d58dac2045 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -4,20 +4,20 @@ use cranelift_codegen::ir::condcodes::*; use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Signature}; -use cranelift_codegen::isa::{self, TargetFrontendConfig}; -use cranelift_entity::{EntityRef, PrimaryMap}; +use cranelift_codegen::isa::{self, TargetFrontendConfig, TargetIsa}; +use cranelift_entity::EntityRef; use cranelift_frontend::FunctionBuilder; use cranelift_frontend::Variable; use cranelift_wasm::{ - self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex, - SignatureIndex, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, + self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex, TableIndex, + TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, }; use std::convert::TryFrom; use std::mem; use wasmparser::Operator; use wasmtime_environ::{ - BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, VMOffsets, - INTERRUPTED, WASM_PAGE_SIZE, + BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, TypeTables, + VMOffsets, INTERRUPTED, WASM_PAGE_SIZE, }; /// Compute an `ir::ExternalName` for a given wasm function index. @@ -109,14 +109,9 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); /// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`. pub struct FuncEnvironment<'module_environment> { - /// Target-specified configuration. - target_config: TargetFrontendConfig, - - /// The module-level environment which this function-level environment belongs to. + isa: &'module_environment (dyn TargetIsa + 'module_environment), module: &'module_environment Module, - - /// The native signatures for each type signature in this module - native_signatures: &'module_environment PrimaryMap, + types: &'module_environment TypeTables, /// The Cranelift global holding the vmctx address. vmctx: Option, @@ -146,27 +141,27 @@ pub struct FuncEnvironment<'module_environment> { impl<'module_environment> FuncEnvironment<'module_environment> { pub fn new( - target_config: TargetFrontendConfig, + isa: &'module_environment (dyn TargetIsa + 'module_environment), module: &'module_environment Module, - native_signatures: &'module_environment PrimaryMap, + types: &'module_environment TypeTables, tunables: &'module_environment Tunables, ) -> Self { let builtin_function_signatures = BuiltinFunctionSignatures::new( - target_config.pointer_type(), - match target_config.pointer_type() { + isa.pointer_type(), + match isa.pointer_type() { ir::types::I32 => ir::types::R32, ir::types::I64 => ir::types::R64, _ => panic!(), }, - target_config.default_call_conv, + crate::wasmtime_call_conv(isa), ); Self { - target_config, + isa, module, - native_signatures, + types, vmctx: None, builtin_function_signatures, - offsets: VMOffsets::new(target_config.pointer_bytes(), module), + offsets: VMOffsets::new(isa.pointer_bytes(), module), tunables, fuel_var: Variable::new(0), vminterrupts_ptr: Variable::new(0), @@ -178,7 +173,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { } fn pointer_type(&self) -> ir::Type { - self.target_config.pointer_type() + self.isa.pointer_type() } fn vmctx(&mut self, func: &mut Function) -> ir::GlobalValue { @@ -680,7 +675,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environment> { fn target_config(&self) -> TargetFrontendConfig { - self.target_config + self.isa.frontend_config() } fn reference_type(&self, ty: WasmType) -> ir::Type { @@ -1339,7 +1334,8 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m index: TypeIndex, ) -> WasmResult { let index = self.module.types[index].unwrap_function(); - Ok(func.import_signature(self.native_signatures[index].clone())) + let sig = crate::indirect_signature(self.isa, self.types, index); + Ok(func.import_signature(sig)) } fn make_direct_func( @@ -1347,8 +1343,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m func: &mut ir::Function, index: FuncIndex, ) -> WasmResult { - let sig_index = self.module.functions[index]; - let sig = self.native_signatures[sig_index].clone(); + let sig = crate::func_signature(self.isa, self.module, self.types, index); let signature = func.import_signature(sig); let name = get_func_name(index); Ok(func.import_function(ir::ExtFuncData { diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index ee330ee6a76c..896294ff39e6 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -90,16 +90,18 @@ use crate::func_environ::{get_func_name, FuncEnvironment}; use cranelift_codegen::ir::{self, ExternalName}; +use cranelift_codegen::isa::{CallConv, TargetIsa}; use cranelift_codegen::machinst::buffer::MachSrcLoc; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::{binemit, isa, Context}; -use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator}; +use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator, SignatureIndex, WasmType}; use std::convert::TryFrom; use std::sync::Mutex; +use target_lexicon::CallingConvention; use wasmtime_environ::{ CompileError, CompiledFunction, Compiler, FunctionAddressMap, FunctionBodyData, - InstructionAddressMap, ModuleTranslation, Relocation, RelocationTarget, StackMapInformation, - TrapInformation, Tunables, TypeTables, + InstructionAddressMap, Module, ModuleTranslation, Relocation, RelocationTarget, + StackMapInformation, TrapInformation, Tunables, TypeTables, }; mod func_environ; @@ -354,18 +356,12 @@ impl Compiler for Cranelift { let func_index = module.func_index(func_index); let mut context = Context::new(); context.func.name = get_func_name(func_index); - let sig_index = module.functions[func_index]; - context.func.signature = types.native_signatures[sig_index].clone(); + context.func.signature = func_signature(isa, module, types, func_index); if tunables.generate_native_debuginfo { context.func.collect_debug_info(); } - let mut func_env = FuncEnvironment::new( - isa.frontend_config(), - module, - &types.native_signatures, - tunables, - ); + let mut func_env = FuncEnvironment::new(isa, module, types, tunables); // We use these as constant offsets below in // `stack_limit_from_arguments`, so assert their values here. This @@ -457,3 +453,83 @@ impl Compiler for Cranelift { }) } } + +pub fn blank_sig(isa: &dyn TargetIsa, call_conv: CallConv) -> ir::Signature { + let pointer_type = isa.pointer_type(); + let mut sig = ir::Signature::new(call_conv); + // Add the caller/callee `vmctx` parameters. + sig.params.push(ir::AbiParam::special( + pointer_type, + ir::ArgumentPurpose::VMContext, + )); + sig.params.push(ir::AbiParam::new(pointer_type)); + return sig; +} + +pub fn wasmtime_call_conv(isa: &dyn TargetIsa) -> CallConv { + match isa.triple().default_calling_convention() { + Ok(CallingConvention::SystemV) | Ok(CallingConvention::AppleAarch64) | Err(()) => { + CallConv::WasmtimeSystemV + } + Ok(CallingConvention::WindowsFastcall) => CallConv::WasmtimeFastcall, + Ok(unimp) => unimplemented!("calling convention: {:?}", unimp), + } +} + +pub fn push_types( + isa: &dyn TargetIsa, + sig: &mut ir::Signature, + types: &TypeTables, + index: SignatureIndex, +) { + let wasm = &types.wasm_signatures[index]; + + let cvt = |ty: &WasmType| { + ir::AbiParam::new(match ty { + WasmType::I32 => ir::types::I32, + WasmType::I64 => ir::types::I64, + WasmType::F32 => ir::types::F32, + WasmType::F64 => ir::types::F64, + WasmType::V128 => ir::types::I8X16, + WasmType::FuncRef | WasmType::ExternRef => { + wasmtime_environ::reference_type(*ty, isa.pointer_type()) + } + WasmType::ExnRef => unimplemented!(), + }) + }; + sig.params.extend(wasm.params.iter().map(&cvt)); + sig.returns.extend(wasm.returns.iter().map(&cvt)); +} + +pub fn indirect_signature( + isa: &dyn TargetIsa, + types: &TypeTables, + index: SignatureIndex, +) -> ir::Signature { + let mut sig = blank_sig(isa, wasmtime_call_conv(isa)); + push_types(isa, &mut sig, types, index); + return sig; +} + +pub fn func_signature( + isa: &dyn TargetIsa, + module: &Module, + types: &TypeTables, + index: FuncIndex, +) -> ir::Signature { + let call_conv = match module.defined_func_index(index) { + // If this is a defined function in the module and it's never possibly + // exported, then we can optimize this function to use the fastest + // calling convention since it's purely an internal implementation + // detail of the module itself. + Some(idx) if !module.possibly_exported_funcs.contains(&idx) => CallConv::Fast, + + // ... otherwise if it's an imported function or if it's a possibly + // exported function then we use the default ABI wasmtime would + // otherwise select. + _ => wasmtime_call_conv(isa), + }; + let mut sig = blank_sig(isa, call_conv); + push_types(isa, &mut sig, types, module.functions[index]); + return sig; +} diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index ee4ff050dc71..3b520475d6f0 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -2,12 +2,11 @@ use crate::tunables::Tunables; use crate::WASM_MAX_PAGES; -use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::*; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; /// Implemenation styles for WebAssembly linear memory. @@ -367,6 +366,10 @@ pub struct Module { /// The type of each nested wasm module this module contains. pub modules: PrimaryMap, + + /// The set of defined functions within this module which are located in + /// element segments. + pub possibly_exported_funcs: HashSet, } /// Initialization routines for creating an instance, encompassing imports, @@ -564,7 +567,6 @@ impl Module { #[allow(missing_docs)] pub struct TypeTables { pub wasm_signatures: PrimaryMap, - pub native_signatures: PrimaryMap, pub module_signatures: PrimaryMap, pub instance_signatures: PrimaryMap, } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 57e2f6bb1e7e..8106c1a3fb72 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -4,14 +4,14 @@ use crate::module::{ }; use crate::tunables::Tunables; use cranelift_codegen::ir; -use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; use cranelift_codegen::isa::TargetFrontendConfig; +use cranelift_codegen::packed_option::ReservedValue; use cranelift_entity::PrimaryMap; use cranelift_wasm::{ self, translate_module, Alias, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, - FuncIndex, Global, GlobalIndex, InstanceIndex, InstanceTypeIndex, Memory, MemoryIndex, - ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, TargetEnvironment, TypeIndex, - WasmError, WasmFuncType, WasmResult, + FuncIndex, Global, GlobalIndex, GlobalInit, InstanceIndex, InstanceTypeIndex, Memory, + MemoryIndex, ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, + TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, }; use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; @@ -357,6 +357,15 @@ impl<'data> ModuleEnvironment<'data> { .module_signatures .push(ModuleSignature { imports, exports }) } + + fn flag_func_possibly_exported(&mut self, func: FuncIndex) { + if func.is_reserved_value() { + return; + } + if let Some(idx) = self.result.module.defined_func_index(func) { + self.result.module.possibly_exported_funcs.insert(idx); + } + } } impl<'data> TargetEnvironment for ModuleEnvironment<'data> { @@ -375,21 +384,17 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn reserve_types(&mut self, num: u32) -> WasmResult<()> { let num = usize::try_from(num).unwrap(); self.result.module.types.reserve(num); - self.types.native_signatures.reserve(num); self.types.wasm_signatures.reserve(num); Ok(()) } - fn declare_type_func(&mut self, wasm: WasmFuncType, sig: ir::Signature) -> WasmResult<()> { + fn declare_type_func(&mut self, wasm: WasmFuncType) -> WasmResult<()> { // Deduplicate wasm function signatures through `interned_func_types`, // which also deduplicates across wasm modules with module linking. let sig_index = match self.interned_func_types.get(&wasm) { Some(idx) => *idx, None => { - let sig = translate_signature(sig, self.pointer_type()); - let sig_index = self.types.native_signatures.push(sig); - let sig_index2 = self.types.wasm_signatures.push(wasm.clone()); - debug_assert_eq!(sig_index, sig_index2); + let sig_index = self.types.wasm_signatures.push(wasm.clone()); self.interned_func_types.insert(wasm, sig_index); sig_index } @@ -641,6 +646,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_global(&mut self, global: Global) -> WasmResult<()> { + if let GlobalInit::RefFunc(index) = global.initializer { + self.flag_func_possibly_exported(index); + } self.result.module.globals.push(global); Ok(()) } @@ -654,6 +662,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_func_export(&mut self, func_index: FuncIndex, name: &str) -> WasmResult<()> { + self.flag_func_possibly_exported(func_index); self.declare_export(EntityIndex::Function(func_index), name) } @@ -678,6 +687,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_start_func(&mut self, func_index: FuncIndex) -> WasmResult<()> { + self.flag_func_possibly_exported(func_index); debug_assert!(self.result.module.start_func.is_none()); self.result.module.start_func = Some(func_index); Ok(()) @@ -698,6 +708,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data offset: usize, elements: Box<[FuncIndex]>, ) -> WasmResult<()> { + for element in elements.iter() { + self.flag_func_possibly_exported(*element); + } self.result .module .table_initializers @@ -715,6 +728,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data elem_index: ElemIndex, segments: Box<[FuncIndex]>, ) -> WasmResult<()> { + for element in segments.iter() { + self.flag_func_possibly_exported(*element); + } let index = self.result.module.passive_elements.len(); self.result.module.passive_elements.push(segments); let old = self @@ -1070,15 +1086,3 @@ and for re-adding support for interface types you can see this issue: Ok(()) } } - -/// Add environment-specific function parameters. -pub fn translate_signature(mut sig: ir::Signature, pointer_type: ir::Type) -> ir::Signature { - // Prepend the vmctx argument. - sig.params.insert( - 0, - AbiParam::special(pointer_type, ArgumentPurpose::VMContext), - ); - // Prepend the caller vmctx argument. - sig.params.insert(1, AbiParam::new(pointer_type)); - sig -} diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 17e6294250c3..12329ea266d7 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -50,6 +50,7 @@ pub use crate::instantiate::{ CompilationArtifacts, CompiledModule, ModuleCode, SetupError, SymbolizeContext, TypeTables, }; pub use crate::link::link_module; +pub use wasmtime_cranelift::{blank_sig, wasmtime_call_conv}; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/jit/src/native.rs b/crates/jit/src/native.rs index afcf83d3ccb4..706cbc04b443 100644 --- a/crates/jit/src/native.rs +++ b/crates/jit/src/native.rs @@ -11,9 +11,4 @@ pub fn builder_without_flags() -> cranelift_codegen::isa::Builder { .expect("host machine is not a supported target") } -pub fn call_conv() -> cranelift_codegen::isa::CallConv { - use target_lexicon::HOST; - cranelift_codegen::isa::CallConv::triple_default(&HOST) -} - pub use cranelift_codegen::isa::lookup; diff --git a/crates/jit/src/object.rs b/crates/jit/src/object.rs index 3516be2b133d..e73fa8cf8185 100644 --- a/crates/jit/src/object.rs +++ b/crates/jit/src/object.rs @@ -8,7 +8,7 @@ use std::collections::BTreeSet; use wasmtime_debug::DwarfSection; use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; -use wasmtime_environ::{CompiledFunctions, ModuleTranslation, ModuleType, TypeTables}; +use wasmtime_environ::{CompiledFunctions, ModuleTranslation, TypeTables}; use wasmtime_obj::{ObjectBuilder, ObjectBuilderTarget}; pub use wasmtime_obj::utils; @@ -42,18 +42,18 @@ pub(crate) fn build_object( // Build trampolines for every signature that can be used by this module. let signatures = translation .module - .types - .values() - .filter_map(|t| match t { - ModuleType::Function(f) => Some(*f), - _ => None, + .functions + .iter() + .filter_map(|(i, sig)| match translation.module.defined_func_index(i) { + Some(i) if !translation.module.possibly_exported_funcs.contains(&i) => None, + _ => Some(*sig), }) .collect::>(); let mut trampolines = Vec::with_capacity(signatures.len()); let mut cx = FunctionBuilderContext::new(); for i in signatures { - let native_sig = &types.native_signatures[i]; - let func = build_trampoline(isa, &mut cx, native_sig, std::mem::size_of::())?; + let native_sig = wasmtime_cranelift::indirect_signature(isa, &types, i); + let func = build_trampoline(isa, &mut cx, &native_sig, std::mem::size_of::())?; // Preserve trampoline function unwind info. if let Some(info) = &func.unwind_info { unwind_info.push(ObjectUnwindInfo::Trampoline(i, info.clone())) diff --git a/crates/jit/src/trampoline.rs b/crates/jit/src/trampoline.rs index bb470dbcb3fc..b8262fd93fb6 100644 --- a/crates/jit/src/trampoline.rs +++ b/crates/jit/src/trampoline.rs @@ -9,7 +9,7 @@ use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline}; pub mod ir { pub(super) use cranelift_codegen::ir::{ - AbiParam, ArgumentPurpose, ConstantOffset, JumpTable, Signature, SourceLoc, + AbiParam, ConstantOffset, JumpTable, Signature, SourceLoc, }; pub use cranelift_codegen::ir::{ ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, @@ -52,16 +52,8 @@ pub(crate) fn build_trampoline( value_size: usize, ) -> Result { let pointer_type = isa.pointer_type(); - let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Add the callee `vmctx` parameter. - wrapper_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the caller `vmctx` parameter. - wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); + let mut wrapper_sig = + wasmtime_cranelift::blank_sig(isa, wasmtime_cranelift::wasmtime_call_conv(isa)); // Add the `callee_address` parameter. wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 43bd809c84aa..55fbcfddca21 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1,6 +1,6 @@ use crate::memory::MemoryCreator; use crate::trampoline::MemoryCreatorProxy; -use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmRet, WasmTy}; +use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmHostResults, WasmTy}; use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; use std::cmp; @@ -339,7 +339,7 @@ macro_rules! generate_wrap_async_host_func { ) where $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { // Defer the check for async support until engine creation time to not introduce an order dependency self.host_funcs.insert( diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 2a70df9e3cd2..924a1002d932 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -6,6 +6,7 @@ use std::cmp::max; use std::fmt; use std::future::Future; use std::mem; +use std::mem::MaybeUninit; use std::panic::{self, AssertUnwindSafe}; use std::pin::Pin; use std::ptr::{self, NonNull}; @@ -350,7 +351,7 @@ macro_rules! generate_wrap_async_func { where T: 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { assert!(store.async_support(), concat!("cannot use `wrap", $num, "_async` without enabling async support on the config")); Func::wrap(store, move |caller: Caller<'_>, $($args: $args),*| { @@ -561,15 +562,15 @@ impl Func { /// Any of the Rust types can be returned from the closure as well, in /// addition to some extra types /// - /// | Rust Return Type | WebAssembly Return Type | Meaning | - /// |-------------------|-------------------------|-------------------| - /// | `()` | nothing | no return value | - /// | `Result` | `T` | function may trap | + /// | Rust Return Type | WebAssembly Return Type | Meaning | + /// |-------------------|-------------------------|-----------------------| + /// | `()` | nothing | no return value | + /// | `T` | `T` | a single return value | + /// | `(T1, T2, ...)` | `T1 T2 ...` | multiple returns | /// - /// At this time multi-value returns are not supported, and supporting this - /// is the subject of [#1178]. - /// - /// [#1178]: https://github.com/bytecodealliance/wasmtime/issues/1178 + /// Note that all return types can also be wrapped in `Result<_, Trap>` to + /// indicate that the host function can generate a trap as well as possibly + /// returning a value. /// /// Finally you can also optionally take [`Caller`] as the first argument of /// your closure. If inserted then you're able to inspect the caller's @@ -1260,10 +1261,12 @@ fn enter_wasm_init<'a>(store: &'a Store) -> Result { /// stable over time. /// /// For more information see [`Func::wrap`] -pub unsafe trait WasmRet { +pub unsafe trait WasmHostResults { // Same as `WasmTy::Abi`. #[doc(hidden)] type Abi: Copy; + #[doc(hidden)] + type Retptr: Copy; // Same as `WasmTy::compatible_with_store`. #[doc(hidden)] @@ -1276,100 +1279,47 @@ pub unsafe trait WasmRet { // `invoke_wasm_and_catch_traps` is on the stack, and therefore this method // is unsafe. #[doc(hidden)] - unsafe fn into_abi_for_ret(self, store: &Store) -> Result; + unsafe fn into_abi_for_ret(self, store: &Store, ptr: Self::Retptr) -> Result; + + #[doc(hidden)] + fn func_type(params: impl Iterator) -> FuncType; - // Same as `WasmTy::push`. #[doc(hidden)] - fn valtype() -> Option; + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi); // Utilities used to convert an instance of this type to a `Result` // explicitly, used when wrapping async functions which always bottom-out // in a function that returns a trap because futures can be cancelled. #[doc(hidden)] - type Fallible: WasmRet; + type Fallible: WasmHostResults; #[doc(hidden)] fn into_fallible(self) -> Self::Fallible; #[doc(hidden)] fn fallible_from_trap(trap: Trap) -> Self::Fallible; } -unsafe impl WasmRet for () { - type Abi = (); - type Fallible = Result<(), Trap>; - - #[inline] - fn compatible_with_store(&self, _store: &Store) -> bool { - true - } - - #[inline] - unsafe fn into_abi_for_ret(self, _store: &Store) -> Result<(), Trap> { - Ok(()) - } - - #[inline] - fn valtype() -> Option { - None - } - - #[inline] - fn into_fallible(self) -> Result<(), Trap> { - Ok(()) - } - - #[inline] - fn fallible_from_trap(trap: Trap) -> Result<(), Trap> { - Err(trap) - } -} - -unsafe impl WasmRet for Result<(), Trap> { - type Abi = (); - type Fallible = Self; - - #[inline] - fn compatible_with_store(&self, _store: &Store) -> bool { - true - } - - #[inline] - unsafe fn into_abi_for_ret(self, _store: &Store) -> Result<(), Trap> { - self - } - - #[inline] - fn valtype() -> Option { - None - } - - #[inline] - fn into_fallible(self) -> Result<(), Trap> { - self - } - - #[inline] - fn fallible_from_trap(trap: Trap) -> Result<(), Trap> { - Err(trap) - } -} - -unsafe impl WasmRet for T +unsafe impl WasmHostResults for T where T: WasmTy, { type Abi = ::Abi; + type Retptr = (); type Fallible = Result; fn compatible_with_store(&self, store: &Store) -> bool { ::compatible_with_store(self, store) } - unsafe fn into_abi_for_ret(self, store: &Store) -> Result { + unsafe fn into_abi_for_ret(self, store: &Store, _retptr: ()) -> Result { Ok(::into_abi(self, store)) } - fn valtype() -> Option { - Some(::valtype()) + fn func_type(params: impl Iterator) -> FuncType { + FuncType::new(params, Some(::valtype())) + } + + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + *ptr.cast::() = f(()); } fn into_fallible(self) -> Result { @@ -1381,26 +1331,35 @@ where } } -unsafe impl WasmRet for Result +unsafe impl WasmHostResults for Result where - T: WasmTy, + T: WasmHostResults, { - type Abi = ::Abi; + type Abi = ::Abi; + type Retptr = ::Retptr; type Fallible = Self; fn compatible_with_store(&self, store: &Store) -> bool { match self { - Ok(x) => ::compatible_with_store(x, store), + Ok(x) => ::compatible_with_store(x, store), Err(_) => true, } } - unsafe fn into_abi_for_ret(self, store: &Store) -> Result { - self.map(|val| ::into_abi(val, store)) + unsafe fn into_abi_for_ret( + self, + store: &Store, + retptr: Self::Retptr, + ) -> Result { + self.and_then(|val| val.into_abi_for_ret(store, retptr)) + } + + fn func_type(params: impl Iterator) -> FuncType { + T::func_type(params) } - fn valtype() -> Option { - Some(::valtype()) + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + T::wrap_trampoline(ptr, f) } fn into_fallible(self) -> Result { @@ -1412,6 +1371,163 @@ where } } +macro_rules! impl_wasm_host_results { + ($n:tt $($t:ident)*) => ( + #[allow(non_snake_case)] + unsafe impl<$($t),*> WasmHostResults for ($($t,)*) + where + $($t: WasmTy,)* + ($($t::Abi,)*): HostAbi, + { + type Abi = <($($t::Abi,)*) as HostAbi>::Abi; + type Retptr = <($($t::Abi,)*) as HostAbi>::Retptr; + type Fallible = Result; + + #[inline] + fn compatible_with_store(&self, _store: &Store) -> bool { + let ($($t,)*) = self; + $( $t.compatible_with_store(_store) && )* true + } + + #[inline] + unsafe fn into_abi_for_ret(self, _store: &Store, ptr: Self::Retptr) -> Result { + let ($($t,)*) = self; + let abi = ($($t.into_abi(_store),)*); + Ok(<($($t::Abi,)*) as HostAbi>::into_abi(abi, ptr)) + } + + fn func_type(params: impl Iterator) -> FuncType { + FuncType::new( + params, + std::array::IntoIter::new([$($t::valtype(),)*]), + ) + } + + #[allow(unused_assignments)] + unsafe fn wrap_trampoline(mut _ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + let ($($t,)*) = <($($t::Abi,)*) as HostAbi>::call(f); + $( + *_ptr.cast() = $t; + _ptr = _ptr.add(1); + )* + } + + #[inline] + fn into_fallible(self) -> Result { + Ok(self) + } + + #[inline] + fn fallible_from_trap(trap: Trap) -> Result { + Err(trap) + } + } + ) +} + +for_each_function_signature!(impl_wasm_host_results); + +// Internal trait representing how to communicate tuples of return values across +// an ABI boundary. This internally corresponds to the "wasmtime" ABI inside of +// cranelift itself. Notably the first element of each tuple is returned via the +// typical system ABI (e.g. systemv or fastcall depending on platform) and all +// other values are returned packed via the stack. +// +// This trait helps to encapsulate all the details of that. +#[doc(hidden)] +pub trait HostAbi { + // A value returned from native functions which return `Self` + type Abi: Copy; + // A return pointer, added to the end of the argument list, for native + // functions that return `Self`. Note that a 0-sized type here should get + // elided at the ABI level. + type Retptr: Copy; + + // Converts a value of `self` into its components. Stores necessary values + // into `ptr` and then returns whatever needs to be returned from the + // function. + unsafe fn into_abi(self, ptr: Self::Retptr) -> Self::Abi; + + // Calls `f` with a suitably sized return area and requires `f` to return + // the raw abi value of the first element of our tuple. This will then + // unpack the `Retptr` and assemble it with `Self::Abi` to return an + // instance of the whole tuple. + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self; +} + +macro_rules! impl_host_abi { + // Base case, everything is `()` + (0) => { + impl HostAbi for () { + type Abi = (); + type Retptr = (); + + unsafe fn into_abi(self, _ptr: Self::Retptr) -> Self::Abi {} + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + f(()) + } + } + }; + + // In the 1-case the retptr is not present, so it's a 0-sized value. + (1 $a:ident) => { + impl<$a: Copy> HostAbi for ($a,) { + type Abi = $a; + type Retptr = (); + + unsafe fn into_abi(self, _ptr: Self::Retptr) -> Self::Abi { + self.0 + } + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + (f(()),) + } + } + }; + + // This is where the more interesting case happens. The first element of the + // tuple is returned via `Abi` and all other elements are returned via + // `Retptr`. We create a `TupleRetNN` structure to represent all of the + // return values here. + ($n:tt $t:ident $($u:ident)*) => {paste::paste!{ + #[doc(hidden)] + #[allow(non_snake_case)] + #[repr(C)] + pub struct []<$($u,)*> { + $($u: $u,)* + } + + #[allow(non_snake_case, unused_assignments)] + impl<$t: Copy, $($u: Copy,)*> HostAbi for ($t, $($u,)*) { + type Abi = $t; + type Retptr = *mut []<$($u,)*>; + + unsafe fn into_abi(self, ptr: Self::Retptr) -> Self::Abi { + let ($t, $($u,)*) = self; + // Store the tail of our tuple into the return pointer... + $((*ptr).$u = $u;)* + // ... and return the head raw. + $t + } + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + // Create space to store all the return values and then invoke + // the function. + let mut space = MaybeUninit::uninit(); + let t = f(space.as_mut_ptr()); + let space = space.assume_init(); + + // Use the return value as the head of the tuple and unpack our + // return area to get the rest of the tuple. + (t, $(space.$u,)*) + } + } + }}; +} + +for_each_function_signature!(impl_host_abi); + /// Internal trait implemented for all arguments that can be passed to /// [`Func::wrap`] and [`Config::wrap_host_func`](crate::Config::wrap_host_func). /// @@ -1533,7 +1649,7 @@ macro_rules! impl_into_func { where F: Fn($($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { let f = move |_: Caller<'_>, $($args:$args),*| { @@ -1549,7 +1665,7 @@ macro_rules! impl_into_func { where F: Fn(Caller<'_>, $($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { /// This shim is called by Wasm code, constructs a `Caller`, @@ -1563,11 +1679,12 @@ macro_rules! impl_into_func { vmctx: *mut VMContext, caller_vmctx: *mut VMContext, $( $args: $args::Abi, )* + retptr: R::Retptr, ) -> R::Abi where F: Fn(Caller<'_>, $( $args ),*) -> R + 'static, $( $args: WasmTy, )* - R: WasmRet, + R: WasmHostResults, { enum CallResult { Ok(T), @@ -1624,7 +1741,7 @@ macro_rules! impl_into_func { raise_cross_store_trap(); } - match ret.into_abi_for_ret(&store) { + match ret.into_abi_for_ret(&store, retptr) { Ok(val) => CallResult::Ok(val), Err(trap) => CallResult::Trap(trap), } @@ -1654,7 +1771,7 @@ macro_rules! impl_into_func { ) where $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { let ptr = mem::transmute::< *const VMFunctionBody, @@ -1662,6 +1779,7 @@ macro_rules! impl_into_func { *mut VMContext, *mut VMContext, $( $args::Abi, )* + R::Retptr, ) -> R::Abi, >(ptr); @@ -1670,15 +1788,14 @@ macro_rules! impl_into_func { let $args = *args.add(_n).cast::<$args::Abi>(); _n += 1; )* - let ret = ptr(callee_vmctx, caller_vmctx, $( $args ),*); - *args.cast::() = ret; + R::wrap_trampoline(args, |retptr| { + ptr(callee_vmctx, caller_vmctx, $( $args, )* retptr) + }); } - let ty = FuncType::new( + let ty = R::func_type( None::.into_iter() $(.chain(Some($args::valtype())))* - , - R::valtype(), ); let trampoline = host_trampoline::<$($args,)* R>; @@ -1686,7 +1803,7 @@ macro_rules! impl_into_func { // If not given a registry, use a default signature index that is guaranteed to trap // if the function is called indirectly without first being associated with a store (a bug condition). let shared_signature_id = registry - .map(|r| r.register(ty.as_wasm_func_type(), trampoline)) + .map(|r| r.register(ty.as_wasm_func_type(), Some(trampoline))) .unwrap_or(VMSharedSignatureIndex::default()); let instance = unsafe { diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index 5ec884c87840..051c91f446c4 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -1,10 +1,10 @@ -use super::invoke_wasm_and_catch_traps; +use super::{invoke_wasm_and_catch_traps, HostAbi}; use crate::{ExternRef, Func, Store, Trap, ValType}; use anyhow::{bail, Result}; use std::marker; use std::mem::{self, MaybeUninit}; use std::ptr; -use wasmtime_runtime::{VMContext, VMFunctionBody, VMTrampoline}; +use wasmtime_runtime::{VMContext, VMFunctionBody}; /// A statically typed WebAssembly function. /// @@ -103,7 +103,6 @@ where let anyfunc = self.func.export.anyfunc.as_ref(); let result = params.invoke::( &self.func.instance.store, - self.func.trampoline, anyfunc.func_ptr.as_ptr(), anyfunc.vmctx, ptr::null_mut(), @@ -181,13 +180,40 @@ macro_rules! primitives { primitives! { i32 => I32 - u32 => I32 i64 => I64 - u64 => I64 f32 => F32 f64 => F64 } +macro_rules! unsigned { + ($($unsigned:ident => $ty:ident/$signed:ident)*) => ($( + unsafe impl WasmTy for $unsigned { + type Abi = $signed; + #[inline] + fn valtype() -> ValType { + ValType::$ty + } + #[inline] + fn compatible_with_store(&self, _: &Store) -> bool { + true + } + #[inline] + fn into_abi(self, _store: &Store) -> Self::Abi { + self as $signed + } + #[inline] + unsafe fn from_abi(abi: Self::Abi, _store: &Store) -> Self { + abi as $unsigned + } + } + )*) +} + +unsigned! { + u32 => I32/i32 + u64 => I64/i64 +} + unsafe impl WasmTy for Option { type Abi = *mut u8; @@ -274,7 +300,6 @@ pub unsafe trait WasmParams { unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, @@ -296,12 +321,11 @@ where unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, ) -> R { - <(T,)>::invoke((self,), store, trampoline, func, vmctx1, vmctx2) + <(T,)>::invoke((self,), store, func, vmctx1, vmctx2) } } @@ -333,66 +357,30 @@ macro_rules! impl_wasm_params { unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, ) -> R { - // Some signatures can go directly into JIT code which uses the - // default platform ABI, but basically only those without - // multiple return values. With multiple return values we can't - // natively in Rust call such a function because there's no way - // to model it (yet). + let fnptr = mem::transmute::< + *const VMFunctionBody, + unsafe extern "C" fn( + *mut VMContext, + *mut VMContext, + $($t::Abi,)* + R::Retptr, + ) -> R::Abi, + >(func); + let ($($t,)*) = self; + // Use the `call` function to acquire a `retptr` which we'll + // forward to the native function. Once we have it we also + // convert all our arguments to abi arguments to go to the raw + // function. // - // To work around that we use the trampoline which passes - // arguments/values via the stack which allows us to match the - // expected ABI. Note that this branch, using the trampoline, - // is slower as a result and has an extra indirect function - // call as well. In the future if this is a problem we should - // consider updating JIT code to use an ABI we can call from - // Rust itself. - if R::uses_trampoline() { - R::with_space(|space1| { - // Figure out whether the parameters or the results - // require more space, and use the bigger one as where - // to store arguments and load return values from. - let mut space2 = [0; $n]; - let space = if space1.len() < space2.len() { - space2.as_mut_ptr() - } else { - space1.as_mut_ptr() - }; - - // ... store the ABI for all values into our storage - // area... - let ($($t,)*) = self; - let mut _n = 0; - $( - *space.add(_n).cast::<$t::Abi>() = $t.into_abi(store); - _n += 1; - )* - - // ... make the indirect call through the trampoline - // which will read from `space` and also write all the - // results to `space`... - trampoline(vmctx1, vmctx2, func, space); - - // ... and then we can decode all the return values - // from `space`. - R::from_storage(space, store) - }) - } else { - let fnptr = mem::transmute::< - *const VMFunctionBody, - unsafe extern "C" fn( - *mut VMContext, - *mut VMContext, - $($t::Abi,)* - ) -> R::Abi, - >(func); - let ($($t,)*) = self; - R::from_abi(fnptr(vmctx1, vmctx2, $($t.into_abi(store),)*), store) - } + // Upon returning `R::call` will convert all the returns back + // into `R`. + R::call(store, |retptr| { + fnptr(vmctx1, vmctx2, $($t.into_abi(store),)* retptr) + }) } } }; @@ -408,80 +396,45 @@ for_each_function_signature!(impl_wasm_params); /// `TypedFunc` is not currently supported. pub unsafe trait WasmResults: WasmParams { #[doc(hidden)] - type Abi; - #[doc(hidden)] - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self; - #[doc(hidden)] - fn uses_trampoline() -> bool; - // Provides a stack-allocated array with enough space to store all these - // result values. - // - // It'd be nice if we didn't have to have this API and could do something - // with const-generics (or something like that), but I couldn't figure it - // out. If a future Rust explorer is able to get something like `const LEN: - // usize` working that'd be great! + type Abi: Copy; #[doc(hidden)] - fn with_space(_: impl FnOnce(&mut [u128]) -> R) -> R; + type Retptr: Copy; #[doc(hidden)] - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self; + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self; } -unsafe impl WasmResults for T { +// Forwards from a bare type `T` to the 1-tuple type `(T,)` +unsafe impl WasmResults for T +where + (T::Abi,): HostAbi, +{ type Abi = <(T,) as WasmResults>::Abi; - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self { - <(T,) as WasmResults>::from_abi(abi, store).0 - } - fn uses_trampoline() -> bool { - <(T,) as WasmResults>::uses_trampoline() - } - fn with_space(f: impl FnOnce(&mut [u128]) -> R) -> R { - <(T,) as WasmResults>::with_space(f) - } - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self { - <(T,) as WasmResults>::from_storage(ptr, store).0 + type Retptr = <(T,) as WasmResults>::Retptr; + + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + <(T,) as WasmResults>::call(store, f).0 } } -#[doc(hidden)] -pub enum Void {} - macro_rules! impl_wasm_results { ($n:tt $($t:ident)*) => { #[allow(non_snake_case, unused_variables)] - unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) { - type Abi = impl_wasm_results!(@abi $n $($t)*); - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self { - impl_wasm_results!(@from_abi abi store $n $($t)*) - } - fn uses_trampoline() -> bool { - $n > 1 - } - fn with_space(f: impl FnOnce(&mut [u128]) -> R) -> R { - f(&mut [0; $n]) - } - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self { - let mut _n = 0; - $( - let $t = $t::from_abi(*ptr.add(_n).cast::<$t::Abi>(), store); - _n += 1; - )* - ($($t,)*) + unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) + where ($($t::Abi,)*): HostAbi + { + type Abi = <($($t::Abi,)*) as HostAbi>::Abi; + type Retptr = <($($t::Abi,)*) as HostAbi>::Retptr; + + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + // Delegate via the host abi to figure out what the actual ABI + // for dealing with this tuple type is, and then we can re-tuple + // everything and create actual values via `from_abi` after the + // call is complete. + let ($($t,)*) = <($($t::Abi,)*) as HostAbi>::call(f); + ($($t::from_abi($t, store),)*) } } }; - - // 0/1 return values we can use natively, everything else isn't expressible - // and won't be used so define the abi type to Void. - (@abi 0) => (()); - (@abi 1 $t:ident) => ($t::Abi); - (@abi $($t:tt)*) => (Void); - - (@from_abi $abi:ident $store:ident 0) => (()); - (@from_abi $abi:ident $store:ident 1 $t:ident) => (($t::from_abi($abi, $store),)); - (@from_abi $abi:ident $store:ident $($t:tt)*) => ({ - debug_assert!(false); - match $abi {} - }); } for_each_function_signature!(impl_wasm_results); diff --git a/crates/wasmtime/src/sig_registry.rs b/crates/wasmtime/src/sig_registry.rs index c9583972c966..99bf127b8d7f 100644 --- a/crates/wasmtime/src/sig_registry.rs +++ b/crates/wasmtime/src/sig_registry.rs @@ -29,7 +29,7 @@ struct Entry { // Note that the code memory for this trampoline is not owned by this // type, but instead it's expected to be owned by the store that this // registry lives within. - trampoline: VMTrampoline, + trampoline: Option, } impl SignatureRegistry { @@ -37,12 +37,19 @@ impl SignatureRegistry { pub fn register( &mut self, wasm: &WasmFuncType, - trampoline: VMTrampoline, + trampoline: Option, ) -> VMSharedSignatureIndex { let len = self.wasm2index.len(); match self.wasm2index.entry(wasm.clone()) { - hash_map::Entry::Occupied(entry) => *entry.get(), + hash_map::Entry::Occupied(entry) => { + let ret = *entry.get(); + let entry = &mut self.index_map[ret.bits() as usize]; + if entry.trampoline.is_none() { + entry.trampoline = trampoline; + } + ret + } hash_map::Entry::Vacant(entry) => { // Keep `signature_hash` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX) // is reserved for VMSharedSignatureIndex::default(). @@ -75,8 +82,10 @@ impl SignatureRegistry { &self, idx: VMSharedSignatureIndex, ) -> Option<(&WasmFuncType, VMTrampoline)> { - self.index_map + let (wasm, trampoline) = self + .index_map .get(idx.bits() as usize) - .map(|e| (&e.wasm, e.trampoline)) + .map(|e| (&e.wasm, e.trampoline))?; + Some((wasm, trampoline?)) } } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index f01c86662f15..6e315462f8c2 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -202,7 +202,7 @@ impl Store { .inner .signatures .borrow_mut() - .register(ty.as_wasm_func_type(), trampoline); + .register(ty.as_wasm_func_type(), Some(trampoline)); Box::new(anyfunc) }); @@ -322,9 +322,15 @@ impl Store { fn register_signatures(&self, module: &Module) { let mut signatures = self.signatures().borrow_mut(); let types = module.types(); + for (_, ty) in module.compiled_module().module().types.iter() { + if let wasmtime_environ::ModuleType::Function(index) = ty { + let wasm = &types.wasm_signatures[*index]; + signatures.register(wasm, None); + } + } for (index, trampoline) in module.compiled_module().trampolines() { let wasm = &types.wasm_signatures[*index]; - signatures.register(wasm, *trampoline); + signatures.register(wasm, Some(*trampoline)); } } diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 7b5eb70e6459..d81ed07bb149 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -18,6 +18,7 @@ use wasmtime_jit::trampoline::{ self, binemit, pretty_error, Context, FunctionBuilder, FunctionBuilderContext, }; use wasmtime_jit::CodeMemory; +use wasmtime_jit::{blank_sig, wasmtime_call_conv}; use wasmtime_runtime::{ Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, VMContext, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, @@ -91,16 +92,7 @@ fn make_trampoline( // Mostly reverse copy of the similar method from wasmtime's // wasmtime-jit/src/compiler.rs. let pointer_type = isa.pointer_type(); - let mut stub_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Add the caller/callee `vmctx` parameters. - stub_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the caller `vmctx` parameter. - stub_sig.params.push(ir::AbiParam::new(pointer_type)); + let mut stub_sig = blank_sig(isa, wasmtime_call_conv(isa)); // Add the `values_vec` parameter. stub_sig.params.push(ir::AbiParam::new(pointer_type)); @@ -220,8 +212,15 @@ fn create_function_trampoline( // reference types which requires safepoints. let isa = config.target_isa_with_reference_types(); - let pointer_type = isa.pointer_type(); - let sig = ft.get_wasmtime_signature(pointer_type); + let mut sig = blank_sig(&*isa, wasmtime_call_conv(&*isa)); + sig.params.extend( + ft.params() + .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), + ); + sig.returns.extend( + ft.results() + .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), + ); let mut fn_builder_ctx = FunctionBuilderContext::new(); let mut module = Module::new(); @@ -271,7 +270,7 @@ pub fn create_function( // If there is no signature registry, use the default signature index which is // guaranteed to trap if there is ever an indirect call on the function (should not happen) let shared_signature_id = registry - .map(|r| r.register(ft.as_wasm_func_type(), trampoline)) + .map(|r| r.register(ft.as_wasm_func_type(), Some(trampoline))) .unwrap_or(VMSharedSignatureIndex::default()); unsafe { diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index 5a090d794cce..2a91af473c91 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -298,28 +298,6 @@ impl FuncType { &self.sig } - /// Get the Cranelift-compatible function signature. - pub(crate) fn get_wasmtime_signature(&self, pointer_type: ir::Type) -> ir::Signature { - use wasmtime_environ::ir::{AbiParam, ArgumentPurpose, Signature}; - use wasmtime_jit::native; - let call_conv = native::call_conv(); - let mut params = vec![ - AbiParam::special(pointer_type, ArgumentPurpose::VMContext), - AbiParam::new(pointer_type), - ]; - params.extend(self.params().map(|p| AbiParam::new(p.get_wasmtime_type()))); - let returns = self - .results() - .map(|p| AbiParam::new(p.get_wasmtime_type())) - .collect::>(); - - Signature { - params, - returns, - call_conv, - } - } - pub(crate) fn from_wasm_func_type(sig: &wasm::WasmFuncType) -> FuncType { FuncType { sig: sig.clone() } } diff --git a/tests/all/func.rs b/tests/all/func.rs index 0e691993843f..4159cafd9a5e 100644 --- a/tests/all/func.rs +++ b/tests/all/func.rs @@ -619,3 +619,162 @@ fn trap_doesnt_leak() -> anyhow::Result<()> { assert!(dtor2_run.get()); Ok(()) } + +#[test] +fn wrap_multiple_results() -> anyhow::Result<()> { + fn test(store: &Store, t: T) -> anyhow::Result<()> + where + T: WasmHostResults + + WasmResults + + PartialEq + + Copy + + std::fmt::Debug + + EqualToValues + + 'static, + { + let f = Func::wrap(store, move || t); + assert_eq!(f.typed::<(), T>()?.call(())?, t); + assert!(t.eq_values(&f.call(&[])?)); + + let module = Module::new(store.engine(), &T::gen_wasm())?; + let instance = Instance::new(store, &module, &[f.into()])?; + let f = instance.get_func("foo").unwrap(); + + assert_eq!(f.typed::<(), T>()?.call(())?, t); + assert!(t.eq_values(&f.call(&[])?)); + Ok(()) + } + + let store = Store::default(); + // 0 element + test(&store, ())?; + + // 1 element + test(&store, (1i32,))?; + test(&store, (2u32,))?; + test(&store, (3i64,))?; + test(&store, (4u64,))?; + test(&store, (5.0f32,))?; + test(&store, (6.0f64,))?; + + // 2 element ... + test(&store, (7i32, 8i32))?; + test(&store, (7i32, 8i64))?; + test(&store, (7i32, 8f32))?; + test(&store, (7i32, 8f64))?; + + test(&store, (7i64, 8i32))?; + test(&store, (7i64, 8i64))?; + test(&store, (7i64, 8f32))?; + test(&store, (7i64, 8f64))?; + + test(&store, (7f32, 8i32))?; + test(&store, (7f32, 8i64))?; + test(&store, (7f32, 8f32))?; + test(&store, (7f32, 8f64))?; + + test(&store, (7f64, 8i32))?; + test(&store, (7f64, 8i64))?; + test(&store, (7f64, 8f32))?; + test(&store, (7f64, 8f64))?; + + // and beyond... + test(&store, (1i32, 2i32, 3i32))?; + test(&store, (1i32, 2f32, 3i32))?; + test(&store, (1f64, 2f32, 3i32))?; + test(&store, (1f64, 2i64, 3i32))?; + test(&store, (1f32, 2f32, 3i64, 4f64))?; + test(&store, (1f64, 2i64, 3i32, 4i64, 5f32))?; + test(&store, (1i32, 2f64, 3i64, 4f64, 5f64, 6f32))?; + test(&store, (1i64, 2i32, 3i64, 4f32, 5f32, 6i32, 7u64))?; + test(&store, (1u32, 2f32, 3u64, 4f64, 5i32, 6f32, 7u64, 8u32))?; + test( + &store, + (1f32, 2f64, 3f32, 4i32, 5u32, 6i64, 7f32, 8i32, 9u64), + )?; + return Ok(()); + + trait EqualToValues { + fn eq_values(&self, values: &[Val]) -> bool; + fn gen_wasm() -> String; + } + + macro_rules! equal_tuples { + ($($cnt:tt ($($a:ident),*))*) => ($( + #[allow(non_snake_case)] + impl<$($a: EqualToValue,)*> EqualToValues for ($($a,)*) { + fn eq_values(&self, values: &[Val]) -> bool { + let ($($a,)*) = self; + let mut _values = values.iter(); + _values.len() == $cnt && + $($a.eq_value(_values.next().unwrap()) &&)* + true + } + + fn gen_wasm() -> String { + let mut wasm = String::new(); + wasm.push_str("(module "); + wasm.push_str("(type $t (func (result "); + $( + wasm.push_str($a::wasm_ty()); + wasm.push_str(" "); + )* + wasm.push_str(")))"); + + wasm.push_str("(import \"\" \"\" (func $host (type $t)))"); + wasm.push_str("(func (export \"foo\") (type $t)"); + wasm.push_str("call $host"); + wasm.push_str(")"); + wasm.push_str(")"); + + wasm + } + } + )*) + } + + equal_tuples! { + 0 () + 1 (A1) + 2 (A1, A2) + 3 (A1, A2, A3) + 4 (A1, A2, A3, A4) + 5 (A1, A2, A3, A4, A5) + 6 (A1, A2, A3, A4, A5, A6) + 7 (A1, A2, A3, A4, A5, A6, A7) + 8 (A1, A2, A3, A4, A5, A6, A7, A8) + 9 (A1, A2, A3, A4, A5, A6, A7, A8, A9) + } + + trait EqualToValue { + fn eq_value(&self, value: &Val) -> bool; + fn wasm_ty() -> &'static str; + } + + macro_rules! equal_values { + ($a:ident $($ty:ident $wasm:tt $variant:ident $e:expr,)*) => ($( + impl EqualToValue for $ty { + fn eq_value(&self, val: &Val) -> bool { + if let Val::$variant($a) = *val { + return *self == $e; + } + false + } + + fn wasm_ty() -> &'static str { + $wasm + } + } + )*) + } + + equal_values! { + a + i32 "i32" I32 a, + u32 "i32" I32 a as u32, + i64 "i64" I64 a, + u64 "i64" I64 a as u64, + f32 "f32" F32 f32::from_bits(a), + f64 "f64" F64 f64::from_bits(a), + } +} From ee18cbd3beb4eb975b9e2fa559e20cb38d407b7b Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 11:45:13 -0700 Subject: [PATCH 02/16] Tweak some names and add docs --- crates/wasmtime/src/config.rs | 4 ++-- crates/wasmtime/src/func.rs | 28 +++++++++++++------------- crates/wasmtime/src/func/typed.rs | 31 ++--------------------------- crates/wasmtime/src/sig_registry.rs | 6 ++++++ crates/wasmtime/src/store.rs | 6 ++++++ tests/all/func.rs | 8 +------- 6 files changed, 31 insertions(+), 52 deletions(-) diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 55fbcfddca21..43bd809c84aa 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1,6 +1,6 @@ use crate::memory::MemoryCreator; use crate::trampoline::MemoryCreatorProxy; -use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmHostResults, WasmTy}; +use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmRet, WasmTy}; use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; use std::cmp; @@ -339,7 +339,7 @@ macro_rules! generate_wrap_async_host_func { ) where $($args: WasmTy,)* - R: WasmHostResults, + R: WasmRet, { // Defer the check for async support until engine creation time to not introduce an order dependency self.host_funcs.insert( diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 924a1002d932..57365ed5ec3e 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -351,7 +351,7 @@ macro_rules! generate_wrap_async_func { where T: 'static, $($args: WasmTy,)* - R: WasmHostResults, + R: WasmRet, { assert!(store.async_support(), concat!("cannot use `wrap", $num, "_async` without enabling async support on the config")); Func::wrap(store, move |caller: Caller<'_>, $($args: $args),*| { @@ -1261,7 +1261,7 @@ fn enter_wasm_init<'a>(store: &'a Store) -> Result { /// stable over time. /// /// For more information see [`Func::wrap`] -pub unsafe trait WasmHostResults { +pub unsafe trait WasmRet { // Same as `WasmTy::Abi`. #[doc(hidden)] type Abi: Copy; @@ -1291,14 +1291,14 @@ pub unsafe trait WasmHostResults { // explicitly, used when wrapping async functions which always bottom-out // in a function that returns a trap because futures can be cancelled. #[doc(hidden)] - type Fallible: WasmHostResults; + type Fallible: WasmRet; #[doc(hidden)] fn into_fallible(self) -> Self::Fallible; #[doc(hidden)] fn fallible_from_trap(trap: Trap) -> Self::Fallible; } -unsafe impl WasmHostResults for T +unsafe impl WasmRet for T where T: WasmTy, { @@ -1331,17 +1331,17 @@ where } } -unsafe impl WasmHostResults for Result +unsafe impl WasmRet for Result where - T: WasmHostResults, + T: WasmRet, { - type Abi = ::Abi; - type Retptr = ::Retptr; + type Abi = ::Abi; + type Retptr = ::Retptr; type Fallible = Self; fn compatible_with_store(&self, store: &Store) -> bool { match self { - Ok(x) => ::compatible_with_store(x, store), + Ok(x) => ::compatible_with_store(x, store), Err(_) => true, } } @@ -1374,7 +1374,7 @@ where macro_rules! impl_wasm_host_results { ($n:tt $($t:ident)*) => ( #[allow(non_snake_case)] - unsafe impl<$($t),*> WasmHostResults for ($($t,)*) + unsafe impl<$($t),*> WasmRet for ($($t,)*) where $($t: WasmTy,)* ($($t::Abi,)*): HostAbi, @@ -1649,7 +1649,7 @@ macro_rules! impl_into_func { where F: Fn($($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmHostResults, + R: WasmRet, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { let f = move |_: Caller<'_>, $($args:$args),*| { @@ -1665,7 +1665,7 @@ macro_rules! impl_into_func { where F: Fn(Caller<'_>, $($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmHostResults, + R: WasmRet, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { /// This shim is called by Wasm code, constructs a `Caller`, @@ -1684,7 +1684,7 @@ macro_rules! impl_into_func { where F: Fn(Caller<'_>, $( $args ),*) -> R + 'static, $( $args: WasmTy, )* - R: WasmHostResults, + R: WasmRet, { enum CallResult { Ok(T), @@ -1771,7 +1771,7 @@ macro_rules! impl_into_func { ) where $($args: WasmTy,)* - R: WasmHostResults, + R: WasmRet, { let ptr = mem::transmute::< *const VMFunctionBody, diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index 051c91f446c4..c0a1d834da42 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -180,40 +180,13 @@ macro_rules! primitives { primitives! { i32 => I32 + u32 => I32 i64 => I64 + u64 => I64 f32 => F32 f64 => F64 } -macro_rules! unsigned { - ($($unsigned:ident => $ty:ident/$signed:ident)*) => ($( - unsafe impl WasmTy for $unsigned { - type Abi = $signed; - #[inline] - fn valtype() -> ValType { - ValType::$ty - } - #[inline] - fn compatible_with_store(&self, _: &Store) -> bool { - true - } - #[inline] - fn into_abi(self, _store: &Store) -> Self::Abi { - self as $signed - } - #[inline] - unsafe fn from_abi(abi: Self::Abi, _store: &Store) -> Self { - abi as $unsigned - } - } - )*) -} - -unsigned! { - u32 => I32/i32 - u64 => I64/i64 -} - unsafe impl WasmTy for Option { type Abi = *mut u8; diff --git a/crates/wasmtime/src/sig_registry.rs b/crates/wasmtime/src/sig_registry.rs index 99bf127b8d7f..0279a7fe34b3 100644 --- a/crates/wasmtime/src/sig_registry.rs +++ b/crates/wasmtime/src/sig_registry.rs @@ -34,6 +34,10 @@ struct Entry { impl SignatureRegistry { /// Register a signature and return its unique index. + /// + /// Note that `trampoline` can be `None` which indicates that an index is + /// desired for this signature but the trampoline for it is not compiled or + /// available. pub fn register( &mut self, wasm: &WasmFuncType, @@ -45,6 +49,8 @@ impl SignatureRegistry { hash_map::Entry::Occupied(entry) => { let ret = *entry.get(); let entry = &mut self.index_map[ret.bits() as usize]; + // If the entry does not previously have a trampoline, then + // overwrite it with whatever was specified by this function. if entry.trampoline.is_none() { entry.trampoline = trampoline; } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index 6e315462f8c2..ae54fee0d25e 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -322,12 +322,18 @@ impl Store { fn register_signatures(&self, module: &Module) { let mut signatures = self.signatures().borrow_mut(); let types = module.types(); + + // Register a unique index for all types in this module, even if they + // don't have a trampoline. for (_, ty) in module.compiled_module().module().types.iter() { if let wasmtime_environ::ModuleType::Function(index) = ty { let wasm = &types.wasm_signatures[*index]; signatures.register(wasm, None); } } + + // Afterwards register all compiled trampolines for this module with the + // signature registry as well. for (index, trampoline) in module.compiled_module().trampolines() { let wasm = &types.wasm_signatures[*index]; signatures.register(wasm, Some(*trampoline)); diff --git a/tests/all/func.rs b/tests/all/func.rs index 4159cafd9a5e..2dbc64c84618 100644 --- a/tests/all/func.rs +++ b/tests/all/func.rs @@ -624,13 +624,7 @@ fn trap_doesnt_leak() -> anyhow::Result<()> { fn wrap_multiple_results() -> anyhow::Result<()> { fn test(store: &Store, t: T) -> anyhow::Result<()> where - T: WasmHostResults - + WasmResults - + PartialEq - + Copy - + std::fmt::Debug - + EqualToValues - + 'static, + T: WasmRet + WasmResults + PartialEq + Copy + std::fmt::Debug + EqualToValues + 'static, { let f = Func::wrap(store, move || t); assert_eq!(f.typed::<(), T>()?.call(())?, t); From 7ae86d92a0893d869f51cdce3ffc13ebba1cb42d Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 11:47:12 -0700 Subject: [PATCH 03/16] "fix" lightbeam compile --- crates/lightbeam/wasmtime/src/lib.rs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/crates/lightbeam/wasmtime/src/lib.rs b/crates/lightbeam/wasmtime/src/lib.rs index a798eee4d219..5174a27c0a5c 100644 --- a/crates/lightbeam/wasmtime/src/lib.rs +++ b/crates/lightbeam/wasmtime/src/lib.rs @@ -174,8 +174,6 @@ struct FuncEnvironment<'module_environment> { /// The module-level environment which this function-level environment belongs to. module: &'module_environment Module, - native_signatures: &'module_environment PrimaryMap, - /// Offsets to struct fields accessed by JIT code. offsets: VMOffsets, } @@ -189,7 +187,6 @@ impl<'module_environment> FuncEnvironment<'module_environment> { Self { module: &translation.module, offsets: VMOffsets::new(pointer_bytes, &translation.module), - native_signatures: &types.native_signatures, } } } @@ -227,8 +224,8 @@ impl lightbeam::ModuleContext for FuncEnvironment<'_> { self.module.functions[FuncIndex::from_u32(func_idx)].as_u32() } - fn signature(&self, index: u32) -> &Self::Signature { - &self.native_signatures[SignatureIndex::from_u32(index)] + fn signature(&self, _index: u32) -> &Self::Signature { + panic!("not impleemnted") } fn defined_table_index(&self, table_index: u32) -> Option { From a6764616c77ade96336f9e16eb1242c79555e127 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 11:52:14 -0700 Subject: [PATCH 04/16] Fix TODO with dummy environ --- cranelift/wasm/src/environ/dummy.rs | 26 ++++++++++++++++++++++---- 1 file changed, 22 insertions(+), 4 deletions(-) diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index 7a56712b7cee..ea9157e55249 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -15,12 +15,13 @@ use crate::translation_utils::{ DataIndex, DefinedFuncIndex, ElemIndex, FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, Table, TableIndex, TypeIndex, }; +use crate::WasmType; use core::convert::TryFrom; use cranelift_codegen::cursor::FuncCursor; use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{self, InstBuilder}; -use cranelift_codegen::isa::TargetFrontendConfig; +use cranelift_codegen::isa::{CallConv, TargetFrontendConfig}; use cranelift_entity::{EntityRef, PrimaryMap, SecondaryMap}; use cranelift_frontend::FunctionBuilder; use std::boxed::Box; @@ -660,9 +661,26 @@ impl TargetEnvironment for DummyEnvironment { } impl<'data> ModuleEnvironment<'data> for DummyEnvironment { - fn declare_type_func(&mut self, _wasm: WasmFuncType) -> WasmResult<()> { - // TODO - // self.info.signatures.push(sig); + fn declare_type_func(&mut self, wasm: WasmFuncType) -> WasmResult<()> { + let mut sig = ir::Signature::new(CallConv::Fast); + let mut cvt = |ty: &WasmType| { + let reference_type = match self.pointer_type() { + ir::types::I32 => ir::types::R32, + ir::types::I64 => ir::types::R64, + _ => panic!("unsupported pointer type"), + }; + ir::AbiParam::new(match ty { + WasmType::I32 => ir::types::I32, + WasmType::I64 => ir::types::I64, + WasmType::F32 => ir::types::F32, + WasmType::F64 => ir::types::F64, + WasmType::V128 => ir::types::I8X16, + WasmType::FuncRef | WasmType::ExternRef | WasmType::ExnRef => reference_type, + }) + }; + sig.params.extend(wasm.params.iter().map(&mut cvt)); + sig.returns.extend(wasm.returns.iter().map(&mut cvt)); + self.info.signatures.push(sig); Ok(()) } From 97bf8079e8fed7c520b4383f12a548f3a8ff1a16 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 12:08:53 -0700 Subject: [PATCH 05/16] Unwind info is a property of the target, not the ABI --- cranelift/codegen/src/context.rs | 2 +- cranelift/codegen/src/isa/call_conv.rs | 14 -------------- cranelift/codegen/src/isa/mod.rs | 16 ++++++++++++++-- cranelift/codegen/src/isa/x86/abi.rs | 2 +- cranelift/codegen/src/isa/x86/unwind/systemv.rs | 2 +- cranelift/codegen/src/machinst/abi.rs | 3 --- cranelift/codegen/src/machinst/abi_impl.rs | 4 ---- 7 files changed, 17 insertions(+), 26 deletions(-) diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index f668fe596a9d..33c205d22604 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -267,7 +267,7 @@ impl Context { isa: &dyn TargetIsa, ) -> CodegenResult> { if let Some(backend) = isa.get_mach_backend() { - let unwind_info_kind = self.func.signature.call_conv.unwind_info_kind(); + let unwind_info_kind = isa.unwind_info_kind(); let result = self.mach_compile_result.as_ref().unwrap(); return backend.emit_unwind_info(result, unwind_info_kind); } diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs index fb0a2ca2635c..ab6c1c2bab2b 100644 --- a/cranelift/codegen/src/isa/call_conv.rs +++ b/cranelift/codegen/src/isa/call_conv.rs @@ -1,4 +1,3 @@ -use crate::machinst::UnwindInfoKind; use crate::settings::{self, LibcallCallConv}; use core::fmt; use core::str; @@ -88,19 +87,6 @@ impl CallConv { _ => false, } } - - /// Returns the kind of unwind info used for this calling convention - pub fn unwind_info_kind(self) -> UnwindInfoKind { - match self { - #[cfg(feature = "unwind")] - CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => { - UnwindInfoKind::SystemV - } - #[cfg(feature = "unwind")] - CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => UnwindInfoKind::Windows, - _ => UnwindInfoKind::None, - } - } } impl fmt::Display for CallConv { diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 9cb0cfab9161..a24f64a25616 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -57,7 +57,7 @@ use crate::flowgraph; use crate::ir; #[cfg(feature = "unwind")] use crate::isa::unwind::systemv::RegisterMappingError; -use crate::machinst::MachBackend; +use crate::machinst::{MachBackend, UnwindInfoKind}; use crate::regalloc; use crate::result::CodegenResult; use crate::settings; @@ -68,7 +68,7 @@ use core::any::Any; use core::fmt; use core::fmt::{Debug, Formatter}; use core::hash::Hasher; -use target_lexicon::{triple, Architecture, PointerWidth, Triple}; +use target_lexicon::{triple, Architecture, OperatingSystem, PointerWidth, Triple}; use thiserror::Error; #[cfg(feature = "riscv")] @@ -476,6 +476,18 @@ pub trait TargetIsa: fmt::Display + Send + Sync { /// IntCC condition for Unsigned Subtraction Overflow (Borrow/Carry). fn unsigned_sub_overflow_condition(&self) -> ir::condcodes::IntCC; + /// Returns the flavor of unwind information emitted for this target. + fn unwind_info_kind(&self) -> UnwindInfoKind { + match self.triple().operating_system { + #[cfg(feature = "unwind")] + OperatingSystem::Windows => UnwindInfoKind::Windows, + #[cfg(feature = "unwind")] + _ => UnwindInfoKind::SystemV, + #[cfg(not(feature = "unwind"))] + _ => UnwindInfoKind::None, + } + } + /// Creates unwind information for the function. /// /// Returns `None` if there is no unwind information for the function. diff --git a/cranelift/codegen/src/isa/x86/abi.rs b/cranelift/codegen/src/isa/x86/abi.rs index 83b44be1b09a..d56d066e8423 100644 --- a/cranelift/codegen/src/isa/x86/abi.rs +++ b/cranelift/codegen/src/isa/x86/abi.rs @@ -1090,7 +1090,7 @@ pub fn create_unwind_info( // Assumption: RBP is being used as the frame pointer for both calling conventions // In the future, we should be omitting frame pointer as an optimization, so this will change - Ok(match func.signature.call_conv.unwind_info_kind() { + Ok(match isa.unwind_info_kind() { UnwindInfoKind::SystemV => { super::unwind::systemv::create_unwind_info(func, isa)?.map(|u| UnwindInfo::SystemV(u)) } diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs index 26e1ccbda7d0..0c3e74976239 100644 --- a/cranelift/codegen/src/isa/x86/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -97,7 +97,7 @@ pub(crate) fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { // Only System V-like calling conventions are supported - match func.signature.call_conv.unwind_info_kind() { + match isa.unwind_info_kind() { crate::machinst::UnwindInfoKind::SystemV => {} _ => return Ok(None), } diff --git a/cranelift/codegen/src/machinst/abi.rs b/cranelift/codegen/src/machinst/abi.rs index 8c7322736c4c..ac4cb7fad2e0 100644 --- a/cranelift/codegen/src/machinst/abi.rs +++ b/cranelift/codegen/src/machinst/abi.rs @@ -189,9 +189,6 @@ pub trait ABICallee { from_slot: SpillSlot, ty: Option, ) -> Self::I; - - /// Desired unwind info type. - fn unwind_info_kind(&self) -> UnwindInfoKind; } /// Trait implemented by an object that tracks ABI-related state and can diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 765712c2cf5c..85a6d6831b20 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -1371,10 +1371,6 @@ impl ABICallee for ABICalleeImpl { .next() .unwrap() } - - fn unwind_info_kind(&self) -> UnwindInfoKind { - self.sig.call_conv.unwind_info_kind() - } } fn abisig_to_uses_and_defs(sig: &ABISig) -> (Vec, Vec>) { From 4f9dbb618a75811dca05010c76495aa7d236fbc5 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 12:10:32 -0700 Subject: [PATCH 06/16] Remove lightbeam unused imports --- crates/lightbeam/wasmtime/src/lib.rs | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/crates/lightbeam/wasmtime/src/lib.rs b/crates/lightbeam/wasmtime/src/lib.rs index 5174a27c0a5c..009acba1b34a 100644 --- a/crates/lightbeam/wasmtime/src/lib.rs +++ b/crates/lightbeam/wasmtime/src/lib.rs @@ -9,12 +9,12 @@ use cranelift_codegen::isa; use lightbeam::{CodeGenSession, NullOffsetSink, Sinks}; use wasmtime_environ::wasm::{ DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, - GlobalIndex, MemoryIndex, SignatureIndex, TableIndex, TypeIndex, + GlobalIndex, MemoryIndex, TableIndex, TypeIndex, }; use wasmtime_environ::{ - entity::PrimaryMap, BuiltinFunctionIndex, CompileError, CompiledFunction, Compiler, - FunctionBodyData, Module, ModuleTranslation, Relocation, RelocationTarget, TrapInformation, - Tunables, TypeTables, VMOffsets, + BuiltinFunctionIndex, CompileError, CompiledFunction, Compiler, FunctionBodyData, Module, + ModuleTranslation, Relocation, RelocationTarget, TrapInformation, Tunables, TypeTables, + VMOffsets, }; /// A compiler that compiles a WebAssembly module with Lightbeam, directly translating the Wasm file. @@ -28,14 +28,14 @@ impl Compiler for Lightbeam { function_body: FunctionBodyData<'_>, isa: &dyn isa::TargetIsa, tunables: &Tunables, - types: &TypeTables, + _types: &TypeTables, ) -> Result { if tunables.generate_native_debuginfo { return Err(CompileError::DebugInfoNotSupported); } let func_index = translation.module.func_index(i); - let env = FuncEnvironment::new(isa.frontend_config().pointer_bytes(), translation, types); + let env = FuncEnvironment::new(isa.frontend_config().pointer_bytes(), translation); let mut codegen_session: CodeGenSession<_> = CodeGenSession::new( translation.function_body_inputs.len() as u32, &env, @@ -179,11 +179,7 @@ struct FuncEnvironment<'module_environment> { } impl<'module_environment> FuncEnvironment<'module_environment> { - fn new( - pointer_bytes: u8, - translation: &'module_environment ModuleTranslation<'_>, - types: &'module_environment TypeTables, - ) -> Self { + fn new(pointer_bytes: u8, translation: &'module_environment ModuleTranslation<'_>) -> Self { Self { module: &translation.module, offsets: VMOffsets::new(pointer_bytes, &translation.module), From f768433fbfa04aefb03f8981f5075a4f80515068 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 12:47:14 -0700 Subject: [PATCH 07/16] Attempt to fix arm64 --- cranelift/codegen/src/isa/aarch64/abi.rs | 25 ++++++++++++------------ 1 file changed, 13 insertions(+), 12 deletions(-) diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index b2a95e314be9..380d2dfe0bc1 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -290,18 +290,19 @@ impl ABIMachineSpec for AArch64MachineDeps { // Compute the stack slot's size. let size = (ty_bits(param.value_type) / 8) as u64; - let size = - if call_conv == isa::CallConv::AppleAarch64 || call_conv.extends_wasmtime() { - // MacOS aarch64 and Wasmtime allow stack slots with - // sizes less than 8 bytes. They still need to be - // properly aligned on their natural data alignment, - // though. - size - } else { - // Every arg takes a minimum slot of 8 bytes. (16-byte stack - // alignment happens separately after all args.) - std::cmp::max(size, 8) - }; + let size = if call_conv == isa::CallConv::AppleAarch64 + || (call_conv.extends_wasmtime() && args_or_rets == ArgsOrRets::Rets) + { + // MacOS aarch64 and Wasmtime allow stack slots with + // sizes less than 8 bytes. They still need to be + // properly aligned on their natural data alignment, + // though. + size + } else { + // Every arg takes a minimum slot of 8 bytes. (16-byte stack + // alignment happens separately after all args.) + std::cmp::max(size, 8) + }; // Align the stack slot. debug_assert!(size.is_power_of_two()); From 386623b62e2596404e59914cf9f0cb85511cbd00 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 12:51:24 -0700 Subject: [PATCH 08/16] Document new ABIs aren't stable --- cranelift/codegen/src/isa/call_conv.rs | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs index ab6c1c2bab2b..fc5856e6e543 100644 --- a/cranelift/codegen/src/isa/call_conv.rs +++ b/cranelift/codegen/src/isa/call_conv.rs @@ -29,9 +29,15 @@ pub enum CallConv { Baldrdash2020, /// Specialized convention for the probestack function. Probestack, - /// Wasmtime equivalent of SystemV, except the multi-return ABI is tweaked. + /// Wasmtime equivalent of SystemV, not ABI-stable. + /// + /// Currently only differs in how multiple return values are handled, + /// returning the first return value in a register and everything else + /// through a return-pointer. WasmtimeSystemV, - /// Wasmtime equivalent of WindowsFastcall, except the multi-return ABI is tweaked. + /// Wasmtime equivalent of WindowsFastcall, not ABI-stable. + /// + /// Differs from fastcall in the same way as `WasmtimeSystemV`. WasmtimeFastcall, } From 5445469bdbcd23107d9a2cb8aaa7bddc4ad5edc9 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Mon, 5 Apr 2021 13:43:23 -0700 Subject: [PATCH 09/16] Fix filetests to use the right target --- cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif | 2 +- .../filetests/isa/x86/windows_fastcall_x64_unwind.clif | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif b/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif index 8ce113512b0b..c5144bfd971d 100644 --- a/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif +++ b/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif @@ -1,7 +1,7 @@ test unwind set opt_level=speed_and_size set is_pic -target x86_64 legacy haswell +target x86_64-linux legacy haswell ; check the unwind information with a function with no args function %no_args() system_v { diff --git a/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif b/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif index 348b37312cde..547e131fbdb0 100644 --- a/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif +++ b/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif @@ -1,7 +1,7 @@ test unwind set opt_level=speed_and_size set is_pic -target x86_64 legacy haswell +target x86_64-windows legacy haswell ; check the unwind information with a leaf function with no args function %no_args_leaf() windows_fastcall { From 95f86be431557253e41baaef0da68ccf0b3bc0fd Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 6 Apr 2021 11:49:29 -0700 Subject: [PATCH 10/16] Don't always do 64-bit stores with cranelift This was overwriting upper bits when 32-bit registers were being stored into return values, so fix the code inline to do a sized store instead of one-size-fits-all store. --- cranelift/codegen/src/isa/x64/inst/mod.rs | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/cranelift/codegen/src/isa/x64/inst/mod.rs b/cranelift/codegen/src/isa/x64/inst/mod.rs index 973052f01ed1..a1556c8b0772 100644 --- a/cranelift/codegen/src/isa/x64/inst/mod.rs +++ b/cranelift/codegen/src/isa/x64/inst/mod.rs @@ -1122,11 +1122,16 @@ impl Inst { pub(crate) fn store(ty: Type, from_reg: Reg, to_addr: impl Into) -> Inst { let rc = from_reg.get_class(); match rc { - RegClass::I64 => { - // Always store the full register, to ensure that the high bits are properly set - // when doing a full reload. - Inst::mov_r_m(OperandSize::Size64, from_reg, to_addr) - } + RegClass::I64 => Inst::mov_r_m( + match ty { + types::B1 => OperandSize::Size8, + types::I32 | types::R32 => OperandSize::Size32, + types::I64 | types::R64 => OperandSize::Size64, + _ => unimplemented!("integer store of type: {}", ty), + }, + from_reg, + to_addr, + ), RegClass::V128 => { let opcode = match ty { types::F32 => SseOpcode::Movss, From 51fe59a812a789836ab8138dc1e29e2a1ddc9acb Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 6 Apr 2021 13:21:31 -0700 Subject: [PATCH 11/16] At least get tests passing on the old backend --- Cargo.toml | 2 +- crates/wasmtime/src/func.rs | 9 +++++++-- tests/all/func.rs | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 20c19899112e..098f8c7282cf 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -98,7 +98,7 @@ all-arch = ["wasmtime/all-arch"] experimental_x64 = [] # Use the old x86 backend. -old-x86-backend = ["wasmtime-jit/old-x86-backend"] +old-x86-backend = ["wasmtime/old-x86-backend"] [badges] maintenance = { status = "actively-developed" } diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 57365ed5ec3e..168d8613fe5b 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -6,7 +6,6 @@ use std::cmp::max; use std::fmt; use std::future::Future; use std::mem; -use std::mem::MaybeUninit; use std::panic::{self, AssertUnwindSafe}; use std::pin::Pin; use std::ptr::{self, NonNull}; @@ -1490,14 +1489,20 @@ macro_rules! impl_host_abi { // tuple is returned via `Abi` and all other elements are returned via // `Retptr`. We create a `TupleRetNN` structure to represent all of the // return values here. + // + // Also note that this isn't implemented for the old backend right now due + // to the original author not really being sure how to implement this in the + // old backend. ($n:tt $t:ident $($u:ident)*) => {paste::paste!{ #[doc(hidden)] #[allow(non_snake_case)] #[repr(C)] + #[cfg(not(feature = "old-x86-backend"))] pub struct []<$($u,)*> { $($u: $u,)* } + #[cfg(not(feature = "old-x86-backend"))] #[allow(non_snake_case, unused_assignments)] impl<$t: Copy, $($u: Copy,)*> HostAbi for ($t, $($u,)*) { type Abi = $t; @@ -1514,7 +1519,7 @@ macro_rules! impl_host_abi { unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { // Create space to store all the return values and then invoke // the function. - let mut space = MaybeUninit::uninit(); + let mut space = std::mem::MaybeUninit::uninit(); let t = f(space.as_mut_ptr()); let space = space.assume_init(); diff --git a/tests/all/func.rs b/tests/all/func.rs index 2dbc64c84618..e3bee038c2d9 100644 --- a/tests/all/func.rs +++ b/tests/all/func.rs @@ -550,6 +550,7 @@ fn trampolines_always_valid() -> anyhow::Result<()> { } #[test] +#[cfg(not(feature = "old-x86-backend"))] fn typed_multiple_results() -> anyhow::Result<()> { let store = Store::default(); let module = Module::new( @@ -621,6 +622,7 @@ fn trap_doesnt_leak() -> anyhow::Result<()> { } #[test] +#[cfg(not(feature = "old-x86-backend"))] fn wrap_multiple_results() -> anyhow::Result<()> { fn test(store: &Store, t: T) -> anyhow::Result<()> where From 102000549a43303a82766d2f09b60d471322e536 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 6 Apr 2021 13:22:12 -0700 Subject: [PATCH 12/16] Fix a typo --- crates/lightbeam/wasmtime/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/crates/lightbeam/wasmtime/src/lib.rs b/crates/lightbeam/wasmtime/src/lib.rs index 009acba1b34a..acb69a11aa5a 100644 --- a/crates/lightbeam/wasmtime/src/lib.rs +++ b/crates/lightbeam/wasmtime/src/lib.rs @@ -221,7 +221,7 @@ impl lightbeam::ModuleContext for FuncEnvironment<'_> { } fn signature(&self, _index: u32) -> &Self::Signature { - panic!("not impleemnted") + panic!("not implemented") } fn defined_table_index(&self, table_index: u32) -> Option { From 5962d4c9118a8816afdfaa0a8a70f854d2ba60ab Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Tue, 6 Apr 2021 15:33:20 -0700 Subject: [PATCH 13/16] Add some filetests with mixed abi calls --- .../filetests/isa/x64/call-conv.clif | 207 ++++++++++++++++++ 1 file changed, 207 insertions(+) create mode 100644 cranelift/filetests/filetests/isa/x64/call-conv.clif diff --git a/cranelift/filetests/filetests/isa/x64/call-conv.clif b/cranelift/filetests/filetests/isa/x64/call-conv.clif new file mode 100644 index 000000000000..66af3e9fca87 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x64/call-conv.clif @@ -0,0 +1,207 @@ +test compile +target x86_64 machinst + +;; system_v has first param in %rdi, fascall in %rcx +function %one_arg(i32) system_v { + sig0 = (i32) windows_fastcall +block0(v0: i32): + ; check: movq %rdi, %rcx + ; nextln: call *%rdi + call_indirect sig0, v0(v0) + return +} + +;; system_v has params in %rdi, %xmm0, fascall in %rcx, %xmm1 +function %two_args(i32, f32) system_v { + sig0 = (i32, f32) windows_fastcall + sig1 = (i32, f32) system_v +block0(v0: i32, v1: f32): + ; check: movq %rdi, %rsi + ; check: movaps %xmm0, %xmm6 + ; check: movq %rsi, %rcx + ; nextln: movaps %xmm6, %xmm1 + ; nextln: call *%rsi + call_indirect sig0, v0(v0, v1) + + ; check: movq %rsi, %rdi + ; nextln: movaps %xmm6, %xmm0 + ; nextln: call *%rsi + call_indirect sig1, v0(v0, v1) + return +} + +;; fastcall preserves xmm6+, rbx, rbp, rdi, rsi, r12-r15 +;; system_v preserves no xmm registers, rbx, rbp, r12-r15 +function %fastcall_to_systemv(i32) windows_fastcall { + sig0 = () system_v +block0(v0: i32): + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: subq $$176, %rsp + ; nextln: movdqu %xmm6, 0(%rsp) + ; nextln: movdqu %xmm7, 16(%rsp) + ; nextln: movdqu %xmm8, 32(%rsp) + ; nextln: movdqu %xmm9, 48(%rsp) + ; nextln: movdqu %xmm10, 64(%rsp) + ; nextln: movdqu %xmm11, 80(%rsp) + ; nextln: movdqu %xmm12, 96(%rsp) + ; nextln: movdqu %xmm13, 112(%rsp) + ; nextln: movdqu %xmm14, 128(%rsp) + ; nextln: movdqu %xmm15, 144(%rsp) + ; nextln: movq %rsi, 160(%rsp) + ; nextln: movq %rdi, 168(%rsp) + ; nextln: call *%rcx + ; nextln: movdqu 0(%rsp), %xmm6 + ; nextln: movdqu 16(%rsp), %xmm7 + ; nextln: movdqu 32(%rsp), %xmm8 + ; nextln: movdqu 48(%rsp), %xmm9 + ; nextln: movdqu 64(%rsp), %xmm10 + ; nextln: movdqu 80(%rsp), %xmm11 + ; nextln: movdqu 96(%rsp), %xmm12 + ; nextln: movdqu 112(%rsp), %xmm13 + ; nextln: movdqu 128(%rsp), %xmm14 + ; nextln: movdqu 144(%rsp), %xmm15 + ; nextln: movq 160(%rsp), %rsi + ; nextln: movq 168(%rsp), %rdi + ; nextln: addq $$176, %rsp + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + call_indirect sig0, v0() + return +} + +function %many_args( + ;; rdi, rsi, rdx, rcx, r8, r9, + i64, i64, i64, i64, i64, i64, + + ;; xmm0-7 + f64, f64, f64, f64, f64, f64, f64, f64, + + ;; stack args + i64, i32, f32, f64 +) system_v { + sig0 = ( + i64, i64, i64, i64, i64, i64, f64, f64, f64, f64, f64, f64, f64, f64, i64, + i32, f32, f64 + ) windows_fastcall +block0( + v0: i64, v1:i64, v2:i64, v3:i64, + v4:i64, v5:i64, + v6: f64, v7: f64, v8:f64, v9:f64, v10:f64, v11:f64, v12:f64, v13:f64, + v14:i64, v15:i32, v16:f32, v17:f64 +): + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: subq $$32, %rsp + ; nextln: movq %r12, 0(%rsp) + ; nextln: movq %r13, 8(%rsp) + ; nextln: movq %r14, 16(%rsp) + ; nextln: movq %rdx, %rax + ; nextln: movq %rcx, %r10 + ; nextln: movq %r8, %r11 + ; nextln: movq %r9, %r12 + ; nextln: movq 16(%rbp), %r13 + ; nextln: movslq 24(%rbp), %r14 + ; nextln: movss 32(%rbp), %xmm8 + ; nextln: movsd 40(%rbp), %xmm9 + ; nextln: subq $$144, %rsp + ; nextln: virtual_sp_offset_adjust 144 + ; nextln: movq %rdi, %rcx + ; nextln: movq %rsi, %rdx + ; nextln: movq %rax, %r8 + ; nextln: movq %r10, %r9 + ; nextln: movq %r11, 32(%rsp) + ; nextln: movq %r12, 40(%rsp) + ; nextln: movsd %xmm0, 48(%rsp) + ; nextln: movsd %xmm1, 56(%rsp) + ; nextln: movsd %xmm2, 64(%rsp) + ; nextln: movsd %xmm3, 72(%rsp) + ; nextln: movsd %xmm4, 80(%rsp) + ; nextln: movsd %xmm5, 88(%rsp) + ; nextln: movsd %xmm6, 96(%rsp) + ; nextln: movsd %xmm7, 104(%rsp) + ; nextln: movq %r13, 112(%rsp) + ; nextln: movl %r14d, 120(%rsp) + ; nextln: movss %xmm8, 128(%rsp) + ; nextln: movsd %xmm9, 136(%rsp) + ; nextln: call *%rdi + ; nextln: addq $$144, %rsp + ; nextln: virtual_sp_offset_adjust -144 + ; nextln: movq 0(%rsp), %r12 + ; nextln: movq 8(%rsp), %r13 + ; nextln: movq 16(%rsp), %r14 + ; nextln: addq $$32, %rsp + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + call_indirect sig0, v0( + v0, v1, v2, v3, + v4, v5, v6, v7, + v8, v9, v10, v11, + v12, v13, v14, v15, + v16, v17 + ) + return +} + +; rdi => rcx +; rsi => rdx +; rdx => r8 +; rcx => r9 +; r8 => stack +function %many_ints(i64, i64, i64, i64, i64) system_v { + sig0 = (i64, i64, i64, i64, i64) windows_fastcall +block0(v0: i64, v1:i64, v2:i64, v3:i64, v4:i64): + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: movq %rdx, %rax + ; nextln: movq %rcx, %r9 + ; nextln: movq %r8, %r10 + ; nextln: subq $$48, %rsp + ; nextln: virtual_sp_offset_adjust 48 + ; nextln: movq %rdi, %rcx + ; nextln: movq %rsi, %rdx + ; nextln: movq %rax, %r8 + ; nextln: movq %r10, 32(%rsp) + ; nextln: call *%rdi + ; nextln: addq $$48, %rsp + ; nextln: virtual_sp_offset_adjust -48 + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + call_indirect sig0, v0(v0, v1, v2, v3, v4) + return +} + +function %many_args2(i32, f32, i64, f64, i32, i32, i32, f32, f64, f32, f64) system_v { + sig0 = (i32, f32, i64, f64, i32, i32, i32, f32, f64, f32, f64) windows_fastcall +block0(v0: i32, v1: f32, v2: i64, v3: f64, v4: i32, v5: i32, v6: i32, v7: f32, v8: f64, v9: f32, v10: f64): + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: movaps %xmm1, %xmm6 + ; nextln: movq %rcx, %rax + ; nextln: movq %r8, %r9 + ; nextln: movaps %xmm3, %xmm7 + ; nextln: subq $$96, %rsp + ; nextln: virtual_sp_offset_adjust 96 + ; nextln: movq %rdi, %rcx + ; nextln: movaps %xmm0, %xmm1 + ; nextln: movq %rsi, %r8 + ; nextln: movaps %xmm6, %xmm3 + ; nextln: movl %edx, 32(%rsp) + ; nextln: movl %eax, 40(%rsp) + ; nextln: movl %r9d, 48(%rsp) + ; nextln: movss %xmm2, 56(%rsp) + ; nextln: movsd %xmm7, 64(%rsp) + ; nextln: movss %xmm4, 72(%rsp) + ; nextln: movsd %xmm5, 80(%rsp) + ; nextln: call *%rdi + ; nextln: addq $$96, %rsp + ; nextln: virtual_sp_offset_adjust -96 + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + call_indirect sig0, v0(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) + return +} From 213613aa65a6faf20dd177c7b15e6eecb1f3b827 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 7 Apr 2021 08:43:34 -0700 Subject: [PATCH 14/16] Get `multi` example working --- examples/multi.rs | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/examples/multi.rs b/examples/multi.rs index cb2d20bf8fb1..6b224c8e8cb8 100644 --- a/examples/multi.rs +++ b/examples/multi.rs @@ -8,9 +8,11 @@ // You can execute this example with `cargo run --example multi` use anyhow::Result; -use wasmtime::*; +#[cfg(not(feature = "old-x86-backend"))] fn main() -> Result<()> { + use wasmtime::*; + println!("Initializing..."); let engine = Engine::default(); let store = Store::new(&engine); @@ -68,3 +70,8 @@ fn main() -> Result<()> { Ok(()) } + +#[cfg(feature = "old-x86-backend")] +fn main() -> Result<()> { + Ok(()) +} From 4a379c25a4459a57d19a2de259925cb98172a355 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 7 Apr 2021 09:05:34 -0700 Subject: [PATCH 15/16] Fix doctests on old x86 backend --- crates/wasmtime/src/func.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 168d8613fe5b..534a4c5638a3 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -1094,7 +1094,9 @@ impl Func { /// and similarly if a function has multiple results you can bind that too /// /// ``` + /// # #[cfg(not(feature = "old-x86-backend"))] /// # use wasmtime::*; + /// # #[cfg(not(feature = "old-x86-backend"))] /// # fn foo(add_with_overflow: &Func) -> anyhow::Result<()> { /// let typed = add_with_overflow.typed::<(u32, u32), (u32, i32)>()?; /// let (result, overflow) = typed.call((u32::max_value(), 2))?; From 9dda311ddf34f0bb1409dec67f22d2d8075abca0 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Wed, 7 Apr 2021 09:29:57 -0700 Subject: [PATCH 16/16] Add a mixture of wasmtime/system_v tests --- .../filetests/isa/x64/call-conv.clif | 137 ++++++++++++++++++ 1 file changed, 137 insertions(+) diff --git a/cranelift/filetests/filetests/isa/x64/call-conv.clif b/cranelift/filetests/filetests/isa/x64/call-conv.clif index 66af3e9fca87..c705193c6b09 100644 --- a/cranelift/filetests/filetests/isa/x64/call-conv.clif +++ b/cranelift/filetests/filetests/isa/x64/call-conv.clif @@ -205,3 +205,140 @@ block0(v0: i32, v1: f32, v2: i64, v3: f64, v4: i32, v5: i32, v6: i32, v7: f32, v call_indirect sig0, v0(v0, v1, v2, v3, v4, v5, v6, v7, v8, v9, v10) return } + +function %wasmtime_mix1(i32) wasmtime_system_v { + sig0 = (i32) system_v +block0(v0: i32): + ; check: movq %rdi, %rsi + ; nextln: movq %rsi, %rdi + ; nextln: call *%rsi + call_indirect sig0, v0(v0) + return +} + +function %wasmtime_mix2(i32) system_v { + sig0 = (i32) wasmtime_system_v +block0(v0: i32): + ; check: movq %rdi, %rsi + ; nextln: movq %rsi, %rdi + ; nextln: call *%rsi + call_indirect sig0, v0(v0) + return +} + +function %wasmtime_mix2() -> i32, i32 system_v { + sig0 = () -> i32, i32 wasmtime_system_v +block0: + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: movl $$1, %esi + ; nextln: subq $$16, %rsp + ; nextln: virtual_sp_offset_adjust 16 + ; nextln: lea 0(%rsp), %rdi + ; nextln: call *%rsi + ; nextln: movslq 0(%rsp), %rsi + ; nextln: addq $$16, %rsp + ; nextln: virtual_sp_offset_adjust -16 + ; nextln: movq %rsi, %rdx + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + v2 = iconst.i32 1 + v0, v1 = call_indirect sig0, v2() + return v0, v1 +} + +function %wasmtime_mix3() -> i32, i32 wasmtime_system_v { + sig0 = () -> i32, i32 system_v +block0: + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: subq $$16, %rsp + ; nextln: movq %r12, 0(%rsp) + ; nextln: movq %rdi, %r12 + ; nextln: movl $$1, %esi + ; nextln: call *%rsi + ; nextln: movl %edx, 0(%r12) + ; nextln: movq 0(%rsp), %r12 + ; nextln: addq $$16, %rsp + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + v2 = iconst.i32 1 + v0, v1 = call_indirect sig0, v2() + return v0, v1 +} + +function %wasmtime_mix4() -> i32, i64, i32 wasmtime_system_v { + sig0 = () -> i32, i64, i32 system_v +block0: + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: subq $$16, %rsp + ; nextln: movq %r12, 0(%rsp) + ; nextln: movq %rdi, %r12 + ; nextln: movl $$1, %esi + ; nextln: subq $$16, %rsp + ; nextln: virtual_sp_offset_adjust 16 + ; nextln: lea 0(%rsp), %rdi + ; nextln: call *%rsi + ; nextln: movslq 0(%rsp), %rsi + ; nextln: addq $$16, %rsp + ; nextln: virtual_sp_offset_adjust -16 + ; nextln: movq %rdx, 0(%r12) + ; nextln: movl %esi, 8(%r12) + ; nextln: movq 0(%rsp), %r12 + ; nextln: addq $$16, %rsp + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + v3 = iconst.i32 1 + v0, v1, v2 = call_indirect sig0, v3() + return v0, v1, v2 +} + +function %wasmtime_mix5() -> f32, i64, i32, f32 wasmtime_system_v { + sig0 = () -> f32, i64, i32, f32 system_v +block0: + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: subq $$16, %rsp + ; nextln: movq %r12, 0(%rsp) + ; nextln: movq %rdi, %r12 + ; nextln: movl $$1, %esi + ; nextln: call *%rsi + ; nextln: movq %rax, 0(%r12) + ; nextln: movl %edx, 8(%r12) + ; nextln: movss %xmm1, 12(%r12) + ; nextln: movq 0(%rsp), %r12 + ; nextln: addq $$16, %rsp + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + v5 = iconst.i32 1 + v0, v1, v2, v3 = call_indirect sig0, v5() + return v0, v1, v2, v3 +} + +function %wasmtime_mix6(f32, i64, i32, f32) -> f32, i64, i32, f32 wasmtime_system_v { + sig0 = (f32, i64, i32, f32) -> f32, i64, i32, f32 system_v +block0(v0: f32, v1: i64, v2: i32, v3: f32): + ; check: pushq %rbp + ; nextln: movq %rsp, %rbp + ; nextln: subq $$16, %rsp + ; nextln: movq %r12, 0(%rsp) + ; nextln: movq %rdx, %r12 + ; nextln: movl $$1, %eax + ; nextln: call *%rax + ; nextln: movq %rax, 0(%r12) + ; nextln: movl %edx, 8(%r12) + ; nextln: movss %xmm1, 12(%r12) + ; nextln: movq 0(%rsp), %r12 + ; nextln: addq $$16, %rsp + ; nextln: movq %rbp, %rsp + ; nextln: popq %rbp + ; nextln: ret + v4 = iconst.i32 1 + v5, v6, v7, v8 = call_indirect sig0, v4(v0, v1, v2, v3) + return v5, v6, v7, v8 +}