diff --git a/Cargo.lock b/Cargo.lock index 148263def6a7..e034571ae584 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -3310,6 +3310,7 @@ dependencies = [ "cranelift-entity", "cranelift-frontend", "cranelift-wasm", + "target-lexicon", "wasmparser", "wasmtime-environ", ] diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index b831f9966a31..f668fe596a9d 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -267,13 +267,7 @@ impl Context { isa: &dyn TargetIsa, ) -> CodegenResult> { if let Some(backend) = isa.get_mach_backend() { - use crate::isa::CallConv; - use crate::machinst::UnwindInfoKind; - let unwind_info_kind = match self.func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => UnwindInfoKind::SystemV, - CallConv::WindowsFastcall => UnwindInfoKind::Windows, - _ => UnwindInfoKind::None, - }; + let unwind_info_kind = self.func.signature.call_conv.unwind_info_kind(); let result = self.mach_compile_result.as_ref().unwrap(); return backend.emit_unwind_info(result, unwind_info_kind); } diff --git a/cranelift/codegen/src/isa/aarch64/abi.rs b/cranelift/codegen/src/isa/aarch64/abi.rs index 8f3b2e6f1eb1..b2a95e314be9 100644 --- a/cranelift/codegen/src/isa/aarch64/abi.rs +++ b/cranelift/codegen/src/isa/aarch64/abi.rs @@ -197,18 +197,26 @@ impl ABIMachineSpec for AArch64MachineDeps { next_stack = 16; } - // Note on return values: on the regular non-baldrdash ABI, we may return values in 8 - // registers for V128 and I64 registers independently of the number of register values - // returned in the other class. That is, we can return values in up to 8 integer and 8 - // vector registers at once. - // In Baldrdash, we can only use one register for return value for all the register - // classes. That is, we can't return values in both one integer and one vector register; - // only one return value may be in a register. - - let (max_per_class_reg_vals, mut remaining_reg_vals) = match (args_or_rets, is_baldrdash) { - (ArgsOrRets::Args, _) => (8, 16), // x0-x7 and v0-v7 - (ArgsOrRets::Rets, false) => (8, 16), // x0-x7 and v0-v7 - (ArgsOrRets::Rets, true) => (1, 1), // x0 or v0, but not both + let (max_per_class_reg_vals, mut remaining_reg_vals) = match args_or_rets { + ArgsOrRets::Args => (8, 16), // x0-x7 and v0-v7 + + // Note on return values: on the regular ABI, we may return values + // in 8 registers for V128 and I64 registers independently of the + // number of register values returned in the other class. That is, + // we can return values in up to 8 integer and + // 8 vector registers at once. + // + // In Baldrdash and Wasmtime, we can only use one register for + // return value for all the register classes. That is, we can't + // return values in both one integer and one vector register; only + // one return value may be in a register. + ArgsOrRets::Rets => { + if is_baldrdash || call_conv.extends_wasmtime() { + (1, 1) // x0 or v0, but not both + } else { + (8, 16) // x0-x7 and v0-v7 + } + } }; for i in 0..params.len() { @@ -282,16 +290,18 @@ impl ABIMachineSpec for AArch64MachineDeps { // Compute the stack slot's size. let size = (ty_bits(param.value_type) / 8) as u64; - let size = if call_conv != isa::CallConv::AppleAarch64 { - // Every arg takes a minimum slot of 8 bytes. (16-byte stack - // alignment happens separately after all args.) - std::cmp::max(size, 8) - } else { - // MacOS aarch64 allows stack slots with sizes less than 8 - // bytes. They still need to be properly aligned on their - // natural data alignment, though. - size - }; + let size = + if call_conv == isa::CallConv::AppleAarch64 || call_conv.extends_wasmtime() { + // MacOS aarch64 and Wasmtime allow stack slots with + // sizes less than 8 bytes. They still need to be + // properly aligned on their natural data alignment, + // though. + size + } else { + // Every arg takes a minimum slot of 8 bytes. (16-byte stack + // alignment happens separately after all args.) + std::cmp::max(size, 8) + }; // Align the stack slot. debug_assert!(size.is_power_of_two()); diff --git a/cranelift/codegen/src/isa/call_conv.rs b/cranelift/codegen/src/isa/call_conv.rs index e764f33c6e37..fb0a2ca2635c 100644 --- a/cranelift/codegen/src/isa/call_conv.rs +++ b/cranelift/codegen/src/isa/call_conv.rs @@ -1,3 +1,4 @@ +use crate::machinst::UnwindInfoKind; use crate::settings::{self, LibcallCallConv}; use core::fmt; use core::str; @@ -29,6 +30,10 @@ pub enum CallConv { Baldrdash2020, /// Specialized convention for the probestack function. Probestack, + /// Wasmtime equivalent of SystemV, except the multi-return ABI is tweaked. + WasmtimeSystemV, + /// Wasmtime equivalent of WindowsFastcall, except the multi-return ABI is tweaked. + WasmtimeFastcall, } impl CallConv { @@ -63,7 +68,7 @@ impl CallConv { /// Is the calling convention extending the Windows Fastcall ABI? pub fn extends_windows_fastcall(self) -> bool { match self { - Self::WindowsFastcall | Self::BaldrdashWindows => true, + Self::WindowsFastcall | Self::BaldrdashWindows | Self::WasmtimeFastcall => true, _ => false, } } @@ -75,6 +80,27 @@ impl CallConv { _ => false, } } + + /// Is the calling convention extending the Wasmtime ABI? + pub fn extends_wasmtime(self) -> bool { + match self { + Self::WasmtimeSystemV | Self::WasmtimeFastcall => true, + _ => false, + } + } + + /// Returns the kind of unwind info used for this calling convention + pub fn unwind_info_kind(self) -> UnwindInfoKind { + match self { + #[cfg(feature = "unwind")] + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => { + UnwindInfoKind::SystemV + } + #[cfg(feature = "unwind")] + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => UnwindInfoKind::Windows, + _ => UnwindInfoKind::None, + } + } } impl fmt::Display for CallConv { @@ -89,6 +115,8 @@ impl fmt::Display for CallConv { Self::BaldrdashWindows => "baldrdash_windows", Self::Baldrdash2020 => "baldrdash_2020", Self::Probestack => "probestack", + Self::WasmtimeSystemV => "wasmtime_system_v", + Self::WasmtimeFastcall => "wasmtime_fastcall", }) } } @@ -106,6 +134,8 @@ impl str::FromStr for CallConv { "baldrdash_windows" => Ok(Self::BaldrdashWindows), "baldrdash_2020" => Ok(Self::Baldrdash2020), "probestack" => Ok(Self::Probestack), + "wasmtime_system_v" => Ok(Self::WasmtimeSystemV), + "wasmtime_fastcall" => Ok(Self::WasmtimeFastcall), _ => Err(()), } } diff --git a/cranelift/codegen/src/isa/x64/abi.rs b/cranelift/codegen/src/isa/x64/abi.rs index f55478a1c8d3..29a0dbd98250 100644 --- a/cranelift/codegen/src/isa/x64/abi.rs +++ b/cranelift/codegen/src/isa/x64/abi.rs @@ -237,10 +237,20 @@ impl ABIMachineSpec for X64ABIMachineSpec { extension: param.extension, }); } else { - // Compute size. Every arg takes a minimum slot of 8 bytes. (16-byte - // stack alignment happens separately after all args.) + // Compute size. For the wasmtime ABI it differs from native + // ABIs in how multiple values are returned, so we take a + // leaf out of arm64's book by not rounding everything up to + // 8 bytes. For all ABI arguments, and other ABI returns, + // though, each slot takes a minimum of 8 bytes. + // + // Note that in all cases 16-byte stack alignment happens + // separately after all args. let size = (reg_ty.bits() / 8) as u64; - let size = std::cmp::max(size, 8); + let size = if args_or_rets == ArgsOrRets::Rets && call_conv.extends_wasmtime() { + size + } else { + std::cmp::max(size, 8) + }; // Align. debug_assert!(size.is_power_of_two()); next_stack = align_to(next_stack, size); @@ -824,15 +834,7 @@ impl From for SyntheticAmode { } fn get_intreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Option { - let is_fastcall = match call_conv { - CallConv::Fast - | CallConv::Cold - | CallConv::SystemV - | CallConv::BaldrdashSystemV - | CallConv::Baldrdash2020 => false, - CallConv::WindowsFastcall => true, - _ => panic!("int args only supported for SysV or Fastcall calling convention"), - }; + let is_fastcall = call_conv.extends_windows_fastcall(); // Fastcall counts by absolute argument number; SysV counts by argument of // this (integer) class. @@ -853,15 +855,7 @@ fn get_intreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Optio } fn get_fltreg_for_arg(call_conv: &CallConv, idx: usize, arg_idx: usize) -> Option { - let is_fastcall = match call_conv { - CallConv::Fast - | CallConv::Cold - | CallConv::SystemV - | CallConv::BaldrdashSystemV - | CallConv::Baldrdash2020 => false, - CallConv::WindowsFastcall => true, - _ => panic!("float args only supported for SysV or Fastcall calling convention"), - }; + let is_fastcall = call_conv.extends_windows_fastcall(); // Fastcall counts by absolute argument number; SysV counts by argument of // this (floating-point) class. @@ -894,7 +888,10 @@ fn get_intreg_for_retval( 1 => Some(regs::rdx()), _ => None, }, - CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => { + CallConv::BaldrdashSystemV + | CallConv::Baldrdash2020 + | CallConv::WasmtimeSystemV + | CallConv::WasmtimeFastcall => { if intreg_idx == 0 && retval_idx == 0 { Some(regs::rax()) } else { @@ -922,7 +919,10 @@ fn get_fltreg_for_retval( 1 => Some(regs::xmm1()), _ => None, }, - CallConv::BaldrdashSystemV | CallConv::Baldrdash2020 => { + CallConv::BaldrdashSystemV + | CallConv::Baldrdash2020 + | CallConv::WasmtimeFastcall + | CallConv::WasmtimeSystemV => { if fltreg_idx == 0 && retval_idx == 0 { Some(regs::xmm0()) } else { @@ -992,12 +992,12 @@ fn get_callee_saves(call_conv: &CallConv, regs: &Set>) -> Vec< CallConv::BaldrdashWindows => { todo!("baldrdash windows"); } - CallConv::Fast | CallConv::Cold | CallConv::SystemV => regs + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => regs .iter() .cloned() .filter(|r| is_callee_save_systemv(r.to_reg())) .collect(), - CallConv::WindowsFastcall => regs + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => regs .iter() .cloned() .filter(|r| is_callee_save_fastcall(r.to_reg())) diff --git a/cranelift/codegen/src/isa/x86/abi.rs b/cranelift/codegen/src/isa/x86/abi.rs index c6df87ae2203..83b44be1b09a 100644 --- a/cranelift/codegen/src/isa/x86/abi.rs +++ b/cranelift/codegen/src/isa/x86/abi.rs @@ -503,10 +503,12 @@ fn callee_saved_regs_used(isa: &dyn TargetIsa, func: &ir::Function) -> RegisterS pub fn prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> { match func.signature.call_conv { // For now, just translate fast and cold as system_v. - CallConv::Fast | CallConv::Cold | CallConv::SystemV => { + CallConv::Fast | CallConv::Cold | CallConv::SystemV | CallConv::WasmtimeSystemV => { system_v_prologue_epilogue(func, isa) } - CallConv::WindowsFastcall => fastcall_prologue_epilogue(func, isa), + CallConv::WindowsFastcall | CallConv::WasmtimeFastcall => { + fastcall_prologue_epilogue(func, isa) + } CallConv::BaldrdashSystemV | CallConv::BaldrdashWindows => { baldrdash_prologue_epilogue(func, isa) } @@ -1084,16 +1086,17 @@ pub fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { use crate::isa::unwind::UnwindInfo; + use crate::machinst::UnwindInfoKind; // Assumption: RBP is being used as the frame pointer for both calling conventions // In the future, we should be omitting frame pointer as an optimization, so this will change - Ok(match func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => { + Ok(match func.signature.call_conv.unwind_info_kind() { + UnwindInfoKind::SystemV => { super::unwind::systemv::create_unwind_info(func, isa)?.map(|u| UnwindInfo::SystemV(u)) } - CallConv::WindowsFastcall => { + UnwindInfoKind::Windows => { super::unwind::winx64::create_unwind_info(func, isa)?.map(|u| UnwindInfo::WindowsX64(u)) } - _ => None, + UnwindInfoKind::None => None, }) } diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs index f3e1cbea84a1..26e1ccbda7d0 100644 --- a/cranelift/codegen/src/isa/x86/unwind/systemv.rs +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -3,7 +3,7 @@ use crate::ir::Function; use crate::isa::{ unwind::systemv::{RegisterMappingError, UnwindInfo}, - CallConv, RegUnit, TargetIsa, + RegUnit, TargetIsa, }; use crate::result::CodegenResult; use gimli::{write::CommonInformationEntry, Encoding, Format, Register, X86_64}; @@ -97,8 +97,8 @@ pub(crate) fn create_unwind_info( isa: &dyn TargetIsa, ) -> CodegenResult> { // Only System V-like calling conventions are supported - match func.signature.call_conv { - CallConv::Fast | CallConv::Cold | CallConv::SystemV => {} + match func.signature.call_conv.unwind_info_kind() { + crate::machinst::UnwindInfoKind::SystemV => {} _ => return Ok(None), } diff --git a/cranelift/codegen/src/machinst/abi_impl.rs b/cranelift/codegen/src/machinst/abi_impl.rs index 17995e18df90..765712c2cf5c 100644 --- a/cranelift/codegen/src/machinst/abi_impl.rs +++ b/cranelift/codegen/src/machinst/abi_impl.rs @@ -647,7 +647,8 @@ impl ABICalleeImpl { || call_conv == isa::CallConv::Cold || call_conv.extends_baldrdash() || call_conv.extends_windows_fastcall() - || call_conv == isa::CallConv::AppleAarch64, + || call_conv == isa::CallConv::AppleAarch64 + || call_conv == isa::CallConv::WasmtimeSystemV, "Unsupported calling convention: {:?}", call_conv ); @@ -1372,15 +1373,7 @@ impl ABICallee for ABICalleeImpl { } fn unwind_info_kind(&self) -> UnwindInfoKind { - match self.sig.call_conv { - #[cfg(feature = "unwind")] - isa::CallConv::Fast | isa::CallConv::Cold | isa::CallConv::SystemV => { - UnwindInfoKind::SystemV - } - #[cfg(feature = "unwind")] - isa::CallConv::WindowsFastcall => UnwindInfoKind::Windows, - _ => UnwindInfoKind::None, - } + self.sig.call_conv.unwind_info_kind() } } diff --git a/cranelift/wasm/src/environ/dummy.rs b/cranelift/wasm/src/environ/dummy.rs index 2bcd004dbe1e..7a56712b7cee 100644 --- a/cranelift/wasm/src/environ/dummy.rs +++ b/cranelift/wasm/src/environ/dummy.rs @@ -660,8 +660,9 @@ impl TargetEnvironment for DummyEnvironment { } impl<'data> ModuleEnvironment<'data> for DummyEnvironment { - fn declare_type_func(&mut self, _wasm: WasmFuncType, sig: ir::Signature) -> WasmResult<()> { - self.info.signatures.push(sig); + fn declare_type_func(&mut self, _wasm: WasmFuncType) -> WasmResult<()> { + // TODO + // self.info.signatures.push(sig); Ok(()) } diff --git a/cranelift/wasm/src/environ/spec.rs b/cranelift/wasm/src/environ/spec.rs index 0bd3c48745f9..3a2389c44d8e 100644 --- a/cranelift/wasm/src/environ/spec.rs +++ b/cranelift/wasm/src/environ/spec.rs @@ -702,11 +702,7 @@ pub trait ModuleEnvironment<'data>: TargetEnvironment { } /// Declares a function signature to the environment. - fn declare_type_func( - &mut self, - wasm_func_type: WasmFuncType, - sig: ir::Signature, - ) -> WasmResult<()>; + fn declare_type_func(&mut self, wasm_func_type: WasmFuncType) -> WasmResult<()>; /// Declares a module type signature to the environment. fn declare_type_module( diff --git a/cranelift/wasm/src/sections_translator.rs b/cranelift/wasm/src/sections_translator.rs index e9f0f784a5b1..c617c9943c89 100644 --- a/cranelift/wasm/src/sections_translator.rs +++ b/cranelift/wasm/src/sections_translator.rs @@ -18,7 +18,6 @@ use crate::wasm_unsupported; use core::convert::TryFrom; use core::convert::TryInto; use cranelift_codegen::ir::immediates::V128Imm; -use cranelift_codegen::ir::{self, AbiParam, Signature}; use cranelift_entity::packed_option::ReservedValue; use cranelift_entity::EntityRef; use std::boxed::Box; @@ -110,18 +109,7 @@ pub fn parse_type_section<'a>( for entry in types { match entry? { TypeDef::Func(wasm_func_ty) => { - let mut sig = Signature::new(environ.target_config().default_call_conv); - sig.params.extend(wasm_func_ty.params.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - sig.returns.extend(wasm_func_ty.returns.iter().map(|ty| { - let cret_arg: ir::Type = type_to_type(*ty, environ) - .expect("only numeric types are supported in function signatures"); - AbiParam::new(cret_arg) - })); - environ.declare_type_func(wasm_func_ty.clone().try_into()?, sig)?; + environ.declare_type_func(wasm_func_ty.clone().try_into()?)?; module_translation_state .wasm_types .push((wasm_func_ty.params, wasm_func_ty.returns)); diff --git a/cranelift/wasm/src/translation_utils.rs b/cranelift/wasm/src/translation_utils.rs index c8493d4b51e5..fffb7b375e27 100644 --- a/cranelift/wasm/src/translation_utils.rs +++ b/cranelift/wasm/src/translation_utils.rs @@ -19,6 +19,7 @@ entity_impl!(FuncIndex); /// Index type of a defined function inside the WebAssembly module. #[derive(Copy, Clone, PartialEq, Eq, Hash, PartialOrd, Ord, Debug)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] pub struct DefinedFuncIndex(u32); entity_impl!(DefinedFuncIndex); diff --git a/crates/cranelift/Cargo.toml b/crates/cranelift/Cargo.toml index f9b763dccc2d..764177223c67 100644 --- a/crates/cranelift/Cargo.toml +++ b/crates/cranelift/Cargo.toml @@ -18,3 +18,4 @@ cranelift-codegen = { path = "../../cranelift/codegen", version = "0.72.0" } cranelift-frontend = { path = "../../cranelift/frontend", version = "0.72.0" } cranelift-entity = { path = "../../cranelift/entity", version = "0.72.0" } wasmparser = "0.77.0" +target-lexicon = "0.12" diff --git a/crates/cranelift/src/func_environ.rs b/crates/cranelift/src/func_environ.rs index a9481aca93ee..29d58dac2045 100644 --- a/crates/cranelift/src/func_environ.rs +++ b/crates/cranelift/src/func_environ.rs @@ -4,20 +4,20 @@ use cranelift_codegen::ir::condcodes::*; use cranelift_codegen::ir::immediates::{Offset32, Uimm64}; use cranelift_codegen::ir::types::*; use cranelift_codegen::ir::{AbiParam, ArgumentPurpose, Function, InstBuilder, Signature}; -use cranelift_codegen::isa::{self, TargetFrontendConfig}; -use cranelift_entity::{EntityRef, PrimaryMap}; +use cranelift_codegen::isa::{self, TargetFrontendConfig, TargetIsa}; +use cranelift_entity::EntityRef; use cranelift_frontend::FunctionBuilder; use cranelift_frontend::Variable; use cranelift_wasm::{ - self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex, - SignatureIndex, TableIndex, TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, + self, FuncIndex, FuncTranslationState, GlobalIndex, GlobalVariable, MemoryIndex, TableIndex, + TargetEnvironment, TypeIndex, WasmError, WasmResult, WasmType, }; use std::convert::TryFrom; use std::mem; use wasmparser::Operator; use wasmtime_environ::{ - BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, VMOffsets, - INTERRUPTED, WASM_PAGE_SIZE, + BuiltinFunctionIndex, MemoryPlan, MemoryStyle, Module, TableStyle, Tunables, TypeTables, + VMOffsets, INTERRUPTED, WASM_PAGE_SIZE, }; /// Compute an `ir::ExternalName` for a given wasm function index. @@ -109,14 +109,9 @@ wasmtime_environ::foreach_builtin_function!(declare_function_signatures); /// The `FuncEnvironment` implementation for use by the `ModuleEnvironment`. pub struct FuncEnvironment<'module_environment> { - /// Target-specified configuration. - target_config: TargetFrontendConfig, - - /// The module-level environment which this function-level environment belongs to. + isa: &'module_environment (dyn TargetIsa + 'module_environment), module: &'module_environment Module, - - /// The native signatures for each type signature in this module - native_signatures: &'module_environment PrimaryMap, + types: &'module_environment TypeTables, /// The Cranelift global holding the vmctx address. vmctx: Option, @@ -146,27 +141,27 @@ pub struct FuncEnvironment<'module_environment> { impl<'module_environment> FuncEnvironment<'module_environment> { pub fn new( - target_config: TargetFrontendConfig, + isa: &'module_environment (dyn TargetIsa + 'module_environment), module: &'module_environment Module, - native_signatures: &'module_environment PrimaryMap, + types: &'module_environment TypeTables, tunables: &'module_environment Tunables, ) -> Self { let builtin_function_signatures = BuiltinFunctionSignatures::new( - target_config.pointer_type(), - match target_config.pointer_type() { + isa.pointer_type(), + match isa.pointer_type() { ir::types::I32 => ir::types::R32, ir::types::I64 => ir::types::R64, _ => panic!(), }, - target_config.default_call_conv, + crate::wasmtime_call_conv(isa), ); Self { - target_config, + isa, module, - native_signatures, + types, vmctx: None, builtin_function_signatures, - offsets: VMOffsets::new(target_config.pointer_bytes(), module), + offsets: VMOffsets::new(isa.pointer_bytes(), module), tunables, fuel_var: Variable::new(0), vminterrupts_ptr: Variable::new(0), @@ -178,7 +173,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { } fn pointer_type(&self) -> ir::Type { - self.target_config.pointer_type() + self.isa.pointer_type() } fn vmctx(&mut self, func: &mut Function) -> ir::GlobalValue { @@ -680,7 +675,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { impl<'module_environment> TargetEnvironment for FuncEnvironment<'module_environment> { fn target_config(&self) -> TargetFrontendConfig { - self.target_config + self.isa.frontend_config() } fn reference_type(&self, ty: WasmType) -> ir::Type { @@ -1339,7 +1334,8 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m index: TypeIndex, ) -> WasmResult { let index = self.module.types[index].unwrap_function(); - Ok(func.import_signature(self.native_signatures[index].clone())) + let sig = crate::indirect_signature(self.isa, self.types, index); + Ok(func.import_signature(sig)) } fn make_direct_func( @@ -1347,8 +1343,7 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m func: &mut ir::Function, index: FuncIndex, ) -> WasmResult { - let sig_index = self.module.functions[index]; - let sig = self.native_signatures[sig_index].clone(); + let sig = crate::func_signature(self.isa, self.module, self.types, index); let signature = func.import_signature(sig); let name = get_func_name(index); Ok(func.import_function(ir::ExtFuncData { diff --git a/crates/cranelift/src/lib.rs b/crates/cranelift/src/lib.rs index ee330ee6a76c..896294ff39e6 100644 --- a/crates/cranelift/src/lib.rs +++ b/crates/cranelift/src/lib.rs @@ -90,16 +90,18 @@ use crate::func_environ::{get_func_name, FuncEnvironment}; use cranelift_codegen::ir::{self, ExternalName}; +use cranelift_codegen::isa::{CallConv, TargetIsa}; use cranelift_codegen::machinst::buffer::MachSrcLoc; use cranelift_codegen::print_errors::pretty_error; use cranelift_codegen::{binemit, isa, Context}; -use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator}; +use cranelift_wasm::{DefinedFuncIndex, FuncIndex, FuncTranslator, SignatureIndex, WasmType}; use std::convert::TryFrom; use std::sync::Mutex; +use target_lexicon::CallingConvention; use wasmtime_environ::{ CompileError, CompiledFunction, Compiler, FunctionAddressMap, FunctionBodyData, - InstructionAddressMap, ModuleTranslation, Relocation, RelocationTarget, StackMapInformation, - TrapInformation, Tunables, TypeTables, + InstructionAddressMap, Module, ModuleTranslation, Relocation, RelocationTarget, + StackMapInformation, TrapInformation, Tunables, TypeTables, }; mod func_environ; @@ -354,18 +356,12 @@ impl Compiler for Cranelift { let func_index = module.func_index(func_index); let mut context = Context::new(); context.func.name = get_func_name(func_index); - let sig_index = module.functions[func_index]; - context.func.signature = types.native_signatures[sig_index].clone(); + context.func.signature = func_signature(isa, module, types, func_index); if tunables.generate_native_debuginfo { context.func.collect_debug_info(); } - let mut func_env = FuncEnvironment::new( - isa.frontend_config(), - module, - &types.native_signatures, - tunables, - ); + let mut func_env = FuncEnvironment::new(isa, module, types, tunables); // We use these as constant offsets below in // `stack_limit_from_arguments`, so assert their values here. This @@ -457,3 +453,83 @@ impl Compiler for Cranelift { }) } } + +pub fn blank_sig(isa: &dyn TargetIsa, call_conv: CallConv) -> ir::Signature { + let pointer_type = isa.pointer_type(); + let mut sig = ir::Signature::new(call_conv); + // Add the caller/callee `vmctx` parameters. + sig.params.push(ir::AbiParam::special( + pointer_type, + ir::ArgumentPurpose::VMContext, + )); + sig.params.push(ir::AbiParam::new(pointer_type)); + return sig; +} + +pub fn wasmtime_call_conv(isa: &dyn TargetIsa) -> CallConv { + match isa.triple().default_calling_convention() { + Ok(CallingConvention::SystemV) | Ok(CallingConvention::AppleAarch64) | Err(()) => { + CallConv::WasmtimeSystemV + } + Ok(CallingConvention::WindowsFastcall) => CallConv::WasmtimeFastcall, + Ok(unimp) => unimplemented!("calling convention: {:?}", unimp), + } +} + +pub fn push_types( + isa: &dyn TargetIsa, + sig: &mut ir::Signature, + types: &TypeTables, + index: SignatureIndex, +) { + let wasm = &types.wasm_signatures[index]; + + let cvt = |ty: &WasmType| { + ir::AbiParam::new(match ty { + WasmType::I32 => ir::types::I32, + WasmType::I64 => ir::types::I64, + WasmType::F32 => ir::types::F32, + WasmType::F64 => ir::types::F64, + WasmType::V128 => ir::types::I8X16, + WasmType::FuncRef | WasmType::ExternRef => { + wasmtime_environ::reference_type(*ty, isa.pointer_type()) + } + WasmType::ExnRef => unimplemented!(), + }) + }; + sig.params.extend(wasm.params.iter().map(&cvt)); + sig.returns.extend(wasm.returns.iter().map(&cvt)); +} + +pub fn indirect_signature( + isa: &dyn TargetIsa, + types: &TypeTables, + index: SignatureIndex, +) -> ir::Signature { + let mut sig = blank_sig(isa, wasmtime_call_conv(isa)); + push_types(isa, &mut sig, types, index); + return sig; +} + +pub fn func_signature( + isa: &dyn TargetIsa, + module: &Module, + types: &TypeTables, + index: FuncIndex, +) -> ir::Signature { + let call_conv = match module.defined_func_index(index) { + // If this is a defined function in the module and it's never possibly + // exported, then we can optimize this function to use the fastest + // calling convention since it's purely an internal implementation + // detail of the module itself. + Some(idx) if !module.possibly_exported_funcs.contains(&idx) => CallConv::Fast, + + // ... otherwise if it's an imported function or if it's a possibly + // exported function then we use the default ABI wasmtime would + // otherwise select. + _ => wasmtime_call_conv(isa), + }; + let mut sig = blank_sig(isa, call_conv); + push_types(isa, &mut sig, types, module.functions[index]); + return sig; +} diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index ee4ff050dc71..3b520475d6f0 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -2,12 +2,11 @@ use crate::tunables::Tunables; use crate::WASM_MAX_PAGES; -use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::*; use indexmap::IndexMap; use serde::{Deserialize, Serialize}; -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; /// Implemenation styles for WebAssembly linear memory. @@ -367,6 +366,10 @@ pub struct Module { /// The type of each nested wasm module this module contains. pub modules: PrimaryMap, + + /// The set of defined functions within this module which are located in + /// element segments. + pub possibly_exported_funcs: HashSet, } /// Initialization routines for creating an instance, encompassing imports, @@ -564,7 +567,6 @@ impl Module { #[allow(missing_docs)] pub struct TypeTables { pub wasm_signatures: PrimaryMap, - pub native_signatures: PrimaryMap, pub module_signatures: PrimaryMap, pub instance_signatures: PrimaryMap, } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 57e2f6bb1e7e..8106c1a3fb72 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -4,14 +4,14 @@ use crate::module::{ }; use crate::tunables::Tunables; use cranelift_codegen::ir; -use cranelift_codegen::ir::{AbiParam, ArgumentPurpose}; use cranelift_codegen::isa::TargetFrontendConfig; +use cranelift_codegen::packed_option::ReservedValue; use cranelift_entity::PrimaryMap; use cranelift_wasm::{ self, translate_module, Alias, DataIndex, DefinedFuncIndex, ElemIndex, EntityIndex, EntityType, - FuncIndex, Global, GlobalIndex, InstanceIndex, InstanceTypeIndex, Memory, MemoryIndex, - ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, TargetEnvironment, TypeIndex, - WasmError, WasmFuncType, WasmResult, + FuncIndex, Global, GlobalIndex, GlobalInit, InstanceIndex, InstanceTypeIndex, Memory, + MemoryIndex, ModuleIndex, ModuleTypeIndex, SignatureIndex, Table, TableIndex, + TargetEnvironment, TypeIndex, WasmError, WasmFuncType, WasmResult, }; use std::collections::{hash_map::Entry, HashMap}; use std::convert::TryFrom; @@ -357,6 +357,15 @@ impl<'data> ModuleEnvironment<'data> { .module_signatures .push(ModuleSignature { imports, exports }) } + + fn flag_func_possibly_exported(&mut self, func: FuncIndex) { + if func.is_reserved_value() { + return; + } + if let Some(idx) = self.result.module.defined_func_index(func) { + self.result.module.possibly_exported_funcs.insert(idx); + } + } } impl<'data> TargetEnvironment for ModuleEnvironment<'data> { @@ -375,21 +384,17 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data fn reserve_types(&mut self, num: u32) -> WasmResult<()> { let num = usize::try_from(num).unwrap(); self.result.module.types.reserve(num); - self.types.native_signatures.reserve(num); self.types.wasm_signatures.reserve(num); Ok(()) } - fn declare_type_func(&mut self, wasm: WasmFuncType, sig: ir::Signature) -> WasmResult<()> { + fn declare_type_func(&mut self, wasm: WasmFuncType) -> WasmResult<()> { // Deduplicate wasm function signatures through `interned_func_types`, // which also deduplicates across wasm modules with module linking. let sig_index = match self.interned_func_types.get(&wasm) { Some(idx) => *idx, None => { - let sig = translate_signature(sig, self.pointer_type()); - let sig_index = self.types.native_signatures.push(sig); - let sig_index2 = self.types.wasm_signatures.push(wasm.clone()); - debug_assert_eq!(sig_index, sig_index2); + let sig_index = self.types.wasm_signatures.push(wasm.clone()); self.interned_func_types.insert(wasm, sig_index); sig_index } @@ -641,6 +646,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_global(&mut self, global: Global) -> WasmResult<()> { + if let GlobalInit::RefFunc(index) = global.initializer { + self.flag_func_possibly_exported(index); + } self.result.module.globals.push(global); Ok(()) } @@ -654,6 +662,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_func_export(&mut self, func_index: FuncIndex, name: &str) -> WasmResult<()> { + self.flag_func_possibly_exported(func_index); self.declare_export(EntityIndex::Function(func_index), name) } @@ -678,6 +687,7 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data } fn declare_start_func(&mut self, func_index: FuncIndex) -> WasmResult<()> { + self.flag_func_possibly_exported(func_index); debug_assert!(self.result.module.start_func.is_none()); self.result.module.start_func = Some(func_index); Ok(()) @@ -698,6 +708,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data offset: usize, elements: Box<[FuncIndex]>, ) -> WasmResult<()> { + for element in elements.iter() { + self.flag_func_possibly_exported(*element); + } self.result .module .table_initializers @@ -715,6 +728,9 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data elem_index: ElemIndex, segments: Box<[FuncIndex]>, ) -> WasmResult<()> { + for element in segments.iter() { + self.flag_func_possibly_exported(*element); + } let index = self.result.module.passive_elements.len(); self.result.module.passive_elements.push(segments); let old = self @@ -1070,15 +1086,3 @@ and for re-adding support for interface types you can see this issue: Ok(()) } } - -/// Add environment-specific function parameters. -pub fn translate_signature(mut sig: ir::Signature, pointer_type: ir::Type) -> ir::Signature { - // Prepend the vmctx argument. - sig.params.insert( - 0, - AbiParam::special(pointer_type, ArgumentPurpose::VMContext), - ); - // Prepend the caller vmctx argument. - sig.params.insert(1, AbiParam::new(pointer_type)); - sig -} diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index 17e6294250c3..12329ea266d7 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -50,6 +50,7 @@ pub use crate::instantiate::{ CompilationArtifacts, CompiledModule, ModuleCode, SetupError, SymbolizeContext, TypeTables, }; pub use crate::link::link_module; +pub use wasmtime_cranelift::{blank_sig, wasmtime_call_conv}; /// Version number of this crate. pub const VERSION: &str = env!("CARGO_PKG_VERSION"); diff --git a/crates/jit/src/native.rs b/crates/jit/src/native.rs index afcf83d3ccb4..706cbc04b443 100644 --- a/crates/jit/src/native.rs +++ b/crates/jit/src/native.rs @@ -11,9 +11,4 @@ pub fn builder_without_flags() -> cranelift_codegen::isa::Builder { .expect("host machine is not a supported target") } -pub fn call_conv() -> cranelift_codegen::isa::CallConv { - use target_lexicon::HOST; - cranelift_codegen::isa::CallConv::triple_default(&HOST) -} - pub use cranelift_codegen::isa::lookup; diff --git a/crates/jit/src/object.rs b/crates/jit/src/object.rs index 3516be2b133d..e73fa8cf8185 100644 --- a/crates/jit/src/object.rs +++ b/crates/jit/src/object.rs @@ -8,7 +8,7 @@ use std::collections::BTreeSet; use wasmtime_debug::DwarfSection; use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; use wasmtime_environ::wasm::{FuncIndex, SignatureIndex}; -use wasmtime_environ::{CompiledFunctions, ModuleTranslation, ModuleType, TypeTables}; +use wasmtime_environ::{CompiledFunctions, ModuleTranslation, TypeTables}; use wasmtime_obj::{ObjectBuilder, ObjectBuilderTarget}; pub use wasmtime_obj::utils; @@ -42,18 +42,18 @@ pub(crate) fn build_object( // Build trampolines for every signature that can be used by this module. let signatures = translation .module - .types - .values() - .filter_map(|t| match t { - ModuleType::Function(f) => Some(*f), - _ => None, + .functions + .iter() + .filter_map(|(i, sig)| match translation.module.defined_func_index(i) { + Some(i) if !translation.module.possibly_exported_funcs.contains(&i) => None, + _ => Some(*sig), }) .collect::>(); let mut trampolines = Vec::with_capacity(signatures.len()); let mut cx = FunctionBuilderContext::new(); for i in signatures { - let native_sig = &types.native_signatures[i]; - let func = build_trampoline(isa, &mut cx, native_sig, std::mem::size_of::())?; + let native_sig = wasmtime_cranelift::indirect_signature(isa, &types, i); + let func = build_trampoline(isa, &mut cx, &native_sig, std::mem::size_of::())?; // Preserve trampoline function unwind info. if let Some(info) = &func.unwind_info { unwind_info.push(ObjectUnwindInfo::Trampoline(i, info.clone())) diff --git a/crates/jit/src/trampoline.rs b/crates/jit/src/trampoline.rs index bb470dbcb3fc..b8262fd93fb6 100644 --- a/crates/jit/src/trampoline.rs +++ b/crates/jit/src/trampoline.rs @@ -9,7 +9,7 @@ use wasmtime_runtime::{InstantiationError, VMFunctionBody, VMTrampoline}; pub mod ir { pub(super) use cranelift_codegen::ir::{ - AbiParam, ArgumentPurpose, ConstantOffset, JumpTable, Signature, SourceLoc, + AbiParam, ConstantOffset, JumpTable, Signature, SourceLoc, }; pub use cranelift_codegen::ir::{ ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, @@ -52,16 +52,8 @@ pub(crate) fn build_trampoline( value_size: usize, ) -> Result { let pointer_type = isa.pointer_type(); - let mut wrapper_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Add the callee `vmctx` parameter. - wrapper_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the caller `vmctx` parameter. - wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); + let mut wrapper_sig = + wasmtime_cranelift::blank_sig(isa, wasmtime_cranelift::wasmtime_call_conv(isa)); // Add the `callee_address` parameter. wrapper_sig.params.push(ir::AbiParam::new(pointer_type)); diff --git a/crates/wasmtime/src/config.rs b/crates/wasmtime/src/config.rs index 43bd809c84aa..55fbcfddca21 100644 --- a/crates/wasmtime/src/config.rs +++ b/crates/wasmtime/src/config.rs @@ -1,6 +1,6 @@ use crate::memory::MemoryCreator; use crate::trampoline::MemoryCreatorProxy; -use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmRet, WasmTy}; +use crate::{func::HostFunc, Caller, FuncType, IntoFunc, Trap, Val, WasmHostResults, WasmTy}; use anyhow::{bail, Result}; use serde::{Deserialize, Serialize}; use std::cmp; @@ -339,7 +339,7 @@ macro_rules! generate_wrap_async_host_func { ) where $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { // Defer the check for async support until engine creation time to not introduce an order dependency self.host_funcs.insert( diff --git a/crates/wasmtime/src/func.rs b/crates/wasmtime/src/func.rs index 2a70df9e3cd2..924a1002d932 100644 --- a/crates/wasmtime/src/func.rs +++ b/crates/wasmtime/src/func.rs @@ -6,6 +6,7 @@ use std::cmp::max; use std::fmt; use std::future::Future; use std::mem; +use std::mem::MaybeUninit; use std::panic::{self, AssertUnwindSafe}; use std::pin::Pin; use std::ptr::{self, NonNull}; @@ -350,7 +351,7 @@ macro_rules! generate_wrap_async_func { where T: 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { assert!(store.async_support(), concat!("cannot use `wrap", $num, "_async` without enabling async support on the config")); Func::wrap(store, move |caller: Caller<'_>, $($args: $args),*| { @@ -561,15 +562,15 @@ impl Func { /// Any of the Rust types can be returned from the closure as well, in /// addition to some extra types /// - /// | Rust Return Type | WebAssembly Return Type | Meaning | - /// |-------------------|-------------------------|-------------------| - /// | `()` | nothing | no return value | - /// | `Result` | `T` | function may trap | + /// | Rust Return Type | WebAssembly Return Type | Meaning | + /// |-------------------|-------------------------|-----------------------| + /// | `()` | nothing | no return value | + /// | `T` | `T` | a single return value | + /// | `(T1, T2, ...)` | `T1 T2 ...` | multiple returns | /// - /// At this time multi-value returns are not supported, and supporting this - /// is the subject of [#1178]. - /// - /// [#1178]: https://github.com/bytecodealliance/wasmtime/issues/1178 + /// Note that all return types can also be wrapped in `Result<_, Trap>` to + /// indicate that the host function can generate a trap as well as possibly + /// returning a value. /// /// Finally you can also optionally take [`Caller`] as the first argument of /// your closure. If inserted then you're able to inspect the caller's @@ -1260,10 +1261,12 @@ fn enter_wasm_init<'a>(store: &'a Store) -> Result { /// stable over time. /// /// For more information see [`Func::wrap`] -pub unsafe trait WasmRet { +pub unsafe trait WasmHostResults { // Same as `WasmTy::Abi`. #[doc(hidden)] type Abi: Copy; + #[doc(hidden)] + type Retptr: Copy; // Same as `WasmTy::compatible_with_store`. #[doc(hidden)] @@ -1276,100 +1279,47 @@ pub unsafe trait WasmRet { // `invoke_wasm_and_catch_traps` is on the stack, and therefore this method // is unsafe. #[doc(hidden)] - unsafe fn into_abi_for_ret(self, store: &Store) -> Result; + unsafe fn into_abi_for_ret(self, store: &Store, ptr: Self::Retptr) -> Result; + + #[doc(hidden)] + fn func_type(params: impl Iterator) -> FuncType; - // Same as `WasmTy::push`. #[doc(hidden)] - fn valtype() -> Option; + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi); // Utilities used to convert an instance of this type to a `Result` // explicitly, used when wrapping async functions which always bottom-out // in a function that returns a trap because futures can be cancelled. #[doc(hidden)] - type Fallible: WasmRet; + type Fallible: WasmHostResults; #[doc(hidden)] fn into_fallible(self) -> Self::Fallible; #[doc(hidden)] fn fallible_from_trap(trap: Trap) -> Self::Fallible; } -unsafe impl WasmRet for () { - type Abi = (); - type Fallible = Result<(), Trap>; - - #[inline] - fn compatible_with_store(&self, _store: &Store) -> bool { - true - } - - #[inline] - unsafe fn into_abi_for_ret(self, _store: &Store) -> Result<(), Trap> { - Ok(()) - } - - #[inline] - fn valtype() -> Option { - None - } - - #[inline] - fn into_fallible(self) -> Result<(), Trap> { - Ok(()) - } - - #[inline] - fn fallible_from_trap(trap: Trap) -> Result<(), Trap> { - Err(trap) - } -} - -unsafe impl WasmRet for Result<(), Trap> { - type Abi = (); - type Fallible = Self; - - #[inline] - fn compatible_with_store(&self, _store: &Store) -> bool { - true - } - - #[inline] - unsafe fn into_abi_for_ret(self, _store: &Store) -> Result<(), Trap> { - self - } - - #[inline] - fn valtype() -> Option { - None - } - - #[inline] - fn into_fallible(self) -> Result<(), Trap> { - self - } - - #[inline] - fn fallible_from_trap(trap: Trap) -> Result<(), Trap> { - Err(trap) - } -} - -unsafe impl WasmRet for T +unsafe impl WasmHostResults for T where T: WasmTy, { type Abi = ::Abi; + type Retptr = (); type Fallible = Result; fn compatible_with_store(&self, store: &Store) -> bool { ::compatible_with_store(self, store) } - unsafe fn into_abi_for_ret(self, store: &Store) -> Result { + unsafe fn into_abi_for_ret(self, store: &Store, _retptr: ()) -> Result { Ok(::into_abi(self, store)) } - fn valtype() -> Option { - Some(::valtype()) + fn func_type(params: impl Iterator) -> FuncType { + FuncType::new(params, Some(::valtype())) + } + + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + *ptr.cast::() = f(()); } fn into_fallible(self) -> Result { @@ -1381,26 +1331,35 @@ where } } -unsafe impl WasmRet for Result +unsafe impl WasmHostResults for Result where - T: WasmTy, + T: WasmHostResults, { - type Abi = ::Abi; + type Abi = ::Abi; + type Retptr = ::Retptr; type Fallible = Self; fn compatible_with_store(&self, store: &Store) -> bool { match self { - Ok(x) => ::compatible_with_store(x, store), + Ok(x) => ::compatible_with_store(x, store), Err(_) => true, } } - unsafe fn into_abi_for_ret(self, store: &Store) -> Result { - self.map(|val| ::into_abi(val, store)) + unsafe fn into_abi_for_ret( + self, + store: &Store, + retptr: Self::Retptr, + ) -> Result { + self.and_then(|val| val.into_abi_for_ret(store, retptr)) + } + + fn func_type(params: impl Iterator) -> FuncType { + T::func_type(params) } - fn valtype() -> Option { - Some(::valtype()) + unsafe fn wrap_trampoline(ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + T::wrap_trampoline(ptr, f) } fn into_fallible(self) -> Result { @@ -1412,6 +1371,163 @@ where } } +macro_rules! impl_wasm_host_results { + ($n:tt $($t:ident)*) => ( + #[allow(non_snake_case)] + unsafe impl<$($t),*> WasmHostResults for ($($t,)*) + where + $($t: WasmTy,)* + ($($t::Abi,)*): HostAbi, + { + type Abi = <($($t::Abi,)*) as HostAbi>::Abi; + type Retptr = <($($t::Abi,)*) as HostAbi>::Retptr; + type Fallible = Result; + + #[inline] + fn compatible_with_store(&self, _store: &Store) -> bool { + let ($($t,)*) = self; + $( $t.compatible_with_store(_store) && )* true + } + + #[inline] + unsafe fn into_abi_for_ret(self, _store: &Store, ptr: Self::Retptr) -> Result { + let ($($t,)*) = self; + let abi = ($($t.into_abi(_store),)*); + Ok(<($($t::Abi,)*) as HostAbi>::into_abi(abi, ptr)) + } + + fn func_type(params: impl Iterator) -> FuncType { + FuncType::new( + params, + std::array::IntoIter::new([$($t::valtype(),)*]), + ) + } + + #[allow(unused_assignments)] + unsafe fn wrap_trampoline(mut _ptr: *mut u128, f: impl FnOnce(Self::Retptr) -> Self::Abi) { + let ($($t,)*) = <($($t::Abi,)*) as HostAbi>::call(f); + $( + *_ptr.cast() = $t; + _ptr = _ptr.add(1); + )* + } + + #[inline] + fn into_fallible(self) -> Result { + Ok(self) + } + + #[inline] + fn fallible_from_trap(trap: Trap) -> Result { + Err(trap) + } + } + ) +} + +for_each_function_signature!(impl_wasm_host_results); + +// Internal trait representing how to communicate tuples of return values across +// an ABI boundary. This internally corresponds to the "wasmtime" ABI inside of +// cranelift itself. Notably the first element of each tuple is returned via the +// typical system ABI (e.g. systemv or fastcall depending on platform) and all +// other values are returned packed via the stack. +// +// This trait helps to encapsulate all the details of that. +#[doc(hidden)] +pub trait HostAbi { + // A value returned from native functions which return `Self` + type Abi: Copy; + // A return pointer, added to the end of the argument list, for native + // functions that return `Self`. Note that a 0-sized type here should get + // elided at the ABI level. + type Retptr: Copy; + + // Converts a value of `self` into its components. Stores necessary values + // into `ptr` and then returns whatever needs to be returned from the + // function. + unsafe fn into_abi(self, ptr: Self::Retptr) -> Self::Abi; + + // Calls `f` with a suitably sized return area and requires `f` to return + // the raw abi value of the first element of our tuple. This will then + // unpack the `Retptr` and assemble it with `Self::Abi` to return an + // instance of the whole tuple. + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self; +} + +macro_rules! impl_host_abi { + // Base case, everything is `()` + (0) => { + impl HostAbi for () { + type Abi = (); + type Retptr = (); + + unsafe fn into_abi(self, _ptr: Self::Retptr) -> Self::Abi {} + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + f(()) + } + } + }; + + // In the 1-case the retptr is not present, so it's a 0-sized value. + (1 $a:ident) => { + impl<$a: Copy> HostAbi for ($a,) { + type Abi = $a; + type Retptr = (); + + unsafe fn into_abi(self, _ptr: Self::Retptr) -> Self::Abi { + self.0 + } + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + (f(()),) + } + } + }; + + // This is where the more interesting case happens. The first element of the + // tuple is returned via `Abi` and all other elements are returned via + // `Retptr`. We create a `TupleRetNN` structure to represent all of the + // return values here. + ($n:tt $t:ident $($u:ident)*) => {paste::paste!{ + #[doc(hidden)] + #[allow(non_snake_case)] + #[repr(C)] + pub struct []<$($u,)*> { + $($u: $u,)* + } + + #[allow(non_snake_case, unused_assignments)] + impl<$t: Copy, $($u: Copy,)*> HostAbi for ($t, $($u,)*) { + type Abi = $t; + type Retptr = *mut []<$($u,)*>; + + unsafe fn into_abi(self, ptr: Self::Retptr) -> Self::Abi { + let ($t, $($u,)*) = self; + // Store the tail of our tuple into the return pointer... + $((*ptr).$u = $u;)* + // ... and return the head raw. + $t + } + + unsafe fn call(f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + // Create space to store all the return values and then invoke + // the function. + let mut space = MaybeUninit::uninit(); + let t = f(space.as_mut_ptr()); + let space = space.assume_init(); + + // Use the return value as the head of the tuple and unpack our + // return area to get the rest of the tuple. + (t, $(space.$u,)*) + } + } + }}; +} + +for_each_function_signature!(impl_host_abi); + /// Internal trait implemented for all arguments that can be passed to /// [`Func::wrap`] and [`Config::wrap_host_func`](crate::Config::wrap_host_func). /// @@ -1533,7 +1649,7 @@ macro_rules! impl_into_func { where F: Fn($($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { let f = move |_: Caller<'_>, $($args:$args),*| { @@ -1549,7 +1665,7 @@ macro_rules! impl_into_func { where F: Fn(Caller<'_>, $($args),*) -> R + 'static, $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { fn into_func(self, registry: Option<&mut SignatureRegistry>) -> (FuncType, InstanceHandle, VMTrampoline) { /// This shim is called by Wasm code, constructs a `Caller`, @@ -1563,11 +1679,12 @@ macro_rules! impl_into_func { vmctx: *mut VMContext, caller_vmctx: *mut VMContext, $( $args: $args::Abi, )* + retptr: R::Retptr, ) -> R::Abi where F: Fn(Caller<'_>, $( $args ),*) -> R + 'static, $( $args: WasmTy, )* - R: WasmRet, + R: WasmHostResults, { enum CallResult { Ok(T), @@ -1624,7 +1741,7 @@ macro_rules! impl_into_func { raise_cross_store_trap(); } - match ret.into_abi_for_ret(&store) { + match ret.into_abi_for_ret(&store, retptr) { Ok(val) => CallResult::Ok(val), Err(trap) => CallResult::Trap(trap), } @@ -1654,7 +1771,7 @@ macro_rules! impl_into_func { ) where $($args: WasmTy,)* - R: WasmRet, + R: WasmHostResults, { let ptr = mem::transmute::< *const VMFunctionBody, @@ -1662,6 +1779,7 @@ macro_rules! impl_into_func { *mut VMContext, *mut VMContext, $( $args::Abi, )* + R::Retptr, ) -> R::Abi, >(ptr); @@ -1670,15 +1788,14 @@ macro_rules! impl_into_func { let $args = *args.add(_n).cast::<$args::Abi>(); _n += 1; )* - let ret = ptr(callee_vmctx, caller_vmctx, $( $args ),*); - *args.cast::() = ret; + R::wrap_trampoline(args, |retptr| { + ptr(callee_vmctx, caller_vmctx, $( $args, )* retptr) + }); } - let ty = FuncType::new( + let ty = R::func_type( None::.into_iter() $(.chain(Some($args::valtype())))* - , - R::valtype(), ); let trampoline = host_trampoline::<$($args,)* R>; @@ -1686,7 +1803,7 @@ macro_rules! impl_into_func { // If not given a registry, use a default signature index that is guaranteed to trap // if the function is called indirectly without first being associated with a store (a bug condition). let shared_signature_id = registry - .map(|r| r.register(ty.as_wasm_func_type(), trampoline)) + .map(|r| r.register(ty.as_wasm_func_type(), Some(trampoline))) .unwrap_or(VMSharedSignatureIndex::default()); let instance = unsafe { diff --git a/crates/wasmtime/src/func/typed.rs b/crates/wasmtime/src/func/typed.rs index 5ec884c87840..051c91f446c4 100644 --- a/crates/wasmtime/src/func/typed.rs +++ b/crates/wasmtime/src/func/typed.rs @@ -1,10 +1,10 @@ -use super::invoke_wasm_and_catch_traps; +use super::{invoke_wasm_and_catch_traps, HostAbi}; use crate::{ExternRef, Func, Store, Trap, ValType}; use anyhow::{bail, Result}; use std::marker; use std::mem::{self, MaybeUninit}; use std::ptr; -use wasmtime_runtime::{VMContext, VMFunctionBody, VMTrampoline}; +use wasmtime_runtime::{VMContext, VMFunctionBody}; /// A statically typed WebAssembly function. /// @@ -103,7 +103,6 @@ where let anyfunc = self.func.export.anyfunc.as_ref(); let result = params.invoke::( &self.func.instance.store, - self.func.trampoline, anyfunc.func_ptr.as_ptr(), anyfunc.vmctx, ptr::null_mut(), @@ -181,13 +180,40 @@ macro_rules! primitives { primitives! { i32 => I32 - u32 => I32 i64 => I64 - u64 => I64 f32 => F32 f64 => F64 } +macro_rules! unsigned { + ($($unsigned:ident => $ty:ident/$signed:ident)*) => ($( + unsafe impl WasmTy for $unsigned { + type Abi = $signed; + #[inline] + fn valtype() -> ValType { + ValType::$ty + } + #[inline] + fn compatible_with_store(&self, _: &Store) -> bool { + true + } + #[inline] + fn into_abi(self, _store: &Store) -> Self::Abi { + self as $signed + } + #[inline] + unsafe fn from_abi(abi: Self::Abi, _store: &Store) -> Self { + abi as $unsigned + } + } + )*) +} + +unsigned! { + u32 => I32/i32 + u64 => I64/i64 +} + unsafe impl WasmTy for Option { type Abi = *mut u8; @@ -274,7 +300,6 @@ pub unsafe trait WasmParams { unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, @@ -296,12 +321,11 @@ where unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, ) -> R { - <(T,)>::invoke((self,), store, trampoline, func, vmctx1, vmctx2) + <(T,)>::invoke((self,), store, func, vmctx1, vmctx2) } } @@ -333,66 +357,30 @@ macro_rules! impl_wasm_params { unsafe fn invoke( self, store: &Store, - trampoline: VMTrampoline, func: *const VMFunctionBody, vmctx1: *mut VMContext, vmctx2: *mut VMContext, ) -> R { - // Some signatures can go directly into JIT code which uses the - // default platform ABI, but basically only those without - // multiple return values. With multiple return values we can't - // natively in Rust call such a function because there's no way - // to model it (yet). + let fnptr = mem::transmute::< + *const VMFunctionBody, + unsafe extern "C" fn( + *mut VMContext, + *mut VMContext, + $($t::Abi,)* + R::Retptr, + ) -> R::Abi, + >(func); + let ($($t,)*) = self; + // Use the `call` function to acquire a `retptr` which we'll + // forward to the native function. Once we have it we also + // convert all our arguments to abi arguments to go to the raw + // function. // - // To work around that we use the trampoline which passes - // arguments/values via the stack which allows us to match the - // expected ABI. Note that this branch, using the trampoline, - // is slower as a result and has an extra indirect function - // call as well. In the future if this is a problem we should - // consider updating JIT code to use an ABI we can call from - // Rust itself. - if R::uses_trampoline() { - R::with_space(|space1| { - // Figure out whether the parameters or the results - // require more space, and use the bigger one as where - // to store arguments and load return values from. - let mut space2 = [0; $n]; - let space = if space1.len() < space2.len() { - space2.as_mut_ptr() - } else { - space1.as_mut_ptr() - }; - - // ... store the ABI for all values into our storage - // area... - let ($($t,)*) = self; - let mut _n = 0; - $( - *space.add(_n).cast::<$t::Abi>() = $t.into_abi(store); - _n += 1; - )* - - // ... make the indirect call through the trampoline - // which will read from `space` and also write all the - // results to `space`... - trampoline(vmctx1, vmctx2, func, space); - - // ... and then we can decode all the return values - // from `space`. - R::from_storage(space, store) - }) - } else { - let fnptr = mem::transmute::< - *const VMFunctionBody, - unsafe extern "C" fn( - *mut VMContext, - *mut VMContext, - $($t::Abi,)* - ) -> R::Abi, - >(func); - let ($($t,)*) = self; - R::from_abi(fnptr(vmctx1, vmctx2, $($t.into_abi(store),)*), store) - } + // Upon returning `R::call` will convert all the returns back + // into `R`. + R::call(store, |retptr| { + fnptr(vmctx1, vmctx2, $($t.into_abi(store),)* retptr) + }) } } }; @@ -408,80 +396,45 @@ for_each_function_signature!(impl_wasm_params); /// `TypedFunc` is not currently supported. pub unsafe trait WasmResults: WasmParams { #[doc(hidden)] - type Abi; - #[doc(hidden)] - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self; - #[doc(hidden)] - fn uses_trampoline() -> bool; - // Provides a stack-allocated array with enough space to store all these - // result values. - // - // It'd be nice if we didn't have to have this API and could do something - // with const-generics (or something like that), but I couldn't figure it - // out. If a future Rust explorer is able to get something like `const LEN: - // usize` working that'd be great! + type Abi: Copy; #[doc(hidden)] - fn with_space(_: impl FnOnce(&mut [u128]) -> R) -> R; + type Retptr: Copy; #[doc(hidden)] - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self; + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self; } -unsafe impl WasmResults for T { +// Forwards from a bare type `T` to the 1-tuple type `(T,)` +unsafe impl WasmResults for T +where + (T::Abi,): HostAbi, +{ type Abi = <(T,) as WasmResults>::Abi; - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self { - <(T,) as WasmResults>::from_abi(abi, store).0 - } - fn uses_trampoline() -> bool { - <(T,) as WasmResults>::uses_trampoline() - } - fn with_space(f: impl FnOnce(&mut [u128]) -> R) -> R { - <(T,) as WasmResults>::with_space(f) - } - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self { - <(T,) as WasmResults>::from_storage(ptr, store).0 + type Retptr = <(T,) as WasmResults>::Retptr; + + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + <(T,) as WasmResults>::call(store, f).0 } } -#[doc(hidden)] -pub enum Void {} - macro_rules! impl_wasm_results { ($n:tt $($t:ident)*) => { #[allow(non_snake_case, unused_variables)] - unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) { - type Abi = impl_wasm_results!(@abi $n $($t)*); - unsafe fn from_abi(abi: Self::Abi, store: &Store) -> Self { - impl_wasm_results!(@from_abi abi store $n $($t)*) - } - fn uses_trampoline() -> bool { - $n > 1 - } - fn with_space(f: impl FnOnce(&mut [u128]) -> R) -> R { - f(&mut [0; $n]) - } - unsafe fn from_storage(ptr: *const u128, store: &Store) -> Self { - let mut _n = 0; - $( - let $t = $t::from_abi(*ptr.add(_n).cast::<$t::Abi>(), store); - _n += 1; - )* - ($($t,)*) + unsafe impl<$($t: WasmTy,)*> WasmResults for ($($t,)*) + where ($($t::Abi,)*): HostAbi + { + type Abi = <($($t::Abi,)*) as HostAbi>::Abi; + type Retptr = <($($t::Abi,)*) as HostAbi>::Retptr; + + unsafe fn call(store: &Store, f: impl FnOnce(Self::Retptr) -> Self::Abi) -> Self { + // Delegate via the host abi to figure out what the actual ABI + // for dealing with this tuple type is, and then we can re-tuple + // everything and create actual values via `from_abi` after the + // call is complete. + let ($($t,)*) = <($($t::Abi,)*) as HostAbi>::call(f); + ($($t::from_abi($t, store),)*) } } }; - - // 0/1 return values we can use natively, everything else isn't expressible - // and won't be used so define the abi type to Void. - (@abi 0) => (()); - (@abi 1 $t:ident) => ($t::Abi); - (@abi $($t:tt)*) => (Void); - - (@from_abi $abi:ident $store:ident 0) => (()); - (@from_abi $abi:ident $store:ident 1 $t:ident) => (($t::from_abi($abi, $store),)); - (@from_abi $abi:ident $store:ident $($t:tt)*) => ({ - debug_assert!(false); - match $abi {} - }); } for_each_function_signature!(impl_wasm_results); diff --git a/crates/wasmtime/src/sig_registry.rs b/crates/wasmtime/src/sig_registry.rs index c9583972c966..99bf127b8d7f 100644 --- a/crates/wasmtime/src/sig_registry.rs +++ b/crates/wasmtime/src/sig_registry.rs @@ -29,7 +29,7 @@ struct Entry { // Note that the code memory for this trampoline is not owned by this // type, but instead it's expected to be owned by the store that this // registry lives within. - trampoline: VMTrampoline, + trampoline: Option, } impl SignatureRegistry { @@ -37,12 +37,19 @@ impl SignatureRegistry { pub fn register( &mut self, wasm: &WasmFuncType, - trampoline: VMTrampoline, + trampoline: Option, ) -> VMSharedSignatureIndex { let len = self.wasm2index.len(); match self.wasm2index.entry(wasm.clone()) { - hash_map::Entry::Occupied(entry) => *entry.get(), + hash_map::Entry::Occupied(entry) => { + let ret = *entry.get(); + let entry = &mut self.index_map[ret.bits() as usize]; + if entry.trampoline.is_none() { + entry.trampoline = trampoline; + } + ret + } hash_map::Entry::Vacant(entry) => { // Keep `signature_hash` len under 2**32 -- VMSharedSignatureIndex::new(std::u32::MAX) // is reserved for VMSharedSignatureIndex::default(). @@ -75,8 +82,10 @@ impl SignatureRegistry { &self, idx: VMSharedSignatureIndex, ) -> Option<(&WasmFuncType, VMTrampoline)> { - self.index_map + let (wasm, trampoline) = self + .index_map .get(idx.bits() as usize) - .map(|e| (&e.wasm, e.trampoline)) + .map(|e| (&e.wasm, e.trampoline))?; + Some((wasm, trampoline?)) } } diff --git a/crates/wasmtime/src/store.rs b/crates/wasmtime/src/store.rs index f01c86662f15..6e315462f8c2 100644 --- a/crates/wasmtime/src/store.rs +++ b/crates/wasmtime/src/store.rs @@ -202,7 +202,7 @@ impl Store { .inner .signatures .borrow_mut() - .register(ty.as_wasm_func_type(), trampoline); + .register(ty.as_wasm_func_type(), Some(trampoline)); Box::new(anyfunc) }); @@ -322,9 +322,15 @@ impl Store { fn register_signatures(&self, module: &Module) { let mut signatures = self.signatures().borrow_mut(); let types = module.types(); + for (_, ty) in module.compiled_module().module().types.iter() { + if let wasmtime_environ::ModuleType::Function(index) = ty { + let wasm = &types.wasm_signatures[*index]; + signatures.register(wasm, None); + } + } for (index, trampoline) in module.compiled_module().trampolines() { let wasm = &types.wasm_signatures[*index]; - signatures.register(wasm, *trampoline); + signatures.register(wasm, Some(*trampoline)); } } diff --git a/crates/wasmtime/src/trampoline/func.rs b/crates/wasmtime/src/trampoline/func.rs index 7b5eb70e6459..d81ed07bb149 100644 --- a/crates/wasmtime/src/trampoline/func.rs +++ b/crates/wasmtime/src/trampoline/func.rs @@ -18,6 +18,7 @@ use wasmtime_jit::trampoline::{ self, binemit, pretty_error, Context, FunctionBuilder, FunctionBuilderContext, }; use wasmtime_jit::CodeMemory; +use wasmtime_jit::{blank_sig, wasmtime_call_conv}; use wasmtime_runtime::{ Imports, InstanceAllocationRequest, InstanceAllocator, InstanceHandle, OnDemandInstanceAllocator, VMContext, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, @@ -91,16 +92,7 @@ fn make_trampoline( // Mostly reverse copy of the similar method from wasmtime's // wasmtime-jit/src/compiler.rs. let pointer_type = isa.pointer_type(); - let mut stub_sig = ir::Signature::new(isa.frontend_config().default_call_conv); - - // Add the caller/callee `vmctx` parameters. - stub_sig.params.push(ir::AbiParam::special( - pointer_type, - ir::ArgumentPurpose::VMContext, - )); - - // Add the caller `vmctx` parameter. - stub_sig.params.push(ir::AbiParam::new(pointer_type)); + let mut stub_sig = blank_sig(isa, wasmtime_call_conv(isa)); // Add the `values_vec` parameter. stub_sig.params.push(ir::AbiParam::new(pointer_type)); @@ -220,8 +212,15 @@ fn create_function_trampoline( // reference types which requires safepoints. let isa = config.target_isa_with_reference_types(); - let pointer_type = isa.pointer_type(); - let sig = ft.get_wasmtime_signature(pointer_type); + let mut sig = blank_sig(&*isa, wasmtime_call_conv(&*isa)); + sig.params.extend( + ft.params() + .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), + ); + sig.returns.extend( + ft.results() + .map(|p| ir::AbiParam::new(p.get_wasmtime_type())), + ); let mut fn_builder_ctx = FunctionBuilderContext::new(); let mut module = Module::new(); @@ -271,7 +270,7 @@ pub fn create_function( // If there is no signature registry, use the default signature index which is // guaranteed to trap if there is ever an indirect call on the function (should not happen) let shared_signature_id = registry - .map(|r| r.register(ft.as_wasm_func_type(), trampoline)) + .map(|r| r.register(ft.as_wasm_func_type(), Some(trampoline))) .unwrap_or(VMSharedSignatureIndex::default()); unsafe { diff --git a/crates/wasmtime/src/types.rs b/crates/wasmtime/src/types.rs index 5a090d794cce..2a91af473c91 100644 --- a/crates/wasmtime/src/types.rs +++ b/crates/wasmtime/src/types.rs @@ -298,28 +298,6 @@ impl FuncType { &self.sig } - /// Get the Cranelift-compatible function signature. - pub(crate) fn get_wasmtime_signature(&self, pointer_type: ir::Type) -> ir::Signature { - use wasmtime_environ::ir::{AbiParam, ArgumentPurpose, Signature}; - use wasmtime_jit::native; - let call_conv = native::call_conv(); - let mut params = vec![ - AbiParam::special(pointer_type, ArgumentPurpose::VMContext), - AbiParam::new(pointer_type), - ]; - params.extend(self.params().map(|p| AbiParam::new(p.get_wasmtime_type()))); - let returns = self - .results() - .map(|p| AbiParam::new(p.get_wasmtime_type())) - .collect::>(); - - Signature { - params, - returns, - call_conv, - } - } - pub(crate) fn from_wasm_func_type(sig: &wasm::WasmFuncType) -> FuncType { FuncType { sig: sig.clone() } } diff --git a/tests/all/func.rs b/tests/all/func.rs index 0e691993843f..4159cafd9a5e 100644 --- a/tests/all/func.rs +++ b/tests/all/func.rs @@ -619,3 +619,162 @@ fn trap_doesnt_leak() -> anyhow::Result<()> { assert!(dtor2_run.get()); Ok(()) } + +#[test] +fn wrap_multiple_results() -> anyhow::Result<()> { + fn test(store: &Store, t: T) -> anyhow::Result<()> + where + T: WasmHostResults + + WasmResults + + PartialEq + + Copy + + std::fmt::Debug + + EqualToValues + + 'static, + { + let f = Func::wrap(store, move || t); + assert_eq!(f.typed::<(), T>()?.call(())?, t); + assert!(t.eq_values(&f.call(&[])?)); + + let module = Module::new(store.engine(), &T::gen_wasm())?; + let instance = Instance::new(store, &module, &[f.into()])?; + let f = instance.get_func("foo").unwrap(); + + assert_eq!(f.typed::<(), T>()?.call(())?, t); + assert!(t.eq_values(&f.call(&[])?)); + Ok(()) + } + + let store = Store::default(); + // 0 element + test(&store, ())?; + + // 1 element + test(&store, (1i32,))?; + test(&store, (2u32,))?; + test(&store, (3i64,))?; + test(&store, (4u64,))?; + test(&store, (5.0f32,))?; + test(&store, (6.0f64,))?; + + // 2 element ... + test(&store, (7i32, 8i32))?; + test(&store, (7i32, 8i64))?; + test(&store, (7i32, 8f32))?; + test(&store, (7i32, 8f64))?; + + test(&store, (7i64, 8i32))?; + test(&store, (7i64, 8i64))?; + test(&store, (7i64, 8f32))?; + test(&store, (7i64, 8f64))?; + + test(&store, (7f32, 8i32))?; + test(&store, (7f32, 8i64))?; + test(&store, (7f32, 8f32))?; + test(&store, (7f32, 8f64))?; + + test(&store, (7f64, 8i32))?; + test(&store, (7f64, 8i64))?; + test(&store, (7f64, 8f32))?; + test(&store, (7f64, 8f64))?; + + // and beyond... + test(&store, (1i32, 2i32, 3i32))?; + test(&store, (1i32, 2f32, 3i32))?; + test(&store, (1f64, 2f32, 3i32))?; + test(&store, (1f64, 2i64, 3i32))?; + test(&store, (1f32, 2f32, 3i64, 4f64))?; + test(&store, (1f64, 2i64, 3i32, 4i64, 5f32))?; + test(&store, (1i32, 2f64, 3i64, 4f64, 5f64, 6f32))?; + test(&store, (1i64, 2i32, 3i64, 4f32, 5f32, 6i32, 7u64))?; + test(&store, (1u32, 2f32, 3u64, 4f64, 5i32, 6f32, 7u64, 8u32))?; + test( + &store, + (1f32, 2f64, 3f32, 4i32, 5u32, 6i64, 7f32, 8i32, 9u64), + )?; + return Ok(()); + + trait EqualToValues { + fn eq_values(&self, values: &[Val]) -> bool; + fn gen_wasm() -> String; + } + + macro_rules! equal_tuples { + ($($cnt:tt ($($a:ident),*))*) => ($( + #[allow(non_snake_case)] + impl<$($a: EqualToValue,)*> EqualToValues for ($($a,)*) { + fn eq_values(&self, values: &[Val]) -> bool { + let ($($a,)*) = self; + let mut _values = values.iter(); + _values.len() == $cnt && + $($a.eq_value(_values.next().unwrap()) &&)* + true + } + + fn gen_wasm() -> String { + let mut wasm = String::new(); + wasm.push_str("(module "); + wasm.push_str("(type $t (func (result "); + $( + wasm.push_str($a::wasm_ty()); + wasm.push_str(" "); + )* + wasm.push_str(")))"); + + wasm.push_str("(import \"\" \"\" (func $host (type $t)))"); + wasm.push_str("(func (export \"foo\") (type $t)"); + wasm.push_str("call $host"); + wasm.push_str(")"); + wasm.push_str(")"); + + wasm + } + } + )*) + } + + equal_tuples! { + 0 () + 1 (A1) + 2 (A1, A2) + 3 (A1, A2, A3) + 4 (A1, A2, A3, A4) + 5 (A1, A2, A3, A4, A5) + 6 (A1, A2, A3, A4, A5, A6) + 7 (A1, A2, A3, A4, A5, A6, A7) + 8 (A1, A2, A3, A4, A5, A6, A7, A8) + 9 (A1, A2, A3, A4, A5, A6, A7, A8, A9) + } + + trait EqualToValue { + fn eq_value(&self, value: &Val) -> bool; + fn wasm_ty() -> &'static str; + } + + macro_rules! equal_values { + ($a:ident $($ty:ident $wasm:tt $variant:ident $e:expr,)*) => ($( + impl EqualToValue for $ty { + fn eq_value(&self, val: &Val) -> bool { + if let Val::$variant($a) = *val { + return *self == $e; + } + false + } + + fn wasm_ty() -> &'static str { + $wasm + } + } + )*) + } + + equal_values! { + a + i32 "i32" I32 a, + u32 "i32" I32 a as u32, + i64 "i64" I64 a, + u64 "i64" I64 a as u64, + f32 "f32" F32 f32::from_bits(a), + f64 "f64" F64 f64::from_bits(a), + } +}