From 5eca0cf82278ca3003bde9f7c5503c89d63d0e88 Mon Sep 17 00:00:00 2001 From: Nick Fitzgerald Date: Fri, 7 Feb 2020 14:05:33 -0800 Subject: [PATCH] Add support for `table.copy` This adds support for the `table.copy` instruction from the bulk memory proposal. It also supports multiple tables, which were introduced by the reference types proposal. Part of #928 --- Cargo.lock | 17 +- build.rs | 2 + crates/api/src/externals.rs | 39 +++- crates/environ/src/func_environ.rs | 187 ++++++++++++++++-- crates/environ/src/module.rs | 6 +- crates/environ/src/module_environ.rs | 36 +++- crates/runtime/src/instance.rs | 46 +++-- crates/runtime/src/lib.rs | 5 +- crates/runtime/src/libcalls.rs | 95 ++++++++- crates/runtime/src/table.rs | 54 ++++- crates/runtime/src/trap_registry.rs | 2 +- crates/runtime/src/traphandlers.rs | 16 ++ crates/runtime/src/vmcontext.rs | 16 ++ tests/misc_testsuite/table_copy.wast | 63 ++++++ .../table_copy_on_imported_tables.wast | 165 ++++++++++++++++ tests/wast_testsuites.rs | 13 ++ 16 files changed, 709 insertions(+), 53 deletions(-) create mode 100644 tests/misc_testsuite/table_copy.wast create mode 100644 tests/misc_testsuite/table_copy_on_imported_tables.wast diff --git a/Cargo.lock b/Cargo.lock index 2323ad36a97d..6a00a4503f48 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2136,31 +2136,18 @@ dependencies = [ "leb128", ] -[[package]] -name = "wast" -version = "6.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3ed3db7029d1d31a15c10126e78b58e51781faefafbc8afb20fb01291b779984" -dependencies = [ - "leb128", -] - [[package]] name = "wast" version = "7.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12a729d076deb29c8509fa71f2d427729f9394f9496844ed8fcab152f35d163d" dependencies = [ "leb128", ] [[package]] name = "wat" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d59ba5b224f5507d55e4f89d0b18cc6452d84640ab11b4c9086d61a3ee62d03" +version = "1.0.8" dependencies = [ - "wast 6.0.0", + "wast 7.0.0", ] [[package]] diff --git a/build.rs b/build.rs index 0678bdc9aa33..718ee96cc55a 100644 --- a/build.rs +++ b/build.rs @@ -181,6 +181,8 @@ fn ignore(testsuite: &str, testname: &str, strategy: &str) -> bool { ("simd", "simd_splat") => return true, // FIXME Unsupported feature: proposed SIMD operator I8x16ShrS ("reference_types", _) => return true, + + ("bulk_memory_operations", "table_copy") => return false, ("bulk_memory_operations", _) => return true, _ => {} diff --git a/crates/api/src/externals.rs b/crates/api/src/externals.rs index 70d796e0479f..948c2da57e8a 100644 --- a/crates/api/src/externals.rs +++ b/crates/api/src/externals.rs @@ -6,8 +6,8 @@ use crate::{Func, Store}; use anyhow::{anyhow, bail, Result}; use std::rc::Rc; use std::slice; -use wasmtime_environ::wasm; -use wasmtime_runtime::InstanceHandle; +use wasmtime_environ::{ir, wasm}; +use wasmtime_runtime::{self as runtime, InstanceHandle}; // Externals @@ -413,6 +413,41 @@ impl Table { } } + /// Copy `len` elements from `src_table[src_index..]` into + /// `dst_table[dst_index..]`. + /// + /// # Errors + /// + /// Returns an error if the range is out of bounds of either the source or + /// destination tables. + pub fn copy( + dst_table: &Table, + dst_index: u32, + src_table: &Table, + src_index: u32, + len: u32, + ) -> Result<()> { + // NB: We must use the `dst_table`'s `wasmtime_handle` for the + // `dst_table_index` and vice versa for `src_table` since each table can + // come from different modules. + + let dst_table_index = dst_table.wasmtime_table_index(); + let dst_table = dst_table.wasmtime_handle.get_defined_table(dst_table_index); + + let src_table_index = src_table.wasmtime_table_index(); + let src_table = src_table.wasmtime_handle.get_defined_table(src_table_index); + + runtime::Table::copy( + dst_table, + src_table, + dst_index, + src_index, + len, + ir::SourceLoc::default(), + )?; + Ok(()) + } + pub(crate) fn wasmtime_export(&self) -> &wasmtime_runtime::Export { &self.wasmtime_export } diff --git a/crates/environ/src/func_environ.rs b/crates/environ/src/func_environ.rs index f8d2ba5ec66d..659314a1ba07 100644 --- a/crates/environ/src/func_environ.rs +++ b/crates/environ/src/func_environ.rs @@ -47,6 +47,7 @@ pub fn get_imported_memory32_size_name() -> ir::ExternalName { } /// An index type for builtin functions. +#[derive(Copy, Clone, Debug)] pub struct BuiltinFunctionIndex(u32); impl BuiltinFunctionIndex { @@ -66,9 +67,28 @@ impl BuiltinFunctionIndex { pub const fn get_imported_memory32_size_index() -> Self { Self(3) } + /// Returns an index for wasm's `table.copy` when both tables are locally + /// defined. + pub const fn get_table_copy_defined_defined_index() -> Self { + Self(4) + } + /// Returns an index for wasm's `table.copy` when the destination table is + /// locally defined and the source table is imported. + pub const fn get_table_copy_defined_imported_index() -> Self { + Self(5) + } + /// Returns an index for wasm's `table.copy` when the destination table is + /// imported and the source table is locally defined. + pub const fn get_table_copy_imported_defined_index() -> Self { + Self(6) + } + /// Returns an index for wasm's `table.copy` when both tables are imported. + pub const fn get_table_copy_imported_imported_index() -> Self { + Self(7) + } /// Returns the total number of builtin functions. pub const fn builtin_functions_total_number() -> u32 { - 4 + 8 } /// Return the index as an u32 number. @@ -96,6 +116,10 @@ pub struct FuncEnvironment<'module_environment> { /// for locally-defined memories. memory_grow_sig: Option, + /// The external function signature for implementing wasm's `table.copy` + /// (it's the same for both local and imported tables). + table_copy_sig: Option, + /// Offsets to struct fields accessed by JIT code. offsets: VMOffsets, } @@ -108,6 +132,7 @@ impl<'module_environment> FuncEnvironment<'module_environment> { vmctx: None, memory32_size_sig: None, memory_grow_sig: None, + table_copy_sig: None, offsets: VMOffsets::new(target_config.pointer_bytes(), module), } } @@ -199,6 +224,97 @@ impl<'module_environment> FuncEnvironment<'module_environment> { } } + // NB: All `table_copy` libcall variants have the same signature. + fn get_table_copy_sig(&mut self, func: &mut Function) -> ir::SigRef { + let sig = self.table_copy_sig.unwrap_or_else(|| { + func.import_signature(Signature { + params: vec![ + AbiParam::special(self.pointer_type(), ArgumentPurpose::VMContext), + // Destination table index. + AbiParam::new(I32), + // Source table index. + AbiParam::new(I32), + // Index within destination table. + AbiParam::new(I32), + // Index within source table. + AbiParam::new(I32), + // Number of elements to copy. + AbiParam::new(I32), + // Source location. + AbiParam::new(I32), + ], + returns: vec![], + call_conv: self.target_config.default_call_conv, + }) + }); + self.table_copy_sig = Some(sig); + sig + } + + fn get_table_copy_func( + &mut self, + func: &mut Function, + dst_table_index: TableIndex, + src_table_index: TableIndex, + ) -> (ir::SigRef, usize, usize, BuiltinFunctionIndex) { + let sig = self.get_table_copy_sig(func); + match ( + self.module.is_imported_table(dst_table_index), + self.module.is_imported_table(src_table_index), + ) { + (false, false) => { + let dst_table_index = self + .module + .defined_table_index(dst_table_index) + .unwrap() + .index(); + let src_table_index = self + .module + .defined_table_index(src_table_index) + .unwrap() + .index(); + ( + sig, + dst_table_index, + src_table_index, + BuiltinFunctionIndex::get_table_copy_defined_defined_index(), + ) + } + (false, true) => { + let dst_table_index = self + .module + .defined_table_index(dst_table_index) + .unwrap() + .index(); + ( + sig, + dst_table_index, + src_table_index.as_u32() as usize, + BuiltinFunctionIndex::get_table_copy_defined_imported_index(), + ) + } + (true, false) => { + let src_table_index = self + .module + .defined_table_index(src_table_index) + .unwrap() + .index(); + ( + sig, + dst_table_index.as_u32() as usize, + src_table_index, + BuiltinFunctionIndex::get_table_copy_imported_defined_index(), + ) + } + (true, true) => ( + sig, + dst_table_index.as_u32() as usize, + src_table_index.as_u32() as usize, + BuiltinFunctionIndex::get_table_copy_imported_imported_index(), + ), + } + } + /// Translates load of builtin function and returns a pair of values `vmctx` /// and address of the loaded function. fn translate_load_builtin_function_address( @@ -803,7 +919,9 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m _src: ir::Value, _len: ir::Value, ) -> WasmResult<()> { - Err(WasmError::Unsupported("bulk memory".to_string())) + Err(WasmError::Unsupported( + "bulk memory: `memory.copy`".to_string(), + )) } fn translate_memory_fill( @@ -815,7 +933,9 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m _val: ir::Value, _len: ir::Value, ) -> WasmResult<()> { - Err(WasmError::Unsupported("bulk memory".to_string())) + Err(WasmError::Unsupported( + "bulk memory: `memory.fill`".to_string(), + )) } fn translate_memory_init( @@ -828,11 +948,15 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m _src: ir::Value, _len: ir::Value, ) -> WasmResult<()> { - Err(WasmError::Unsupported("bulk memory".to_string())) + Err(WasmError::Unsupported( + "bulk memory: `memory.copy`".to_string(), + )) } fn translate_data_drop(&mut self, _pos: FuncCursor, _seg_index: u32) -> WasmResult<()> { - Err(WasmError::Unsupported("bulk memory".to_string())) + Err(WasmError::Unsupported( + "bulk memory: `data.drop`".to_string(), + )) } fn translate_table_size( @@ -841,21 +965,50 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m _index: TableIndex, _table: ir::Table, ) -> WasmResult { - Err(WasmError::Unsupported("bulk memory".to_string())) + Err(WasmError::Unsupported( + "bulk memory: `table.size`".to_string(), + )) } fn translate_table_copy( &mut self, - _pos: FuncCursor, - _dst_table_index: TableIndex, + mut pos: FuncCursor, + dst_table_index: TableIndex, _dst_table: ir::Table, - _src_table_index: TableIndex, + src_table_index: TableIndex, _src_table: ir::Table, - _dst: ir::Value, - _src: ir::Value, - _len: ir::Value, + dst: ir::Value, + src: ir::Value, + len: ir::Value, ) -> WasmResult<()> { - Err(WasmError::Unsupported("bulk memory".to_string())) + use cranelift_codegen::cursor::Cursor; + + let (func_sig, dst_table_index_arg, src_table_index_arg, func_idx) = + self.get_table_copy_func(&mut pos.func, dst_table_index, src_table_index); + + let dst_table_index_arg = pos.ins().iconst(I32, dst_table_index_arg as i64); + let src_table_index_arg = pos.ins().iconst(I32, src_table_index_arg as i64); + + let src_loc = pos.srcloc(); + let src_loc_arg = pos.ins().iconst(I32, src_loc.bits() as i64); + + let (vmctx, func_addr) = self.translate_load_builtin_function_address(&mut pos, func_idx); + + pos.ins().call_indirect( + func_sig, + func_addr, + &[ + vmctx, + dst_table_index_arg, + src_table_index_arg, + dst, + src, + len, + src_loc_arg, + ], + ); + + Ok(()) } fn translate_table_init( @@ -868,10 +1021,14 @@ impl<'module_environment> cranelift_wasm::FuncEnvironment for FuncEnvironment<'m _src: ir::Value, _len: ir::Value, ) -> WasmResult<()> { - Err(WasmError::Unsupported("bulk memory".to_string())) + Err(WasmError::Unsupported( + "bulk memory: `table.init`".to_string(), + )) } fn translate_elem_drop(&mut self, _pos: FuncCursor, _seg_index: u32) -> WasmResult<()> { - Err(WasmError::Unsupported("bulk memory".to_string())) + Err(WasmError::Unsupported( + "bulk memory: `elem.drop`".to_string(), + )) } } diff --git a/crates/environ/src/module.rs b/crates/environ/src/module.rs index 127e3eb87493..b7cc2f1709ff 100644 --- a/crates/environ/src/module.rs +++ b/crates/environ/src/module.rs @@ -6,7 +6,7 @@ use cranelift_codegen::ir; use cranelift_entity::{EntityRef, PrimaryMap}; use cranelift_wasm::{ DefinedFuncIndex, DefinedGlobalIndex, DefinedMemoryIndex, DefinedTableIndex, FuncIndex, Global, - GlobalIndex, Memory, MemoryIndex, SignatureIndex, Table, TableIndex, + GlobalIndex, Memory, MemoryIndex, PassiveElemIndex, SignatureIndex, Table, TableIndex, }; use indexmap::IndexMap; use more_asserts::assert_ge; @@ -175,6 +175,9 @@ pub struct Module { /// WebAssembly table initializers. pub table_elements: Vec, + /// WebAssembly passive elements. + pub passive_elements: HashMap>, + /// WebAssembly table initializers. pub func_names: HashMap, } @@ -198,6 +201,7 @@ impl Module { exports: IndexMap::new(), start_func: None, table_elements: Vec::new(), + passive_elements: HashMap::new(), func_names: HashMap::new(), } } diff --git a/crates/environ/src/module_environ.rs b/crates/environ/src/module_environ.rs index 9273c6f9917d..efac1650f3f8 100644 --- a/crates/environ/src/module_environ.rs +++ b/crates/environ/src/module_environ.rs @@ -7,7 +7,8 @@ use cranelift_codegen::isa::TargetFrontendConfig; use cranelift_entity::PrimaryMap; use cranelift_wasm::{ self, translate_module, DefinedFuncIndex, FuncIndex, Global, GlobalIndex, Memory, MemoryIndex, - ModuleTranslationState, SignatureIndex, Table, TableIndex, TargetEnvironment, WasmResult, + ModuleTranslationState, PassiveDataIndex, PassiveElemIndex, SignatureIndex, Table, TableIndex, + TargetEnvironment, WasmError, WasmResult, }; use std::convert::TryFrom; @@ -324,6 +325,24 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data Ok(()) } + fn declare_passive_element( + &mut self, + elem_index: PassiveElemIndex, + segments: Box<[FuncIndex]>, + ) -> WasmResult<()> { + let old = self + .result + .module + .passive_elements + .insert(elem_index, segments); + debug_assert!( + old.is_none(), + "should never get duplicate element indices, that would be a bug in `cranelift_wasm`'s \ + translation" + ); + Ok(()) + } + fn define_function_body( &mut self, _module_translation: &ModuleTranslationState, @@ -362,6 +381,21 @@ impl<'data> cranelift_wasm::ModuleEnvironment<'data> for ModuleEnvironment<'data Ok(()) } + fn reserve_passive_data(&mut self, count: u32) -> WasmResult<()> { + self.result.module.passive_elements.reserve(count as usize); + Ok(()) + } + + fn declare_passive_data( + &mut self, + _data_index: PassiveDataIndex, + _data: &'data [u8], + ) -> WasmResult<()> { + Err(WasmError::Unsupported( + "bulk memory: passive data".to_string(), + )) + } + fn declare_func_name(&mut self, func_index: FuncIndex, name: &'data str) -> WasmResult<()> { self.result .module diff --git a/crates/runtime/src/instance.rs b/crates/runtime/src/instance.rs index 656b10ba74eb..e5fd1fff7671 100644 --- a/crates/runtime/src/instance.rs +++ b/crates/runtime/src/instance.rs @@ -516,6 +516,30 @@ impl Instance { .unwrap_or_else(|| panic!("no table for index {}", table_index.index())) .set(index, val) } + + /// Get a table by index regardless of whether it is locally-defined or an + /// imported, foreign table. + pub(crate) fn get_table(&self, table_index: TableIndex) -> &Table { + if let Some(defined_table_index) = self.module.defined_table_index(table_index) { + &self.tables[defined_table_index] + } else { + self.get_foreign_table(table_index) + } + } + + /// Get a locally-defined table. + pub(crate) fn get_defined_table(&self, index: DefinedTableIndex) -> &Table { + &self.tables[index] + } + + /// Get an imported, foreign table. + pub(crate) fn get_foreign_table(&self, index: TableIndex) -> &Table { + let import = self.imported_table(index); + let foreign_instance = unsafe { (&mut *(import).vmctx).instance() }; + let foreign_table = unsafe { &mut *(import).from }; + let foreign_index = foreign_instance.table_index(foreign_table); + &foreign_instance.tables[foreign_index] + } } /// A handle holding an `Instance` of a WebAssembly module. @@ -773,6 +797,11 @@ impl InstanceHandle { self.instance().table_set(table_index, index, val) } + /// Get a table defined locally within this module. + pub fn get_defined_table(&self, index: DefinedTableIndex) -> &Table { + self.instance().get_defined_table(index) + } + /// Return a reference to the contained `Instance`. pub(crate) fn instance(&self) -> &Instance { unsafe { &*(self.instance as *const Instance) } @@ -806,7 +835,7 @@ fn check_table_init_bounds(instance: &Instance) -> Result<(), InstantiationError let module = Arc::clone(&instance.module); for init in &module.table_elements { let start = get_table_init_start(init, instance); - let table = get_table(init, instance); + let table = instance.get_table(init.table_index); let size = usize::try_from(table.size()).unwrap(); if size < start + init.elements.len() { @@ -905,26 +934,13 @@ fn get_table_init_start(init: &TableElements, instance: &Instance) -> usize { start } -/// Return a byte-slice view of a table's data. -fn get_table<'instance>(init: &TableElements, instance: &'instance Instance) -> &'instance Table { - if let Some(defined_table_index) = instance.module.defined_table_index(init.table_index) { - &instance.tables[defined_table_index] - } else { - let import = instance.imported_table(init.table_index); - let foreign_instance = unsafe { (&mut *(import).vmctx).instance() }; - let foreign_table = unsafe { &mut *(import).from }; - let foreign_index = foreign_instance.table_index(foreign_table); - &foreign_instance.tables[foreign_index] - } -} - /// Initialize the table memory from the provided initializers. fn initialize_tables(instance: &Instance) -> Result<(), InstantiationError> { let vmctx = instance.vmctx_ptr(); let module = Arc::clone(&instance.module); for init in &module.table_elements { let start = get_table_init_start(init, instance); - let table = get_table(init, instance); + let table = instance.get_table(init.table_index); for (i, func_idx) in init.elements.iter().enumerate() { let callee_sig = instance.module.functions[*func_idx]; diff --git a/crates/runtime/src/lib.rs b/crates/runtime/src/lib.rs index 188265a74736..9a09e3dbe91b 100644 --- a/crates/runtime/src/lib.rs +++ b/crates/runtime/src/lib.rs @@ -42,9 +42,12 @@ pub use crate::instance::{InstanceHandle, InstantiationError, LinkError}; pub use crate::jit_int::GdbJitImageRegistration; pub use crate::mmap::Mmap; pub use crate::sig_registry::SignatureRegistry; +pub use crate::table::Table; pub use crate::trap_registry::{TrapDescription, TrapRegistration, TrapRegistry}; pub use crate::traphandlers::resume_panic; -pub use crate::traphandlers::{raise_user_trap, wasmtime_call, wasmtime_call_trampoline, Trap}; +pub use crate::traphandlers::{ + raise_lib_trap, raise_user_trap, wasmtime_call, wasmtime_call_trampoline, Trap, +}; pub use crate::vmcontext::{ VMCallerCheckedAnyfunc, VMContext, VMFunctionBody, VMFunctionImport, VMGlobalDefinition, VMGlobalImport, VMInvokeArgument, VMMemoryDefinition, VMMemoryImport, VMSharedSignatureIndex, diff --git a/crates/runtime/src/libcalls.rs b/crates/runtime/src/libcalls.rs index 9349bca791fd..aa4b04f448fc 100644 --- a/crates/runtime/src/libcalls.rs +++ b/crates/runtime/src/libcalls.rs @@ -2,8 +2,11 @@ //! inline rather than calling them, particularly when CPUs have special //! instructions which compute them directly. +use crate::table::Table; +use crate::traphandlers::raise_lib_trap; use crate::vmcontext::VMContext; -use wasmtime_environ::wasm::{DefinedMemoryIndex, MemoryIndex}; +use wasmtime_environ::ir; +use wasmtime_environ::wasm::{DefinedMemoryIndex, DefinedTableIndex, MemoryIndex, TableIndex}; /// Implementation of f32.ceil pub extern "C" fn wasmtime_f32_ceil(x: f32) -> f32 { @@ -137,3 +140,93 @@ pub unsafe extern "C" fn wasmtime_imported_memory32_size( instance.imported_memory_size(memory_index) } + +/// Implementation of `table.copy` when both tables are locally defined. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_table_copy_defined_defined( + vmctx: *mut VMContext, + dst_table_index: u32, + src_table_index: u32, + dst: u32, + src: u32, + len: u32, + source_loc: u32, +) { + let dst_table_index = DefinedTableIndex::from_u32(dst_table_index); + let src_table_index = DefinedTableIndex::from_u32(src_table_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + let dst_table = instance.get_defined_table(dst_table_index); + let src_table = instance.get_defined_table(src_table_index); + if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) { + raise_lib_trap(trap); + } +} + +/// Implementation of `table.copy` when the destination table is locally defined +/// and the source table is imported. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_table_copy_defined_imported( + vmctx: *mut VMContext, + dst_table_index: u32, + src_table_index: u32, + dst: u32, + src: u32, + len: u32, + source_loc: u32, +) { + let dst_table_index = DefinedTableIndex::from_u32(dst_table_index); + let src_table_index = TableIndex::from_u32(src_table_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + let dst_table = instance.get_defined_table(dst_table_index); + let src_table = instance.get_foreign_table(src_table_index); + if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) { + raise_lib_trap(trap); + } +} + +/// Implementation of `table.copy` when the destination table is imported +/// and the source table is locally defined. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_table_copy_imported_defined( + vmctx: *mut VMContext, + dst_table_index: u32, + src_table_index: u32, + dst: u32, + src: u32, + len: u32, + source_loc: u32, +) { + let dst_table_index = TableIndex::from_u32(dst_table_index); + let src_table_index = DefinedTableIndex::from_u32(src_table_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + let dst_table = instance.get_foreign_table(dst_table_index); + let src_table = instance.get_defined_table(src_table_index); + if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) { + raise_lib_trap(trap); + } +} + +/// Implementation of `table.copy` when both tables are imported. +#[no_mangle] +pub unsafe extern "C" fn wasmtime_table_copy_imported_imported( + vmctx: *mut VMContext, + dst_table_index: u32, + src_table_index: u32, + dst: u32, + src: u32, + len: u32, + source_loc: u32, +) { + let dst_table_index = TableIndex::from_u32(dst_table_index); + let src_table_index = TableIndex::from_u32(src_table_index); + let source_loc = ir::SourceLoc::new(source_loc); + let instance = (&mut *vmctx).instance(); + let dst_table = instance.get_foreign_table(dst_table_index); + let src_table = instance.get_foreign_table(src_table_index); + if let Err(trap) = Table::copy(dst_table, src_table, dst, src, len, source_loc) { + raise_lib_trap(trap); + } +} diff --git a/crates/runtime/src/table.rs b/crates/runtime/src/table.rs index 843fd630e48e..45672a75fa7f 100644 --- a/crates/runtime/src/table.rs +++ b/crates/runtime/src/table.rs @@ -3,10 +3,12 @@ //! `Table` is to WebAssembly tables what `LinearMemory` is to WebAssembly linear memories. use crate::vmcontext::{VMCallerCheckedAnyfunc, VMTableDefinition}; +use crate::{Trap, TrapDescription}; +use backtrace::Backtrace; use std::cell::RefCell; use std::convert::{TryFrom, TryInto}; use wasmtime_environ::wasm::TableElementType; -use wasmtime_environ::{TablePlan, TableStyle}; +use wasmtime_environ::{ir, TablePlan, TableStyle}; /// A table instance. #[derive(Debug)] @@ -87,6 +89,56 @@ impl Table { } } + /// Copy `len` elements from `src_table[src_index..]` into `dst_table[dst_index..]`. + /// + /// # Errors + /// + /// Returns an error if the range is out of bounds of either the source or + /// destination tables. + pub fn copy( + dst_table: &Self, + src_table: &Self, + dst_index: u32, + src_index: u32, + len: u32, + source_loc: ir::SourceLoc, + ) -> Result<(), Trap> { + // https://webassembly.github.io/bulk-memory-operations/core/exec/instructions.html#exec-table-copy + + if src_index + .checked_add(len) + .map_or(true, |n| n > src_table.size()) + || dst_index + .checked_add(len) + .map_or(true, |m| m > dst_table.size()) + { + return Err(Trap::Wasm { + desc: TrapDescription { + source_loc, + trap_code: ir::TrapCode::TableOutOfBounds, + }, + backtrace: Backtrace::new(), + }); + } + + let srcs = src_index..src_index + len; + let dsts = dst_index..dst_index + len; + + // Note on the unwraps: the bounds check above means that these will + // never panic. + if dst_index <= src_index { + for (s, d) in (srcs).zip(dsts) { + dst_table.set(d, src_table.get(s).unwrap()).unwrap(); + } + } else { + for (s, d) in srcs.rev().zip(dsts.rev()) { + dst_table.set(d, src_table.get(s).unwrap()).unwrap(); + } + } + + Ok(()) + } + /// Return a `VMTableDefinition` for exposing the table to compiled wasm code. pub fn vmtable(&self) -> VMTableDefinition { let mut vec = self.vec.borrow_mut(); diff --git a/crates/runtime/src/trap_registry.rs b/crates/runtime/src/trap_registry.rs index 317fa920aa4b..f8f4c8b56dc3 100644 --- a/crates/runtime/src/trap_registry.rs +++ b/crates/runtime/src/trap_registry.rs @@ -84,7 +84,7 @@ fn trap_code_to_expected_string(trap_code: ir::TrapCode) -> String { match trap_code { StackOverflow => "call stack exhausted".to_string(), HeapOutOfBounds => "out of bounds memory access".to_string(), - TableOutOfBounds => "undefined element".to_string(), + TableOutOfBounds => "undefined element: out of bounds".to_string(), OutOfBounds => "out of bounds".to_string(), // Note: not covered by the test suite IndirectCallToNull => "uninitialized element".to_string(), BadSignature => "indirect call type mismatch".to_string(), diff --git a/crates/runtime/src/traphandlers.rs b/crates/runtime/src/traphandlers.rs index 04ba4b2c3654..cc741f0bdc4b 100644 --- a/crates/runtime/src/traphandlers.rs +++ b/crates/runtime/src/traphandlers.rs @@ -80,6 +80,20 @@ pub unsafe fn raise_user_trap(data: Box) -> ! { tls::with(|info| info.unwrap().unwind_with(UnwindReason::UserTrap(data))) } +/// Raises a trap from inside library code immediately. +/// +/// This function performs as-if a wasm trap was just executed. This trap +/// payload is then returned from `wasmtime_call` and `wasmtime_call_trampoline` +/// below. +/// +/// # Safety +/// +/// Only safe to call when wasm code is on the stack, aka `wasmtime_call` or +/// `wasmtime_call_trampoline` must have been previously called. +pub unsafe fn raise_lib_trap(trap: Trap) -> ! { + tls::with(|info| info.unwrap().unwind_with(UnwindReason::LibTrap(trap))) +} + /// Carries a Rust panic across wasm code and resumes the panic on the other /// side. /// @@ -186,6 +200,7 @@ enum UnwindReason { None, Panic(Box), UserTrap(Box), + LibTrap(Trap), Trap { backtrace: Backtrace, pc: usize }, } @@ -213,6 +228,7 @@ impl CallThreadState { debug_assert_eq!(ret, 0); Err(Trap::User(data)) } + UnwindReason::LibTrap(trap) => Err(trap), UnwindReason::Trap { backtrace, pc } => { debug_assert_eq!(ret, 0); let instance = unsafe { InstanceHandle::from_vmctx(self.vmctx) }; diff --git a/crates/runtime/src/vmcontext.rs b/crates/runtime/src/vmcontext.rs index 7567261eaa07..38e7186af3d2 100644 --- a/crates/runtime/src/vmcontext.rs +++ b/crates/runtime/src/vmcontext.rs @@ -537,6 +537,7 @@ impl VMBuiltinFunctionsArray { pub fn initialized() -> Self { use crate::libcalls::*; let mut ptrs = [0; Self::len()]; + ptrs[BuiltinFunctionIndex::get_memory32_grow_index().index() as usize] = wasmtime_memory32_grow as usize; ptrs[BuiltinFunctionIndex::get_imported_memory32_grow_index().index() as usize] = @@ -545,6 +546,21 @@ impl VMBuiltinFunctionsArray { wasmtime_memory32_size as usize; ptrs[BuiltinFunctionIndex::get_imported_memory32_size_index().index() as usize] = wasmtime_imported_memory32_size as usize; + + ptrs[BuiltinFunctionIndex::get_table_copy_defined_defined_index().index() as usize] = + wasmtime_table_copy_defined_defined as usize; + + ptrs[BuiltinFunctionIndex::get_table_copy_defined_imported_index().index() as usize] = + wasmtime_table_copy_defined_imported as usize; + + ptrs[BuiltinFunctionIndex::get_table_copy_imported_defined_index().index() as usize] = + wasmtime_table_copy_imported_defined as usize; + + ptrs[BuiltinFunctionIndex::get_table_copy_imported_imported_index().index() as usize] = + wasmtime_table_copy_imported_imported as usize; + + debug_assert!(ptrs.iter().cloned().all(|p| p != 0)); + Self { ptrs } } } diff --git a/tests/misc_testsuite/table_copy.wast b/tests/misc_testsuite/table_copy.wast new file mode 100644 index 000000000000..f06258ea249d --- /dev/null +++ b/tests/misc_testsuite/table_copy.wast @@ -0,0 +1,63 @@ +(module + (func $f (param i32 i32 i32) (result i32) (local.get 0)) + (func $g (param i32 i32 i32) (result i32) (local.get 1)) + (func $h (param i32 i32 i32) (result i32) (local.get 2)) + + ;; Indices: 0 1 2 3 4 5 6 7 8 + (table funcref (elem $f $g $h $f $g $h $f $g $h)) + ;; After table.copy: $g $h $f + + (func (export "copy") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + table.copy) + + (func (export "call") (param i32 i32 i32 i32) (result i32) + local.get 0 + local.get 1 + local.get 2 + local.get 3 + call_indirect (param i32 i32 i32) (result i32)) +) + +;; Call $f at 0 +(assert_return + (invoke "call" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0)) + (i32.const 1)) + +;; Call $g at 1 +(assert_return + (invoke "call" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 1)) + (i32.const 1)) + +;; Call $h at 2 +(assert_return + (invoke "call" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 2)) + (i32.const 1)) + +;; Do a `table.copy` to rearrange the elements. Copy from 4..7 to 0..3. +(invoke "copy" (i32.const 0) (i32.const 4) (i32.const 3)) + +;; Call $g at 0 +(assert_return + (invoke "call" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0)) + (i32.const 1)) + +;; Call $h at 1 +(assert_return + (invoke "call" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 1)) + (i32.const 1)) + +;; Call $f at 2 +(assert_return + (invoke "call" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 2)) + (i32.const 1)) + +;; Copying up to the end does not trap. +(invoke "copy" (i32.const 7) (i32.const 0) (i32.const 2)) + +;; Copying past the end traps. +(assert_trap + (invoke "copy" (i32.const 7) (i32.const 0) (i32.const 3)) + "undefined element") diff --git a/tests/misc_testsuite/table_copy_on_imported_tables.wast b/tests/misc_testsuite/table_copy_on_imported_tables.wast new file mode 100644 index 000000000000..dae846c2d9e5 --- /dev/null +++ b/tests/misc_testsuite/table_copy_on_imported_tables.wast @@ -0,0 +1,165 @@ +(module $m + (func $f (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 0)) + (func $g (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 1)) + (func $h (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 2)) + + (table $t (export "t") funcref (elem $f $g $h $f $g $h))) + +(register "m" $m) + +(module $n + (table $t (import "m" "t") 6 funcref) + + (func $i (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 3)) + (func $j (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 4)) + (func $k (param i32 i32 i32 i32 i32 i32) (result i32) (local.get 5)) + + (table $u (export "u") funcref (elem $i $j $k $i $j $k)) + + (func (export "copy_into_t_from_u") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + table.copy $t $u) + + (func (export "copy_into_u_from_t") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + table.copy $u $t) + + (func (export "call_t") (param i32 i32 i32 i32 i32 i32 i32) (result i32) + local.get 0 + local.get 1 + local.get 2 + local.get 3 + local.get 4 + local.get 5 + local.get 6 + call_indirect $t (param i32 i32 i32 i32 i32 i32) (result i32)) + + (func (export "call_u") (param i32 i32 i32 i32 i32 i32 i32) (result i32) + local.get 0 + local.get 1 + local.get 2 + local.get 3 + local.get 4 + local.get 5 + local.get 6 + call_indirect $u (param i32 i32 i32 i32 i32 i32) (result i32))) + +;; Everything has what we initially expect. +(assert_return + (invoke "call_t" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) + (i32.const 0)) + (i32.const 1)) +(assert_return + (invoke "call_t" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) + (i32.const 1)) + (i32.const 1)) +(assert_return + (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) + (i32.const 2)) + (i32.const 1)) +(assert_return + (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) + (i32.const 0)) + (i32.const 1)) +(assert_return + (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) + (i32.const 1)) + (i32.const 1)) +(assert_return + (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) + (i32.const 2)) + (i32.const 1)) + +;; Now test copying between a local and an imported table. + +;; Copy $i $j $k into $t at 3..6 from $u at 0..3. +(invoke "copy_into_t_from_u" (i32.const 3) (i32.const 0) (i32.const 3)) + +(assert_return + (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) + (i32.const 3)) + (i32.const 1)) +(assert_return + (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) + (i32.const 4)) + (i32.const 1)) +(assert_return + (invoke "call_t" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) + (i32.const 5)) + (i32.const 1)) + +;; Copy $f $g $h into $u at 0..3 from $t at 0..3. +(invoke "copy_into_u_from_t" (i32.const 0) (i32.const 0) (i32.const 3)) + +(assert_return + (invoke "call_u" (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) + (i32.const 0)) + (i32.const 1)) +(assert_return + (invoke "call_u" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) + (i32.const 1)) + (i32.const 1)) +(assert_return + (invoke "call_u" (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) + (i32.const 2)) + (i32.const 1)) + +(register "n" $n) + +(module $o + (table $t (import "m" "t") 6 funcref) + (table $u (import "n" "u") 6 funcref) + + (func (export "copy_into_t_from_u_2") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + table.copy $t $u) + + (func (export "copy_into_u_from_t_2") (param i32 i32 i32) + local.get 0 + local.get 1 + local.get 2 + table.copy $u $t) + + (func (export "call_t_2") (param i32 i32 i32 i32 i32 i32 i32) (result i32) + local.get 0 + local.get 1 + local.get 2 + local.get 3 + local.get 4 + local.get 5 + local.get 6 + call_indirect $t (param i32 i32 i32 i32 i32 i32) (result i32)) + + (func (export "call_u_2") (param i32 i32 i32 i32 i32 i32 i32) (result i32) + local.get 0 + local.get 1 + local.get 2 + local.get 3 + local.get 4 + local.get 5 + local.get 6 + call_indirect $u (param i32 i32 i32 i32 i32 i32) (result i32))) + +;; Now test copying between two imported tables. + +;; Copy $i into $t at 0 from $u at 3. +(invoke "copy_into_t_from_u_2" (i32.const 0) (i32.const 3) (i32.const 1)) + +(assert_return + (invoke "call_t_2" (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) + (i32.const 0)) + (i32.const 1)) + +;; Copy $g into $u at 4 from $t at 1. +(invoke "copy_into_u_from_t_2" (i32.const 4) (i32.const 1) (i32.const 1)) + +(assert_return + (invoke "call_u_2" (i32.const 0) (i32.const 1) (i32.const 0) (i32.const 0) (i32.const 0) (i32.const 0) + (i32.const 4)) + (i32.const 1)) diff --git a/tests/wast_testsuites.rs b/tests/wast_testsuites.rs index 543bed93775f..f22514941d2e 100644 --- a/tests/wast_testsuites.rs +++ b/tests/wast_testsuites.rs @@ -16,12 +16,25 @@ fn run_wast(wast: &str, strategy: Strategy) -> anyhow::Result<()> { // by reference types. let reftypes = simd || wast.iter().any(|s| s == "reference-types"); + // Reference types assumes support for bulk memory. + let bulk_mem = reftypes + || wast.iter().any(|s| s == "bulk-memory-operations") + || wast.iter().any(|s| s == "table_copy.wast") + || wast + .iter() + .any(|s| s == "table_copy_on_imported_tables.wast"); + + // And bulk memory also assumes support for reference types (e.g. multiple + // tables). + let reftypes = reftypes || bulk_mem; + let multi_val = wast.iter().any(|s| s == "multi-value"); let mut cfg = Config::new(); cfg.wasm_simd(simd) .wasm_reference_types(reftypes) .wasm_multi_value(multi_val) + .wasm_bulk_memory(bulk_mem) .strategy(strategy)? .cranelift_debug_verifier(true); let store = Store::new(&Engine::new(&cfg));