Add support for funcrefs inside GC objects (#9341)

crates/cranelift/src/gc/enabled.rs

@@ -7,11 +7,12 @@ use cranelift_codegen::{
     cursor::FuncCursor,
     ir::{self, condcodes::IntCC, InstBuilder},
 };
+use cranelift_entity::packed_option::ReservedValue;
 use cranelift_frontend::FunctionBuilder;
 use wasmtime_environ::{
-    wasm_unsupported, GcArrayLayout, GcLayout, GcStructLayout, ModuleInternedTypeIndex, PtrSize,
-    TypeIndex, WasmCompositeType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult,
-    WasmStorageType, WasmValType, I31_DISCRIMINANT, NON_NULL_NON_I31_MASK,
+    GcArrayLayout, GcLayout, GcStructLayout, ModuleInternedTypeIndex, PtrSize, TypeIndex,
+    WasmCompositeType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult, WasmStorageType,
+    WasmValType, I31_DISCRIMINANT, NON_NULL_NON_I31_MASK,
 };
 
 mod drc;
@@ -86,9 +87,40 @@ fn read_field_at_addr(
                 WasmHeapTopType::Any | WasmHeapTopType::Extern => gc_compiler(func_env)?
                     .translate_read_gc_reference(func_env, builder, r, addr, flags)?,
                 WasmHeapTopType::Func => {
-                    return Err(wasm_unsupported!(
-                        "funcrefs inside the GC heap are not yet implemented"
-                    ));
+                    let expected_ty = match r.heap_type {
+                        WasmHeapType::Func => ModuleInternedTypeIndex::reserved_value(),
+                        WasmHeapType::ConcreteFunc(ty) => ty.unwrap_module_type_index(),
+                        WasmHeapType::NoFunc => {
+                            let null = builder.ins().iconst(func_env.pointer_type(), 0);
+                            if !r.nullable {
+                                // Because `nofunc` is uninhabited, and this
+                                // reference is non-null, this is unreachable
+                                // code. Unconditionally trap via conditional
+                                // trap instructions to avoid inserting block
+                                // terminators in the middle of this block.
+                                builder
+                                    .ins()
+                                    .trapz(null, ir::TrapCode::User(DEBUG_ASSERT_TRAP_CODE));
+                            }
+                            return Ok(null);
+                        }
+                        _ => unreachable!("not a function heap type"),
+                    };
+                    let expected_ty = builder
+                        .ins()
+                        .iconst(ir::types::I32, i64::from(expected_ty.as_bits()));
+
+                    let vmctx = func_env.vmctx_val(&mut builder.cursor());
+
+                    let func_ref_id = builder.ins().load(ir::types::I32, flags, addr, 0);
+                    let get_interned_func_ref = func_env
+                        .builtin_functions
+                        .get_interned_func_ref(builder.func);
+
+                    let call_inst = builder
+                        .ins()
+                        .call(get_interned_func_ref, &[vmctx, func_ref_id, expected_ty]);
+                    builder.func.dfg.first_result(call_inst)
                 }
             },
         },
@@ -103,6 +135,51 @@ fn read_field_at_addr(
     Ok(value)
 }
 
+fn write_func_ref_at_addr(
+    func_env: &mut FuncEnvironment<'_>,
+    builder: &mut FunctionBuilder<'_>,
+    ref_type: WasmRefType,
+    flags: ir::MemFlags,
+    field_addr: ir::Value,
+    func_ref: ir::Value,
+) -> WasmResult<()> {
+    assert_eq!(ref_type.heap_type.top(), WasmHeapTopType::Func);
+
+    let vmctx = func_env.vmctx_val(&mut builder.cursor());
+
+    let intern_func_ref_for_gc_heap = func_env
+        .builtin_functions
+        .intern_func_ref_for_gc_heap(builder.func);
+
+    let func_ref = if ref_type.heap_type == WasmHeapType::NoFunc {
+        let null = builder.ins().iconst(func_env.pointer_type(), 0);
+        if !ref_type.nullable {
+            // Because `nofunc` is uninhabited, and this reference is
+            // non-null, this is unreachable code. Unconditionally trap
+            // via conditional trap instructions to avoid inserting
+            // block terminators in the middle of this block.
+            builder
+                .ins()
+                .trapz(null, ir::TrapCode::User(DEBUG_ASSERT_TRAP_CODE));
+        }
+        null
+    } else {
+        func_ref
+    };
+
+    // Convert the raw `funcref` into a `FuncRefTableId` for use in the
+    // GC heap.
+    let call_inst = builder
+        .ins()
+        .call(intern_func_ref_for_gc_heap, &[vmctx, func_ref]);
+    let func_ref_id = builder.func.dfg.first_result(call_inst);
+
+    // Store the id in the field.
+    builder.ins().store(flags, func_ref_id, field_addr, 0);
+
+    Ok(())
+}
+
 fn write_field_at_addr(
     func_env: &mut FuncEnvironment<'_>,
     builder: &mut FunctionBuilder<'_>,
@@ -121,9 +198,7 @@ fn write_field_at_addr(
             builder.ins().istore16(flags, new_val, field_addr, 0);
         }
         WasmStorageType::Val(WasmValType::Ref(r)) if r.heap_type.top() == WasmHeapTopType::Func => {
-            return Err(wasm_unsupported!(
-                "funcrefs inside the GC heap are not yet implemented"
-            ))
+            write_func_ref_at_addr(func_env, builder, r, flags, field_addr, new_val)?;
         }
         WasmStorageType::Val(WasmValType::Ref(r)) => {
             gc_compiler(func_env)?

crates/cranelift/src/gc/enabled/drc.rs

@@ -3,7 +3,7 @@
 
 use super::{
     emit_array_fill_impl, uextend_i32_to_pointer_type, unbarriered_load_gc_ref,
-    unbarriered_store_gc_ref, BoundsCheck, Offset,
+    unbarriered_store_gc_ref, write_func_ref_at_addr, BoundsCheck, Offset,
 };
 use crate::gc::{gc_compiler, ArrayInit};
 use crate::translate::TargetEnvironment;
@@ -13,9 +13,9 @@ use cranelift_codegen::ir::{self, InstBuilder};
 use cranelift_frontend::FunctionBuilder;
 use smallvec::SmallVec;
 use wasmtime_environ::{
-    drc::DrcTypeLayouts, wasm_unsupported, GcArrayLayout, GcTypeLayouts, ModuleInternedTypeIndex,
-    PtrSize, TypeIndex, VMGcKind, WasmCompositeType, WasmHeapTopType, WasmHeapType, WasmRefType,
-    WasmResult, WasmStorageType, WasmValType,
+    drc::DrcTypeLayouts, GcArrayLayout, GcTypeLayouts, ModuleInternedTypeIndex, PtrSize, TypeIndex,
+    VMGcKind, WasmCompositeType, WasmHeapTopType, WasmHeapType, WasmRefType, WasmResult,
+    WasmStorageType, WasmValType,
 };
 
 #[derive(Default)]
@@ -135,9 +135,7 @@ impl DrcCompiler {
             WasmStorageType::Val(WasmValType::Ref(r))
                 if r.heap_type.top() == WasmHeapTopType::Func =>
             {
-                return Err(wasm_unsupported!(
-                    "funcrefs inside the GC heap are not yet implemented"
-                ));
+                write_func_ref_at_addr(func_env, builder, r, flags, field_addr, val)?;
             }
             WasmStorageType::Val(WasmValType::Ref(r)) => {
                 self.translate_init_gc_reference(func_env, builder, r, field_addr, val, flags)?;

crates/environ/src/builtin.rs

@@ -90,6 +90,40 @@ macro_rules! foreach_builtin_function {
                 align: i32
             ) -> reference;
 
+            // Intern a `funcref` into the GC heap, returning its
+            // `FuncRefTableId`.
+            //
+            // This libcall may not GC.
+            #[cfg(feature = "gc")]
+            intern_func_ref_for_gc_heap(
+                vmctx: vmctx,
+                func_ref: pointer
+            ) -> i32;
+
+            // Get the raw `VMFuncRef` pointer associated with a
+            // `FuncRefTableId` from an earlier `intern_func_ref_for_gc_heap`
+            // call.
+            //
+            // This libcall may not GC.
+            //
+            // Passes in the `ModuleInternedTypeIndex` of the funcref's expected
+            // type, or `ModuleInternedTypeIndex::reserved_value()` if we are
+            // getting the function reference as an untyped `funcref` rather
+            // than a typed `(ref $ty)`.
+            //
+            // TODO: We will want to eventually expose the table directly to
+            // Wasm code, so that it doesn't need to make a libcall to go from
+            // id to `VMFuncRef`. That will be a little tricky: it will also
+            // require updating the pointer to the slab in the `VMContext` (or
+            // `VMRuntimeLimits` or wherever we put it) when the slab is
+            // resized.
+            #[cfg(feature = "gc")]
+            get_interned_func_ref(
+                vmctx: vmctx,
+                func_ref_id: i32,
+                module_interned_type_index: i32
+            ) -> pointer;
+
             // Builtin implementation of the `array.new_data` instruction.
             #[cfg(feature = "gc")]
             array_new_data(
@@ -141,8 +175,9 @@ macro_rules! foreach_builtin_function {
                 vmctx: vmctx,
                 array_interned_type_index: i32,
                 array: reference,
-                dst_index: i32,
+                dst: i32,
                 elem_index: i32,
+                src: i32,
                 len: i32
             );
 

crates/wasmtime/src/runtime/vm/gc.rs

@@ -8,11 +8,13 @@ mod disabled;
 #[cfg(not(feature = "gc"))]
 pub use disabled::*;
 
+mod func_ref;
 mod gc_ref;
 mod gc_runtime;
 mod host_data;
 mod i31;
 
+pub use func_ref::*;
 pub use gc_ref::*;
 pub use gc_runtime::*;
 pub use host_data::*;
@@ -42,16 +44,21 @@ pub struct GcStore {
 
     /// The `externref` host data table for this GC heap.
     pub host_data_table: ExternRefHostDataTable,
+
+    /// The function-references table for this GC heap.
+    pub func_ref_table: FuncRefTable,
 }
 
 impl GcStore {
     /// Create a new `GcStore`.
     pub fn new(allocation_index: GcHeapAllocationIndex, gc_heap: Box<dyn GcHeap>) -> Self {
         let host_data_table = ExternRefHostDataTable::default();
+        let func_ref_table = FuncRefTable::default();
         Self {
             allocation_index,
             gc_heap,
             host_data_table,
+            func_ref_table,
         }
     }
 

crates/wasmtime/src/runtime/vm/gc/enabled/arrayref.rs

@@ -3,7 +3,8 @@ use crate::{
     prelude::*,
     runtime::vm::{GcHeap, GcStore, VMGcRef},
     store::{AutoAssertNoGc, StoreOpaque},
-    AnyRef, ExternRef, HeapType, RootedGcRefImpl, StorageType, Val, ValType,
+    vm::{FuncRefTableId, SendSyncPtr},
+    AnyRef, ExternRef, Func, HeapType, RootedGcRefImpl, StorageType, Val, ValType,
 };
 use core::fmt;
 use wasmtime_environ::{GcArrayLayout, VMGcKind};
@@ -163,7 +164,20 @@ impl VMArrayRef {
                     let raw = data.read_u32(offset);
                     Val::AnyRef(AnyRef::_from_raw(store, raw))
                 }
-                HeapType::Func => todo!("funcrefs inside gc objects not yet implemented"),
+                HeapType::Func => {
+                    let func_ref_id = data.read_u32(offset);
+                    let func_ref_id = FuncRefTableId::from_raw(func_ref_id);
+                    let func_ref = store
+                        .unwrap_gc_store()
+                        .func_ref_table
+                        .get_untyped(func_ref_id);
+                    Val::FuncRef(unsafe {
+                        Func::from_vm_func_ref(
+                            store,
+                            func_ref.map_or(core::ptr::null_mut(), |f| f.as_ptr()),
+                        )
+                    })
+                }
                 otherwise => unreachable!("not a top type: {otherwise:?}"),
             },
         }
@@ -236,7 +250,17 @@ impl VMArrayRef {
                 data.write_u32(offset, gc_ref.map_or(0, |r| r.as_raw_u32()));
             }
 
-            Val::FuncRef(_) => todo!("funcrefs inside gc objects not yet implemented"),
+            Val::FuncRef(f) => {
+                let func_ref = match f {
+                    Some(f) => Some(SendSyncPtr::new(f.vm_func_ref(store))),
+                    None => None,
+                };
+                let id = unsafe { store.gc_store_mut()?.func_ref_table.intern(func_ref) };
+                store
+                    .gc_store_mut()?
+                    .gc_object_data(self.as_gc_ref())
+                    .write_u32(offset, id.into_raw());
+            }
         }
         Ok(())
     }
@@ -329,12 +353,16 @@ impl VMArrayRef {
                     .write_u32(offset, x);
             }
 
-            Val::FuncRef(_) => {
-                // TODO: we can't trust the GC heap, which means we can't read
-                // native VMFuncRef pointers out of it and trust them. That
-                // means we need to do the same side table kind of thing we do
-                // with `externref` host data here. This isn't implemented yet.
-                bail!("funcrefs in GC objects are not yet implemented")
+            Val::FuncRef(f) => {
+                let func_ref = match f {
+                    Some(f) => Some(SendSyncPtr::new(f.vm_func_ref(store))),
+                    None => None,
+                };
+                let id = unsafe { store.gc_store_mut()?.func_ref_table.intern(func_ref) };
+                store
+                    .gc_store_mut()?
+                    .gc_object_data(self.as_gc_ref())
+                    .write_u32(offset, id.into_raw());
             }
         }
         Ok(())