Trace ptr_or_offset in jl_genericmemoryref_t

.github/scripts/ci-test-other.sh

@@ -19,18 +19,14 @@ moving_feature=${is_moving,,}
 
 declare -a tests_to_skip=(
     # see https://github.com/mmtk/mmtk-julia/issues/259
-    "atomics"
     "abstractarray"
-    "Artifacts"
     "cmdlineargs"
     "Downloads"
     "read"
     "threads"
     "LibCURL"
-    "subarray"
     "rounding"
     "loading"
-    "compileall"
     "misc"
 )
 

.github/scripts/ci-test-stdlib.sh

@@ -32,14 +32,11 @@ declare -a tests_to_skip=(
 
     # Skipping tests that fail for max moving Immix
     # see https://github.com/mmtk/mmtk-julia/issues/259
-    "Artifacts"
     "Downloads"
     "REPL"
     "TOML"
     "Random"
     "LibCURL"
-    "Mmap"
-    "SharedArrays"
     "LazyArtifacts"
 )
 # These tests need multiple workers.

mmtk/Cargo.toml

@@ -3,7 +3,7 @@ name = "mmtk-julia"
 version = "0.30.3"
 authors = ["Eduardo Souza <ledusou@gmail.com>", "Yi Lin <qinsoon@gmail.com>"]
 build = "build.rs"
-edition = "2018"
+edition = "2021"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 

mmtk/src/julia_scanning.rs

@@ -62,6 +62,17 @@ pub unsafe fn mmtk_jl_to_typeof(t: Address) -> *const jl_datatype_t {
     t.to_ptr::<jl_datatype_t>()
 }
 
+fn trace_internal_pointer(slot: Address, closure: &mut impl SlotVisitor<JuliaVMSlot>) {
+    let internal_pointer = unsafe { slot.load::<Address>() };
+    // find the beginning of the object and trace it since the object may have moved
+    if let Some(object) =
+        memory_manager::find_object_from_internal_pointer(internal_pointer, usize::MAX)
+    {
+        let offset = internal_pointer - object.to_raw_address();
+        process_offset_slot(closure, slot, offset);
+    }
+}
+
 const PRINT_OBJ_TYPE: bool = false;
 
 trait ValidOffset: Copy {
@@ -102,6 +113,27 @@ unsafe fn scan_julia_obj_n<T>(
     }
 }
 
+unsafe fn scan_julia_hidden_ptr_n<T>(
+    obj: Address,
+    begin: Address,
+    end: Address,
+    closure: &mut impl SlotVisitor<JuliaVMSlot>,
+) where
+    T: ValidOffset,
+{
+    let mut ptr = begin;
+    while ptr < end {
+        let offset = ptr.load::<T>();
+        let slot = obj.shift::<Address>(offset.to_isize());
+        let hidden_ptr = slot.load::<Address>();
+        // hidden pointers are internal pointers, or just integers. We must check if it is a real pointer, then trace it.
+        if mmtk_object_is_managed_by_mmtk(hidden_ptr.as_usize()) {
+            trace_internal_pointer(slot, closure);
+        }
+        ptr = ptr.shift::<T>(1);
+    }
+}
+
 // This function is a rewrite of `gc_mark_outrefs()` in `gc.c`
 // INFO: *_custom() functions are acessors to bitfields that do not use bindgen generated code.
 #[inline(always)]
@@ -233,26 +265,6 @@ pub unsafe fn scan_julia_object<SV: SlotVisitor<JuliaVMSlot>>(obj: Address, clos
         crate::object_model::assert_generic_datatype(obj);
     }
     let vt = vtag.to_ptr::<jl_datatype_t>();
-    if (*vt).name == jl_array_typename {
-        let a = obj.to_ptr::<jl_array_t>();
-        let memref = (*a).ref_;
-
-        let ptr_or_offset = memref.ptr_or_offset;
-        // if the object moves its pointer inside the array object (void* ptr_or_offset) needs to be updated as well
-        if mmtk_object_is_managed_by_mmtk(ptr_or_offset as usize) {
-            let ptr_or_ref_slot = Address::from_ptr(::std::ptr::addr_of!((*a).ref_.ptr_or_offset));
-            let mem_addr_as_usize = memref.mem as usize;
-            let ptr_or_offset_as_usize = ptr_or_offset as usize;
-            if ptr_or_offset_as_usize > mem_addr_as_usize {
-                let offset = ptr_or_offset_as_usize - mem_addr_as_usize;
-
-                // Only update the offset pointer if the offset is valid (> 0)
-                if offset > 0 {
-                    process_offset_slot(closure, ptr_or_ref_slot, offset);
-                }
-            }
-        }
-    }
     if (*vt).name == jl_genericmemory_typename {
         if PRINT_OBJ_TYPE {
             println!("scan_julia_obj {}: genericmemory\n", obj);
@@ -403,7 +415,37 @@ pub unsafe fn scan_julia_object<SV: SlotVisitor<JuliaVMSlot>>(obj: Address, clos
             }
             _ => {
                 debug_assert!((*layout).fielddesc_type_custom() == 3);
-                unimplemented!();
+                panic!(
+                    "Unexpected field desc type custom: {}",
+                    (*layout).fielddesc_type_custom()
+                );
+            }
+        }
+    }
+
+    let nhidden_pointers = (*layout).nhidden_pointers;
+    if nhidden_pointers != 0 {
+        let obj_begin = mmtk_jl_dt_layout_hidden_ptrs(layout);
+        let obj_end;
+        match (*layout).fielddesc_type_custom() {
+            0 => {
+                obj_end = obj_begin.shift::<u8>(nhidden_pointers as isize);
+                scan_julia_hidden_ptr_n::<u8>(obj, obj_begin, obj_end, closure);
+            }
+            1 => {
+                obj_end = obj_begin.shift::<u16>(nhidden_pointers as isize);
+                scan_julia_hidden_ptr_n::<u16>(obj, obj_begin, obj_end, closure);
+            }
+            2 => {
+                obj_end = obj_begin.shift::<u32>(nhidden_pointers as isize);
+                scan_julia_hidden_ptr_n::<u32>(obj, obj_begin, obj_end, closure);
+            }
+            _ => {
+                debug_assert!((*layout).fielddesc_type_custom() == 3);
+                panic!(
+                    "Unexpected field desc type custom: {}",
+                    (*layout).fielddesc_type_custom()
+                );
             }
         }
     }
@@ -740,15 +782,26 @@ pub unsafe fn mmtk_jl_svecref(vt: *mut jl_svec_t, i: usize) -> *const jl_datatyp
     result as *const _jl_datatype_t
 }
 
+#[inline(always)]
+pub unsafe fn mmtk_jl_dt_layout_fields(l: *const jl_datatype_layout_t) -> Address {
+    Address::from_ptr(l) + std::mem::size_of::<jl_datatype_layout_t>()
+}
+
 #[inline(always)]
 pub unsafe fn mmtk_jl_dt_layout_ptrs(l: *const jl_datatype_layout_t) -> Address {
     mmtk_jl_dt_layout_fields(l)
         + (mmtk_jl_fielddesc_size((*l).fielddesc_type_custom()) * (*l).nfields) as usize
 }
 
 #[inline(always)]
-pub unsafe fn mmtk_jl_dt_layout_fields(l: *const jl_datatype_layout_t) -> Address {
-    Address::from_ptr(l) + std::mem::size_of::<jl_datatype_layout_t>()
+pub unsafe fn mmtk_jl_dt_layout_hidden_ptrs(l: *const jl_datatype_layout_t) -> Address {
+    let ptrs = mmtk_jl_dt_layout_ptrs(l);
+    let direct_ptrs_size = if (*l).npointers == 0 {
+        0
+    } else {
+        ((*l).npointers as usize) << ((*l).fielddesc_type_custom())
+    };
+    ptrs + direct_ptrs_size
 }
 
 #[inline(always)]
@@ -789,6 +842,12 @@ pub unsafe fn mmtk_jl_bt_entry_jlvalue(
     i: usize,
 ) -> ObjectReference {
     let entry = unsafe { (*bt_entry.add(2 + i)).__bindgen_anon_1.jlvalue };
-    debug_assert!(!entry.is_null());
-    unsafe { ObjectReference::from_raw_address_unchecked(Address::from_mut_ptr(entry)) }
+    debug_assert!(!entry.ptr.is_null());
+    unsafe { ObjectReference::from_raw_address_unchecked(Address::from_mut_ptr(entry.ptr)) }
+}
+
+pub unsafe fn mmtk_jl_is_datatype(vt: *const jl_datatype_t) -> bool {
+    let type_tag = mmtk_jl_typetagof(Address::from_ptr(vt));
+
+    type_tag.as_usize() == ((jl_small_typeof_tags_jl_datatype_tag as usize) << 4)
 }

mmtk/src/julia_types.rs

@@ -1081,10 +1081,11 @@ const _: () = {
         [::std::mem::offset_of!(_jl_genericmemory_t, ptr) - 8usize];
 };
 pub type jl_genericmemory_t = _jl_genericmemory_t;
+pub type jl_hidden_ptr_ptr_or_offset_t = *mut ::std::os::raw::c_void;
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
 pub struct jl_genericmemoryref_t {
-    pub ptr_or_offset: *mut ::std::os::raw::c_void,
+    pub ptr_or_offset: jl_hidden_ptr_ptr_or_offset_t,
     pub mem: *mut jl_genericmemory_t,
 }
 #[allow(clippy::unnecessary_operation, clippy::identity_op)]
@@ -1883,10 +1884,11 @@ pub struct jl_typename_t {
     pub _bitfield_1: __BindgenBitfieldUnit<[u8; 1usize]>,
     pub max_methods: u8,
     pub constprop_heustic: u8,
+    pub hiddenptrfields: *const u32,
 }
 #[allow(clippy::unnecessary_operation, clippy::identity_op)]
 const _: () = {
-    ["Size of jl_typename_t"][::std::mem::size_of::<jl_typename_t>() - 104usize];
+    ["Size of jl_typename_t"][::std::mem::size_of::<jl_typename_t>() - 112usize];
     ["Alignment of jl_typename_t"][::std::mem::align_of::<jl_typename_t>() - 8usize];
     ["Offset of field: jl_typename_t::name"][::std::mem::offset_of!(jl_typename_t, name) - 0usize];
     ["Offset of field: jl_typename_t::module"]
@@ -1915,6 +1917,8 @@ const _: () = {
         [::std::mem::offset_of!(jl_typename_t, max_methods) - 101usize];
     ["Offset of field: jl_typename_t::constprop_heustic"]
         [::std::mem::offset_of!(jl_typename_t, constprop_heustic) - 102usize];
+    ["Offset of field: jl_typename_t::hiddenptrfields"]
+        [::std::mem::offset_of!(jl_typename_t, hiddenptrfields) - 104usize];
 };
 impl jl_typename_t {
     #[inline]
@@ -2082,7 +2086,9 @@ pub struct jl_datatype_layout_t {
     pub size: u32,
     pub nfields: u32,
     pub npointers: u32,
+    pub nhidden_pointers: u32,
     pub first_ptr: i32,
+    pub first_hidden_ptr: i32,
     pub alignment: u16,
     pub flags: jl_datatype_layout_t__bindgen_ty_1,
 }
@@ -2337,20 +2343,24 @@ impl jl_datatype_layout_t__bindgen_ty_1 {
 }
 #[allow(clippy::unnecessary_operation, clippy::identity_op)]
 const _: () = {
-    ["Size of jl_datatype_layout_t"][::std::mem::size_of::<jl_datatype_layout_t>() - 20usize];
+    ["Size of jl_datatype_layout_t"][::std::mem::size_of::<jl_datatype_layout_t>() - 28usize];
     ["Alignment of jl_datatype_layout_t"][::std::mem::align_of::<jl_datatype_layout_t>() - 4usize];
     ["Offset of field: jl_datatype_layout_t::size"]
         [::std::mem::offset_of!(jl_datatype_layout_t, size) - 0usize];
     ["Offset of field: jl_datatype_layout_t::nfields"]
         [::std::mem::offset_of!(jl_datatype_layout_t, nfields) - 4usize];
     ["Offset of field: jl_datatype_layout_t::npointers"]
         [::std::mem::offset_of!(jl_datatype_layout_t, npointers) - 8usize];
+    ["Offset of field: jl_datatype_layout_t::nhidden_pointers"]
+        [::std::mem::offset_of!(jl_datatype_layout_t, nhidden_pointers) - 12usize];
     ["Offset of field: jl_datatype_layout_t::first_ptr"]
-        [::std::mem::offset_of!(jl_datatype_layout_t, first_ptr) - 12usize];
+        [::std::mem::offset_of!(jl_datatype_layout_t, first_ptr) - 16usize];
+    ["Offset of field: jl_datatype_layout_t::first_hidden_ptr"]
+        [::std::mem::offset_of!(jl_datatype_layout_t, first_hidden_ptr) - 20usize];
     ["Offset of field: jl_datatype_layout_t::alignment"]
-        [::std::mem::offset_of!(jl_datatype_layout_t, alignment) - 16usize];
+        [::std::mem::offset_of!(jl_datatype_layout_t, alignment) - 24usize];
     ["Offset of field: jl_datatype_layout_t::flags"]
-        [::std::mem::offset_of!(jl_datatype_layout_t, flags) - 18usize];
+        [::std::mem::offset_of!(jl_datatype_layout_t, flags) - 26usize];
 };
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
@@ -3013,6 +3023,12 @@ const _: () = {
 };
 #[repr(C)]
 #[derive(Debug, Copy, Clone)]
+pub struct pinned_ref<T> {
+    pub _phantom_0: ::std::marker::PhantomData<::std::cell::UnsafeCell<T>>,
+    pub ptr: *mut T,
+}
+#[repr(C)]
+#[derive(Debug, Copy, Clone)]
 pub struct _jl_handler_t {
     pub eh_ctx: sigjmp_buf,
     pub gcstack: *mut jl_gcframe_t,
@@ -3056,7 +3072,7 @@ pub struct _jl_bt_element_t {
 #[derive(Copy, Clone)]
 pub union _jl_bt_element_t__bindgen_ty_1 {
     pub uintptr: usize,
-    pub jlvalue: *mut jl_value_t,
+    pub jlvalue: pinned_ref<jl_value_t>,
 }
 #[allow(clippy::unnecessary_operation, clippy::identity_op)]
 const _: () = {
@@ -3425,3 +3441,10 @@ const _: () = {
     ["Align of template specialization: std_atomic_open0_intptr_t_close0"]
         [::std::mem::align_of::<std_atomic<isize>>() - 8usize];
 };
+#[allow(clippy::unnecessary_operation, clippy::identity_op)]
+const _: () = {
+    ["Size of template specialization: pinned_ref_open0_jl_value_t_close0"]
+        [::std::mem::size_of::<pinned_ref<jl_value_t>>() - 8usize];
+    ["Align of template specialization: pinned_ref_open0_jl_value_t_close0"]
+        [::std::mem::align_of::<pinned_ref<jl_value_t>>() - 8usize];
+};

mmtk/src/lib.rs

@@ -134,6 +134,7 @@ extern "C" {
     pub fn jl_log_pinning_event(pinned_object: Address, filename: *const i8, lineno: i32);
     pub fn jl_gc_log();
     pub static jl_true: *mut crate::julia_types::jl_value_t;
+    pub fn jl_symbol(name: *const i8) -> *mut crate::julia_types::jl_sym_t;
 }
 
 #[macro_export]

mmtk/src/object_model.rs

@@ -54,8 +54,9 @@ pub(crate) const LOS_METADATA_SPEC: VMLocalLOSMarkNurserySpec =
 
 impl ObjectModel<JuliaVM> for VMObjectModel {
     const GLOBAL_LOG_BIT_SPEC: VMGlobalLogBitSpec = LOGGING_SIDE_METADATA_SPEC;
+    // See https://github.com/mmtk/mmtk-core/issues/1331
     const LOCAL_FORWARDING_POINTER_SPEC: VMLocalForwardingPointerSpec =
-        VMLocalForwardingPointerSpec::in_header(-64);
+        VMLocalForwardingPointerSpec::in_header(0);
 
     const LOCAL_PINNING_BIT_SPEC: VMLocalPinningBitSpec = LOCAL_PINNING_METADATA_BITS_SPEC;
     const LOCAL_FORWARDING_BITS_SPEC: VMLocalForwardingBitsSpec =
@@ -66,6 +67,8 @@ impl ObjectModel<JuliaVM> for VMObjectModel {
     const UNIFIED_OBJECT_REFERENCE_ADDRESS: bool = false;
     const OBJECT_REF_OFFSET_LOWER_BOUND: isize = 0;
 
+    const NEED_VO_BITS_DURING_TRACING: bool = true;
+
     fn copy(
         from: ObjectReference,
         semantics: CopySemantics,
@@ -126,7 +129,7 @@ impl ObjectModel<JuliaVM> for VMObjectModel {
             debug_assert!(test_hash_state(to_obj, UNHASHED));
             to_obj
         } else if test_hash_state(from, HASHED) {
-            info!(
+            debug!(
                 "Moving a hashed object {} with size = {}. New size = {}",
                 from, cur_bytes, new_bytes
             );
@@ -141,7 +144,7 @@ impl ObjectModel<JuliaVM> for VMObjectModel {
             unsafe {
                 dst.store::<usize>(hash);
             }
-            info!("Store hash {:x} into {}", hash, dst);
+            debug!("Store hash {:x} into {}", hash, dst);
             dst += STORED_HASH_BYTES;
 
             // Copy the object
@@ -153,13 +156,13 @@ impl ObjectModel<JuliaVM> for VMObjectModel {
                 unsafe { ObjectReference::from_raw_address_unchecked(dst + header_offset) };
             copy_context.post_copy(to_obj, new_bytes, semantics);
 
-            info!("old object {}, new objectt {}", from, to_obj);
+            debug!("old object {}, new objectt {}", from, to_obj);
 
             // set_hash_state(from, UNHASHED);
             set_hash_state(to_obj, HASHED_AND_MOVED);
             to_obj
         } else if test_hash_state(from, HASHED_AND_MOVED) {
-            info!("Moving a hashed+moved object {}", from);
+            debug!("Moving a hashed+moved object {}", from);
             debug_assert_eq!(cur_bytes, new_bytes);
             debug_assert_eq!(from.to_raw_address(), from_start + 16usize);
             debug_assert_eq!(header_offset, 16);
@@ -203,12 +206,15 @@ impl ObjectModel<JuliaVM> for VMObjectModel {
         }
 
         // zero from_obj (for debugging purposes)
+        // We cannot zero from_obj. We use find_object_from_internal_pointer during trace.
+        // So we will need to access from_obj after it is being moved to calculate its size.
+        // We cannot zero from_obj. See https://github.com/mmtk/mmtk-core/issues/1331
         #[cfg(debug_assertions)]
         {
             use atomic::Ordering;
-            unsafe {
-                libc::memset(from_start.to_mut_ptr(), 0, cur_bytes);
-            }
+            // unsafe {
+            //     libc::memset(from_start.to_mut_ptr(), 0, cur_bytes);
+            // }
 
             Self::LOCAL_FORWARDING_BITS_SPEC.store_atomic::<JuliaVM, u8>(
                 from,