Supporting moving immix

.github/scripts/ci-test-patching.sh

@@ -24,6 +24,10 @@ declare -a tests_to_skip=(
  # This test checks GC logging
  '@test occursin("GC: pause", read(tmppath, String))' "$JULIA_PATH/test/misc.jl"
 
+ # These tests check for the number of stock GC threads (which we set to 0 with mmtk)
+ '@test (cpu_threads == 1 ? "1" : string(div(cpu_threads, 2))) ==' "$JULIA_PATH/test/cmdlineargs.jl"
+ '@test read(`$exename --gcthreads=2 -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
+ '@test read(`$exename -e $code`, String) == "2"' "$JULIA_PATH/test/cmdlineargs.jl"
  # This seems to be a regression from upstream when we merge with upstream 43bf2c8.
  # The required string int.jl does not appear in the output even if I test with the stock Julia code.
  # I do not know what is wrong, but at this point, I dont want to spend time on it.

julia/mmtk_julia.c

@@ -21,6 +21,8 @@ extern void mmtk_store_obj_size_c(void* obj, size_t size);
 extern void jl_gc_free_array(jl_array_t *a);
 extern size_t mmtk_get_obj_size(void* obj);
 extern void jl_rng_split(uint64_t to[JL_RNG_SIZE], uint64_t from[JL_RNG_SIZE]);
+extern void _jl_free_stack(jl_ptls_t ptls, void *stkbuf, size_t bufsz);
+extern void free_stack(void *stkbuf, size_t bufsz);
 extern jl_mutex_t finalizers_lock;
 extern void jl_gc_wait_for_the_world(jl_ptls_t* gc_all_tls_states, int gc_n_threads);
 extern void mmtk_block_thread_for_gc(void);
@@ -114,6 +116,9 @@ static void mmtk_sweep_malloced_arrays(void) JL_NOTSAFEPOINT
  continue;
  }
  if (mmtk_is_live_object(ma->a)) {
+ // if the array has been forwarded, the reference needs to be updated
+ jl_array_t *maybe_forwarded = (jl_array_t*)mmtk_get_possibly_forwared(ma->a);
+ ma->a = maybe_forwarded;
  pma = &ma->next;
  }
  else {
@@ -274,6 +279,19 @@ static void add_node_to_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer*
  }
 }
 
+static void add_node_to_tpinned_roots_buffer(RootsWorkClosure* closure, RootsWorkBuffer* buf, size_t* buf_len, void* root) {
+ if (root == NULL)
+ return;
+
+ buf->ptr[*buf_len] = root;
+ *buf_len += 1;
+ if (*buf_len >= buf->cap) {
+ RootsWorkBuffer new_buf = (closure->report_tpinned_nodes_func)(buf->ptr, *buf_len, buf->cap, closure->data, true);
+ *buf = new_buf;
+ *buf_len = 0;
+ }
+}
+
 void scan_vm_specific_roots(RootsWorkClosure* closure)
 {
  // Create a new buf
@@ -302,10 +320,15 @@ void scan_vm_specific_roots(RootsWorkClosure* closure)
  // constants
  add_node_to_roots_buffer(closure, &buf, &len, jl_emptytuple_type);
  add_node_to_roots_buffer(closure, &buf, &len, cmpswap_names);
- add_node_to_roots_buffer(closure, &buf, &len, jl_global_roots_table);
+
+ // jl_global_roots_table must be transitively pinned 
+ RootsWorkBuffer tpinned_buf = (closure->report_tpinned_nodes_func)((void**)0, 0, 0, closure->data, true);
+ size_t tpinned_len = 0;
+ add_node_to_tpinned_roots_buffer(closure, &tpinned_buf, &tpinned_len, jl_global_roots_table);
 
  // Push the result of the work.
  (closure->report_nodes_func)(buf.ptr, len, buf.cap, closure->data, false);
+ (closure->report_tpinned_nodes_func)(tpinned_buf.ptr, tpinned_len, tpinned_buf.cap, closure->data, false);
 }
 
 JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn process_edge) {
@@ -349,6 +372,107 @@ JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, void* closure, ProcessEdgeFn
  }
 }
 
+// number of stacks to always keep available per pool - from gc-stacks.c
+#define MIN_STACK_MAPPINGS_PER_POOL 5
+
+// if data is inlined inside the array object --- to->data needs to be updated when copying the array
+void update_inlined_array(void* from, void* to) {
+ jl_value_t* jl_from = (jl_value_t*) from;
+ jl_value_t* jl_to = (jl_value_t*) to;
+
+ uintptr_t tag_to = (uintptr_t)jl_typeof(jl_to);
+ jl_datatype_t *vt = (jl_datatype_t*)tag_to;
+
+ if(vt->name == jl_array_typename) {
+ jl_array_t *a = (jl_array_t*)jl_from;
+ jl_array_t *b = (jl_array_t*)jl_to;
+ if (a->flags.how == 0 && mmtk_object_is_managed_by_mmtk(a->data)) { // a is inlined (a->data is an mmtk object)
+ size_t offset_of_data = ((size_t)a->data - a->offset*a->elsize) - (size_t)a;
+ if (offset_of_data > 0 && offset_of_data <= ARRAY_INLINE_NBYTES) {
+ b->data = (void*)((size_t) b + offset_of_data);
+ }
+ }
+ }
+}
+
+// modified sweep_stack_pools from gc-stacks.c
+void mmtk_sweep_stack_pools(void)
+{
+ // Stack sweeping algorithm:
+ // // deallocate stacks if we have too many sitting around unused
+ // for (stk in halfof(free_stacks))
+ // free_stack(stk, pool_sz);
+ // // then sweep the task stacks
+ // for (t in live_tasks)
+ // if (!gc-marked(t))
+ // stkbuf = t->stkbuf
+ // bufsz = t->bufsz
+ // if (stkbuf)
+ // push(free_stacks[sz], stkbuf)
+ for (int i = 0; i < jl_n_threads; i++) {
+ jl_ptls_t ptls2 = jl_all_tls_states[i];
+
+ // free half of stacks that remain unused since last sweep
+ for (int p = 0; p < JL_N_STACK_POOLS; p++) {
+ arraylist_t *al = &ptls2->heap.free_stacks[p];
+ size_t n_to_free;
+ if (al->len > MIN_STACK_MAPPINGS_PER_POOL) {
+ n_to_free = al->len / 2;
+ if (n_to_free > (al->len - MIN_STACK_MAPPINGS_PER_POOL))
+ n_to_free = al->len - MIN_STACK_MAPPINGS_PER_POOL;
+ }
+ else {
+ n_to_free = 0;
+ }
+ for (int n = 0; n < n_to_free; n++) {
+ void *stk = arraylist_pop(al);
+ free_stack(stk, pool_sizes[p]);
+ }
+ }
+
+ arraylist_t *live_tasks = &ptls2->heap.live_tasks;
+ size_t n = 0;
+ size_t ndel = 0;
+ size_t l = live_tasks->len;
+ void **lst = live_tasks->items;
+ if (l == 0)
+ continue;
+ while (1) {
+ jl_task_t *t = (jl_task_t*)lst[n];
+ if (mmtk_is_live_object(t)) {
+ jl_task_t *maybe_forwarded = (jl_task_t*)mmtk_get_possibly_forwared(t);
+ live_tasks->items[n] = maybe_forwarded;
+ t = maybe_forwarded;
+ assert(jl_is_task(t));
+ if (t->stkbuf == NULL)
+ ndel++; // jl_release_task_stack called
+ else
+ n++;
+ } else {
+ ndel++;
+ void *stkbuf = t->stkbuf;
+ size_t bufsz = t->bufsz;
+ if (stkbuf) {
+ t->stkbuf = NULL;
+ _jl_free_stack(ptls2, stkbuf, bufsz);
+ }
+#ifdef _COMPILER_TSAN_ENABLED_
+ if (t->ctx.tsan_state) {
+ __tsan_destroy_fiber(t->ctx.tsan_state);
+ t->ctx.tsan_state = NULL;
+ }
+#endif
+ }
+ if (n >= l - ndel)
+ break;
+ void *tmp = lst[n];
+ lst[n] = lst[n + ndel];
+ lst[n + ndel] = tmp;
+ }
+ live_tasks->len -= ndel;
+ }
+}
+
 #define jl_array_data_owner_addr(a) (((jl_value_t**)((char*)a + jl_array_data_owner_offset(jl_array_ndims(a)))))
 
 JL_DLLEXPORT void* get_stackbase(int16_t tid) {
@@ -431,6 +555,7 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
  .mmtk_jl_run_finalizers = mmtk_jl_run_finalizers,
  .jl_throw_out_of_memory_error = jl_throw_out_of_memory_error,
  .sweep_malloced_array = mmtk_sweep_malloced_arrays,
+ .sweep_stack_pools = mmtk_sweep_stack_pools,
  .wait_in_a_safepoint = mmtk_wait_in_a_safepoint,
  .exit_from_safepoint = mmtk_exit_from_safepoint,
  .jl_hrtime = jl_hrtime,
@@ -442,5 +567,6 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
  .arraylist_grow = (void (*)(void*, long unsigned int))arraylist_grow,
  .get_jl_gc_have_pending_finalizers = get_jl_gc_have_pending_finalizers,
  .scan_vm_specific_roots = scan_vm_specific_roots,
+ .update_inlined_array = update_inlined_array,
  .prepare_to_collect = jl_gc_prepare_to_collect,
 };

mmtk/Cargo.toml

@@ -9,8 +9,8 @@ edition = "2018"
 # Metadata for the Julia repository
 [package.metadata.julia]
 # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works.
-julia_repo = "https://github.com/mmtk/julia.git"
-julia_version = "5c406d9bb20d76e2298a6101f171cfac491f651c"
+julia_repo = "https://github.com/udesou/julia.git"
+julia_version = "cfb6d90e1fcba32a088f073ebf71ac6a98f2cf1d"
 
 [lib]
 crate-type = ["cdylib"]
@@ -39,17 +39,19 @@ enum-map = ">=2.1"
 atomic = "0.4.6"
 chrono = "*"
 thread-id = "*"
+memoffset = "*"
 
 # ykstackmaps = { git = "https://github.com/udesou/ykstackmaps.git", branch = "udesou-master", version = "*" }
 
 [features]
-default = ["mmtk/vm_space", "julia_copy_stack"]
+default = ["mmtk/vm_space", "julia_copy_stack", "object_pinning"]
 
 # Plans
 nogc = []
-immix = ["non_moving_immix"]
-stickyimmix = ["non_moving_immix"]
+immix = []
+stickyimmix = ["mmtk/sticky_immix_non_moving_nursery", "mmtk/immix_smaller_block"]
 marksweep = []
+object_pinning = ["mmtk/object_pinning"]
 
 # TODO remove this when we properly support moving
 non_moving_immix = ["mmtk/immix_non_moving", "mmtk/immix_smaller_block"]

mmtk/api/mmtk.h

@@ -22,6 +22,7 @@ typedef struct {
 typedef struct {
  RootsWorkBuffer (*report_edges_func)(void** buf, size_t size, size_t cap, void* data, bool renew);
  RootsWorkBuffer (*report_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew);
+ RootsWorkBuffer (*report_tpinned_nodes_func)(void** buf, size_t size, size_t cap, void* data, bool renew);
  void* data;
 } RootsWorkClosure;
 
@@ -47,6 +48,8 @@ extern bool mmtk_is_mapped_address(void* addr);
 extern int mmtk_object_is_managed_by_mmtk(void* addr);
 extern void mmtk_runtime_panic(void);
 extern void mmtk_unreachable(void);
+extern unsigned char mmtk_pin_object(void* obj);
+extern bool mmtk_is_pinned(void* obj);
 
 extern void mmtk_set_vm_space(void* addr, size_t size);
 extern void mmtk_immortal_region_post_alloc(void* addr, size_t size);
@@ -72,6 +75,7 @@ typedef struct {
  void (* mmtk_jl_run_finalizers) (void* tls);
  void (* jl_throw_out_of_memory_error) (void);
  void (* sweep_malloced_array) (void);
+ void (* sweep_stack_pools) (void);
  void (* wait_in_a_safepoint) (void);
  void (* exit_from_safepoint) (int8_t old_state);
  uint64_t (* jl_hrtime) (void);
@@ -83,6 +87,7 @@ typedef struct {
  void (*arraylist_grow)(void* a, size_t n);
  int* (*get_jl_gc_have_pending_finalizers)(void);
  void (*scan_vm_specific_roots)(RootsWorkClosure* closure);
+ void (*update_inlined_array) (void* from, void* to);
  void (*prepare_to_collect)(void);
 } Julia_Upcalls;
 
@@ -105,6 +110,7 @@ extern void mmtk_run_finalizers_for_obj(void* obj);
 extern void mmtk_run_finalizers(bool at_exit);
 extern void mmtk_gc_poll(void *tls);
 extern void mmtk_julia_copy_stack_check(int copy_stack);
+extern void* mmtk_get_possibly_forwared(void* object);
 
 /**
  * VM Accounting

mmtk/src/api.rs

@@ -508,3 +508,29 @@ pub extern "C" fn mmtk_get_obj_size(obj: ObjectReference) -> usize {
  addr_size.load::<u64>() as usize
  }
 }
+
+#[cfg(feature = "object_pinning")]
+#[no_mangle]
+pub extern "C" fn mmtk_pin_object(object: ObjectReference) -> bool {
+ debug_assert!(
+ mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()),
+ "Object is not managed by mmtk - pinning it via this function isn't supported."
+ );
+ memory_manager::pin_object::<JuliaVM>(object)
+}
+
+#[cfg(feature = "object_pinning")]
+#[no_mangle]
+pub extern "C" fn mmtk_unpin_object(object: ObjectReference) -> bool {
+ debug_assert!(
+ mmtk_object_is_managed_by_mmtk(object.to_raw_address().as_usize()),
+ "Object is not managed by mmtk - pinning it via this function isn't supported."
+ );
+ memory_manager::unpin_object::<JuliaVM>(object)
+}
+
+#[cfg(feature = "object_pinning")]
+#[no_mangle]
+pub extern "C" fn mmtk_is_pinned(object: ObjectReference) -> bool {
+ memory_manager::is_pinned::<JuliaVM>(object)
+}

mmtk/src/edges.rs

@@ -215,6 +215,13 @@ pub struct RootsWorkClosure {
  factory_ptr: *mut libc::c_void,
  renew: bool,
  ) -> RootsWorkBuffer<ObjectReference>,
+ pub report_tpinned_nodes_func: extern "C" fn(
+ buf: *mut ObjectReference,
+ size: usize,
+ cap: usize,
+ factory_ptr: *mut libc::c_void,
+ renew: bool,
+ ) -> RootsWorkBuffer<ObjectReference>,
  pub factory_ptr: *mut libc::c_void,
 }
 
@@ -262,10 +269,31 @@ impl RootsWorkClosure {
  }
  }
 
+ extern "C" fn report_tpinned_nodes<F: RootsWorkFactory<JuliaVMEdge>>(
+ buf: *mut ObjectReference,
+ size: usize,
+ cap: usize,
+ factory_ptr: *mut libc::c_void,
+ renew: bool,
+ ) -> RootsWorkBuffer<ObjectReference> {
+ if !buf.is_null() {
+ let buf = unsafe { Vec::<ObjectReference>::from_raw_parts(buf, size, cap) };
+ let factory: &mut F = unsafe { &mut *(factory_ptr as *mut F) };
+ factory.create_process_tpinning_roots_work(buf);
+ }
+
+ if renew {
+ RootsWorkBuffer::new()
+ } else {
+ RootsWorkBuffer::empty()
+ }
+ }
+
  pub fn from_roots_work_factory<F: RootsWorkFactory<JuliaVMEdge>>(factory: &mut F) -> Self {
  RootsWorkClosure {
  report_edges_func: Self::report_simple_edges::<F>,
  report_nodes_func: Self::report_nodes::<F>,
+ report_tpinned_nodes_func: Self::report_tpinned_nodes::<F>,
  factory_ptr: factory as *mut F as *mut libc::c_void,
  }
  }

mmtk/src/julia_finalizer.rs

@@ -186,6 +186,9 @@ fn mark_finlist<T: ObjectTracer>(list: &mut ArrayListT, start: usize, tracer: &m
  let mut i = start;
  while i < list.len {
  let cur = list.get(i);
+ let cur_i = i;
+ let mut cur_tag: usize = 0;
+
  if cur.is_zero() {
  i += 1;
  continue;
@@ -195,6 +198,7 @@ fn mark_finlist<T: ObjectTracer>(list: &mut ArrayListT, start: usize, tracer: &m
  // Skip next
  i += 1;
  debug_assert!(i < list.len);
+ cur_tag = 1;
  gc_ptr_clear_tag(cur, 1)
  } else {
  ObjectReference::from_raw_address(cur)
@@ -205,10 +209,11 @@ fn mark_finlist<T: ObjectTracer>(list: &mut ArrayListT, start: usize, tracer: &m
  }
 
  let traced = tracer.trace_object(new_obj);
- debug_assert_eq!(
- traced, new_obj,
- "Object is moved -- we need to save the new object back to the finalizer list"
- );
+ // if object has moved, update the list applying the tag
+ list.set(cur_i, unsafe {
+ Address::from_usize(traced.to_raw_address() | cur_tag)
+ });
+
  i += 1;
  }
 }