Embed mutator in _jl_tls_states_t (#64)

This PR updates the bindings to work with
mmtk/julia#16.
* Update the layout of `MMTkMutatorContext`
* Do not include `gc.h` in `mmtk.h`
* Properly maintain the `MUTATORS` table. Add `mmtk_post_bind_mutator`

julia/mmtk_julia.c

@@ -1,5 +1,6 @@
 #include "mmtk_julia.h"
 #include "mmtk.h"
+#include "mmtkMutator.h"
 #include <stdbool.h>
 #include <stddef.h>
 #include "gc.h"
@@ -36,23 +37,6 @@ JL_DLLEXPORT void (jl_mmtk_harness_end)(void)
  mmtk_harness_end();
 }
 
-STATIC_INLINE void* alloc_default_object(jl_ptls_t ptls, size_t size, int offset) {
- int64_t delta = (-offset -(int64_t)(ptls->cursor)) & 15; // aligned to 16
- uint64_t aligned_addr = (uint64_t)ptls->cursor + delta;
-
- if(__unlikely(aligned_addr+size > (uint64_t)ptls->limit)) {
- jl_ptls_t ptls2 = jl_current_task->ptls;
- ptls2->mmtk_mutator_ptr->allocators.immix[0].cursor = ptls2->cursor;
- void* res = mmtk_alloc(ptls2->mmtk_mutator_ptr, size, 16, offset, 0);
- ptls2->cursor = ptls2->mmtk_mutator_ptr->allocators.immix[0].cursor;
- ptls2->limit = ptls2->mmtk_mutator_ptr->allocators.immix[0].limit;
- return res;
- } else {
- ptls->cursor = (void*) (aligned_addr+size);
- return (void*) aligned_addr;
- }
-}
-
 JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int pool_offset,
  int osize, void *ty)
 {
@@ -62,16 +46,16 @@ JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_default(jl_ptls_t ptls, int pool_offse
  jl_value_t *v;
  if ((uintptr_t)ty != jl_buff_tag) {
  // v needs to be 16 byte aligned, therefore v_tagged needs to be offset accordingly to consider the size of header
- jl_taggedvalue_t *v_tagged = (jl_taggedvalue_t *)mmtk_alloc(ptls->mmtk_mutator_ptr, osize, 16, sizeof(jl_taggedvalue_t), 0); // (jl_taggedvalue_t *) alloc_default_object(ptls, osize, sizeof(jl_taggedvalue_t));
+ jl_taggedvalue_t *v_tagged = (jl_taggedvalue_t *)mmtk_alloc(&ptls->mmtk_mutator, osize, 16, sizeof(jl_taggedvalue_t), 0);
  v = jl_valueof(v_tagged);
- mmtk_post_alloc(ptls->mmtk_mutator_ptr, v, osize, 0);
+ mmtk_post_alloc(&ptls->mmtk_mutator, v, osize, 0);
  } else {
  // allocating an extra word to store the size of buffer objects
- jl_taggedvalue_t *v_tagged = (jl_taggedvalue_t *)mmtk_alloc(ptls->mmtk_mutator_ptr, osize + sizeof(jl_taggedvalue_t), 16, 0, 0); // (jl_taggedvalue_t *) alloc_default_object(ptls, osize + sizeof(jl_taggedvalue_t), 0);
+ jl_taggedvalue_t *v_tagged = (jl_taggedvalue_t *)mmtk_alloc(&ptls->mmtk_mutator, osize + sizeof(jl_taggedvalue_t), 16, 0, 0);
  jl_value_t* v_tagged_aligned = ((jl_value_t*)((char*)(v_tagged) + sizeof(jl_taggedvalue_t)));
  v = jl_valueof(v_tagged_aligned);
  mmtk_store_obj_size_c(v, osize + sizeof(jl_taggedvalue_t));
- mmtk_post_alloc(ptls->mmtk_mutator_ptr, v, osize + sizeof(jl_taggedvalue_t), 0);
+ mmtk_post_alloc(&ptls->mmtk_mutator, v, osize + sizeof(jl_taggedvalue_t), 0);
  }
 
  ptls->gc_num.allocd += osize;
@@ -95,7 +79,7 @@ JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_big(jl_ptls_t ptls, size_t sz)
  jl_throw(jl_memory_exception);
  }
 
- bigval_t *v = (bigval_t*)mmtk_alloc_large(ptls->mmtk_mutator_ptr, allocsz, JL_CACHE_BYTE_ALIGNMENT, 0, 2);
+ bigval_t *v = (bigval_t*)mmtk_alloc_large(&ptls->mmtk_mutator, allocsz, JL_CACHE_BYTE_ALIGNMENT, 0, 2);
 
  if (v == NULL) {
  assert(0 && "Allocation failed");
@@ -107,7 +91,7 @@ JL_DLLEXPORT jl_value_t *jl_mmtk_gc_alloc_big(jl_ptls_t ptls, size_t sz)
  ptls->gc_num.bigalloc++;
 
  jl_value_t *result = jl_valueof(&v->header);
- mmtk_post_alloc(ptls->mmtk_mutator_ptr, result, allocsz, 2);
+ mmtk_post_alloc(&ptls->mmtk_mutator, result, allocsz, 2);
 
  return result;
 }
@@ -147,7 +131,7 @@ static void mmtk_sweep_malloced_arrays(void) JL_NOTSAFEPOINT
 extern void mark_metadata_scanned(jl_value_t* obj);
 extern int8_t check_metadata_scanned(jl_value_t* obj);
 
-int8_t object_has_been_scanned(jl_value_t* obj)
+int8_t object_has_been_scanned(void* obj)
 {
  uintptr_t tag = (uintptr_t)jl_typeof(obj);
  jl_datatype_t *vt = (jl_datatype_t*)tag;
@@ -164,7 +148,7 @@ int8_t object_has_been_scanned(jl_value_t* obj)
  return 0;
 }
 
-void mark_object_as_scanned(jl_value_t* obj) {
+void mark_object_as_scanned(void* obj) {
  if (jl_object_in_image((jl_value_t *)obj)) {
  jl_taggedvalue_t *o = jl_astaggedvalue(obj);
  o->bits.gc = GC_MARKED;
@@ -185,8 +169,9 @@ void mmtk_exit_from_safepoint(int8_t old_state) {
 // it will block until GC is done
 // that thread simply exits from block_for_gc without executing finalizers
 // when executing finalizers do not let another thread do GC (set a variable such that while that variable is true, no GC can be done)
-int8_t set_gc_initial_state(jl_ptls_t ptls) 
+int8_t set_gc_initial_state(void* ptls_raw) 
 {
+ jl_ptls_t ptls = (jl_ptls_t) ptls_raw;
  int8_t old_state = jl_atomic_load_relaxed(&((jl_ptls_t)ptls)->gc_state);
  jl_atomic_store_release(&((jl_ptls_t)ptls)->gc_state, JL_GC_STATE_WAITING);
  if (!jl_safepoint_start_gc()) {
@@ -231,8 +216,9 @@ void wait_for_the_world(void)
  }
 }
 
-size_t get_lo_size(jl_value_t* obj) 
+size_t get_lo_size(void* obj_raw) 
 {
+ jl_value_t* obj = (jl_value_t*) obj_raw;
  jl_taggedvalue_t *v = jl_astaggedvalue(obj);
  // bigval_header: but we cannot access the function here. So use container_of instead.
  bigval_t* hdr = container_of(v, bigval_t, header);
@@ -246,18 +232,12 @@ void set_jl_last_err(int e)
 
 int get_jl_last_err(void) 
 {
- gc_n_threads = jl_atomic_load_acquire(&jl_n_threads);
- gc_all_tls_states = jl_atomic_load_relaxed(&jl_all_tls_states);
- for (int t_i = 0; t_i < gc_n_threads; t_i++) {
- jl_ptls_t ptls = gc_all_tls_states[t_i];
- ptls->cursor = 0;
- ptls->limit = 0;
- }
  return errno;
 }
 
-void* get_obj_start_ref(jl_value_t* obj) 
+void* get_obj_start_ref(void* obj_raw) 
 {
+ jl_value_t* obj = (jl_value_t*) obj_raw;
  uintptr_t tag = (uintptr_t)jl_typeof(obj);
  jl_datatype_t *vt = (jl_datatype_t*)tag;
  void* obj_start_ref; 
@@ -271,8 +251,9 @@ void* get_obj_start_ref(jl_value_t* obj)
  return obj_start_ref;
 }
 
-size_t get_so_size(jl_value_t* obj) 
+size_t get_so_size(void* obj_raw) 
 {
+ jl_value_t* obj = (jl_value_t*) obj_raw;
  uintptr_t tag = (uintptr_t)jl_typeof(obj);
  jl_datatype_t *vt = (jl_datatype_t*)tag;
 
@@ -392,8 +373,10 @@ size_t get_so_size(jl_value_t* obj)
  return 0;
 }
 
-void run_finalizer_function(jl_value_t *o, jl_value_t *ff, bool is_ptr)
+void run_finalizer_function(void *o_raw, void *ff_raw, bool is_ptr)
 {
+ jl_value_t *o = (jl_value_t*) o_raw;
+ jl_value_t *ff = (jl_value_t*) ff_raw;
  if (is_ptr) {
  run_finalizer(jl_current_task, (jl_value_t *)(((uintptr_t)o) | 1), (jl_value_t *)ff);
  } else {
@@ -423,7 +406,7 @@ void mmtk_jl_run_pending_finalizers(void* ptls) {
  }
 }
 
-void mmtk_jl_run_finalizers(jl_ptls_t ptls) {
+void mmtk_jl_run_finalizers(void* ptls) {
  // Only disable finalizers on current thread
  // Doing this on all threads is racy (it's impossible to check
  // or wait for finalizers on other threads without dead lock).
@@ -663,7 +646,7 @@ static void jl_gc_queue_remset_mmtk(jl_ptls_t ptls2)
  }
 }
 
-void calculate_roots(jl_ptls_t ptls)
+void calculate_roots(void* ptls_raw)
 {
  for (int t_i = 0; t_i < gc_n_threads; t_i++)
  gc_premark(gc_all_tls_states[t_i]);
@@ -682,8 +665,8 @@ void calculate_roots(jl_ptls_t ptls)
  queue_roots();
 }
 
-JL_DLLEXPORT void scan_julia_exc_obj(jl_task_t* obj, closure_pointer closure, ProcessEdgeFn process_edge) {
- jl_task_t *ta = (jl_task_t*)obj;
+JL_DLLEXPORT void scan_julia_exc_obj(void* obj_raw, closure_pointer closure, ProcessEdgeFn process_edge) {
+ jl_task_t *ta = (jl_task_t*)obj_raw;
 
  if (ta->excstack) { // inlining label `excstack` from mark_loop
  // if it is not managed by MMTk, nothing needs to be done because the object does not need to be scanned
@@ -745,8 +728,9 @@ const bool PRINT_OBJ_TYPE = false;
  * directly (not an edge), specifying whether to scan the object or not; and only scan the object 
  * (necessary for boot image / non-MMTk objects)
 **/
-JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, ProcessEdgeFn process_edge, ProcessOffsetEdgeFn process_offset_edge) 
+JL_DLLEXPORT void scan_julia_obj(void* obj_raw, closure_pointer closure, ProcessEdgeFn process_edge, ProcessOffsetEdgeFn process_offset_edge) 
 {
+ jl_value_t* obj = (jl_value_t*) obj_raw;
  uintptr_t tag = (uintptr_t)jl_typeof(obj);
  jl_datatype_t *vt = (jl_datatype_t*)tag; // type of obj
 
@@ -962,6 +946,10 @@ void update_gc_time(uint64_t inc) {
  gc_num.total_time += inc;
 }
 
+uintptr_t get_abi_structs_checksum_c(void) {
+ return sizeof(MMTkMutatorContext);
+}
+
 Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
  .scan_julia_obj = scan_julia_obj,
  .scan_julia_exc_obj = scan_julia_exc_obj,
@@ -986,4 +974,5 @@ Julia_Upcalls mmtk_upcalls = (Julia_Upcalls) {
  .exit_from_safepoint = mmtk_exit_from_safepoint,
  .jl_hrtime = jl_hrtime,
  .update_gc_time = update_gc_time,
+ .get_abi_structs_checksum_c = get_abi_structs_checksum_c,
 };

julia/mmtk_julia.h

@@ -3,32 +3,16 @@
 
 extern Julia_Upcalls mmtk_upcalls;
 
-void calculate_roots(jl_ptls_t ptls);
-
-void run_finalizer_function(jl_value_t *o, jl_value_t *ff, bool is_ptr);
-
 int get_jl_last_err(void);
 
 void set_jl_last_err(int e);
 
-size_t get_lo_size(jl_value_t* obj);
-
-int8_t set_gc_initial_state(jl_ptls_t ptls);
-
 void set_gc_final_state(int8_t old_state);
 
 int set_gc_running_state(jl_ptls_t ptls);
 
 void set_gc_old_state(int8_t old_state);
 
-void mark_object_as_scanned(jl_value_t* obj);
-
-int8_t object_has_been_scanned(jl_value_t* obj);
-
 void mmtk_jl_gc_run_all_finalizers(void);
 
-void mmtk_jl_run_finalizers(jl_ptls_t tls);
-
 void mmtk_jl_run_pending_finalizers(void* tls);
-
-JL_DLLEXPORT void scan_julia_obj(jl_value_t* obj, closure_pointer closure, ProcessEdgeFn process_edge, ProcessOffsetEdgeFn process_offset_edge);

mmtk/Cargo.toml

@@ -10,7 +10,7 @@ edition = "2018"
 [package.metadata.julia]
 # Our CI matches the following line and extract mmtk/julia. If this line is updated, please check ci yaml files and make sure it works.
 julia_repo = "https://github.com/mmtk/julia.git"
-julia_version = "0eeb64b0191b08b0ce7b59a66dea6139db60dbd2"
+julia_version = "98a66ba3c0925ea21bfe051a191210eeae7df0f2"
 
 [lib]
 crate-type = ["staticlib", "rlib", "dylib"]

mmtk/api/mmtk.h

@@ -4,7 +4,6 @@
 #include <stdbool.h>
 #include <stddef.h>
 #include <stdint.h>
-#include "gc.h"
 
 #ifdef __cplusplus
 extern "C" {
@@ -24,7 +23,7 @@ typedef void (*ProcessOffsetEdgeFn)(closure_pointer closure, void* slot, int off
  * Allocation
  */
 extern MMTk_Mutator mmtk_bind_mutator(void *tls, int tid);
-extern void mmtk_add_mutator_ref(void* mutator_ref);
+extern void mmtk_post_bind_mutator(MMTk_Mutator mutator, MMTk_Mutator original_mutator);
 extern void mmtk_destroy_mutator(MMTk_Mutator mutator);
 
 extern void* mmtk_alloc(MMTk_Mutator mutator, size_t size,
@@ -78,29 +77,30 @@ extern const void* MMTK_SIDE_LOG_BIT_BASE_ADDRESS;
 // * int is 4 bytes
 // * size_t is 8 bytes
 typedef struct {
- void (* scan_julia_obj) (jl_value_t* obj, closure_pointer closure, ProcessEdgeFn process_edge, ProcessOffsetEdgeFn process_offset_edge);
- void (* scan_julia_exc_obj) (jl_task_t* obj, closure_pointer closure, ProcessEdgeFn process_edge);
+ void (* scan_julia_obj) (void* obj, closure_pointer closure, ProcessEdgeFn process_edge, ProcessOffsetEdgeFn process_offset_edge);
+ void (* scan_julia_exc_obj) (void* obj, closure_pointer closure, ProcessEdgeFn process_edge);
  void* (* get_stackbase) (int16_t tid);
- void (* calculate_roots) (jl_ptls_t tls);
- void (* run_finalizer_function) (jl_value_t* obj, jl_value_t* function, bool is_ptr);
+ void (* calculate_roots) (void* tls);
+ void (* run_finalizer_function) (void* obj, void* function, bool is_ptr);
  int (* get_jl_last_err) (void);
  void (* set_jl_last_err) (int e);
- size_t (* get_lo_size) (jl_value_t* obj);
- size_t (* get_so_size) (jl_value_t* obj);
- void* (* get_obj_start_ref) (jl_value_t* obj);
+ size_t (* get_lo_size) (void* obj);
+ size_t (* get_so_size) (void* obj);
+ void* (* get_obj_start_ref) (void* obj);
  void (* wait_for_the_world) (void);
- int8_t (* set_gc_initial_state) (jl_ptls_t tls);
+ int8_t (* set_gc_initial_state) (void* tls);
  void (* set_gc_final_state) (int8_t old_state);
  void (* set_gc_old_state) (int8_t old_state);
- void (* mmtk_jl_run_finalizers) (jl_ptls_t tls);
+ void (* mmtk_jl_run_finalizers) (void* tls);
  void (* jl_throw_out_of_memory_error) (void);
- void (* mark_object_as_scanned) (jl_value_t* obj);
- int8_t (* object_has_been_scanned) (jl_value_t* obj);
+ void (* mark_object_as_scanned) (void* obj);
+ int8_t (* object_has_been_scanned) (void* obj);
  void (* sweep_malloced_array) (void);
  void (* wait_in_a_safepoint) (void);
  void (* exit_from_safepoint) (int8_t old_state);
  uint64_t (* jl_hrtime) (void);
  void (* update_gc_time) (uint64_t);
+ uintptr_t (* get_abi_structs_checksum_c) (void);
 } Julia_Upcalls;
 
 /**

mmtk/api/mmtkMutator.h

@@ -15,11 +15,12 @@ typedef struct {
 } RustDynPtr;
 
 // These constants should match the constants defind in mmtk::util::alloc::allocators
-// const int MAX_BUMP_ALLOCATORS = 6;
-// const int MAX_LARGE_OBJECT_ALLOCATORS = 2;
-// const int MAX_MALLOC_ALLOCATORS = 1;
-// const int MAX_IMMIX_ALLOCATORS = 1;
-// const int MAX_MARK_COMPACT_ALLOCATORS = 1;
+#define MAX_BUMP_ALLOCATORS 6
+#define MAX_LARGE_OBJECT_ALLOCATORS 2
+#define MAX_MALLOC_ALLOCATORS 1
+#define MAX_IMMIX_ALLOCATORS 1
+#define MAX_FREE_LIST_ALLOCATORS 2
+#define MAX_MARK_COMPACT_ALLOCATORS 1
 
 // The following types should have the same layout as the types with the same name in MMTk core (Rust)
 
@@ -51,25 +52,46 @@ typedef struct {
  uint8_t _align[7];
  uint8_t line_opt_tag;
  uintptr_t line_opt;
- uint8_t alloc_slow_for_stress;
 } ImmixAllocator;
 
+typedef struct {
+ void* Address;
+} FLBlock;
+
+typedef struct {
+ FLBlock first;
+ FLBlock last;
+ size_t size;
+ char lock;
+} FLBlockList;
+
+typedef struct {
+ void* tls;
+ void* space;
+ RustDynPtr plan;
+ FLBlockList* available_blocks;
+ FLBlockList* available_blocks_stress;
+ FLBlockList* unswept_blocks;
+ FLBlockList* consumed_blocks;
+} FreeListAllocator;
+
 typedef struct {
  void* tls;
  void* space;
  RustDynPtr plan;
-} MMTkMallocAllocator;
+} MMTkMallocAllocator; // Prefix with MMTk to avoid name clash
 
 typedef struct {
  BumpAllocator bump_allocator;
 } MarkCompactAllocator;
 
 typedef struct {
- BumpAllocator bump_pointer[6];
- LargeObjectAllocator large_object[2];
- MMTkMallocAllocator malloc[1];
- ImmixAllocator immix[1];
- MarkCompactAllocator markcompact[1];
+ BumpAllocator bump_pointer[MAX_BUMP_ALLOCATORS];
+ LargeObjectAllocator large_object[MAX_LARGE_OBJECT_ALLOCATORS];
+ MMTkMallocAllocator malloc[MAX_MALLOC_ALLOCATORS];
+ ImmixAllocator immix[MAX_IMMIX_ALLOCATORS];
+ FreeListAllocator free_list[MAX_FREE_LIST_ALLOCATORS];
+ MarkCompactAllocator markcompact[MAX_MARK_COMPACT_ALLOCATORS];
 } Allocators;
 
 typedef struct {
@@ -81,9 +103,10 @@ typedef struct {
 
 typedef struct {
  Allocators allocators;
- void* barrier;
+ RustDynPtr barrier;
  void* mutator_tls;
  RustDynPtr plan;
  MutatorConfig config;
 } MMTkMutatorContext;
+
 #endif // MMTK_MUTATOR_HPP