Embed mutator in _jl_tls_states_t

src/gc.c

@@ -3501,6 +3501,11 @@ void jl_init_thread_heap(jl_ptls_t ptls)
  jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval);
 }
 
+void jl_deinit_thread_heap(jl_ptls_t ptls)
+{
+ // Do nothing
+}
+
 // System-wide initializations
 void jl_gc_init(void)
 {

src/julia.h

@@ -2391,7 +2391,7 @@ STATIC_INLINE void mmtk_gc_wb_full(const void *parent, const void *ptr) JL_NOTSA
 {
  jl_task_t *ct = jl_current_task;
  jl_ptls_t ptls = ct->ptls;
- mmtk_object_reference_write_post(ptls->mmtk_mutator_ptr, parent, ptr);
+ mmtk_object_reference_write_post(&ptls->mmtk_mutator, parent, ptr);
 }
 
 // Inlined fastpath
@@ -2405,7 +2405,7 @@ STATIC_INLINE void mmtk_gc_wb_fast(const void *parent, const void *ptr) JL_NOTSA
  if (((byte_val >> shift) & 1) == 1) {
  jl_task_t *ct = jl_current_task;
  jl_ptls_t ptls = ct->ptls;
- mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, ptr);
+ mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, ptr);
  }
  }
 }

src/julia_internal.h

@@ -550,7 +550,7 @@ STATIC_INLINE jl_value_t *jl_gc_permobj(size_t sz, void *ty) JL_NOTSAFEPOINT
  o->header = tag | GC_OLD_MARKED;
 #ifdef MMTK_GC
  jl_ptls_t ptls = jl_current_task->ptls;
- mmtk_post_alloc(ptls->mmtk_mutator_ptr, jl_valueof(o), allocsz, 1);
+ mmtk_post_alloc(&ptls->mmtk_mutator, jl_valueof(o), allocsz, 1);
 #endif
  return jl_valueof(o);
 }
@@ -918,6 +918,7 @@ void jl_init_serializer(void);
 void jl_gc_init(void);
 void jl_init_uv(void);
 void jl_init_thread_heap(jl_ptls_t ptls) JL_NOTSAFEPOINT;
+void jl_deinit_thread_heap(jl_ptls_t ptls) JL_NOTSAFEPOINT;
 void jl_init_int32_int64_cache(void);
 JL_DLLEXPORT void jl_init_options(void);
 

src/julia_threads.h

@@ -282,9 +282,7 @@ typedef struct _jl_tls_states_t {
  )
 
 #ifdef MMTK_GC
- MMTkMutatorContext* mmtk_mutator_ptr;
- void* cursor;
- void* limit;
+ MMTkMutatorContext mmtk_mutator;
 #endif
 
  // some hidden state (usually just because we don't have the type's size declaration)

src/llvm-final-gc-lowering.cpp

@@ -280,17 +280,23 @@ Value *FinalLowerGC::lowerGCAllocBytes(CallInst *target, Function &F)
  #else // MMTK_GC
  auto pool_osize_i32 = ConstantInt::get(Type::getInt32Ty(F.getContext()), osize);
  auto pool_osize = ConstantInt::get(Type::getInt64Ty(F.getContext()), osize);
- auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, cursor));
- auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), offsetof(jl_tls_states_t, limit));
+
+ // Assuming we use the first immix allocator.
+ // FIXME: We should get the allocator index and type from MMTk.
+ auto allocator_offset = offsetof(jl_tls_states_t, mmtk_mutator) + offsetof(MMTkMutatorContext, allocators) + offsetof(Allocators, immix);
+
+ auto cursor_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), allocator_offset + offsetof(ImmixAllocator, cursor));
+ auto limit_pos = ConstantInt::get(Type::getInt64Ty(target->getContext()), allocator_offset + offsetof(ImmixAllocator, limit));
 
  auto cursor_tls_i8 = builder.CreateGEP(Type::getInt8Ty(target->getContext()), ptls, cursor_pos);
  auto cursor_ptr = builder.CreateBitCast(cursor_tls_i8, PointerType::get(Type::getInt64Ty(target->getContext()), 0), "cursor_ptr");
  auto cursor = builder.CreateLoad(Type::getInt64Ty(target->getContext()), cursor_ptr, "cursor");
 
-
+ // offset = 8
  auto delta_offset = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), ConstantInt::get(Type::getInt64Ty(target->getContext()), 8));
  auto delta_cursor = builder.CreateNSWSub(ConstantInt::get(Type::getInt64Ty(target->getContext()), 0), cursor);
  auto delta_op = builder.CreateNSWAdd(delta_offset, delta_cursor);
+ // alignment 16 (15 = 16 - 1)
  auto delta = builder.CreateAnd(delta_op, ConstantInt::get(Type::getInt64Ty(target->getContext()), 15), "delta");
  auto result = builder.CreateNSWAdd(cursor, delta, "result");
 

src/mmtk-gc.c

@@ -266,8 +266,17 @@ void jl_init_thread_heap(jl_ptls_t ptls)
  memset(&ptls->gc_num, 0, sizeof(ptls->gc_num));
  jl_atomic_store_relaxed(&ptls->gc_num.allocd, -(int64_t)gc_num.interval);
 
+ // Create mutator
  MMTk_Mutator mmtk_mutator = mmtk_bind_mutator((void *)ptls, ptls->tid);
- ptls->mmtk_mutator_ptr = ((MMTkMutatorContext*)mmtk_mutator);
+ // Copy the mutator to the thread local storage
+ memcpy(&ptls->mmtk_mutator, mmtk_mutator, sizeof(MMTkMutatorContext));
+ // Call post_bind to maintain a list of active mutators and to reclaim the old mutator (which is no longer needed)
+ mmtk_post_bind_mutator(&ptls->mmtk_mutator, mmtk_mutator);
+}
+
+void jl_deinit_thread_heap(jl_ptls_t ptls)
+{
+ mmtk_destroy_mutator(&ptls->mmtk_mutator);
 }
 
 // System-wide initialization
@@ -506,7 +515,7 @@ void disable_collection(void)
 JL_DLLEXPORT void jl_gc_array_ptr_copy(jl_array_t *dest, void **dest_p, jl_array_t *src, void **src_p, ssize_t n) JL_NOTSAFEPOINT
 {
  jl_ptls_t ptls = jl_current_task->ptls;
- mmtk_memory_region_copy(ptls->mmtk_mutator_ptr, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n);
+ mmtk_memory_region_copy(&ptls->mmtk_mutator, jl_array_owner(src), src_p, jl_array_owner(dest), dest_p, n);
 }
 
 // No inline write barrier -- only used for debugging
@@ -524,20 +533,20 @@ JL_DLLEXPORT void jl_gc_wb1_slow(const void *parent) JL_NOTSAFEPOINT
 {
  jl_task_t *ct = jl_current_task;
  jl_ptls_t ptls = ct->ptls;
- mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, (const void*) 0);
+ mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, (const void*) 0);
 }
 
 JL_DLLEXPORT void jl_gc_wb2_slow(const void *parent, const void* ptr) JL_NOTSAFEPOINT
 {
  jl_task_t *ct = jl_current_task;
  jl_ptls_t ptls = ct->ptls;
- mmtk_object_reference_write_slow(ptls->mmtk_mutator_ptr, parent, ptr);
+ mmtk_object_reference_write_slow(&ptls->mmtk_mutator, parent, ptr);
 }
 
 void *jl_gc_perm_alloc_nolock(size_t sz, int zero, unsigned align, unsigned offset)
 {
  jl_ptls_t ptls = jl_current_task->ptls;
- void* addr = mmtk_alloc(ptls->mmtk_mutator_ptr, sz, align, offset, 1);
+ void* addr = mmtk_alloc(&ptls->mmtk_mutator, sz, align, offset, 1);
  return addr;
 }
 

src/symbol.c

@@ -41,7 +41,7 @@ static jl_sym_t *mk_symbol(const char *str, size_t len) JL_NOTSAFEPOINT
  jl_set_typetagof(sym, jl_symbol_tag, GC_OLD_MARKED);
 #ifdef MMTK_GC
  jl_ptls_t ptls = jl_current_task->ptls;
- mmtk_post_alloc(ptls->mmtk_mutator_ptr, jl_valueof(tag), nb, 1);
+ mmtk_post_alloc(&ptls->mmtk_mutator, jl_valueof(tag), nb, 1);
 #endif
  jl_atomic_store_relaxed(&sym->left, NULL);
  jl_atomic_store_relaxed(&sym->right, NULL);

src/threading.c

@@ -478,6 +478,9 @@ static void jl_delete_thread(void *value) JL_NOTSAFEPOINT_ENTER
 #else
  pthread_mutex_unlock(&in_signal_lock);
 #endif
+
+ jl_deinit_thread_heap(ptls);
+
  // then park in safe-region
  (void)jl_gc_safe_enter(ptls);
 }