From b893e026cce4d8064f3a559465c05575367200e5 Mon Sep 17 00:00:00 2001 From: Yichao Yu Date: Sun, 23 Apr 2017 13:31:36 -0400 Subject: [PATCH] Use mark loop in GC --- src/gc-debug.c | 26 +-- src/gc.c | 454 +++++-------------------------------------------- src/gc.h | 4 +- 3 files changed, 64 insertions(+), 420 deletions(-) diff --git a/src/gc-debug.c b/src/gc-debug.c index 4767750b92814..69f02b0dab3ab 100644 --- a/src/gc-debug.c +++ b/src/gc-debug.c @@ -198,18 +198,21 @@ static void restore(void) static void gc_verify_track(jl_ptls_t ptls) { + jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; do { + gc_mark_sp_t sp; + gc_mark_sp_init(gc_cache, &sp); arraylist_push(&lostval_parents_done, lostval); jl_printf(JL_STDERR, "Now looking for %p =======\n", lostval); clear_mark(GC_CLEAN); - mark_all_roots(ptls); - gc_mark_object_list(ptls, &to_finalize, 0); + gc_mark_queue_all_roots(ptls, &sp); + gc_mark_queue_finlist(gc_cache, &sp, &to_finalize, 0); for (int i = 0;i < jl_n_threads;i++) { jl_ptls_t ptls2 = jl_all_tls_states[i]; - gc_mark_object_list(ptls, &ptls2->finalizers, 0); + gc_mark_queue_finlist(gc_cache, &sp, &ptls2->finalizers, 0); } - gc_mark_object_list(ptls, &finalizer_list_marked, 0); - visit_mark_stack(ptls); + gc_mark_queue_finlist(gc_cache, &sp, &finalizer_list_marked, 0); + gc_mark_loop(ptls, sp); if (lostval_parents.len == 0) { jl_printf(JL_STDERR, "Could not find the missing link. We missed a toplevel root. This is odd.\n"); break; @@ -243,19 +246,22 @@ static void gc_verify_track(jl_ptls_t ptls) void gc_verify(jl_ptls_t ptls) { + jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; + gc_mark_sp_t sp; + gc_mark_sp_init(gc_cache, &sp); lostval = NULL; lostval_parents.len = 0; lostval_parents_done.len = 0; clear_mark(GC_CLEAN); gc_verifying = 1; - mark_all_roots(ptls); - gc_mark_object_list(ptls, &to_finalize, 0); + gc_mark_queue_all_roots(ptls, &sp); + gc_mark_queue_finlist(gc_cache, &sp, &to_finalize, 0); for (int i = 0;i < jl_n_threads;i++) { jl_ptls_t ptls2 = jl_all_tls_states[i]; - gc_mark_object_list(ptls, &ptls2->finalizers, 0); + gc_mark_queue_finlist(gc_cache, &sp, &ptls2->finalizers, 0); } - gc_mark_object_list(ptls, &finalizer_list_marked, 0); - visit_mark_stack(ptls); + gc_mark_queue_finlist(gc_cache, &sp, &finalizer_list_marked, 0); + gc_mark_loop(ptls, sp); int clean_len = bits_save[GC_CLEAN].len; for(int i = 0; i < clean_len + bits_save[GC_OLD].len; i++) { jl_taggedvalue_t *v = (jl_taggedvalue_t*)bits_save[i >= clean_len ? GC_OLD : GC_CLEAN].items[i >= clean_len ? i - clean_len : i]; diff --git a/src/gc.c b/src/gc.c index 07101cc63d590..f8e4e3ea6a257 100644 --- a/src/gc.c +++ b/src/gc.c @@ -1256,25 +1256,6 @@ static void gc_sweep_pool(int sweep_full) // mark phase -static jl_value_t **mark_stack = NULL; -static jl_value_t **mark_stack_base = NULL; -static size_t mark_stack_size = 0; -static size_t mark_sp = 0; - -static void grow_mark_stack(void) -{ - size_t newsz = mark_stack_size>0 ? mark_stack_size*2 : 32000; - size_t offset = mark_stack - mark_stack_base; - mark_stack_base = (jl_value_t**)realloc(mark_stack_base, newsz*sizeof(void*)); - if (mark_stack_base == NULL) { - jl_printf(JL_STDERR, "Couldn't grow mark stack to : %" PRIuPTR "\n", - (uintptr_t)newsz); - exit(1); - } - mark_stack = mark_stack_base + offset; - mark_stack_size = newsz; -} - JL_DLLEXPORT void jl_gc_queue_root(jl_value_t *ptr) { jl_ptls_t ptls = jl_get_ptls_states(); @@ -1306,36 +1287,9 @@ void gc_queue_binding(jl_binding_t *bnd) arraylist_push(&ptls->heap.rem_bindings, bnd); } -static void gc_scan_obj_(jl_ptls_t ptls, jl_value_t *v, int d, - uintptr_t tag, uint8_t bits); -static void gc_mark_obj(jl_ptls_t ptls, jl_value_t *v, - uintptr_t tag, uint8_t bits); #ifdef JL_DEBUG_BUILD static void *volatile gc_findval; // for usage from gdb, for finding the gc-root for a value #endif -// Returns whether the object is young -STATIC_INLINE int gc_push_root(jl_ptls_t ptls, void *v, int d) // v isa jl_value_t* -{ -#ifdef JL_DEBUG_BUILD - if (v == gc_findval) - jl_raise_debugger(); -#endif - assert(v != NULL); - jl_taggedvalue_t *o = jl_astaggedvalue(v); - verify_val(v); - uintptr_t tag = o->header; - if (!gc_marked(tag)) { - uint8_t bits; - if (__likely(gc_setmark_tag(o, GC_MARKED, tag, &bits))) { - tag = tag & ~(uintptr_t)15; - if (!gc_verifying) - gc_mark_obj(ptls, (jl_value_t*)v, tag, bits); - gc_scan_obj_(ptls, (jl_value_t*)v, d, tag, bits); - } - return !gc_old(bits); - } - return !gc_old(tag); -} // TODO rename this as it is misleading now void jl_gc_setmark(jl_ptls_t ptls, jl_value_t *v) @@ -1350,43 +1304,6 @@ void jl_gc_setmark(jl_ptls_t ptls, jl_value_t *v) } } -NOINLINE static int gc_mark_module(jl_ptls_t ptls, jl_module_t *m, - int d, int8_t bits) -{ - size_t i; - int refyoung = 0; - void **table = m->bindings.table; - for(i=1; i < m->bindings.size; i+=2) { - if (table[i] != HT_NOTFOUND) { - jl_binding_t *b = (jl_binding_t*)table[i]; - gc_setmark_buf_(ptls, b, bits, sizeof(jl_binding_t)); - void *vb = jl_astaggedvalue(b); - verify_parent1("module", m, &vb, "binding_buff"); - (void)vb; - if (b->value != NULL) { - verify_parent2("module", m, &b->value, "binding(%s)", - jl_symbol_name(b->name)); - refyoung |= gc_push_root(ptls, b->value, d); - } - if (b->globalref != NULL) - refyoung |= gc_push_root(ptls, b->globalref, d); - } - } - // this is only necessary because bindings for "using" modules - // are added only when accessed. therefore if a module is replaced - // after "using" it but before accessing it, this array might - // contain the only reference. - for(i=0; i < m->usings.len; i++) { - refyoung |= gc_push_root(ptls, m->usings.items[i], d); - } - - if (m->parent) { - refyoung |= gc_push_root(ptls, m->parent, d); - } - - return refyoung; -} - // Handle the case where the stack is only partially copied. STATIC_INLINE uintptr_t gc_get_stack_addr(void *_addr, uintptr_t offset, uintptr_t lb, uintptr_t ub) @@ -1404,113 +1321,17 @@ STATIC_INLINE uintptr_t gc_read_stack(void *_addr, uintptr_t offset, return *(uintptr_t*)real_addr; } -static void gc_mark_stack(jl_ptls_t ptls, jl_gcframe_t *s, uintptr_t offset, - uintptr_t lb, uintptr_t ub, int d) -{ - while (s != NULL) { - jl_value_t ***rts = (jl_value_t***)(((void**)s) + 2); - size_t nroots = gc_read_stack(&s->nroots, offset, lb, ub); - size_t nr = nroots >> 1; - if (nroots & 1) { - for (size_t i = 0; i < nr; i++) { - void **slot = (void**)gc_read_stack(&rts[i], offset, lb, ub); - void *obj = (void*)gc_read_stack(slot, offset, lb, ub); - if (obj != NULL) { - gc_push_root(ptls, obj, d); - } - } - } - else { - for (size_t i=0; i < nr; i++) { - void *obj = (void*)gc_read_stack(&rts[i], offset, lb, ub); - if (obj) { - gc_push_root(ptls, obj, d); - } - } - } - s = (jl_gcframe_t*)gc_read_stack(&s->prev, offset, lb, ub); - } -} - -static void gc_mark_task_stack(jl_ptls_t ptls, jl_task_t *ta, int d, int8_t bits) -{ - gc_scrub_record_task(ta); - int stkbuf = (ta->stkbuf != (void*)(intptr_t)-1 && ta->stkbuf != NULL); - int16_t tid = ta->tid; - jl_ptls_t ptls2 = jl_all_tls_states[tid]; - if (stkbuf) { -#ifdef COPY_STACKS - gc_setmark_buf_(ptls, ta->stkbuf, bits, ta->bufsz); -#else - // stkbuf isn't owned by julia for the root task - if (ta != ptls2->root_task) { - gc_setmark_buf_(ptls, ta->stkbuf, bits, ta->ssize); - } -#endif - } - if (ta == ptls2->current_task) { - gc_mark_stack(ptls, ptls2->pgcstack, 0, 0, (uintptr_t)-1, d); - } - else if (stkbuf) { - uintptr_t offset = 0; - uintptr_t lb = 0; - uintptr_t ub = (uintptr_t)-1; -#ifdef COPY_STACKS - ub = (uintptr_t)ptls2->stackbase; - lb = ub - ta->ssize; - offset = (uintptr_t)ta->stkbuf - lb; -#endif - gc_mark_stack(ptls, ta->gcstack, offset, lb, ub, d); - } -} - -NOINLINE static void gc_mark_task(jl_ptls_t ptls, jl_task_t *ta, - int d, int8_t bits) -{ - if (ta->parent) gc_push_root(ptls, ta->parent, d); - gc_push_root(ptls, ta->tls, d); - gc_push_root(ptls, ta->consumers, d); - gc_push_root(ptls, ta->donenotify, d); - gc_push_root(ptls, ta->exception, d); - if (ta->backtrace) gc_push_root(ptls, ta->backtrace, d); - if (ta->start) gc_push_root(ptls, ta->start, d); - if (ta->result) gc_push_root(ptls, ta->result, d); - gc_mark_task_stack(ptls, ta, d, bits); -} - -void gc_mark_object_list(jl_ptls_t ptls, arraylist_t *list, size_t start) -{ - void **items = list->items; - size_t len = list->len; - for (size_t i = start;i < len;i++) { - void *v = items[i]; - if (__unlikely(!v)) - continue; - if (gc_ptr_tag(v, 1)) { - v = gc_ptr_clear_tag(v, 1); - i++; - assert(i < len); - } - gc_push_root(ptls, v, 0); - } -} - -JL_NORETURN NOINLINE void gc_assert_datatype_fail(jl_datatype_t *vt) +JL_NORETURN NOINLINE void gc_assert_datatype_fail(jl_ptls_t ptls, jl_datatype_t *vt, + gc_mark_sp_t sp) { jl_printf(JL_STDOUT, "GC error (probable corruption) :\n"); gc_debug_print_status(); jl_(vt); gc_debug_critical_error(); + gc_mark_loop_unwind(ptls, sp, 0); abort(); } -STATIC_INLINE void gc_assert_datatype(jl_datatype_t *vt) -{ - if (__likely(jl_is_datatype(vt))) - return; - gc_assert_datatype_fail(vt); -} - // This stores the label address in the mark loop function. // We can't directly store that to a global array so we need some hack to get that. // See the call to `gc_mark_loop` in init with a `NULL` `ptls`. @@ -2286,7 +2107,8 @@ mark: { goto pop; } else { - gc_assert_datatype(vt); + if (__unlikely(!jl_is_datatype(vt))) + gc_assert_datatype_fail(ptls, vt, sp); if (update_meta) gc_setmark(ptls, o, bits, jl_datatype_size(vt)); if (vt == jl_weakref_type) @@ -2341,236 +2163,50 @@ mark: { } } -#define MAX_MARK_DEPTH 400 -// Scan a marked object `v` and recursively mark its children. -// The object will be queued on the mark stack when recursion depth -// becomes too high. -// It does so assuming that the tag of the (marked) object is `tag`. -// If `v` is `GC_OLD_MARKED` and some of its children are `GC_MARKED` (young), -// `v` is added to the remset -static void gc_scan_obj_(jl_ptls_t ptls, jl_value_t *v, int d, - uintptr_t tag, uint8_t bits) -{ - assert(v != NULL); - assert(gc_marked(bits)); - jl_datatype_t *vt = (jl_datatype_t*)tag; -#ifdef JL_DEBUG_BUILD - gc_assert_datatype(vt); // should have checked in `gc_mark_obj` -#endif - int refyoung = 0, nptr = 0; - - assert(vt != jl_symbol_type); - // weakref should not be marked - if (vt == jl_weakref_type) - return; - // fast path - uint32_t npointers = vt->layout->npointers; - if (npointers == 0) - return; - d++; - if (d >= MAX_MARK_DEPTH) - goto queue_the_root; - - // some values have special representations - if (vt == jl_simplevector_type) { - size_t l = jl_svec_len(v); - jl_value_t **data = jl_svec_data(v); - nptr += l; - for(size_t i=0; i < l; i++) { - jl_value_t *elt = data[i]; - if (elt != NULL) { - verify_parent2("svec", v, &data[i], "elem(%d)", (int)i); - refyoung |= gc_push_root(ptls, elt, d); - } - } - } - else if (vt->name == jl_array_typename) { - jl_array_t *a = (jl_array_t*)v; - jl_array_flags_t flags = a->flags; - if (flags.how == 3) { - jl_value_t *owner = jl_array_data_owner(a); - refyoung |= gc_push_root(ptls, owner, d); - goto ret; - } - else if (flags.how == 1) { - void *val_buf = jl_astaggedvalue((char*)a->data - - a->offset * a->elsize); - verify_parent1("array", v, &val_buf, - "buffer ('loc' addr is meaningless)"); - (void)val_buf; - gc_setmark_buf_(ptls, (char*)a->data - a->offset * a->elsize, - bits, array_nbytes(a)); - } - if (flags.ptrarray && a->data != NULL) { - size_t l = jl_array_len(a); - if (l > 100000 && d > MAX_MARK_DEPTH-10) { - // don't mark long arrays at high depth, to try to avoid - // copying the whole array into the mark queue - goto queue_the_root; - } - else { - nptr += l; - void *data = a->data; - for (size_t i=0; i < l; i++) { - jl_value_t *elt = ((jl_value_t**)data)[i]; - if (elt != NULL) { - verify_parent2("array", v, &((jl_value_t**)data)[i], "elem(%d)", (int)i); - refyoung |= gc_push_root(ptls, elt, d); - } - } - } - } - } - else if (vt == jl_module_type) { - // should increase nptr here - refyoung |= gc_mark_module(ptls, (jl_module_t*)v, d, bits); - } - else if (vt == jl_task_type) { - // ditto nptr - gc_mark_task(ptls, (jl_task_t*)v, d, bits); - // tasks should always be remarked since we do not trigger the write barrier - // for stores to stack slots - refyoung = 1; - } - else { - int nf = (int)jl_datatype_nfields(vt); - nptr += (npointers & 0xff) << (npointers & 0x300); - uint32_t offsets = jl_datatype_layout_n_nonptr(vt->layout); - nf -= offsets & 0xffff; - for (int i = (offsets >> 16); i < nf; i++) { - if (jl_field_isptr(vt, i)) { - jl_value_t **slot = (jl_value_t**)((char*)v + jl_field_offset(vt, i)); - jl_value_t *fld = *slot; - if (fld) { - verify_parent2("object", v, slot, "field(%d)", i); - refyoung |= gc_push_root(ptls, fld, d); - } - } - } - } - -ret: - if ((bits == GC_OLD_MARKED) && refyoung && !gc_verifying) { - ptls->heap.remset_nptr += nptr; - // v is an old object referencing young objects - arraylist_push(ptls->heap.remset, v); - } - return; - -queue_the_root: - if (mark_sp >= mark_stack_size) - grow_mark_stack(); - mark_stack[mark_sp++] = (jl_value_t*)v; -} - -STATIC_INLINE void gc_scan_obj(jl_ptls_t ptls, jl_value_t *v, int d, - uintptr_t tag) -{ - gc_scan_obj_(ptls, v, d, tag & ~(uintptr_t)15, tag & 0xf); -} - -// Update the metadata of a marked object (without scanning it). -static void gc_mark_obj(jl_ptls_t ptls, jl_value_t *v, - uintptr_t tag, uint8_t bits) -{ - assert(v != NULL); - assert(gc_marked(bits)); - jl_taggedvalue_t *o = jl_astaggedvalue(v); - jl_datatype_t *vt = (jl_datatype_t*)(tag & ~(uintptr_t)15); - gc_assert_datatype(vt); - // Symbols are always marked - assert(vt != jl_symbol_type); - - // some values have special representations - if (vt == jl_simplevector_type) { - size_t l = jl_svec_len(v); - gc_setmark(ptls, o, bits, l * sizeof(void*) + sizeof(jl_svec_t)); - } - else if (vt->name == jl_array_typename) { - jl_array_t *a = (jl_array_t*)v; - jl_array_flags_t flags = a->flags; - if (flags.pooled) - gc_setmark_pool(ptls, o, bits); - else - gc_setmark_big(ptls, o, bits); - if (flags.how == 2) { - objprofile_count(jl_malloc_tag, bits == GC_OLD_MARKED, - array_nbytes(a)); - if (bits == GC_OLD_MARKED) { - ptls->gc_cache.perm_scanned_bytes += array_nbytes(a); - } - else { - ptls->gc_cache.scanned_bytes += array_nbytes(a); - } - } - } - else if (vt == jl_module_type) { - gc_setmark(ptls, o, bits, sizeof(jl_module_t)); - } - else if (vt == jl_task_type) { - gc_setmark(ptls, o, bits, sizeof(jl_task_t)); - } - else if (vt == jl_string_type) { - gc_setmark(ptls, o, bits, jl_string_len(v) + sizeof(size_t) + 1); - } - else { - gc_setmark(ptls, o, bits, jl_datatype_size(vt)); - } -} - -void visit_mark_stack(jl_ptls_t ptls) -{ - while (mark_sp > 0 && !should_timeout()) { - jl_value_t *v = mark_stack[--mark_sp]; - assert(jl_astaggedvalue(v)->bits.gc); - gc_scan_obj(ptls, v, 0, jl_astaggedvalue(v)->header); - } - assert(!mark_sp); -} - extern jl_array_t *jl_module_init_order; extern jl_typemap_entry_t *call_cache[N_CALL_CACHE]; extern jl_array_t *jl_all_methods; -static void jl_gc_mark_thread_local(jl_ptls_t ptls, jl_ptls_t ptls2) +static void jl_gc_queue_thread_local(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, + jl_ptls_t ptls2) { // `current_module` might not have a value when the thread is not // running. if (ptls2->current_module) - gc_push_root(ptls, ptls2->current_module, 0); - gc_push_root(ptls, ptls2->current_task, 0); - gc_push_root(ptls, ptls2->root_task, 0); - gc_push_root(ptls, ptls2->exception_in_transit, 0); + gc_mark_queue_obj(gc_cache, sp, ptls2->current_module); + gc_mark_queue_obj(gc_cache, sp, ptls2->current_task); + gc_mark_queue_obj(gc_cache, sp, ptls2->root_task); + gc_mark_queue_obj(gc_cache, sp, ptls2->exception_in_transit); } // mark the initial root set -static void mark_roots(jl_ptls_t ptls) +static void mark_roots(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp) { // modules - gc_push_root(ptls, jl_main_module, 0); - gc_push_root(ptls, jl_internal_main_module, 0); + gc_mark_queue_obj(gc_cache, sp, jl_main_module); + gc_mark_queue_obj(gc_cache, sp, jl_internal_main_module); // invisible builtin values if (jl_an_empty_vec_any != NULL) - gc_push_root(ptls, jl_an_empty_vec_any, 0); + gc_mark_queue_obj(gc_cache, sp, jl_an_empty_vec_any); if (jl_module_init_order != NULL) - gc_push_root(ptls, jl_module_init_order, 0); - gc_push_root(ptls, jl_cfunction_list.unknown, 0); - gc_push_root(ptls, jl_anytuple_type_type, 0); - gc_push_root(ptls, jl_ANY_flag, 0); + gc_mark_queue_obj(gc_cache, sp, jl_module_init_order); + gc_mark_queue_obj(gc_cache, sp, jl_cfunction_list.unknown); + gc_mark_queue_obj(gc_cache, sp, jl_anytuple_type_type); + gc_mark_queue_obj(gc_cache, sp, jl_ANY_flag); for (size_t i = 0; i < N_CALL_CACHE; i++) if (call_cache[i]) - gc_push_root(ptls, call_cache[i], 0); + gc_mark_queue_obj(gc_cache, sp, call_cache[i]); if (jl_all_methods != NULL) - gc_push_root(ptls, jl_all_methods, 0); + gc_mark_queue_obj(gc_cache, sp, jl_all_methods); #ifndef COPY_STACKS - gc_push_root(ptls, jl_unprotect_stack_func, 0); + gc_mark_queue_obj(gc_cache, sp, jl_unprotect_stack_func); #endif // constants - gc_push_root(ptls, jl_typetype_type, 0); - gc_push_root(ptls, jl_emptytuple_type, 0); + gc_mark_queue_obj(gc_cache, sp, jl_typetype_type); + gc_mark_queue_obj(gc_cache, sp, jl_emptytuple_type); } // find unmarked objects that need to be finalized from the finalizer list "list". @@ -2704,14 +2340,12 @@ static void jl_gc_premark(jl_ptls_t ptls2) } } -static void jl_gc_mark_remset(jl_ptls_t ptls, jl_ptls_t ptls2) +static void jl_gc_queue_remset(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, jl_ptls_t ptls2) { size_t len = ptls2->heap.last_remset->len; void **items = ptls2->heap.last_remset->items; - for (size_t i = 0; i < len; i++) { - jl_value_t *item = (jl_value_t*)items[i]; - gc_scan_obj(ptls, item, 0, jl_astaggedvalue(item)->header); - } + for (size_t i = 0; i < len; i++) + gc_mark_queue_scan_obj(gc_cache, sp, (jl_value_t*)items[i]); int n_bnd_refyoung = 0; len = ptls2->heap.rem_bindings.len; items = ptls2->heap.rem_bindings.items; @@ -2720,7 +2354,7 @@ static void jl_gc_mark_remset(jl_ptls_t ptls, jl_ptls_t ptls2) // A null pointer can happen here when the binding is cleaned up // as an exception is thrown after it was already queued (#10221) if (!ptr->value) continue; - if (gc_push_root(ptls, ptr->value, 0)) { + if (gc_mark_queue_obj(gc_cache, sp, ptr->value)) { items[n_bnd_refyoung] = ptr; n_bnd_refyoung++; } @@ -2741,9 +2375,12 @@ static void jl_gc_mark_ptrfree(jl_ptls_t ptls) // Only one thread should be running in this function static int _jl_gc_collect(jl_ptls_t ptls, int full) { + jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; + gc_mark_sp_t sp; + gc_mark_sp_init(gc_cache, &sp); + uint64_t t0 = jl_hrtime(); int64_t last_perm_scanned_bytes = perm_scanned_bytes; - assert(mark_sp == 0); // 1. fix GC bits of objects in the remset. for (int t_i = 0; t_i < jl_n_threads; t_i++) @@ -2752,14 +2389,15 @@ static int _jl_gc_collect(jl_ptls_t ptls, int full) for (int t_i = 0; t_i < jl_n_threads; t_i++) { jl_ptls_t ptls2 = jl_all_tls_states[t_i]; // 2.1. mark every object in the `last_remsets` and `rem_binding` - jl_gc_mark_remset(ptls, ptls2); + jl_gc_queue_remset(gc_cache, &sp, ptls2); // 2.2. mark every thread local root - jl_gc_mark_thread_local(ptls, ptls2); + jl_gc_queue_thread_local(gc_cache, &sp, ptls2); } // 3. walk roots - mark_roots(ptls); - visit_mark_stack(ptls); + mark_roots(gc_cache, &sp); + gc_mark_loop(ptls, sp); + gc_mark_sp_init(gc_cache, &sp); gc_num.since_sweep += gc_num.allocd + (int64_t)gc_num.interval; gc_settime_premark_end(); gc_time_mark_pause(t0, scanned_bytes, perm_scanned_bytes); @@ -2781,19 +2419,20 @@ static int _jl_gc_collect(jl_ptls_t ptls, int full) } for (int i = 0;i < jl_n_threads;i++) { jl_ptls_t ptls2 = jl_all_tls_states[i]; - gc_mark_object_list(ptls, &ptls2->finalizers, 0); + gc_mark_queue_finlist(gc_cache, &sp, &ptls2->finalizers, 0); } - gc_mark_object_list(ptls, &finalizer_list_marked, orig_marked_len); + gc_mark_queue_finlist(gc_cache, &sp, &finalizer_list_marked, orig_marked_len); // "Flush" the mark stack before flipping the reset_age bit // so that the objects are not incorrectly resetted. - visit_mark_stack(ptls); + gc_mark_loop(ptls, sp); + gc_mark_sp_init(gc_cache, &sp); mark_reset_age = 1; // Reset the age and old bit for any unmarked objects referenced by the // `to_finalize` list. These objects are only reachable from this list // and should not be referenced by any old objects so this won't break // the GC invariant. - gc_mark_object_list(ptls, &to_finalize, 0); - visit_mark_stack(ptls); + gc_mark_queue_finlist(gc_cache, &sp, &to_finalize, 0); + gc_mark_loop(ptls, sp); mark_reset_age = 0; gc_settime_postmark_end(); @@ -2944,11 +2583,12 @@ JL_DLLEXPORT void jl_gc_collect(int full) } } -void mark_all_roots(jl_ptls_t ptls) +void gc_mark_queue_all_roots(jl_ptls_t ptls, gc_mark_sp_t *sp) { + jl_gc_mark_cache_t *gc_cache = &ptls->gc_cache; for (size_t i = 0; i < jl_n_threads; i++) - jl_gc_mark_thread_local(ptls, jl_all_tls_states[i]); - mark_roots(ptls); + jl_gc_queue_thread_local(gc_cache, sp, jl_all_tls_states[i]); + mark_roots(gc_cache, sp); jl_gc_mark_ptrfree(ptls); } diff --git a/src/gc.h b/src/gc.h index 0026ae47d64b1..44785fda43298 100644 --- a/src/gc.h +++ b/src/gc.h @@ -486,9 +486,7 @@ STATIC_INLINE void gc_mark_sp_init(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *s sp->pc_end = gc_cache->pc_stack_end; } -void mark_all_roots(jl_ptls_t ptls); -void gc_mark_object_list(jl_ptls_t ptls, arraylist_t *list, size_t start); -void visit_mark_stack(jl_ptls_t ptls); +void gc_mark_queue_all_roots(jl_ptls_t ptls, gc_mark_sp_t *sp); void gc_mark_queue_finlist(jl_gc_mark_cache_t *gc_cache, gc_mark_sp_t *sp, arraylist_t *list, size_t start); void gc_mark_loop(jl_ptls_t ptls, gc_mark_sp_t sp);