diff --git a/gc.c b/gc.c index 1e901657a990fe..cf9ab2bd9e6aec 100644 --- a/gc.c +++ b/gc.c @@ -2531,6 +2531,64 @@ gc_event_hook_body(rb_execution_context_t *ec, rb_objspace_t *objspace, const rb #define gc_event_hook(objspace, event, data) gc_event_hook_prep(objspace, event, data, (void)0) +#if USE_MMTK +static inline void +maybe_register_finalizable(VALUE obj) { + // Any object that has non-trivial cleaning-up code in `obj_free` + // should be registered as "finalizable" to MMTk. + switch (RB_BUILTIN_TYPE(obj)) { + case T_OBJECT: + // FIXME: Ordinary objects can be non-embedded, too, + // but there are just too many such objects, + // and few of them have large buffers. + // Just let them leak for now. + // We'll prioritize eliminating the underlying buffer of ordinary objects. + break; + case T_MODULE: + case T_CLASS: + case T_STRING: + case T_ARRAY: + case T_HASH: + case T_REGEXP: + case T_DATA: + case T_MATCH: + case T_FILE: + case T_ICLASS: + case T_BIGNUM: + case T_STRUCT: + case T_SYMBOL: + case T_IMEMO: + mmtk_add_finalizer((void*)obj); + RUBY_DEBUG_LOG("Object registered for finalization: %p: %s %s", + (void*)obj, + rb_type_str(RB_BUILTIN_TYPE(obj)), + klass==0?"(null)":rb_class2name(klass) + ); + break; + case T_RATIONAL: + case T_COMPLEX: + case T_FLOAT: + // There are only counters increments for these types in `obj_free` + break; + case T_NIL: + case T_FIXNUM: + case T_TRUE: + case T_FALSE: + // These are non-heap value types. + case T_MOVED: + // Should not see this when object is just created. + case T_NODE: + // GC doesn't handle T_NODE. + rb_bug("maybe_register_finalizable: unexpected data type 0x%x(%p) 0x%"PRIxVALUE, + BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags); + break; + default: + rb_bug("maybe_register_finalizable: unknown data type 0x%x(%p) 0x%"PRIxVALUE, + BUILTIN_TYPE(obj), (void*)obj, RBASIC(obj)->flags); + } +} +#endif + static inline VALUE newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, VALUE obj) { @@ -2546,20 +2604,7 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace, #if USE_MMTK if (rb_mmtk_enabled_p()) { - switch (RB_BUILTIN_TYPE(obj)) { - case T_DATA: - case T_FILE: - case T_SYMBOL: - mmtk_register_finalizable((void*)obj); - RUBY_DEBUG_LOG("Object registered for finalization: %p: %s %s", - (void*)obj, - rb_type_str(RB_BUILTIN_TYPE(obj)), - klass==0?"(null)":rb_class2name(klass) - ); - break; - default: - break; // Do nothing. - } + maybe_register_finalizable(obj); } #endif @@ -4698,6 +4743,70 @@ force_chain_object(st_data_t key, st_data_t val, st_data_t arg) bool rb_obj_is_main_ractor(VALUE gv); +#if USE_MMTK +void +rb_mmtk_call_finalizer_inner(rb_objspace_t *objspace, VALUE obj, bool on_exit) { + if (USE_RUBY_DEBUG_LOG) { + RUBY_DEBUG_LOG("Resurrected for obj_free: %p: %s %s", + resurrected, + rb_type_str(RB_BUILTIN_TYPE(obj)), + CLASS_OF(obj)==0?"(null)":rb_class2name(CLASS_OF(obj)) + ); + } + if (on_exit) { + if (rb_obj_is_thread(obj)) { + RUBY_DEBUG_LOG("Skipped thread: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); + return; + } + if (rb_obj_is_mutex(obj)) { + RUBY_DEBUG_LOG("Skipped mutex: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); + return; + } + if (rb_obj_is_fiber(obj)) { + RUBY_DEBUG_LOG("Skipped fiber: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); + return; + } + if (rb_obj_is_main_ractor(obj)) { + RUBY_DEBUG_LOG("Skipped main ractor: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); + return; + } + } + obj_free(objspace, obj); + + // The object may contain dangling pointers after `obj_free`. + // Clear its flags field to ensure the GC does not attempt to scan it. + // TODO: We can instead clear the VO bit (a.k.a. alloc-bit) when mmtk-core supports that. + RVALUE *v = RANY(obj); + v->as.free.flags = 0; + v->as.free.next = NULL; + + RUBY_DEBUG_LOG("Object freed: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); +} + +void +rb_mmtk_call_finalizer(rb_objspace_t *objspace, bool on_exit) +{ + if (on_exit) { + struct RawVecOfObjRef resurrrected_objs = mmtk_get_all_finalizers(); + + for (size_t i = 0; i < resurrrected_objs.len; i++) { + void *resurrected = resurrrected_objs.ptr[resurrrected_objs.len - i - 1]; + + VALUE obj = (VALUE)resurrected; + rb_mmtk_call_finalizer_inner(objspace, obj, on_exit); + } + + mmtk_free_raw_vec_of_obj_ref(resurrrected_objs); + } else { + void *resurrected; + while ((resurrected = mmtk_get_finalized_object()) != NULL) { + VALUE obj = (VALUE)resurrected; + rb_mmtk_call_finalizer_inner(objspace, obj, on_exit); + } + } +} +#endif + void rb_objspace_call_finalizer(rb_objspace_t *objspace) { @@ -4794,35 +4903,7 @@ rb_objspace_call_finalizer(rb_objspace_t *objspace) #if USE_MMTK if (rb_mmtk_enabled_p()) { - void *resurrected; - while ((resurrected = mmtk_poll_finalizable(true)) != NULL) { - VALUE obj = (VALUE)resurrected; - if (USE_RUBY_DEBUG_LOG) { - RUBY_DEBUG_LOG("Resurrected for obj_free: %p: %s %s", - resurrected, - rb_type_str(RB_BUILTIN_TYPE(obj)), - CLASS_OF(obj)==0?"(null)":rb_class2name(CLASS_OF(obj)) - ); - } - if (rb_obj_is_thread(obj)) { - RUBY_DEBUG_LOG("Skipped thread: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); - continue; - } - if (rb_obj_is_mutex(obj)) { - RUBY_DEBUG_LOG("Skipped mutex: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); - continue; - } - if (rb_obj_is_fiber(obj)) { - RUBY_DEBUG_LOG("Skipped fiber: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); - continue; - } - if (rb_obj_is_main_ractor(obj)) { - RUBY_DEBUG_LOG("Skipped main ractor: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); - continue; - } - obj_free(objspace, obj); - RUBY_DEBUG_LOG("Object freed: %p: %s", resurrected, rb_type_str(RB_BUILTIN_TYPE(obj))); - } + rb_mmtk_call_finalizer(objspace, true); } #endif @@ -12433,11 +12514,21 @@ objspace_malloc_increase_body(rb_objspace_t *objspace, void *mem, size_t new_siz if (type == MEMOP_TYPE_MALLOC) { retry: if (malloc_increase > malloc_limit && ruby_native_thread_p() && !dont_gc_val()) { +#if USE_MMTK + if (rb_mmtk_enabled_p()) { + mmtk_handle_user_collection_request((MMTk_VMMutatorThread)GET_THREAD()); + gc_reset_malloc_info(objspace, true); + rb_mmtk_call_finalizer(objspace, false); + } else { +#endif if (ruby_thread_has_gvl_p() && is_lazy_sweeping(objspace)) { gc_rest(objspace); /* gc_rest can reduce malloc_increase */ goto retry; } garbage_collect_with_gvl(objspace, GPR_FLAG_MALLOC); +#if USE_MMTK + } +#endif } } @@ -15276,6 +15367,11 @@ rb_mmtk_scan_object_ruby_style(void *object) VALUE obj = (VALUE)object; + // TODO: When mmtk-core can clear the VO bit (a.k.a. alloc-bit), we can remove this. + if (RB_BUILTIN_TYPE(obj) == T_NONE) { + return; + } + rb_objspace_t *objspace = &rb_objspace; gc_mark_children(objspace, obj); } diff --git a/mmtk.h b/mmtk.h index d5dfcabf6a7890..5009031d078897 100644 --- a/mmtk.h +++ b/mmtk.h @@ -24,6 +24,17 @@ typedef void* MMTk_VMMutatorThread; #define MMTK_GC_THREAD_KIND_CONTROLLER 0 #define MMTK_GC_THREAD_KIND_WORKER 1 +struct ObjectClosure { + void* (*c_function)(void* rust_closure, void* worker, void *data); + void* rust_closure; +}; + +struct RawVecOfObjRef { + void **ptr; + size_t len; + size_t capa; +}; + typedef struct { void (*init_gc_worker_thread)(MMTk_VMWorkerThread worker_tls); MMTk_VMWorkerThread (*get_gc_thread_tls)(void); @@ -110,13 +121,10 @@ extern void mmtk_add_phantom_candidate(void* ref); extern void mmtk_harness_begin(void *tls); extern void mmtk_harness_end(void *tls); -extern void mmtk_register_finalizable(void *reff); -extern void* mmtk_poll_finalizable(bool include_live); - -struct ObjectClosure { - void* (*c_function)(void* rust_closure, void* worker, void *data); - void* rust_closure; -}; +extern void mmtk_add_finalizer(void *reff); +extern void* mmtk_get_finalized_object(); +extern struct RawVecOfObjRef mmtk_get_all_finalizers(); +extern void mmtk_free_raw_vec_of_obj_ref(struct RawVecOfObjRef raw_vec); #ifdef __cplusplus } diff --git a/string.c b/string.c index 73cb9fc25e6e58..538104e67e0624 100644 --- a/string.c +++ b/string.c @@ -475,12 +475,6 @@ fstr_update_callback(st_data_t *key, st_data_t *value, st_data_t data, int exist } RBASIC(str)->flags |= RSTRING_FSTR; -#if USE_MMTK - if (rb_mmtk_enabled_p()) { - mmtk_register_finalizable((void *)str); - } -#endif - *key = *value = arg->fstr = str; return ST_CONTINUE; }