From b58f9ea91ea5a20db8bda509a185e9ae7ca0a9a0 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 27 Feb 2024 00:44:26 +0000 Subject: [PATCH 1/5] Implement fixed heap size for Julia --- base/options.jl | 1 + src/gc.c | 55 ++++++++++++++++++++++++++++++++++++++++++------- src/jloptions.c | 35 +++++++++++++++++++++++++++++++ src/jloptions.h | 1 + 4 files changed, 85 insertions(+), 7 deletions(-) diff --git a/base/options.jl b/base/options.jl index fb043672dc19a..bde8b1b3be564 100644 --- a/base/options.jl +++ b/base/options.jl @@ -56,6 +56,7 @@ struct JLOptions strip_ir::Int8 permalloc_pkgimg::Int8 heap_size_hint::UInt64 + fixed_heap_size::UInt64 end # This runs early in the sysimage != is not defined yet diff --git a/src/gc.c b/src/gc.c index 8a856c9a003fb..e6c32b6add87e 100644 --- a/src/gc.c +++ b/src/gc.c @@ -25,6 +25,11 @@ _Atomic(int) gc_master_tid; uv_mutex_t gc_threads_lock; uv_cond_t gc_threads_cond; +// Globally allocated bytes by malloc - used for fixed heap size +_Atomic(uint64_t) malloc_bytes; +// Globally allocated pool pages - used for fixed heap size +extern uint64_t jl_current_pg_count(void); + // Linked list of callback functions typedef void (*jl_gc_cb_func_t)(void); @@ -393,6 +398,7 @@ extern int64_t live_bytes; static int64_t perm_scanned_bytes; // old bytes scanned while marking int prev_sweep_full = 1; int current_sweep_full = 0; +int next_sweep_full = 0; // force next sweep to be a full sweep - used by fixed heap size // Full collection heuristics static int64_t promoted_bytes = 0; @@ -574,11 +580,22 @@ void gc_setmark_buf(jl_ptls_t ptls, void *o, uint8_t mark_mode, size_t minsz) JL inline void maybe_collect(jl_ptls_t ptls) { - if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || jl_gc_debug_check_other()) { - jl_gc_collect(JL_GC_AUTO); - } - else { - jl_gc_safepoint_(ptls); + if (jl_options.fixed_heap_size) { + uint64_t current_heap_size = ((uint64_t)jl_current_pg_count()) << (uint64_t)14; + current_heap_size += jl_atomic_load_relaxed(&malloc_bytes); + if (current_heap_size >= jl_options.fixed_heap_size) { + printf("GC: fixed heap size = %ld, current heap size = %ld\n", jl_options.fixed_heap_size, current_heap_size); + jl_gc_collect(JL_GC_AUTO); + } else { + jl_gc_safepoint_(ptls); + } + } else { + if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || jl_gc_debug_check_other()) { + jl_gc_collect(JL_GC_AUTO); + } + else { + jl_gc_safepoint_(ptls); + } } } @@ -2698,7 +2715,12 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) sweep_full = 1; recollect = 1; } + if (next_sweep_full) { + next_sweep_full = 0; + sweep_full = 1; + } if (sweep_full) { + printf("Full sweep\n"); // these are the difference between the number of gc-perm bytes scanned // on the first collection after sweep_full, and the current scan perm_scanned_bytes = 0; @@ -2814,6 +2836,13 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) } } + if (jl_options.fixed_heap_size) { + uint64_t current_heap_size = ((uint64_t)jl_current_pg_count()) << ((uint64_t)14); + if (current_heap_size > (jl_options.fixed_heap_size * 4 / 5)) { + next_sweep_full = 1; + } + } + gc_time_summary(sweep_full, t_start, gc_end_time, gc_num.freed, live_bytes, gc_num.interval, pause, gc_num.time_to_safepoint, @@ -3019,6 +3048,12 @@ void jl_gc_init(void) #endif if (jl_options.heap_size_hint) jl_gc_set_max_memory(jl_options.heap_size_hint); + + if (jl_options.fixed_heap_size) { + // This guarantees that we will not trigger a GC before reaching heap limit + gc_num.interval = jl_options.fixed_heap_size; + } + t_start = jl_hrtime(); } @@ -3035,6 +3070,7 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); + jl_atomic_fetch_add_relaxed(&malloc_bytes, sz); } return malloc(sz); } @@ -3050,6 +3086,7 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + nm*sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); + jl_atomic_fetch_add_relaxed(&malloc_bytes, nm * sz); } return calloc(nm, sz); } @@ -3065,6 +3102,7 @@ JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.freed) + sz); jl_atomic_store_relaxed(&ptls->gc_num.freecall, jl_atomic_load_relaxed(&ptls->gc_num.freecall) + 1); + jl_atomic_fetch_add_relaxed(&malloc_bytes, -sz); } } @@ -3075,12 +3113,15 @@ JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size if (pgcstack != NULL && ct->world_age) { jl_ptls_t ptls = ct->ptls; maybe_collect(ptls); - if (sz < old) + if (sz < old) { jl_atomic_store_relaxed(&ptls->gc_num.freed, jl_atomic_load_relaxed(&ptls->gc_num.freed) + (old - sz)); - else + jl_atomic_fetch_add_relaxed(&malloc_bytes, old - sz); + } else { jl_atomic_store_relaxed(&ptls->gc_num.allocd, jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (sz - old)); + jl_atomic_fetch_add_relaxed(&malloc_bytes, sz - old); + } jl_atomic_store_relaxed(&ptls->gc_num.realloc, jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); } diff --git a/src/jloptions.c b/src/jloptions.c index f325452e19e41..d23869a7330e3 100644 --- a/src/jloptions.c +++ b/src/jloptions.c @@ -255,6 +255,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) opt_strip_metadata, opt_strip_ir, opt_heap_size_hint, + opt_fixed_heap_size, opt_permalloc_pkgimg, opt_gc_threads, }; @@ -318,6 +319,7 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) { "strip-ir", no_argument, 0, opt_strip_ir }, { "permalloc-pkgimg",required_argument, 0, opt_permalloc_pkgimg }, { "heap-size-hint", required_argument, 0, opt_heap_size_hint }, + { "fixed-heap-size", required_argument, 0, opt_fixed_heap_size }, { 0, 0, 0, 0 } }; @@ -823,6 +825,39 @@ JL_DLLEXPORT void jl_parse_opts(int *argcp, char ***argvp) jl_errorf("julia: invalid argument to --heap-size-hint without memory size specified"); break; + case opt_fixed_heap_size: + if (optarg != NULL) { + size_t endof = strlen(optarg); + long double value = 0.0; + if (sscanf(optarg, "%Lf", &value) == 1 && value > 1e-7) { + char unit = optarg[endof - 1]; + uint64_t multiplier = 1ull; + switch (unit) { + case 'k': + case 'K': + multiplier <<= 10; + break; + case 'm': + case 'M': + multiplier <<= 20; + break; + case 'g': + case 'G': + multiplier <<= 30; + break; + case 't': + case 'T': + multiplier <<= 40; + break; + default: + break; + } + jl_options.fixed_heap_size = (uint64_t)(value * multiplier); + } + } + if (jl_options.fixed_heap_size == 0) + jl_errorf("julia: invalid argument to --fixed-heap-size without memory size specified"); + break; case opt_permalloc_pkgimg: if (!strcmp(optarg,"yes")) jl_options.permalloc_pkgimg = 1; diff --git a/src/jloptions.h b/src/jloptions.h index 93f6d321f38d6..dc46e42a0220b 100644 --- a/src/jloptions.h +++ b/src/jloptions.h @@ -60,6 +60,7 @@ typedef struct { int8_t strip_ir; int8_t permalloc_pkgimg; uint64_t heap_size_hint; + uint64_t fixed_heap_size; } jl_options_t; #endif From 793e6153ba71fc37288469c1b01bcd653b9ca84a Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Thu, 29 Feb 2024 03:40:51 +0000 Subject: [PATCH 2/5] Remove prints --- src/gc.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/gc.c b/src/gc.c index e6c32b6add87e..f76eae768628b 100644 --- a/src/gc.c +++ b/src/gc.c @@ -584,7 +584,6 @@ inline void maybe_collect(jl_ptls_t ptls) uint64_t current_heap_size = ((uint64_t)jl_current_pg_count()) << (uint64_t)14; current_heap_size += jl_atomic_load_relaxed(&malloc_bytes); if (current_heap_size >= jl_options.fixed_heap_size) { - printf("GC: fixed heap size = %ld, current heap size = %ld\n", jl_options.fixed_heap_size, current_heap_size); jl_gc_collect(JL_GC_AUTO); } else { jl_gc_safepoint_(ptls); @@ -2720,7 +2719,6 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) sweep_full = 1; } if (sweep_full) { - printf("Full sweep\n"); // these are the difference between the number of gc-perm bytes scanned // on the first collection after sweep_full, and the current scan perm_scanned_bytes = 0; From f1185aa63edcf1ba355434c1d6eaa95095801cb1 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Mon, 4 Mar 2024 23:30:12 +0000 Subject: [PATCH 3/5] disable other heuristics to trigger full heap gc --- src/gc.c | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/gc.c b/src/gc.c index 627acb2b7dc4e..b1a1fa09289a0 100644 --- a/src/gc.c +++ b/src/gc.c @@ -2724,6 +2724,10 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) sweep_full = 1; recollect = 1; } + if (jl_options.fixed_heap_size) { + // For fixed heap size, do not trigger full sweep for any other heuristics + sweep_full = 0; + } if (next_sweep_full) { next_sweep_full = 0; sweep_full = 1; From e29cbc264ac3d33e5f8f5f6a1af122544df7b55a Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Mon, 25 Mar 2024 04:37:25 +0000 Subject: [PATCH 4/5] Add a build-time flag WITH_GC_FIXED_HEAP --- Make.inc | 6 ++++++ src/gc.c | 36 +++++++++++++++++++++++++++++------- 2 files changed, 35 insertions(+), 7 deletions(-) diff --git a/Make.inc b/Make.inc index b80ddf8873a9e..4414f1a6911ba 100644 --- a/Make.inc +++ b/Make.inc @@ -85,6 +85,7 @@ HAVE_SSP := 0 # GC debugging options WITH_GC_VERIFY := 0 WITH_GC_DEBUG_ENV := 0 +WITH_GC_FIXED_HEAP := 0 # MMTk GC WITH_MMTK ?= 0 @@ -738,6 +739,11 @@ JCXXFLAGS += -DGC_DEBUG_ENV JCFLAGS += -DGC_DEBUG_ENV endif +ifeq ($(WITH_GC_FIXED_HEAP), 1) +JCXXFLAGS += -DGC_FIXED_HEAP +JCFLAGS += -DGC_FIXED_HEAP +endif + ifeq ($(WITH_MMTK), 1) ifeq (${MMTK_JULIA_DIR},) $(error MMTK_JULIA_DIR must be set to use MMTk) diff --git a/src/gc.c b/src/gc.c index b1a1fa09289a0..f063d23558313 100644 --- a/src/gc.c +++ b/src/gc.c @@ -25,10 +25,12 @@ _Atomic(int) gc_master_tid; uv_mutex_t gc_threads_lock; uv_cond_t gc_threads_cond; +#ifdef GC_FIXED_HEAP // Globally allocated bytes by malloc - used for fixed heap size _Atomic(uint64_t) malloc_bytes; // Globally allocated pool pages - used for fixed heap size extern uint64_t jl_current_pg_count(void); +#endif // Linked list of callback functions @@ -398,7 +400,9 @@ extern int64_t live_bytes; static int64_t perm_scanned_bytes; // old bytes scanned while marking int prev_sweep_full = 1; int current_sweep_full = 0; +#ifdef GC_FIXED_HEAP int next_sweep_full = 0; // force next sweep to be a full sweep - used by fixed heap size +#endif // Full collection heuristics static int64_t promoted_bytes = 0; @@ -580,6 +584,7 @@ void gc_setmark_buf(jl_ptls_t ptls, void *o, uint8_t mark_mode, size_t minsz) JL inline void maybe_collect(jl_ptls_t ptls) { +#ifdef GC_FIXED_HEAP if (jl_options.fixed_heap_size) { uint64_t current_heap_size = ((uint64_t)jl_current_pg_count()) << (uint64_t)14; current_heap_size += jl_atomic_load_relaxed(&malloc_bytes); @@ -588,13 +593,14 @@ inline void maybe_collect(jl_ptls_t ptls) } else { jl_gc_safepoint_(ptls); } - } else { - if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || jl_gc_debug_check_other()) { - jl_gc_collect(JL_GC_AUTO); - } - else { - jl_gc_safepoint_(ptls); - } + return; + } +#endif + if (jl_atomic_load_relaxed(&ptls->gc_num.allocd) >= 0 || jl_gc_debug_check_other()) { + jl_gc_collect(JL_GC_AUTO); + } + else { + jl_gc_safepoint_(ptls); } } @@ -2724,6 +2730,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) sweep_full = 1; recollect = 1; } +#ifdef GC_FIXED_HEAP if (jl_options.fixed_heap_size) { // For fixed heap size, do not trigger full sweep for any other heuristics sweep_full = 0; @@ -2732,6 +2739,7 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) next_sweep_full = 0; sweep_full = 1; } +#endif if (sweep_full) { // these are the difference between the number of gc-perm bytes scanned // on the first collection after sweep_full, and the current scan @@ -2848,12 +2856,14 @@ static int _jl_gc_collect(jl_ptls_t ptls, jl_gc_collection_t collection) } } +#ifdef GC_FIXED_HEAP if (jl_options.fixed_heap_size) { uint64_t current_heap_size = ((uint64_t)jl_current_pg_count()) << ((uint64_t)14); if (current_heap_size > (jl_options.fixed_heap_size * 4 / 5)) { next_sweep_full = 1; } } +#endif gc_time_summary(sweep_full, t_start, gc_end_time, gc_num.freed, live_bytes, gc_num.interval, pause, @@ -3061,10 +3071,12 @@ void jl_gc_init(void) if (jl_options.heap_size_hint) jl_gc_set_max_memory(jl_options.heap_size_hint); +#ifdef GC_FIXED_HEAP if (jl_options.fixed_heap_size) { // This guarantees that we will not trigger a GC before reaching heap limit gc_num.interval = jl_options.fixed_heap_size; } +#endif t_start = jl_hrtime(); } @@ -3082,7 +3094,9 @@ JL_DLLEXPORT void *jl_gc_counted_malloc(size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); +#ifdef GC_FIXED_HEAP jl_atomic_fetch_add_relaxed(&malloc_bytes, sz); +#endif } return malloc(sz); } @@ -3098,7 +3112,9 @@ JL_DLLEXPORT void *jl_gc_counted_calloc(size_t nm, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.allocd) + nm*sz); jl_atomic_store_relaxed(&ptls->gc_num.malloc, jl_atomic_load_relaxed(&ptls->gc_num.malloc) + 1); +#ifdef GC_FIXED_HEAP jl_atomic_fetch_add_relaxed(&malloc_bytes, nm * sz); +#endif } return calloc(nm, sz); } @@ -3114,7 +3130,9 @@ JL_DLLEXPORT void jl_gc_counted_free_with_size(void *p, size_t sz) jl_atomic_load_relaxed(&ptls->gc_num.freed) + sz); jl_atomic_store_relaxed(&ptls->gc_num.freecall, jl_atomic_load_relaxed(&ptls->gc_num.freecall) + 1); +#ifdef GC_FIXED_HEAP jl_atomic_fetch_add_relaxed(&malloc_bytes, -sz); +#endif } } @@ -3128,11 +3146,15 @@ JL_DLLEXPORT void *jl_gc_counted_realloc_with_old_size(void *p, size_t old, size if (sz < old) { jl_atomic_store_relaxed(&ptls->gc_num.freed, jl_atomic_load_relaxed(&ptls->gc_num.freed) + (old - sz)); +#ifdef GC_FIXED_HEAP jl_atomic_fetch_add_relaxed(&malloc_bytes, old - sz); +#endif } else { jl_atomic_store_relaxed(&ptls->gc_num.allocd, jl_atomic_load_relaxed(&ptls->gc_num.allocd) + (sz - old)); +#ifdef GC_FIXED_HEAP jl_atomic_fetch_add_relaxed(&malloc_bytes, sz - old); +#endif } jl_atomic_store_relaxed(&ptls->gc_num.realloc, jl_atomic_load_relaxed(&ptls->gc_num.realloc) + 1); From 767b207c7dd79b46a1f39c85404f7372c9079390 Mon Sep 17 00:00:00 2001 From: Yi Lin Date: Tue, 26 Mar 2024 03:45:40 +0000 Subject: [PATCH 5/5] Add a warning if fixed-heap-size is not set. --- Make.inc | 4 +++- src/gc.c | 2 ++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Make.inc b/Make.inc index 4414f1a6911ba..5a0d97ac4843d 100644 --- a/Make.inc +++ b/Make.inc @@ -85,7 +85,9 @@ HAVE_SSP := 0 # GC debugging options WITH_GC_VERIFY := 0 WITH_GC_DEBUG_ENV := 0 -WITH_GC_FIXED_HEAP := 0 + +# Overwrite Julia's GC heuristics and only trigger a GC if the heap is full (fixed_heap_size needs to be set in this build) +WITH_GC_FIXED_HEAP ?= 0 # MMTk GC WITH_MMTK ?= 0 diff --git a/src/gc.c b/src/gc.c index f063d23558313..a8b032a540c24 100644 --- a/src/gc.c +++ b/src/gc.c @@ -3075,6 +3075,8 @@ void jl_gc_init(void) if (jl_options.fixed_heap_size) { // This guarantees that we will not trigger a GC before reaching heap limit gc_num.interval = jl_options.fixed_heap_size; + } else { + jl_printf(JL_STDERR, "Warning: The option fixed-heap-size is not set for a build with WITH_GC_FIXED_HEAP\n"); } #endif