From f1c218fdbf130b03ce45a27e8960ec0b459aa22a Mon Sep 17 00:00:00 2001 From: Christophe Nasarre Date: Thu, 4 May 2023 09:55:31 +0200 Subject: [PATCH 1/3] The AllocationTick threshold is computed by a Poisson process with a 100 KB mean. --- src/coreclr/gc/gc.cpp | 57 +++++++++++++++++++++++++++++++---------- src/coreclr/gc/gcpriv.h | 1 + 2 files changed, 45 insertions(+), 13 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index b67815536cf829..a096672d0c46f3 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -17,6 +17,8 @@ // #include "gcpriv.h" +#include +#include #if defined(TARGET_AMD64) && defined(TARGET_WINDOWS) #define USE_VXSORT @@ -1892,7 +1894,7 @@ size_t align_on_segment_hard_limit (size_t add) #endif //SERVER_GC -const size_t etw_allocation_tick = 100*1024; +const size_t etw_allocation_tick_mean = 100*1024; const size_t low_latency_alloc = 256*1024; @@ -2480,6 +2482,7 @@ uint8_t* gc_heap::last_gen1_pin_end; gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons; size_t gc_heap::etw_allocation_running_amount[total_oh_count]; +size_t gc_heap::etw_allocation_running_threshold[total_oh_count]; uint64_t gc_heap::total_alloc_bytes_soh = 0; @@ -14424,6 +14427,10 @@ gc_heap::init_gc_heap (int h_number) #endif //MULTIPLE_HEAPS memset (etw_allocation_running_amount, 0, sizeof (etw_allocation_running_amount)); + for (int i = 0; i < total_oh_count; i++) + { + etw_allocation_running_threshold[i] = etw_allocation_tick_mean; + } memset (allocated_since_last_gc, 0, sizeof (allocated_since_last_gc)); memset (&oom_info, 0, sizeof (oom_info)); memset (&fgm_result, 0, sizeof (fgm_result)); @@ -18009,6 +18016,23 @@ void gc_heap::trigger_gc_for_alloc (int gen_number, gc_reason gr, #endif //BACKGROUND_GC } +// The code of the helper function is a replicate of CRT rand() implementation. +inline +int FastRNG(int iMaxValue) +{ + static BOOL bisRandInit = FALSE; + static int lHoldrand = 1L; + + if (!bisRandInit) + { + lHoldrand = (int)time(NULL); + bisRandInit = TRUE; + } + int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff); + return randValue % iMaxValue; +} + + inline bool gc_heap::update_alloc_info (int gen_number, size_t allocated_size, size_t* etw_allocation_amount) { @@ -18018,11 +18042,26 @@ bool gc_heap::update_alloc_info (int gen_number, size_t allocated_size, size_t* size_t& etw_allocated = etw_allocation_running_amount[oh_index]; etw_allocated += allocated_size; - if (etw_allocated > etw_allocation_tick) + + size_t& etw_threshold = etw_allocation_running_threshold[oh_index]; + if (etw_allocated > etw_threshold) { *etw_allocation_amount = etw_allocated; exceeded_p = true; etw_allocated = 0; + +// avoid computing if not needed +#ifdef FEATURE_EVENT_TRACE + #ifdef FEATURE_NATIVEAOT + if (EVENT_ENABLED(GCAllocationTick_V1)) + #else + if (EVENT_ENABLED(GCAllocationTick_V4)) + #endif + { + // compute the next threshold based on a Poisson process with a etw_allocation_tick_mean average + etw_threshold = (size_t)(-log(1 - ((double)FastRNG(RAND_MAX)/(double)RAND_MAX)) * etw_allocation_tick_mean) + 1; + } +#endif } return exceeded_p; @@ -46228,22 +46267,14 @@ void StressHeapDummy (); // the test/application run by CLR is enabling any FPU exceptions. // We want to avoid any unexpected exception coming from stress // infrastructure, so CLRRandom is not an option. -// The code below is a replicate of CRT rand() implementation. +// The code of the helper function is a replicate of CRT rand() implementation. // Using CRT rand() is not an option because we will interfere with the user application // that may also use it. int StressRNG(int iMaxValue) { - static BOOL bisRandInit = FALSE; - static int lHoldrand = 1L; - - if (!bisRandInit) - { - lHoldrand = (int)time(NULL); - bisRandInit = TRUE; - } - int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff); - return randValue % iMaxValue; + return FastRNG(iMaxValue); } + #endif // STRESS_HEAP #endif // !FEATURE_NATIVEAOT diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 3ca72125148d22..66b7e7ef0e89f5 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -3854,6 +3854,7 @@ class gc_heap PER_HEAP_FIELD_DIAG_ONLY gen_to_condemn_tuning gen_to_condemn_reasons; PER_HEAP_FIELD_DIAG_ONLY size_t etw_allocation_running_amount[total_oh_count]; + PER_HEAP_FIELD_DIAG_ONLY size_t etw_allocation_running_threshold[total_oh_count]; PER_HEAP_FIELD_DIAG_ONLY uint64_t total_alloc_bytes_soh; PER_HEAP_FIELD_DIAG_ONLY uint64_t total_alloc_bytes_uoh; From 0dbfd06c0df72a4ab650d6ec4475ee675009465f Mon Sep 17 00:00:00 2001 From: Christophe Nasarre Date: Tue, 16 May 2023 08:39:47 +0200 Subject: [PATCH 2/3] Use different randomizer --- src/coreclr/gc/gc.cpp | 33 ++++++++++++--------------------- 1 file changed, 12 insertions(+), 21 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index a096672d0c46f3..4d1e94d91bf5db 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -18,7 +18,6 @@ #include "gcpriv.h" #include -#include #if defined(TARGET_AMD64) && defined(TARGET_WINDOWS) #define USE_VXSORT @@ -18016,23 +18015,6 @@ void gc_heap::trigger_gc_for_alloc (int gen_number, gc_reason gr, #endif //BACKGROUND_GC } -// The code of the helper function is a replicate of CRT rand() implementation. -inline -int FastRNG(int iMaxValue) -{ - static BOOL bisRandInit = FALSE; - static int lHoldrand = 1L; - - if (!bisRandInit) - { - lHoldrand = (int)time(NULL); - bisRandInit = TRUE; - } - int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff); - return randValue % iMaxValue; -} - - inline bool gc_heap::update_alloc_info (int gen_number, size_t allocated_size, size_t* etw_allocation_amount) { @@ -18059,7 +18041,7 @@ bool gc_heap::update_alloc_info (int gen_number, size_t allocated_size, size_t* #endif { // compute the next threshold based on a Poisson process with a etw_allocation_tick_mean average - etw_threshold = (size_t)(-log(1 - ((double)FastRNG(RAND_MAX)/(double)RAND_MAX)) * etw_allocation_tick_mean) + 1; + etw_threshold = (size_t)(-log(1 - ((double)gc_rand::get_rand(RAND_MAX)/(double)RAND_MAX)) * etw_allocation_tick_mean) + 1; } #endif } @@ -46267,12 +46249,21 @@ void StressHeapDummy (); // the test/application run by CLR is enabling any FPU exceptions. // We want to avoid any unexpected exception coming from stress // infrastructure, so CLRRandom is not an option. -// The code of the helper function is a replicate of CRT rand() implementation. +// The code below is a replicate of CRT rand() implementation. // Using CRT rand() is not an option because we will interfere with the user application // that may also use it. int StressRNG(int iMaxValue) { - return FastRNG(iMaxValue); + static BOOL bisRandInit = FALSE; + static int lHoldrand = 1L; + + if (!bisRandInit) + { + lHoldrand = (int)time(NULL); + bisRandInit = TRUE; + } + int randValue = (((lHoldrand = lHoldrand * 214013L + 2531011L) >> 16) & 0x7fff); + return randValue % iMaxValue; } #endif // STRESS_HEAP From d074c90be744e00e30c04aa9f00a74be75fb5b3e Mon Sep 17 00:00:00 2001 From: Christophe Nasarre Date: Fri, 26 May 2023 11:59:12 +0200 Subject: [PATCH 3/3] Allow AllocationTick to be triggered when threshold is inside an allocation context. --- src/coreclr/gc/gc.cpp | 100 ++++++++++++++++++++++++++++++++------ src/coreclr/gc/gcconfig.h | 4 +- src/coreclr/gc/gcpriv.h | 9 ++++ 3 files changed, 96 insertions(+), 17 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 4d1e94d91bf5db..899b759672f805 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -2480,8 +2480,10 @@ uint8_t* gc_heap::last_gen1_pin_end; gen_to_condemn_tuning gc_heap::gen_to_condemn_reasons; +uint64_t gc_heap::etw_allocationTickMode; size_t gc_heap::etw_allocation_running_amount[total_oh_count]; size_t gc_heap::etw_allocation_running_threshold[total_oh_count]; +size_t gc_heap::etw_allocation_next_threshold[total_oh_count]; uint64_t gc_heap::total_alloc_bytes_soh = 0; @@ -14425,10 +14427,12 @@ gc_heap::init_gc_heap (int h_number) heap_number = h_number; #endif //MULTIPLE_HEAPS + etw_allocationTickMode = GCConfig::GetAllocationTickMode(); memset (etw_allocation_running_amount, 0, sizeof (etw_allocation_running_amount)); for (int i = 0; i < total_oh_count; i++) { etw_allocation_running_threshold[i] = etw_allocation_tick_mean; + etw_allocation_next_threshold[i] = etw_allocation_tick_mean; } memset (allocated_since_last_gc, 0, sizeof (allocated_since_last_gc)); memset (&oom_info, 0, sizeof (oom_info)); @@ -16177,6 +16181,30 @@ size_t gc_heap::limit_from_size (size_t size, uint32_t flags, size_t physical_li size_t desired_size_to_allocate = max (padded_size, min_size_to_allocate); size_t new_physical_limit = min (physical_limit, desired_size_to_allocate); +#ifdef FEATURE_EVENT_TRACE + // If the AllocationTick threshold will be reached, check if the next one + // will be within the currently calculated limit. + // In that case, shrink the limit to the next threshold + if (etw_allocationTickMode == 3) + { + #ifdef FEATURE_NATIVEAOT + if (EVENT_ENABLED(GCAllocationTick_V1)) + #else + if (EVENT_ENABLED(GCAllocationTick_V4)) + #endif + { + if (is_alloc_beyond_threshold(gen_number, size)) + { + size_t nextThreshold = get_alloc_next_threshold(gen_number); + if (nextThreshold <= (new_physical_limit - padded_size)) + { + new_physical_limit = size + nextThreshold + Align(min_obj_size, align_const); + } + } + } + } +#endif + size_t new_limit = new_allocation_limit (padded_size, new_physical_limit, gen_number); @@ -18016,21 +18044,9 @@ void gc_heap::trigger_gc_for_alloc (int gen_number, gc_reason gr, } inline -bool gc_heap::update_alloc_info (int gen_number, size_t allocated_size, size_t* etw_allocation_amount) +size_t gc_heap::compute_alloc_threshold () { - bool exceeded_p = false; - int oh_index = gen_to_oh (gen_number); - allocated_since_last_gc[oh_index] += allocated_size; - - size_t& etw_allocated = etw_allocation_running_amount[oh_index]; - etw_allocated += allocated_size; - - size_t& etw_threshold = etw_allocation_running_threshold[oh_index]; - if (etw_allocated > etw_threshold) - { - *etw_allocation_amount = etw_allocated; - exceeded_p = true; - etw_allocated = 0; + size_t threshold = etw_allocation_tick_mean; // avoid computing if not needed #ifdef FEATURE_EVENT_TRACE @@ -18040,10 +18056,62 @@ bool gc_heap::update_alloc_info (int gen_number, size_t allocated_size, size_t* if (EVENT_ENABLED(GCAllocationTick_V4)) #endif { - // compute the next threshold based on a Poisson process with a etw_allocation_tick_mean average - etw_threshold = (size_t)(-log(1 - ((double)gc_rand::get_rand(RAND_MAX)/(double)RAND_MAX)) * etw_allocation_tick_mean) + 1; + if ((etw_allocationTickMode == 2) || (etw_allocationTickMode == 3)) + { + // compute the next threshold based on a Poisson process with a etw_allocation_tick_mean average + threshold = (size_t)(-log((double)gc_rand::get_rand(RAND_MAX)/(double)RAND_MAX) * etw_allocation_tick_mean) + 1; + } + else + if (etw_allocationTickMode == 1) + { + // compute the next threshold as mean +/- mean/2 (i.e. from mean/2 + 1 to mean + mean/2) + threshold = (etw_allocation_tick_mean / 2) + (size_t)(gc_rand::get_rand(etw_allocation_tick_mean) + 1); + } + + // nothing to do for fixed mode: threshold is defined as 100 KB by default } #endif + + return threshold; +} + +inline +size_t gc_heap::get_alloc_current_threshold (int gen_number) +{ + int oh_index = gen_to_oh (gen_number); + return etw_allocation_running_threshold[oh_index]; +} + +inline +size_t gc_heap::get_alloc_next_threshold (int gen_number) +{ + int oh_index = gen_to_oh (gen_number); + return etw_allocation_next_threshold[oh_index]; +} + +inline +bool gc_heap::is_alloc_beyond_threshold(int gen_number, size_t size) +{ + int oh_index = gen_to_oh (gen_number); + return (etw_allocation_running_amount[oh_index] + size > etw_allocation_running_threshold[oh_index]); +} + +inline +bool gc_heap::update_alloc_info (int gen_number, size_t allocated_size, size_t* etw_allocation_amount) +{ + bool exceeded_p = is_alloc_beyond_threshold(gen_number, allocated_size); + + int oh_index = gen_to_oh (gen_number); + allocated_since_last_gc[oh_index] += allocated_size; + etw_allocation_running_amount[oh_index] += allocated_size; + + if (exceeded_p) + { + *etw_allocation_amount = etw_allocation_running_amount[oh_index]; + etw_allocation_running_amount[oh_index] = 0; + + etw_allocation_running_threshold[oh_index] = etw_allocation_next_threshold[oh_index]; + etw_allocation_next_threshold[oh_index] = compute_alloc_threshold(); } return exceeded_p; diff --git a/src/coreclr/gc/gcconfig.h b/src/coreclr/gc/gcconfig.h index cf10191056d4b1..fc45a42c81210c 100644 --- a/src/coreclr/gc/gcconfig.h +++ b/src/coreclr/gc/gcconfig.h @@ -137,7 +137,9 @@ class GCConfigStringHolder INT_CONFIG (GCConserveMem, "GCConserveMemory", "System.GC.ConserveMemory", 0, "Specifies how hard GC should try to conserve memory - values 0-9") \ INT_CONFIG (GCWriteBarrier, "GCWriteBarrier", NULL, 0, "Specifies whether GC should use more precise but slower write barrier") \ STRING_CONFIG(GCName, "GCName", "System.GC.Name", "Specifies the path of the standalone GC implementation.") \ - INT_CONFIG (GCSpinCountUnit, "GCSpinCountUnit", 0, 0, "Specifies the spin count unit used by the GC.") + INT_CONFIG (GCSpinCountUnit, "GCSpinCountUnit", 0, 0, "Specifies the spin count unit used by the GC.") \ + INT_CONFIG (AllocationTickMode, "GCAllocationTickMode", "GCAllocationTickMode", 0, "Specifies the AllocationTick mode (0=fixed, 1=variable, 2=Poisson+AC, 3=Poisson in AC") + // This class is responsible for retreiving configuration information // for how the GC should operate. class GCConfig diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 66b7e7ef0e89f5..849f014557ce2e 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -2967,6 +2967,10 @@ class gc_heap uint64_t* available_page_file=NULL); PER_HEAP_METHOD size_t generation_size (int gen_number); PER_HEAP_ISOLATED_METHOD size_t get_total_survived_size(); + PER_HEAP_METHOD size_t get_alloc_current_threshold (int gen_number); + PER_HEAP_METHOD size_t get_alloc_next_threshold (int gen_number); + PER_HEAP_METHOD bool is_alloc_beyond_threshold (int gen_number, size_t size); + PER_HEAP_METHOD size_t compute_alloc_threshold (); PER_HEAP_METHOD bool update_alloc_info (int gen_number, size_t allocated_size, size_t* etw_allocation_amount); @@ -3853,8 +3857,13 @@ class gc_heap #endif //HEAP_ANALYZE PER_HEAP_FIELD_DIAG_ONLY gen_to_condemn_tuning gen_to_condemn_reasons; + PER_HEAP_FIELD_DIAG_ONLY uint64_t etw_allocationTickMode; PER_HEAP_FIELD_DIAG_ONLY size_t etw_allocation_running_amount[total_oh_count]; + // it is needed to know what will be the next threshold after the running one + // to compute the limit of an allocation context: if it is smaller than this + // next threshold, then the limit is adjusted to the next threshold PER_HEAP_FIELD_DIAG_ONLY size_t etw_allocation_running_threshold[total_oh_count]; + PER_HEAP_FIELD_DIAG_ONLY size_t etw_allocation_next_threshold[total_oh_count]; PER_HEAP_FIELD_DIAG_ONLY uint64_t total_alloc_bytes_soh; PER_HEAP_FIELD_DIAG_ONLY uint64_t total_alloc_bytes_uoh;