diff --git a/src/coreclr/src/gc/gc.cpp b/src/coreclr/src/gc/gc.cpp index c3e52c7540bee..3c26bd5b60e2e 100644 --- a/src/coreclr/src/gc/gc.cpp +++ b/src/coreclr/src/gc/gc.cpp @@ -192,6 +192,25 @@ BOOL is_induced_blocking (gc_reason reason) (reason == reason_lowmemory_host_blocking)); } +gc_oh_num gen_to_oh(int gen) +{ + switch (gen) + { + case soh_gen0: + return gc_oh_num::soh; + case soh_gen1: + return gc_oh_num::soh; + case soh_gen2: + return gc_oh_num::soh; + case loh_generation: + return gc_oh_num::loh; + case poh_generation: + return gc_oh_num::poh; + default: + return gc_oh_num::none; + } +} + #ifndef DACCESS_COMPILE int64_t qpf; size_t start_time; @@ -2067,7 +2086,6 @@ void qsort1(uint8_t** low, uint8_t** high, unsigned int depth); void* virtual_alloc (size_t size); void* virtual_alloc (size_t size, bool use_large_pages_p); -void virtual_free (void* add, size_t size); /* per heap static initialization */ #if defined(BACKGROUND_GC) && !defined(MULTIPLE_HEAPS) @@ -2151,6 +2169,8 @@ CLRCriticalSection gc_heap::check_commit_cs; size_t gc_heap::current_total_committed = 0; +size_t gc_heap::committed_by_oh[total_oh_count] = {0, 0, 0, 0}; + size_t gc_heap::current_total_committed_bookkeeping = 0; #ifdef SHORT_PLUGS @@ -3950,7 +3970,7 @@ struct initial_memory_details initial_memory_details memory_details; -BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t pinned_size, int num_heaps, bool use_large_pages_p) +BOOL gc_heap::reserve_initial_memory (size_t normal_size, size_t large_size, size_t pinned_size, int num_heaps, bool use_large_pages_p) { BOOL reserve_success = FALSE; @@ -4089,7 +4109,7 @@ BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t pinne return reserve_success; } -void destroy_initial_memory() +void gc_heap::destroy_initial_memory() { if (memory_details.initial_memory != NULL) { @@ -4109,7 +4129,7 @@ void destroy_initial_memory() virtual_free (memory_details.initial_pinned_heap[0].memory_base, memory_details.block_count*memory_details.block_size_pinned); - } + } else { assert (memory_details.allocation_pattern == initial_memory_details::EACH_BLOCK); @@ -4136,7 +4156,8 @@ heap_segment* make_initial_segment (int gen, int h_number) { void* mem = memory_details.get_initial_memory (gen, h_number); size_t size = memory_details.get_initial_size (gen); - heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size , h_number); + gc_oh_num oh = gen_to_oh (gen); + heap_segment* res = gc_heap::make_heap_segment ((uint8_t*)mem, size, oh, h_number); return res; } @@ -4203,14 +4224,6 @@ void* virtual_alloc (size_t size, bool use_large_pages_p) return aligned_mem; } -void virtual_free (void* add, size_t size) -{ - GCToOSInterface::VirtualRelease (add, size); - gc_heap::reserved_memory -= size; - dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[", - size, (size_t)add, (size_t)((uint8_t*)add+size))); -} - static size_t get_valid_segment_size (BOOL large_seg=FALSE) { size_t seg_size, initial_seg_size; @@ -4384,7 +4397,7 @@ gc_heap::soh_get_segment_to_expand() } } - heap_segment* result = get_segment (size, FALSE); + heap_segment* result = get_segment (size, gc_oh_num::soh); if(result) { @@ -4425,8 +4438,10 @@ gc_heap::soh_get_segment_to_expand() //returns 0 in case of allocation failure heap_segment* -gc_heap::get_segment (size_t size, BOOL loh_p) +gc_heap::get_segment (size_t size, gc_oh_num oh) { + assert(oh != gc_oh_num::none); + BOOL uoh_p = (oh == gc_oh_num::loh) || (oh == gc_oh_num::poh); if (heap_hard_limit) return NULL; @@ -4495,11 +4510,11 @@ gc_heap::get_segment (size_t size, BOOL loh_p) void* mem = virtual_alloc (size); if (!mem) { - fgm_result.set_fgm (fgm_reserve_segment, size, loh_p); + fgm_result.set_fgm (fgm_reserve_segment, size, uoh_p); return 0; } - result = gc_heap::make_heap_segment ((uint8_t*)mem, size, heap_number); + result = gc_heap::make_heap_segment ((uint8_t*)mem, size, oh, heap_number); if (result) { @@ -4523,7 +4538,7 @@ gc_heap::get_segment (size_t size, BOOL loh_p) end = (uint8_t*)g_gc_highest_address; } - if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, loh_p) != 0) + if (gc_heap::grow_brick_card_tables (start, end, size, result, __this, uoh_p) != 0) { virtual_free (mem, size); return 0; @@ -4531,7 +4546,7 @@ gc_heap::get_segment (size_t size, BOOL loh_p) } else { - fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, loh_p); + fgm_result.set_fgm (fgm_commit_segment_beg, SEGMENT_INITIAL_COMMIT, uoh_p); virtual_free (mem, size); } @@ -4555,11 +4570,11 @@ gc_heap::get_segment (size_t size, BOOL loh_p) return result; } -void release_segment (heap_segment* sg) +void gc_heap::release_segment (heap_segment* sg) { ptrdiff_t delta = 0; FIRE_EVENT(GCFreeSegment_V1, heap_segment_mem(sg)); - virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg); + virtual_free (sg, (uint8_t*)heap_segment_reserved (sg)-(uint8_t*)sg, sg); } heap_segment* gc_heap::get_segment_for_uoh (int gen_number, size_t size @@ -4571,7 +4586,8 @@ heap_segment* gc_heap::get_segment_for_uoh (int gen_number, size_t size #ifndef MULTIPLE_HEAPS gc_heap* hp = 0; #endif //MULTIPLE_HEAPS - heap_segment* res = hp->get_segment (size, TRUE); + gc_oh_num oh = gen_to_oh (gen_number); + heap_segment* res = hp->get_segment (size, oh); if (res != 0) { #ifdef MULTIPLE_HEAPS @@ -5541,7 +5557,7 @@ bool gc_heap::virtual_alloc_commit_for_heap (void* addr, size_t size, int h_numb return GCToOSInterface::VirtualCommit(addr, size); } -bool gc_heap::virtual_commit (void* address, size_t size, int h_number, bool* hard_limit_exceeded_p) +bool gc_heap::virtual_commit (void* address, size_t size, gc_oh_num oh, int h_number, bool* hard_limit_exceeded_p) { #ifndef HOST_64BIT assert (heap_hard_limit == 0); @@ -5549,9 +5565,9 @@ bool gc_heap::virtual_commit (void* address, size_t size, int h_number, bool* ha if (heap_hard_limit) { - bool exceeded_p = false; - check_commit_cs.Enter(); + committed_by_oh[oh] += size; + bool exceeded_p = false; if ((current_total_committed + size) > heap_hard_limit) { @@ -5590,6 +5606,8 @@ bool gc_heap::virtual_commit (void* address, size_t size, int h_number, bool* ha if (!commit_succeeded_p && heap_hard_limit) { check_commit_cs.Enter(); + committed_by_oh[oh] -= size; + dprintf (1, ("commit failed, updating %Id to %Id", current_total_committed, (current_total_committed - size))); current_total_committed -= size; @@ -5598,11 +5616,10 @@ bool gc_heap::virtual_commit (void* address, size_t size, int h_number, bool* ha check_commit_cs.Leave(); } - return commit_succeeded_p; } -bool gc_heap::virtual_decommit (void* address, size_t size, int h_number) +bool gc_heap::virtual_decommit (void* address, size_t size, gc_oh_num oh, int h_number) { #ifndef HOST_64BIT assert (heap_hard_limit == 0); @@ -5613,6 +5630,7 @@ bool gc_heap::virtual_decommit (void* address, size_t size, int h_number) if (decommit_succeeded_p && heap_hard_limit) { check_commit_cs.Enter(); + committed_by_oh[oh] -= size; current_total_committed -= size; if (h_number < 0) current_total_committed_bookkeeping -= size; @@ -5622,6 +5640,18 @@ bool gc_heap::virtual_decommit (void* address, size_t size, int h_number) return decommit_succeeded_p; } +void gc_heap::virtual_free (void* add, size_t allocated_size, heap_segment* sg) +{ + assert(!heap_hard_limit); + bool release_succeeded_p = GCToOSInterface::VirtualRelease (add, allocated_size); + if (release_succeeded_p) + { + reserved_memory -= allocated_size; + dprintf (2, ("Virtual Free size %Id: [%Ix, %Ix[", + allocated_size, (size_t)add, (size_t)((uint8_t*)add + allocated_size))); + } +} + class mark { public: @@ -7225,7 +7255,7 @@ uint32_t* gc_heap::make_card_table (uint8_t* start, uint8_t* end) // mark array will be committed separately (per segment). size_t commit_size = alloc_size - ms; - if (!virtual_commit (mem, commit_size)) + if (!virtual_commit (mem, commit_size, gc_oh_num::none)) { dprintf (1, ("Card table commit failed")); GCToOSInterface::VirtualRelease (mem, alloc_size); @@ -7295,7 +7325,7 @@ int gc_heap::grow_brick_card_tables (uint8_t* start, size_t size, heap_segment* new_seg, gc_heap* hp, - BOOL loh_p) + BOOL uoh_p) { uint8_t* la = g_gc_lowest_address; uint8_t* ha = g_gc_highest_address; @@ -7420,7 +7450,7 @@ int gc_heap::grow_brick_card_tables (uint8_t* start, if (!mem) { - set_fgm_result (fgm_grow_table, alloc_size, loh_p); + set_fgm_result (fgm_grow_table, alloc_size, uoh_p); goto fail; } @@ -7431,10 +7461,10 @@ int gc_heap::grow_brick_card_tables (uint8_t* start, // mark array will be committed separately (per segment). size_t commit_size = alloc_size - ms; - if (!virtual_commit (mem, commit_size)) + if (!virtual_commit (mem, commit_size, gc_oh_num::none)) { dprintf (GC_TABLE_LOG, ("Table commit failed")); - set_fgm_result (fgm_commit_table, commit_size, loh_p); + set_fgm_result (fgm_commit_table, commit_size, uoh_p); goto fail; } } @@ -7501,14 +7531,14 @@ int gc_heap::grow_brick_card_tables (uint8_t* start, if (!commit_new_mark_array_global (new_mark_array)) { dprintf (GC_TABLE_LOG, ("failed to commit portions in the mark array for existing segments")); - set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p); + set_fgm_result (fgm_commit_table, logging_ma_commit_size, uoh_p); goto fail; } if (!commit_mark_array_new_seg (hp, new_seg, translated_ct, saved_g_lowest_address)) { dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg")); - set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p); + set_fgm_result (fgm_commit_table, logging_ma_commit_size, uoh_p); goto fail; } } @@ -7625,7 +7655,7 @@ int gc_heap::grow_brick_card_tables (uint8_t* start, if (!commit_mark_array_new_seg (hp, new_seg)) { dprintf (GC_TABLE_LOG, ("failed to commit mark array for the new seg in range")); - set_fgm_result (fgm_commit_table, logging_ma_commit_size, loh_p); + set_fgm_result (fgm_commit_table, logging_ma_commit_size, uoh_p); return -1; } } @@ -9158,11 +9188,12 @@ int gc_heap::object_gennum_plan (uint8_t* o) #pragma optimize("", on) // Go back to command line default optimizations #endif //_MSC_VER && TARGET_X86 -heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, int h_number) +heap_segment* gc_heap::make_heap_segment (uint8_t* new_pages, size_t size, gc_oh_num oh, int h_number) { + assert(oh != gc_oh_num::none); size_t initial_commit = SEGMENT_INITIAL_COMMIT; - if (!virtual_commit (new_pages, initial_commit, h_number)) + if (!virtual_commit (new_pages, initial_commit, oh, h_number)) { return 0; } @@ -9277,15 +9308,22 @@ size_t gc_heap::decommit_heap_segment_pages_worker (heap_segment* seg, size_t size = heap_segment_committed (seg) - page_start; if (size > 0) { - virtual_decommit (page_start, size, heap_number); - dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)", - (size_t)page_start, - (size_t)(page_start + size), - size)); - heap_segment_committed (seg) = page_start; - if (heap_segment_used (seg) > heap_segment_committed (seg)) + bool decommit_succeeded_p = virtual_decommit (page_start, size, heap_segment_oh (seg), heap_number); + if (decommit_succeeded_p) { - heap_segment_used (seg) = heap_segment_committed (seg); + dprintf (3, ("Decommitting heap segment [%Ix, %Ix[(%d)", + (size_t)page_start, + (size_t)(page_start + size), + size)); + heap_segment_committed (seg) = page_start; + if (heap_segment_used (seg) > heap_segment_committed (seg)) + { + heap_segment_used (seg) = heap_segment_committed (seg); + } + } + else + { + dprintf (3, ("Decommitting heap segment failed")); } } return size; @@ -9303,13 +9341,16 @@ void gc_heap::decommit_heap_segment (heap_segment* seg) #endif //BACKGROUND_GC size_t size = heap_segment_committed (seg) - page_start; - virtual_decommit (page_start, size, heap_number); + bool decommit_succeeded_p = virtual_decommit (page_start, size, heap_segment_oh (seg), heap_number); - //re-init the segment object - heap_segment_committed (seg) = page_start; - if (heap_segment_used (seg) > heap_segment_committed (seg)) + if (decommit_succeeded_p) { - heap_segment_used (seg) = heap_segment_committed (seg); + //re-init the segment object + heap_segment_committed (seg) = page_start; + if (heap_segment_used (seg) > heap_segment_committed (seg)) + { + heap_segment_used (seg) = heap_segment_committed (seg); + } } } @@ -10935,7 +10976,7 @@ BOOL gc_heap::grow_heap_segment (heap_segment* seg, uint8_t* high_address, bool* "Growing heap_segment: %Ix high address: %Ix\n", (size_t)seg, (size_t)high_address); - bool ret = virtual_commit (heap_segment_committed (seg), c_size, heap_number, hard_limit_exceeded_p); + bool ret = virtual_commit (heap_segment_committed (seg), c_size, heap_segment_oh (seg), heap_number, hard_limit_exceeded_p); if (ret) { heap_segment_committed (seg) += c_size; @@ -17010,7 +17051,9 @@ BOOL gc_heap::expand_soh_with_minimal_gc() return TRUE; } else + { return FALSE; + } } // Only to be done on the thread that calls restart in a join for server GC @@ -25826,7 +25869,7 @@ BOOL gc_heap::commit_mark_array_by_range (uint8_t* begin, uint8_t* end, uint32_t size)); #endif //SIMPLE_DPRINTF - if (virtual_commit (commit_start, size)) + if (virtual_commit (commit_start, size, gc_oh_num::none)) { // We can only verify the mark array is cleared from begin to end, the first and the last // page aren't necessarily all cleared 'cause they could be used by other segments or @@ -26050,7 +26093,7 @@ void gc_heap::decommit_mark_array_by_seg (heap_segment* seg) if (decommit_start < decommit_end) { - if (!virtual_decommit (decommit_start, size)) + if (!virtual_decommit (decommit_start, size, gc_oh_num::none)) { dprintf (GC_TABLE_LOG, ("decommit on %Ix for %Id bytes failed", decommit_start, size)); diff --git a/src/coreclr/src/gc/gcpriv.h b/src/coreclr/src/gc/gcpriv.h index e9ac7a7bfd15f..ac6e746317889 100644 --- a/src/coreclr/src/gc/gcpriv.h +++ b/src/coreclr/src/gc/gcpriv.h @@ -223,6 +223,7 @@ const int policy_expand = 2; #define JOIN_LOG (MIN_CUSTOM_LOG_LEVEL + 6) #define SPINLOCK_LOG (MIN_CUSTOM_LOG_LEVEL + 7) #define SNOOP_LOG (MIN_CUSTOM_LOG_LEVEL + 8) +#define COMMIT_ACCOUNTING_LOG (MIN_CUSTOM_LOG_LEVEL + 9) // NOTE! This is for HEAP_BALANCE_INSTRUMENTATION // This particular one is special and needs to be well formatted because we @@ -398,6 +399,17 @@ enum gc_tuning_point tuning_deciding_short_on_seg = 5 }; +enum gc_oh_num +{ + soh = 0, + loh = 1, + poh = 2, + none = 3, + total_oh_count = 4 +}; + +gc_oh_num gen_to_oh (int gen); + #if defined(TRACE_GC) && defined(BACKGROUND_GC) static const char * const str_bgc_state[] = { @@ -1138,6 +1150,7 @@ class gc_heap static heap_segment* make_heap_segment (uint8_t* new_pages, size_t size, + gc_oh_num oh, int h_number); static @@ -1269,6 +1282,11 @@ class gc_heap #endif // FEATURE_BASICFREEZE protected: + PER_HEAP_ISOLATED + BOOL reserve_initial_memory (size_t normal_size, size_t large_size, size_t pinned_size, int num_heaps, bool use_large_pages_p); + + PER_HEAP_ISOLATED + void destroy_initial_memory(); PER_HEAP_ISOLATED void walk_heap (walk_fn fn, void* context, int gen_number, BOOL walk_large_object_heap_p); @@ -1631,7 +1649,9 @@ class gc_heap PER_HEAP heap_segment* soh_get_segment_to_expand(); PER_HEAP - heap_segment* get_segment (size_t size, BOOL loh_p); + heap_segment* get_segment (size_t size, gc_oh_num oh); + PER_HEAP_ISOLATED + void release_segment (heap_segment* sg); PER_HEAP_ISOLATED void seg_mapping_table_add_segment (heap_segment* seg, gc_heap* hp); PER_HEAP_ISOLATED @@ -1661,9 +1681,11 @@ class gc_heap PER_HEAP_ISOLATED bool virtual_alloc_commit_for_heap (void* addr, size_t size, int h_number); PER_HEAP_ISOLATED - bool virtual_commit (void* address, size_t size, int h_number=-1, bool* hard_limit_exceeded_p=NULL); + bool virtual_commit (void* address, size_t size, gc_oh_num oh, int h_number=-1, bool* hard_limit_exceeded_p=NULL); + PER_HEAP_ISOLATED + bool virtual_decommit (void* address, size_t size, gc_oh_num oh, int h_number=-1); PER_HEAP_ISOLATED - bool virtual_decommit (void* address, size_t size, int h_number=-1); + void virtual_free (void* add, size_t size, heap_segment* sg=NULL); PER_HEAP void clear_gen0_bricks(); #ifdef BACKGROUND_GC @@ -3367,6 +3389,9 @@ class gc_heap PER_HEAP_ISOLATED size_t current_total_committed; + PER_HEAP_ISOLATED + size_t committed_by_oh[total_oh_count]; + // This is what GC uses for its own bookkeeping. PER_HEAP_ISOLATED size_t current_total_committed_bookkeeping; @@ -4790,6 +4815,22 @@ BOOL heap_segment_uoh_p (heap_segment * inst) return !!(inst->flags & (heap_segment_flags_loh | heap_segment_flags_poh)); } +inline gc_oh_num heap_segment_oh (heap_segment * inst) +{ + if ((inst->flags & heap_segment_flags_loh) != 0) + { + return gc_oh_num::loh; + } + else if ((inst->flags & heap_segment_flags_poh) != 0) + { + return gc_oh_num::poh; + } + else + { + return gc_oh_num::soh; + } +} + #ifdef BACKGROUND_GC inline BOOL heap_segment_decommitted_p (heap_segment * inst)