From 34e0b9e80b46c78f91ebb433d14d549db7d7460a Mon Sep 17 00:00:00 2001 From: Manish Godse <61718172+mangod9@users.noreply.github.com> Date: Thu, 27 Jul 2023 22:50:40 -0700 Subject: [PATCH 1/6] adding clrgc variant with regions enabled (#89129) * adding clrgc variant with regions enabled * only build regions for 64 bit platforms * rename to clrgcexp for experimental --- src/coreclr/gc/CMakeLists.txt | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/coreclr/gc/CMakeLists.txt b/src/coreclr/gc/CMakeLists.txt index 71d6c42f6adec0..12426bd27dc337 100644 --- a/src/coreclr/gc/CMakeLists.txt +++ b/src/coreclr/gc/CMakeLists.txt @@ -104,6 +104,16 @@ list(APPEND GC_SOURCES ${GC_HEADERS}) convert_to_absolute_path(GC_SOURCES ${GC_SOURCES}) +# clrgcexp is build with standalone+regions +if (CLR_CMAKE_TARGET_ARCH_ARM64 OR CLR_CMAKE_TARGET_ARCH_AMD64) + add_library_clr(clrgcexp SHARED ${GC_SOURCES}) + add_dependencies(clrgcexp eventing_headers) + target_link_libraries(clrgcexp PRIVATE ${GC_LINK_LIBRARIES}) + target_compile_definitions(clrgcexp PRIVATE -DUSE_REGIONS) + install_clr(TARGETS clrgcexp DESTINATIONS . COMPONENT runtime) +endif (CLR_CMAKE_TARGET_ARCH_ARM64 OR CLR_CMAKE_TARGET_ARCH_AMD64) + +# clrgc is build with standalone+segments add_library_clr(clrgc SHARED ${GC_SOURCES}) add_dependencies(clrgc eventing_headers) target_link_libraries(clrgc PRIVATE ${GC_LINK_LIBRARIES}) From 9a96694a15849e437cbee993e756105df09bfafb Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Wed, 23 Aug 2023 14:57:18 -0700 Subject: [PATCH 2/6] draft --- src/coreclr/gc/gc.cpp | 180 +++++++++++++++++++++++++++-------- src/coreclr/gc/gcinterface.h | 2 +- src/coreclr/gc/gcpriv.h | 36 +++++-- 3 files changed, 172 insertions(+), 46 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 28bae97336bd72..82622871d5dac1 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -50738,24 +50738,29 @@ bool CFinalize::Initialize() GC_NOTRIGGER; } CONTRACTL_END; - m_Array = new (nothrow)(Object*[100]); + const int INITIAL_FINALIZER_ARRAY_SIZE = 100; + m_Array = new (nothrow)(Object*[INITIAL_FINALIZER_ARRAY_SIZE]); if (!m_Array) { ASSERT (m_Array); - STRESS_LOG_OOM_STACK(sizeof(Object*[100])); + STRESS_LOG_OOM_STACK(sizeof(Object*[INITIAL_FINALIZER_ARRAY_SIZE])); if (GCConfig::GetBreakOnOOM()) { GCToOSInterface::DebugBreak(); } return false; } - m_EndArray = &m_Array[100]; + m_EndArray = &m_Array[INITIAL_FINALIZER_ARRAY_SIZE]; - for (int i =0; i < FreeList; i++) + for (int i = 0; i < FreeListSeg; i++) { SegQueueLimit (i) = m_Array; } + for (int i = FreeListSeg; i < MaxSeg; i++) + { + SegQueueLimit (i) = m_EndArray; + } m_PromotedCount = 0; lock = -1; #ifdef _DEBUG @@ -50840,8 +50845,8 @@ CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size) unsigned int dest = gen_segment (gen); // Adjust boundary for segments so that GC will keep objects alive. - Object*** s_i = &SegQueue (FreeList); - if ((*s_i) == m_EndArray) + Object*** s_i = &SegQueue (FreeListSeg); + if ((*s_i) == SegQueueLimit(FreeListSeg)) { if (!GrowArray()) { @@ -50861,20 +50866,31 @@ CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size) return false; } } + + // | loop: /->-->-\ /->->-\ | + // | / \ / \ | + // Start: | ... | dest |a other |b gen0 |_ free | ... + // | / | + // final store: o -->-/ | + // | | + // Result: | ... | dest o| other a| gen0 b| free | ... + + assert (dest < FreeListSeg); Object*** end_si = &SegQueueLimit (dest); - do + while (s_i > end_si) { + assert (s_i > &SegQueue(0)); //is the segment empty? if (!(*s_i == *(s_i-1))) { - //no, swap the end elements. + //no, move the first element of the segment to the (new) last location in the segment *(*s_i) = *(*(s_i-1)); } //increment the fill pointer (*s_i)++; //go to the next segment. s_i--; - } while (s_i > end_si); + } // We have reached the destination segment // store the object @@ -50895,14 +50911,14 @@ CFinalize::GetNextFinalizableObject (BOOL only_non_critical) if (!IsSegEmpty(FinalizerListSeg)) { - obj = *(--SegQueueLimit (FinalizerListSeg)); + obj = *(SegQueue (FinalizerListSeg)++); } else if (!only_non_critical && !IsSegEmpty(CriticalFinalizerListSeg)) { //the FinalizerList is empty, we can adjust both // limit instead of moving the object to the free list - obj = *(--SegQueueLimit (CriticalFinalizerListSeg)); - --SegQueueLimit (FinalizerListSeg); + obj = *(SegQueue (CriticalFinalizerListSeg)++); + SegQueue (FinalizerListSeg)++; } if (obj) { @@ -50915,7 +50931,7 @@ CFinalize::GetNextFinalizableObject (BOOL only_non_critical) size_t CFinalize::GetNumberFinalizableObjects() { - return SegQueueLimit(FinalizerListSeg) - SegQueue(FinalizerListSeg); + return SegQueueLimit(FinalizerMaxSeg) - SegQueue(FinalizerStartSeg); } void @@ -50930,11 +50946,16 @@ CFinalize::MoveItem (Object** fromIndex, step = -1; else step = +1; - // Place the element at the boundary closest to dest + // Each iteration places the element at the boundary closest to dest + // and then adjusts the boundary to move that element one segment closer + // to dest. Object** srcIndex = fromIndex; for (unsigned int i = fromSeg; i != toSeg; i+= step) { + // Select SegQueue[i] for step==-1, SegQueueLimit[i] for step==1 Object**& destFill = m_FillPointers[i+(step - 1 )/2]; + // Select SegQueue[i] for step==-1, SegQueueLimit[i]-1 for step==1 + // (SegQueueLimit[i]-1 is the last entry in segment i) Object** destIndex = destFill - (step + 1)/2; if (srcIndex != destIndex) { @@ -50957,8 +50978,8 @@ CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC) pSC->thread_number = hn; //scan the finalization queue - Object** startIndex = SegQueue (CriticalFinalizerListSeg); - Object** stopIndex = SegQueueLimit (FinalizerListSeg); + Object** startIndex = SegQueue (FinalizerStartSeg); + Object** stopIndex = SegQueueLimit (FinalizerMaxSeg); for (Object** po = startIndex; po < stopIndex; po++) { @@ -50972,12 +50993,20 @@ CFinalize::GcScanRoots (promote_func* fn, int hn, ScanContext *pSC) void CFinalize::WalkFReachableObjects (fq_walk_fn fn) { - Object** startIndex = SegQueue (CriticalFinalizerListSeg); - Object** stopCriticalIndex = SegQueueLimit (CriticalFinalizerListSeg); - Object** stopIndex = SegQueueLimit (FinalizerListSeg); + Object** startIndex = SegQueue (FinalizerListSeg); + Object** stopIndex = SegQueueLimit (FinalizerListSeg); for (Object** po = startIndex; po < stopIndex; po++) { - fn(po < stopCriticalIndex, *po); + bool isCriticalFinalizer = false; + fn(isCriticalFinalizer, *po); + } + + startIndex = SegQueue (CriticalFinalizerListSeg); + stopIndex = SegQueueLimit (CriticalFinalizerListSeg); + for (Object** po = startIndex; po < stopIndex; po++) + { + bool isCriticalFinalizer = true; + fn(isCriticalFinalizer, *po); } } @@ -51016,13 +51045,13 @@ CFinalize::ScanForFinalization (promote_func* pfn, int gen, BOOL mark_only_p, if (GCToEEInterface::EagerFinalized(obj)) { - MoveItem (i, Seg, FreeList); + MoveItem (i, Seg, FreeListSeg); } else if ((obj->GetHeader()->GetBits()) & BIT_SBLK_FINALIZER_RUN) { //remove the object because we don't want to //run the finalizer - MoveItem (i, Seg, FreeList); + MoveItem (i, Seg, FreeListSeg); //Reset the bit so it will be put back on the queue //if resurrected and re-registered. @@ -51106,10 +51135,21 @@ CFinalize::RelocateFinalizationData (int gen, gc_heap* hp) unsigned int Seg = gen_segment (gen); Object** startIndex = SegQueue (Seg); + Object** endIndex = SegQueue (FreeListSeg); - dprintf (3, ("RelocateFinalizationData gen=%d, [%p,%p[", gen, startIndex, SegQueue (FreeList))); + dprintf (3, ("RelocateFinalizationData gen=%d, [%p,%p[", gen, startIndex, endIndex)); - for (Object** po = startIndex; po < SegQueue (FreeList);po++) + for (Object** po = startIndex; po < endIndex; po++) + { + GCHeap::Relocate (po, &sc); + } + + startIndex = SegQueueLimit (FreeListSeg); + endIndex = SegQueueLimit (MaxSeg); + + dprintf (3, ("RelocateFinalizationData gen=%d, [%p,%p[", gen, startIndex, endIndex)); + + for (Object** po = startIndex; po < endIndex; po++) { GCHeap::Relocate (po, &sc); } @@ -51178,15 +51218,28 @@ CFinalize::GrowArray() { return FALSE; } - memcpy (newArray, m_Array, oldArraySize*sizeof(Object*)); + + // Start: | genX | genY | free | final | crit | + // + // Result: | genX | genY | free | final | crit | + + size_t prefixCount = SegQueue(FreeListSeg) - m_Array; + memcpy (newArray, m_Array, prefixCount * sizeof(Object*)); + size_t postfixCount = m_EndArray - SegQueueLimit(FreeListSeg); + memcpy (newArray + newArraySize - postfixCount, SegQueueLimit(FreeListSeg), postfixCount * sizeof(Object*)); dprintf (3, ("Grow finalizer array [%p,%p[ -> [%p,%p[", m_Array, m_EndArray, newArray, &m_Array[newArraySize])); //adjust the fill pointers - for (int i = 0; i < FreeList; i++) + for (int i = 0; i < FreeListSeg; i++) { m_FillPointers [i] += (newArray - m_Array); } + //additional space in new array is added to FreeListSeg, not the end + for (int i = FreeListSeg; i < MaxSeg; ++i) + { + m_FillPointers [i] += (newArray - m_Array) + (newArraySize - oldArraySize); + } delete[] m_Array; m_Array = newArray; m_EndArray = &m_Array [newArraySize]; @@ -51199,15 +51252,16 @@ CFinalize::GrowArray() bool CFinalize::MergeFinalizationData (CFinalize* other_fq) { // compute how much space we will need for the merged data - size_t otherNeededArraySize = other_fq->SegQueue (FreeList) - other_fq->m_Array; + size_t otherNeededArraySize = other_fq->UsedCount(); if (otherNeededArraySize == 0) { // the other queue is empty - nothing to do! return true; } size_t thisArraySize = (m_EndArray - m_Array); - size_t thisNeededArraySize = SegQueue (FreeList) - m_Array; + size_t thisNeededArraySize = UsedCount(); size_t neededArraySize = thisNeededArraySize + otherNeededArraySize; + size_t growthCount = 0; Object ** newArray = m_Array; @@ -51216,6 +51270,7 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) { // if not allocate new array newArray = new (nothrow) Object*[neededArraySize]; + growthCount = neededArraySize - thisArraySize; // if unsuccessful, return false without changing anything if (!newArray) @@ -51225,8 +51280,13 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) } } - // copy the finalization data from this and the other finalize queue - for (int i = FreeList - 1; i >= 0; i--) + // Since the target might be the original array (with the original data), + // the order of copying must not overwrite any data until it has been + // copied. Both loops skip 'FreeListSeg' because there is no need to copy + // the unused data in the freelist. + + // copy the generation data from this and the other finalize queue + for (int i = FreeListSeg - 1; i >= 0; i--) { size_t thisIndex = SegQueue (i) - m_Array; size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array; @@ -51239,14 +51299,35 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) memmove (&newArray[thisLimit + otherIndex], &other_fq->m_Array[otherIndex], sizeof(newArray[0])*otherSize); } + // copy the finalization data from this and the other finalize queue + // + // note reverse order from above to preserve the existing data + for (int i = FreeListSeg + 1; i <= MaxSeg; i++) + { + size_t thisIndex = SegQueue (i) - m_Array; + size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array; + size_t thisLimit = SegQueueLimit (i) - m_Array; + size_t otherLimit = other_fq->SegQueueLimit (i) - other_fq->m_Array; + size_t thisSize = thisLimit - thisIndex; + size_t otherSize = otherLimit - otherIndex; + + memmove (&newArray[thisIndex + otherIndex + growthCount], &m_Array[thisIndex ], sizeof(newArray[0])*thisSize ); + memmove (&newArray[thisLimit + otherIndex + growthCount], &other_fq->m_Array[otherIndex], sizeof(newArray[0])*otherSize); + } + // adjust the m_FillPointers to reflect the sum of both queues on this queue, // and reflect that the other queue is now empty - for (int i = FreeList - 1; i >= 0; i--) + // + // unlike copying, this loop does include 'FreeListSeg' since the + // boundary needs to be set correctly, but including MaxSeg itself would + // set m_EndArray + for (int i = MaxSeg - 1; i >= 0; i--) { size_t thisLimit = SegQueueLimit (i) - m_Array; size_t otherLimit = other_fq->SegQueueLimit (i) - other_fq->m_Array; + size_t offset = ((i >= FreeListSeg) ? growthCount : 0); - SegQueueLimit (i) = &newArray[thisLimit + otherLimit]; + SegQueueLimit (i) = &newArray[thisLimit + otherLimit + offset]; other_fq->SegQueueLimit (i) = other_fq->m_Array; } @@ -51264,10 +51345,10 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) bool CFinalize::SplitFinalizationData (CFinalize* other_fq) { // the other finalization queue is assumed to be empty at this point - size_t otherCurrentArraySize = other_fq->SegQueue (FreeList) - other_fq->m_Array; + size_t otherCurrentArraySize = other_fq->UsedCount(); assert (otherCurrentArraySize == 0); - size_t thisCurrentArraySize = SegQueue (FreeList) - m_Array; + size_t thisCurrentArraySize = UsedCount(); if (thisCurrentArraySize == 0) { // this queue is empty - nothing to split! @@ -51293,9 +51374,9 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) } // move half of the items in each section over to the other queue - PTR_PTR_Object newFillPointers[FreeList]; + PTR_PTR_Object newFillPointers[MaxSeg]; PTR_PTR_Object segQueue = m_Array; - for (int i = 0; i < FreeList; i++) + for (int i = 0; i < FreeListSeg; i++) { size_t thisIndex = SegQueue (i) - m_Array; size_t thisLimit = SegQueueLimit (i) - m_Array; @@ -51309,14 +51390,37 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) memmove (&other_fq->m_Array[otherIndex], &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0])*otherSize); other_fq->SegQueueLimit (i) = &other_fq->m_Array[otherIndex + otherSize]; - // we delete the moved half from this queue + // slide the unmoved half to its new position in the queue + // (this will delete the moved half once copies and m_FillPointers updates are completed) memmove (segQueue, &m_Array[thisIndex], sizeof(m_Array[0])*thisNewSize); segQueue += thisNewSize; newFillPointers[i] = segQueue; } + segQueue = m_EndArray; + for (int i = MaxSeg; i > FreeListSeg; i++) + { + size_t thisIndex = SegQueue (i) - m_Array; + size_t thisLimit = SegQueueLimit (i) - m_Array; + size_t thisSize = thisLimit - thisIndex; + + // we move half to the other queue + size_t otherSize = thisSize / 2; + size_t otherIndex = other_fq->SegQueueLimit (i) - other_fq->m_Array; + size_t thisNewSize = thisSize - otherSize; + + memmove (&other_fq->m_Array[otherIndex], &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0])*otherSize); + other_fq->SegQueue (i) = &other_fq->m_Array[otherIndex]; + + // slide the unmoved half to its new position in the queue + // (this will delete the moved half once copies and m_FillPointers updates are completed) + segQueue -= thisNewSize; + memmove (segQueue, &m_Array[thisIndex], sizeof(m_Array[0])*thisNewSize); + newFillPointers[i - 1] = segQueue; + } + // finally update the fill pointers from the new copy we generated - for (int i = 0; i < FreeList; i++) + for (int i = 0; i < MaxSeg; i++) { m_FillPointers[i] = newFillPointers[i]; } diff --git a/src/coreclr/gc/gcinterface.h b/src/coreclr/gc/gcinterface.h index 53512895427b27..b83965aceb2de8 100644 --- a/src/coreclr/gc/gcinterface.h +++ b/src/coreclr/gc/gcinterface.h @@ -521,7 +521,7 @@ typedef bool (* walk_fn)(Object*, void*); typedef bool (* walk_fn2)(Object*, uint8_t**, void*); typedef void (* gen_walk_fn)(void* context, int generation, uint8_t* range_start, uint8_t* range_end, uint8_t* range_reserved); typedef void (* record_surv_fn)(uint8_t* begin, uint8_t* end, ptrdiff_t reloc, void* context, bool compacting_p, bool bgc_p); -typedef void (* fq_walk_fn)(bool, void*); +typedef void (* fq_walk_fn)(bool isCritical, void* pObject); typedef void (* fq_scan_fn)(Object** ppObject, ScanContext *pSC, uint32_t dwFlags); typedef void (* handle_scan_fn)(Object** pRef, Object* pSec, uint32_t flags, ScanContext* context, bool isDependent); typedef bool (* async_pin_enum_fn)(Object* object, void* context); diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index d970fa97566616..395b343840ede4 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -4636,12 +4636,30 @@ class CFinalize private: - //adjust the count and add a constant to add a segment + // Segments are bounded by m_Array (the overall start), each element of + // m_FillPointers, and then m_EndArray (the overall end). m_Array could + // be considered the first element of (i.e., before all of) m_FillPointers + // and m_EndArray the last. + // + // Therefore, the lower bound on segment X is m_FillPointers[x-1] with a + // special case for the first, and the upper bound on segment X is + // m_FillPointers[x] with special cases for the last. + // + // The "generation" segments [0, FreeListSeg) grow/shrink on the 'End' side + // (high index), and the "finalizer" segments grow/shrink on the 'Start' + // side (low index). + + // Adjust the count and add a constant to add a segment static const int ExtraSegCount = 2; + static const int FreeListSeg = total_generation_count; static const int FinalizerListSeg = total_generation_count + 1; - static const int CriticalFinalizerListSeg = total_generation_count; - //Does not correspond to a segment - static const int FreeList = total_generation_count + ExtraSegCount; + // The end of this segment is m_EndArray, not an entry in m_FillPointers. + static const int CriticalFinalizerListSeg = total_generation_count + 2; + + static const int FinalizerStartSeg = FinalizerListSeg; + static const int FinalizerMaxSeg = CriticalFinalizerListSeg; + + static const int MaxSeg = CriticalFinalizerListSeg; PTR_PTR_Object m_FillPointers[total_generation_count + ExtraSegCount]; PTR_PTR_Object m_Array; @@ -4664,14 +4682,18 @@ class CFinalize } inline PTR_PTR_Object& SegQueueLimit (unsigned int Seg) { - return m_FillPointers [Seg]; + return (Seg == MaxSeg ? m_EndArray : m_FillPointers[Seg]); + } + + size_t UsedCount () + { + return (SegQueue(FreeListSeg) - m_Array) + (m_EndArray - SegQueueLimit(FreeListSeg)); } BOOL IsSegEmpty ( unsigned int i) { - ASSERT ( (int)i < FreeList); + ASSERT ((int)i <= MaxSeg); return (SegQueueLimit(i) == SegQueue (i)); - } public: From da63587eaba0ac3520a2c07d6e0f07b229cf8bf7 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Sat, 9 Sep 2023 02:39:41 -0700 Subject: [PATCH 3/6] cleanup, refactor, untested --- src/coreclr/gc/gc.cpp | 157 +++++++++++++++++++++++++--------------- src/coreclr/gc/gcpriv.h | 3 + 2 files changed, 100 insertions(+), 60 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 71527b28d1a434..1445f867fa41f8 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -51147,19 +51147,15 @@ CFinalize::RegisterForFinalization (int gen, Object* obj, size_t size) } } - // | loop: /->-->-\ /->->-\ | - // | / \ / \ | // Start: | ... | dest |a other |b gen0 |_ free | ... - // | / | - // final store: o -->-/ | - // | | // Result: | ... | dest o| other a| gen0 b| free | ... + // + // The loop shuffles 'a' to 'b' to _, and the final store after the loop puts 'o' in place. assert (dest < FreeListSeg); Object*** end_si = &SegQueueLimit (dest); while (s_i > end_si) { - assert (s_i > &SegQueue(0)); //is the segment empty? if (!(*s_i == *(s_i-1))) { @@ -51497,9 +51493,9 @@ CFinalize::GrowArray() return FALSE; } - // Start: | genX | genY | free | final | crit | + // Start: | gen2 | gen1 | gen0 | free | final | crit | // - // Result: | genX | genY | free | final | crit | + // Result: | gen2 | gen1 | gen0 | free | final | crit | size_t prefixCount = SegQueue(FreeListSeg) - m_Array; memcpy (newArray, m_Array, prefixCount * sizeof(Object*)); @@ -51566,15 +51562,7 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) // copy the generation data from this and the other finalize queue for (int i = FreeListSeg - 1; i >= 0; i--) { - size_t thisIndex = SegQueue (i) - m_Array; - size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array; - size_t thisLimit = SegQueueLimit (i) - m_Array; - size_t otherLimit = other_fq->SegQueueLimit (i) - other_fq->m_Array; - size_t thisSize = thisLimit - thisIndex; - size_t otherSize = otherLimit - otherIndex; - - memmove (&newArray[thisIndex + otherIndex], &m_Array[thisIndex ], sizeof(newArray[0])*thisSize ); - memmove (&newArray[thisLimit + otherIndex], &other_fq->m_Array[otherIndex], sizeof(newArray[0])*otherSize); + MergeSegment(newArray, 0, this, other_fq, i); } // copy the finalization data from this and the other finalize queue @@ -51582,15 +51570,7 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) // note reverse order from above to preserve the existing data for (int i = FreeListSeg + 1; i <= MaxSeg; i++) { - size_t thisIndex = SegQueue (i) - m_Array; - size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array; - size_t thisLimit = SegQueueLimit (i) - m_Array; - size_t otherLimit = other_fq->SegQueueLimit (i) - other_fq->m_Array; - size_t thisSize = thisLimit - thisIndex; - size_t otherSize = otherLimit - otherIndex; - - memmove (&newArray[thisIndex + otherIndex + growthCount], &m_Array[thisIndex ], sizeof(newArray[0])*thisSize ); - memmove (&newArray[thisLimit + otherIndex + growthCount], &other_fq->m_Array[otherIndex], sizeof(newArray[0])*otherSize); + MergeSegment(newArray, growthCount, this, other_fq, i); } // adjust the m_FillPointers to reflect the sum of both queues on this queue, @@ -51618,6 +51598,28 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) return true; } +// merge finalization data for one segment +// +// Start: fq1: | ... (size X) ... | segment1 | ... | +// fq2: | ... (size Y) ... | segment2 | ... | +// +// Result: dest: | ... (size X+Y+destExtraOffset) ... | segment1 segment2 | ... | +// +// dest might be unique or m_Array in fq1. +// destExtraOffset is used to keep the segments after the FreeListSeg at the end of the array. +void CFinalize::MergeSegment (Object** dest, size_t destExtraOffset, CFinalize* fq1, CFinalize* fq2, unsigned int segment) +{ + size_t index1 = fq1->SegQueue (segment) - fq1->m_Array; + size_t index2 = fq2->SegQueue (segment) - fq2->m_Array; + size_t limit1 = fq1->SegQueueLimit (segment) - fq1->m_Array; + size_t limit2 = fq2->SegQueueLimit (segment) - fq2->m_Array; + size_t size1 = limit1 - index1; + size_t size2 = limit2 - index2; + + memmove (&dest[index1 + index2 + destExtraOffset], &fq1->m_Array[index1], sizeof(dest[0]) * size1); + memmove (&dest[limit1 + index2 + destExtraOffset], &fq2->m_Array[index2], sizeof(dest[0]) * size2); +} + // split finalization data from this queue with another queue // return false in case of failure - in this case, move no items bool CFinalize::SplitFinalizationData (CFinalize* other_fq) @@ -51656,45 +51658,13 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) PTR_PTR_Object segQueue = m_Array; for (int i = 0; i < FreeListSeg; i++) { - size_t thisIndex = SegQueue (i) - m_Array; - size_t thisLimit = SegQueueLimit (i) - m_Array; - size_t thisSize = thisLimit - thisIndex; - - // we move half to the other queue - size_t otherSize = thisSize / 2; - size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array; - size_t thisNewSize = thisSize - otherSize; - - memmove (&other_fq->m_Array[otherIndex], &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0])*otherSize); - other_fq->SegQueueLimit (i) = &other_fq->m_Array[otherIndex + otherSize]; - - // slide the unmoved half to its new position in the queue - // (this will delete the moved half once copies and m_FillPointers updates are completed) - memmove (segQueue, &m_Array[thisIndex], sizeof(m_Array[0])*thisNewSize); - segQueue += thisNewSize; - newFillPointers[i] = segQueue; + segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i, TRUE); } segQueue = m_EndArray; for (int i = MaxSeg; i > FreeListSeg; i++) { - size_t thisIndex = SegQueue (i) - m_Array; - size_t thisLimit = SegQueueLimit (i) - m_Array; - size_t thisSize = thisLimit - thisIndex; - - // we move half to the other queue - size_t otherSize = thisSize / 2; - size_t otherIndex = other_fq->SegQueueLimit (i) - other_fq->m_Array; - size_t thisNewSize = thisSize - otherSize; - - memmove (&other_fq->m_Array[otherIndex], &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0])*otherSize); - other_fq->SegQueue (i) = &other_fq->m_Array[otherIndex]; - - // slide the unmoved half to its new position in the queue - // (this will delete the moved half once copies and m_FillPointers updates are completed) - segQueue -= thisNewSize; - memmove (segQueue, &m_Array[thisIndex], sizeof(m_Array[0])*thisNewSize); - newFillPointers[i - 1] = segQueue; + segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i, FALSE); } // finally update the fill pointers from the new copy we generated @@ -51706,6 +51676,73 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) return true; } +// split finalization data for one segment +// +// Case: copying_left_to_right == true +// Start: +// this: | copied | ... | half1 half2 | ... | +// ^ +// thisDest +// +// other_fq: | copied | ........... | +// +// Result: this: | copied | half1 | ............... | +// ^ +// return value +// +// other_fq: | copied | half2 | ... | +// +// Case: copying_left_to_right == false +// Start: +// this: | ... | half1 half2 | ... | copied | +// ^ +// thisDest +// +// other_fq: | ........... | copied | +// +// Result: this: | ............... | half1 | copied | +// ^ +// return value +// +// other_fq: | ... | half2 | copied | +// +// dest might be unique or m_Array in fq1 +Object** CFinalize::SplitSegment(Object** thisDest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment, BOOL copying_left_to_right) +{ + size_t thisIndex = SegQueue (segment) - m_Array; + size_t thisLimit = SegQueueLimit (segment) - m_Array; + size_t thisSize = thisLimit - thisIndex; + + // we move half to the other queue + // and slide the other half to its new position in the queue + // (this will delete the moved half once copies and m_FillPointers updates are completed) + size_t otherSize = thisSize / 2; + Object** otherDest = other_fq->SegQueue (segment); + size_t thisNewSize = thisSize - otherSize; + + if (copying_left_to_right) + { + memmove (otherDest, &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0]) * otherSize); + other_fq->SegQueueLimit (segment) = otherDest + otherSize; + + memmove (thisDest, &m_Array[thisIndex], sizeof(m_Array[0]) * thisNewSize); + thisDest += thisNewSize; + newFillPointers[segment] = thisDest; + } + else + { + otherDest -= otherSize; + memmove (otherDest, &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0]) * otherSize); + other_fq->SegQueue (segment) = otherDest; + + thisDest -= thisNewSize; + memmove (thisDest, &m_Array[thisIndex], sizeof(m_Array[0]) * thisNewSize); + newFillPointers[segment - 1] = thisDest; + } + + return thisDest; +} + #ifdef VERIFY_HEAP void CFinalize::CheckFinalizerObjects() { diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index 35ecb926ca7836..c4baab436576bc 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -4742,6 +4742,9 @@ class CFinalize return (SegQueueLimit(i) == SegQueue (i)); } + static void MergeSegment (Object** dest, size_t destExtraOffset, CFinalize* fq1, CFinalize* fq2, unsigned int segment); + Object** SplitSegment (Object** dest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment, BOOL copying_left_to_right); + public: ~CFinalize(); bool Initialize(); From e2cc0beb6e5f48ec1e4946f89ffcd673194499a9 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Mon, 11 Sep 2023 13:46:03 -0700 Subject: [PATCH 4/6] Fix Merge/Split --- src/coreclr/gc/gc.cpp | 81 +++++++++++++++++++++-------------------- src/coreclr/gc/gcpriv.h | 3 +- 2 files changed, 43 insertions(+), 41 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 1445f867fa41f8..09444a35e10c98 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -51562,7 +51562,15 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) // copy the generation data from this and the other finalize queue for (int i = FreeListSeg - 1; i >= 0; i--) { - MergeSegment(newArray, 0, this, other_fq, i); + size_t index1 = SegQueue (i) - m_Array; + size_t index2 = other_fq->SegQueue (i) - other_fq->m_Array; + size_t limit1 = SegQueueLimit (i) - m_Array; + size_t limit2 = other_fq->SegQueueLimit (i) - other_fq->m_Array; + size_t size1 = limit1 - index1; + size_t size2 = limit2 - index2; + + memmove (&newArray[index1 + index2], &m_Array[index1], sizeof(newArray[0]) * size1); + memmove (&newArray[limit1 + index2], &other_fq->m_Array[index2], sizeof(newArray[0]) * size2); } // copy the finalization data from this and the other finalize queue @@ -51570,25 +51578,40 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) // note reverse order from above to preserve the existing data for (int i = FreeListSeg + 1; i <= MaxSeg; i++) { - MergeSegment(newArray, growthCount, this, other_fq, i); + size_t indexFromEnd1 = m_EndArray - SegQueue (i); + size_t indexFromEnd2 = other_fq->m_EndArray - other_fq->SegQueue (i); + size_t limitFromEnd1 = m_EndArray - SegQueueLimit (i); + size_t limitFromEnd2 = other_fq->m_EndArray - other_fq->SegQueueLimit (i); + size_t size1 = indexFromEnd1 - limitFromEnd1; + size_t size2 = indexFromEnd2 - limitFromEnd2; + + memmove (&newArray[thisArraySize + growthCount - indexFromEnd1 - indexFromEnd2], m_EndArray - indexFromEnd1, sizeof(newArray[0]) * size1); + memmove (&newArray[thisArraySize + growthCount - limitFromEnd1 - indexFromEnd2], other_fq->m_EndArray - indexFromEnd2, sizeof(newArray[0]) * size2); } // adjust the m_FillPointers to reflect the sum of both queues on this queue, // and reflect that the other queue is now empty // - // unlike copying, this loop does include 'FreeListSeg' since the - // boundary needs to be set correctly, but including MaxSeg itself would - // set m_EndArray - for (int i = MaxSeg - 1; i >= 0; i--) + // unlike copying, these loops do need to set the 'FreeListSeg' boundaries, + // but including MaxSeg itself would set m_EndArray + for (int i = 0; i < FreeListSeg; ++i) { size_t thisLimit = SegQueueLimit (i) - m_Array; size_t otherLimit = other_fq->SegQueueLimit (i) - other_fq->m_Array; - size_t offset = ((i >= FreeListSeg) ? growthCount : 0); - SegQueueLimit (i) = &newArray[thisLimit + otherLimit + offset]; + SegQueueLimit (i) = &newArray[thisLimit + otherLimit]; other_fq->SegQueueLimit (i) = other_fq->m_Array; } + for (int i = MaxSeg; i > FreeListSeg; i--) + { + size_t thisLimit = m_EndArray - SegQueue (i); + size_t otherLimit = other_fq->m_EndArray - other_fq->SegQueue (i); + + SegQueue (i) = &newArray[(thisArraySize + growthCount) - thisLimit - otherLimit]; + + other_fq->SegQueue (i) = other_fq->m_EndArray; + } if (m_Array != newArray) { delete[] m_Array; @@ -51598,28 +51621,6 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) return true; } -// merge finalization data for one segment -// -// Start: fq1: | ... (size X) ... | segment1 | ... | -// fq2: | ... (size Y) ... | segment2 | ... | -// -// Result: dest: | ... (size X+Y+destExtraOffset) ... | segment1 segment2 | ... | -// -// dest might be unique or m_Array in fq1. -// destExtraOffset is used to keep the segments after the FreeListSeg at the end of the array. -void CFinalize::MergeSegment (Object** dest, size_t destExtraOffset, CFinalize* fq1, CFinalize* fq2, unsigned int segment) -{ - size_t index1 = fq1->SegQueue (segment) - fq1->m_Array; - size_t index2 = fq2->SegQueue (segment) - fq2->m_Array; - size_t limit1 = fq1->SegQueueLimit (segment) - fq1->m_Array; - size_t limit2 = fq2->SegQueueLimit (segment) - fq2->m_Array; - size_t size1 = limit1 - index1; - size_t size2 = limit2 - index2; - - memmove (&dest[index1 + index2 + destExtraOffset], &fq1->m_Array[index1], sizeof(dest[0]) * size1); - memmove (&dest[limit1 + index2 + destExtraOffset], &fq2->m_Array[index2], sizeof(dest[0]) * size2); -} - // split finalization data from this queue with another queue // return false in case of failure - in this case, move no items bool CFinalize::SplitFinalizationData (CFinalize* other_fq) @@ -51658,13 +51659,13 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) PTR_PTR_Object segQueue = m_Array; for (int i = 0; i < FreeListSeg; i++) { - segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i, TRUE); + segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i); } segQueue = m_EndArray; - for (int i = MaxSeg; i > FreeListSeg; i++) + for (int i = MaxSeg; i > FreeListSeg; i--) { - segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i, FALSE); + segQueue = SplitSegment(segQueue, &newFillPointers[0], other_fq, i); } // finally update the fill pointers from the new copy we generated @@ -51678,7 +51679,7 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) // split finalization data for one segment // -// Case: copying_left_to_right == true +// Case: segment < FreeListSeg // Start: // this: | copied | ... | half1 half2 | ... | // ^ @@ -51692,7 +51693,7 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) // // other_fq: | copied | half2 | ... | // -// Case: copying_left_to_right == false +// Case: segment > FreeListSeg // Start: // this: | ... | half1 half2 | ... | copied | // ^ @@ -51707,8 +51708,10 @@ bool CFinalize::SplitFinalizationData (CFinalize* other_fq) // other_fq: | ... | half2 | copied | // // dest might be unique or m_Array in fq1 -Object** CFinalize::SplitSegment(Object** thisDest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment, BOOL copying_left_to_right) +Object** CFinalize::SplitSegment(Object** thisDest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment) { + ASSERT(segment != FreeListSeg); + size_t thisIndex = SegQueue (segment) - m_Array; size_t thisLimit = SegQueueLimit (segment) - m_Array; size_t thisSize = thisLimit - thisIndex; @@ -51717,11 +51720,11 @@ Object** CFinalize::SplitSegment(Object** thisDest, Object*** newFillPointers, C // and slide the other half to its new position in the queue // (this will delete the moved half once copies and m_FillPointers updates are completed) size_t otherSize = thisSize / 2; - Object** otherDest = other_fq->SegQueue (segment); size_t thisNewSize = thisSize - otherSize; - if (copying_left_to_right) + if (segment < FreeListSeg) { + Object** otherDest = other_fq->SegQueue(segment); memmove (otherDest, &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0]) * otherSize); other_fq->SegQueueLimit (segment) = otherDest + otherSize; @@ -51731,7 +51734,7 @@ Object** CFinalize::SplitSegment(Object** thisDest, Object*** newFillPointers, C } else { - otherDest -= otherSize; + Object** otherDest = other_fq->SegQueueLimit(segment) - otherSize; memmove (otherDest, &m_Array[thisIndex + thisNewSize], sizeof(other_fq->m_Array[0]) * otherSize); other_fq->SegQueue (segment) = otherDest; diff --git a/src/coreclr/gc/gcpriv.h b/src/coreclr/gc/gcpriv.h index c4baab436576bc..e8475ac7cfaf72 100644 --- a/src/coreclr/gc/gcpriv.h +++ b/src/coreclr/gc/gcpriv.h @@ -4742,8 +4742,7 @@ class CFinalize return (SegQueueLimit(i) == SegQueue (i)); } - static void MergeSegment (Object** dest, size_t destExtraOffset, CFinalize* fq1, CFinalize* fq2, unsigned int segment); - Object** SplitSegment (Object** dest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment, BOOL copying_left_to_right); + Object** SplitSegment (Object** dest, Object*** newFillPointers, CFinalize* other_fq, unsigned int segment); public: ~CFinalize(); From 58ac9659ed16ab9267c5a4494b10574f8c195116 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Mon, 11 Sep 2023 14:07:06 -0700 Subject: [PATCH 5/6] restore names --- src/coreclr/gc/gc.cpp | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 09444a35e10c98..22bcdf70a6b074 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -51562,15 +51562,15 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) // copy the generation data from this and the other finalize queue for (int i = FreeListSeg - 1; i >= 0; i--) { - size_t index1 = SegQueue (i) - m_Array; - size_t index2 = other_fq->SegQueue (i) - other_fq->m_Array; - size_t limit1 = SegQueueLimit (i) - m_Array; - size_t limit2 = other_fq->SegQueueLimit (i) - other_fq->m_Array; - size_t size1 = limit1 - index1; - size_t size2 = limit2 - index2; + size_t thisIndex = SegQueue (i) - m_Array; + size_t otherIndex = other_fq->SegQueue (i) - other_fq->m_Array; + size_t thisLimit = SegQueueLimit (i) - m_Array; + size_t otherLimit = other_fq->SegQueueLimit (i) - other_fq->m_Array; + size_t thisSize = thisLimit - thisIndex; + size_t otherSize = otherLimit - otherIndex; - memmove (&newArray[index1 + index2], &m_Array[index1], sizeof(newArray[0]) * size1); - memmove (&newArray[limit1 + index2], &other_fq->m_Array[index2], sizeof(newArray[0]) * size2); + memmove (&newArray[thisIndex + otherIndex], &m_Array[thisIndex ], sizeof(newArray[0]) * thisSize); + memmove (&newArray[thisLimit + otherIndex], &other_fq->m_Array[otherIndex], sizeof(newArray[0]) * otherSize); } // copy the finalization data from this and the other finalize queue @@ -51578,15 +51578,15 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) // note reverse order from above to preserve the existing data for (int i = FreeListSeg + 1; i <= MaxSeg; i++) { - size_t indexFromEnd1 = m_EndArray - SegQueue (i); - size_t indexFromEnd2 = other_fq->m_EndArray - other_fq->SegQueue (i); - size_t limitFromEnd1 = m_EndArray - SegQueueLimit (i); - size_t limitFromEnd2 = other_fq->m_EndArray - other_fq->SegQueueLimit (i); - size_t size1 = indexFromEnd1 - limitFromEnd1; - size_t size2 = indexFromEnd2 - limitFromEnd2; + size_t thisIndexFromEnd = m_EndArray - SegQueue (i); + size_t otherIndexFromEnd = other_fq->m_EndArray - other_fq->SegQueue (i); + size_t thisLimitFromEnd = m_EndArray - SegQueueLimit (i); + size_t otherLimitFromEnd = other_fq->m_EndArray - other_fq->SegQueueLimit (i); + size_t thisSize = thisIndexFromEnd - thisLimitFromEnd; + size_t otherSize = otherIndexFromEnd - otherLimitFromEnd; - memmove (&newArray[thisArraySize + growthCount - indexFromEnd1 - indexFromEnd2], m_EndArray - indexFromEnd1, sizeof(newArray[0]) * size1); - memmove (&newArray[thisArraySize + growthCount - limitFromEnd1 - indexFromEnd2], other_fq->m_EndArray - indexFromEnd2, sizeof(newArray[0]) * size2); + memmove (&newArray[thisArraySize + growthCount - thisIndexFromEnd - otherIndexFromEnd], m_EndArray - thisIndexFromEnd, sizeof(newArray[0]) * thisSize); + memmove (&newArray[thisArraySize + growthCount - thisLimitFromEnd - otherIndexFromEnd], other_fq->m_EndArray - otherIndexFromEnd, sizeof(newArray[0]) * otherSize); } // adjust the m_FillPointers to reflect the sum of both queues on this queue, From 7d83dea9c98d0924d50b8da0bb04afea23f0a6d9 Mon Sep 17 00:00:00 2001 From: Mark Plesko Date: Mon, 11 Sep 2023 14:10:02 -0700 Subject: [PATCH 6/6] restore spacing --- src/coreclr/gc/gc.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/coreclr/gc/gc.cpp b/src/coreclr/gc/gc.cpp index 22bcdf70a6b074..147128439998d0 100644 --- a/src/coreclr/gc/gc.cpp +++ b/src/coreclr/gc/gc.cpp @@ -51569,8 +51569,8 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) size_t thisSize = thisLimit - thisIndex; size_t otherSize = otherLimit - otherIndex; - memmove (&newArray[thisIndex + otherIndex], &m_Array[thisIndex ], sizeof(newArray[0]) * thisSize); - memmove (&newArray[thisLimit + otherIndex], &other_fq->m_Array[otherIndex], sizeof(newArray[0]) * otherSize); + memmove (&newArray[thisIndex + otherIndex], &m_Array[thisIndex ], sizeof(newArray[0])*thisSize ); + memmove (&newArray[thisLimit + otherIndex], &other_fq->m_Array[otherIndex], sizeof(newArray[0])*otherSize); } // copy the finalization data from this and the other finalize queue @@ -51585,8 +51585,8 @@ bool CFinalize::MergeFinalizationData (CFinalize* other_fq) size_t thisSize = thisIndexFromEnd - thisLimitFromEnd; size_t otherSize = otherIndexFromEnd - otherLimitFromEnd; - memmove (&newArray[thisArraySize + growthCount - thisIndexFromEnd - otherIndexFromEnd], m_EndArray - thisIndexFromEnd, sizeof(newArray[0]) * thisSize); - memmove (&newArray[thisArraySize + growthCount - thisLimitFromEnd - otherIndexFromEnd], other_fq->m_EndArray - otherIndexFromEnd, sizeof(newArray[0]) * otherSize); + memmove (&newArray[thisArraySize + growthCount - thisIndexFromEnd - otherIndexFromEnd], m_EndArray - thisIndexFromEnd, sizeof(newArray[0])*thisSize ); + memmove (&newArray[thisArraySize + growthCount - thisLimitFromEnd - otherIndexFromEnd], other_fq->m_EndArray - otherIndexFromEnd, sizeof(newArray[0])*otherSize); } // adjust the m_FillPointers to reflect the sum of both queues on this queue,