diff --git a/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs index afc3fca603c1f7..354245e98d342f 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/GC.CoreCLR.cs @@ -124,34 +124,6 @@ internal enum GC_ALLOC_FLAGS [MethodImpl(MethodImplOptions.InternalCall)] internal static extern ulong GetGenerationSize(int gen); - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GCInterface_AddMemoryPressure")] - private static partial void _AddMemoryPressure(ulong bytesAllocated); - - [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GCInterface_RemoveMemoryPressure")] - private static partial void _RemoveMemoryPressure(ulong bytesAllocated); - - public static void AddMemoryPressure(long bytesAllocated) - { - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytesAllocated); - if (IntPtr.Size == 4) - { - ArgumentOutOfRangeException.ThrowIfGreaterThan(bytesAllocated, int.MaxValue); - } - - _AddMemoryPressure((ulong)bytesAllocated); - } - - public static void RemoveMemoryPressure(long bytesAllocated) - { - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytesAllocated); - if (IntPtr.Size == 4) - { - ArgumentOutOfRangeException.ThrowIfGreaterThan(bytesAllocated, int.MaxValue); - } - - _RemoveMemoryPressure((ulong)bytesAllocated); - } - // Returns the generation that obj is currently in. // public static int GetGeneration(object obj) diff --git a/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs b/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs index 594ecbf3c55227..eb092a1cac61af 100644 --- a/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs +++ b/src/coreclr/System.Private.CoreLib/src/System/Threading/Thread.CoreCLR.cs @@ -177,10 +177,19 @@ private void Initialize() private static partial void Initialize(ObjectHandleOnStack thread); /// Clean up the thread when it goes away. - ~Thread() => InternalFinalize(); // Delegate to the unmanaged portion. + ~Thread() + { + InternalFinalize(this); + } + + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "ThreadNative_Finalize")] + private static partial void InternalFinalize(ObjectHandleOnStack thread); + + private static void InternalFinalize(Thread thread) + { + InternalFinalize(ObjectHandleOnStack.Create(ref thread)); + } - [MethodImpl(MethodImplOptions.InternalCall)] - private extern void InternalFinalize(); private void ThreadNameChanged(string? value) { diff --git a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs index 6043e60afd02f9..863525c708c46d 100644 --- a/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs +++ b/src/coreclr/nativeaot/System.Private.CoreLib/src/System/GC.NativeAot.cs @@ -513,140 +513,6 @@ public static int CollectionCount(int generation) return RuntimeImports.RhGetGcCollectionCount(generation, false); } - // Support for AddMemoryPressure and RemoveMemoryPressure below. - private const uint PressureCount = 4; -#if TARGET_64BIT - private const uint MinGCMemoryPressureBudget = 4 * 1024 * 1024; -#else - private const uint MinGCMemoryPressureBudget = 3 * 1024 * 1024; -#endif - - private const uint MaxGCMemoryPressureRatio = 10; - - private static int[] s_gcCounts = new int[] { 0, 0, 0 }; - - private static long[] s_addPressure = new long[] { 0, 0, 0, 0 }; - private static long[] s_removePressure = new long[] { 0, 0, 0, 0 }; - private static uint s_iteration; - - /// - /// Resets the pressure accounting after a gen2 GC has occurred. - /// - private static void CheckCollectionCount() - { - if (s_gcCounts[2] != CollectionCount(2)) - { - for (int i = 0; i < 3; i++) - { - s_gcCounts[i] = CollectionCount(i); - } - - s_iteration++; - - uint p = s_iteration % PressureCount; - - s_addPressure[p] = 0; - s_removePressure[p] = 0; - } - } - - private static long InterlockedAddMemoryPressure(ref long pAugend, long addend) - { - long oldMemValue; - long newMemValue; - - do - { - oldMemValue = pAugend; - newMemValue = oldMemValue + addend; - - // check for overflow - if (newMemValue < oldMemValue) - { - newMemValue = long.MaxValue; - } - } while (Interlocked.CompareExchange(ref pAugend, newMemValue, oldMemValue) != oldMemValue); - - return newMemValue; - } - - /// - /// New AddMemoryPressure implementation (used by RCW and the CLRServicesImpl class) - /// 1. Less sensitive than the original implementation (start budget 3 MB) - /// 2. Focuses more on newly added memory pressure - /// 3. Budget adjusted by effectiveness of last 3 triggered GC (add / remove ratio, max 10x) - /// 4. Budget maxed with 30% of current managed GC size - /// 5. If Gen2 GC is happening naturally, ignore past pressure - /// - /// Here's a brief description of the ideal algorithm for Add/Remove memory pressure: - /// Do a GC when (HeapStart is less than X * MemPressureGrowth) where - /// - HeapStart is GC Heap size after doing the last GC - /// - MemPressureGrowth is the net of Add and Remove since the last GC - /// - X is proportional to our guess of the ummanaged memory death rate per GC interval, - /// and would be calculated based on historic data using standard exponential approximation: - /// Xnew = UMDeath/UMTotal * 0.5 + Xprev - /// - /// - public static void AddMemoryPressure(long bytesAllocated) - { - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytesAllocated); -#if !TARGET_64BIT - ArgumentOutOfRangeException.ThrowIfGreaterThan(bytesAllocated, int.MaxValue); -#endif - - CheckCollectionCount(); - uint p = s_iteration % PressureCount; - long newMemValue = InterlockedAddMemoryPressure(ref s_addPressure[p], bytesAllocated); - - Debug.Assert(PressureCount == 4, "GC.AddMemoryPressure contains unrolled loops which depend on the PressureCount"); - - if (newMemValue >= MinGCMemoryPressureBudget) - { - long add = s_addPressure[0] + s_addPressure[1] + s_addPressure[2] + s_addPressure[3] - s_addPressure[p]; - long rem = s_removePressure[0] + s_removePressure[1] + s_removePressure[2] + s_removePressure[3] - s_removePressure[p]; - - long budget = MinGCMemoryPressureBudget; - - if (s_iteration >= PressureCount) // wait until we have enough data points - { - // Adjust according to effectiveness of GC - // Scale budget according to past m_addPressure / m_remPressure ratio - if (add >= rem * MaxGCMemoryPressureRatio) - { - budget = MinGCMemoryPressureBudget * MaxGCMemoryPressureRatio; - } - else if (add > rem) - { - Debug.Assert(rem != 0); - - // Avoid overflow by calculating addPressure / remPressure as fixed point (1 = 1024) - budget = (add * 1024 / rem) * budget / 1024; - } - } - - // If still over budget, check current managed heap size - if (newMemValue >= budget) - { - long heapOver3 = RuntimeImports.RhGetCurrentObjSize() / 3; - - if (budget < heapOver3) //Max - { - budget = heapOver3; - } - - if (newMemValue >= budget) - { - // last check - if we would exceed 20% of GC "duty cycle", do not trigger GC at this time - if ((RuntimeImports.RhGetGCNow() - RuntimeImports.RhGetLastGCStartTime(2)) > (RuntimeImports.RhGetLastGCDuration(2) * 5)) - { - RuntimeImports.RhCollect(2, InternalGCCollectionMode.NonBlocking); - CheckCollectionCount(); - } - } - } - } - } - internal struct GCConfigurationContext { internal Dictionary Configurations; @@ -705,18 +571,6 @@ public static unsafe IReadOnlyDictionary GetConfigurationVariabl return context.Configurations!; } - public static void RemoveMemoryPressure(long bytesAllocated) - { - ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytesAllocated); -#if !TARGET_64BIT - ArgumentOutOfRangeException.ThrowIfGreaterThan(bytesAllocated, int.MaxValue); -#endif - - CheckCollectionCount(); - uint p = s_iteration % PressureCount; - InterlockedAddMemoryPressure(ref s_removePressure[p], bytesAllocated); - } - public static long GetTotalMemory(bool forceFullCollection) { long size = RuntimeImports.RhGetGcTotalMemory(); diff --git a/src/coreclr/runtime/GCNativeHelpers.cpp b/src/coreclr/runtime/GCNativeHelpers.cpp new file mode 100644 index 00000000000000..7238f7037e2567 --- /dev/null +++ b/src/coreclr/runtime/GCNativeHelpers.cpp @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#include "common.h" +#include "GCNativeHelpers.h" +#include "gcheaputilities.h" + +extern "C" INT64 QCALLTYPE GC_GetCurrentObjSize() +{ + return (INT64)GCHeapUtilities::GetGCHeap()->GetCurrentObjSize(); +} + +extern "C" INT64 QCALLTYPE GC_GetNow() +{ + return (INT64)GCHeapUtilities::GetGCHeap()->GetNow(); +} + +extern "C" INT64 QCALLTYPE GC_GetLastGCStartTime(INT32 generation) +{ + return (INT64)GCHeapUtilities::GetGCHeap()->GetLastGCStartTime(generation); +} + +extern "C" INT64 QCALLTYPE GC_GetLastGCDuration(INT32 generation) +{ + return (INT64)GCHeapUtilities::GetGCHeap()->GetLastGCDuration(generation); +} + +extern "C" void QCALLTYPE GC_SendEtwAddMemoryPressureEvent(UINT64 bytesAllocated) +{ + FireEtwIncreaseMemoryPressure(bytesAllocated, GetClrInstanceId()); +} + +extern "C" void QCALLTYPE GC_SendEtwRemoveMemoryPressureEvent(UINT64 bytesAllocated) +{ + FireEtwDecreaseMemoryPressure(bytesAllocated, GetClrInstanceId()); +} diff --git a/src/coreclr/runtime/GCNativeHelpers.h b/src/coreclr/runtime/GCNativeHelpers.h new file mode 100644 index 00000000000000..6da9587cc08f3c --- /dev/null +++ b/src/coreclr/runtime/GCNativeHelpers.h @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +#ifndef __GCNATIVEHELPERS_H__ +#define __GCNATIVEHELPERS_H__ + +extern "C" INT64 QCALLTYPE GC_GetCurrentObjSize(); +extern "C" INT64 QCALLTYPE GC_GetNow(); +extern "C" INT64 QCALLTYPE GC_GetLastGCStartTime(INT32 generation); +extern "C" INT64 QCALLTYPE GC_GetLastGCDuration(INT32 generation); +extern "C" void QCALLTYPE GC_SendEtwAddMemoryPressureEvent(UINT64 bytesAllocated); +extern "C" void QCALLTYPE GC_SendEtwRemoveMemoryPressureEvent(UINT64 bytesAllocated); + +#endif // __GCNATIVEHELPERS_H__ diff --git a/src/coreclr/vm/CMakeLists.txt b/src/coreclr/vm/CMakeLists.txt index 4966617f6afe23..e3f79773b534f0 100644 --- a/src/coreclr/vm/CMakeLists.txt +++ b/src/coreclr/vm/CMakeLists.txt @@ -293,6 +293,7 @@ set(VM_SOURCES_WKS bundle.cpp ${RUNTIME_DIR}/CachedInterfaceDispatch.cpp CachedInterfaceDispatch_Coreclr.cpp + ${RUNTIME_DIR}/GCNativeHelpers.cpp ${RUNTIME_DIR}/MiscNativeHelpers.cpp cachelinealloc.cpp callconvbuilder.cpp diff --git a/src/coreclr/vm/comsynchronizable.cpp b/src/coreclr/vm/comsynchronizable.cpp index 643373a5207b2b..76a5679655306e 100644 --- a/src/coreclr/vm/comsynchronizable.cpp +++ b/src/coreclr/vm/comsynchronizable.cpp @@ -595,31 +595,36 @@ void ThreadBaseObject::InitExisting() } } -FCIMPL1(void, ThreadNative::Finalize, ThreadBaseObject* pThisUNSAFE) +extern "C" void QCALLTYPE ThreadNative_Finalize(QCall::ObjectHandleOnStack thread) { - FCALL_CONTRACT; + QCALL_CONTRACT; - THREADBASEREF refThis = (THREADBASEREF)pThisUNSAFE; - Thread* thread = refThis->GetInternal(); + BEGIN_QCALL; + + GCX_COOP(); + + THREADBASEREF refThis = (THREADBASEREF)thread.Get(); + Thread* pThread = refThis->GetInternal(); // Prevent multiple calls to Finalize // Objects can be resurrected after being finalized. However, there is no // race condition here. We always check whether an exposed thread object is // still attached to the internal Thread object, before proceeding. - if (thread) + if (pThread) { refThis->ResetStartHelper(); - if (GetThreadNULLOk() != thread) + if (GetThreadNULLOk() != pThread) { refThis->ClearInternal(); } - thread->SetThreadState(Thread::TS_Finalized); + pThread->SetThreadState(Thread::TS_Finalized); Thread::SetCleanupNeededForFinalizedThread(); } + + END_QCALL; } -FCIMPLEND FCIMPL0(FC_BOOL_RET, ThreadNative::CatchAtSafePoint) { diff --git a/src/coreclr/vm/comsynchronizable.h b/src/coreclr/vm/comsynchronizable.h index 16b1fe6d898b7a..4ca0240333a188 100644 --- a/src/coreclr/vm/comsynchronizable.h +++ b/src/coreclr/vm/comsynchronizable.h @@ -40,11 +40,11 @@ class ThreadNative }; static FCDECL0(INT32, GetOptimalMaxSpinWaitsPerSpinIteration); - static FCDECL1(void, Finalize, ThreadBaseObject* pThis); static FCDECL0(FC_BOOL_RET, CatchAtSafePoint); static FCDECL0(FC_BOOL_RET, CurrentThreadIsFinalizerThread); }; +extern "C" void QCALLTYPE ThreadNative_Finalize(QCall::ObjectHandleOnStack thread); extern "C" void QCALLTYPE ThreadNative_Start(QCall::ThreadHandle thread, int threadStackSize, int priority, BOOL isThreadPool, PCWSTR pThreadName); extern "C" void QCALLTYPE ThreadNative_SetPriority(QCall::ObjectHandleOnStack thread, INT32 iPriority); extern "C" void QCALLTYPE ThreadNative_GetCurrentThread(QCall::ObjectHandleOnStack thread); diff --git a/src/coreclr/vm/comutilnative.cpp b/src/coreclr/vm/comutilnative.cpp index 27ea6fa75b4b6e..a9e0e9bf25c279 100644 --- a/src/coreclr/vm/comutilnative.cpp +++ b/src/coreclr/vm/comutilnative.cpp @@ -1099,26 +1099,6 @@ extern "C" void QCALLTYPE GCInterface_ReRegisterForFinalize(QCall::ObjectHandleO END_QCALL; } -FORCEINLINE UINT64 GCInterface::InterlockedAdd (UINT64 *pAugend, UINT64 addend) { - WRAPPER_NO_CONTRACT; - - UINT64 oldMemValue; - UINT64 newMemValue; - - do { - oldMemValue = *pAugend; - newMemValue = oldMemValue + addend; - - // check for overflow - if (newMemValue < oldMemValue) - { - newMemValue = UINT64_MAX; - } - } while (InterlockedCompareExchange64((LONGLONG*) pAugend, (LONGLONG) newMemValue, (LONGLONG) oldMemValue) != (LONGLONG) oldMemValue); - - return newMemValue; -} - FORCEINLINE UINT64 GCInterface::InterlockedSub(UINT64 *pMinuend, UINT64 subtrahend) { WRAPPER_NO_CONTRACT; @@ -1138,15 +1118,6 @@ FORCEINLINE UINT64 GCInterface::InterlockedSub(UINT64 *pMinuend, UINT64 subtrahe return newMemValue; } -extern "C" void QCALLTYPE GCInterface_AddMemoryPressure(UINT64 bytesAllocated) -{ - QCALL_CONTRACT; - - BEGIN_QCALL; - GCInterface::AddMemoryPressure(bytesAllocated); - END_QCALL; -} - extern "C" void QCALLTYPE GCInterface_EnumerateConfigurationValues(void* configurationContext, EnumerateConfigurationValuesCallback callback) { QCALL_CONTRACT; @@ -1252,54 +1223,7 @@ uint64_t GCInterface::GetGenerationBudget(int generation) return GCHeapUtilities::GetGCHeap()->GetGenerationBudget(generation); } -#ifdef HOST_64BIT -const unsigned MIN_MEMORYPRESSURE_BUDGET = 4 * 1024 * 1024; // 4 MB -#else // HOST_64BIT -const unsigned MIN_MEMORYPRESSURE_BUDGET = 3 * 1024 * 1024; // 3 MB -#endif // HOST_64BIT - -const unsigned MAX_MEMORYPRESSURE_RATIO = 10; // 40 MB or 30 MB - - -// Resets pressure accounting after a gen2 GC has occurred. -void GCInterface::CheckCollectionCount() -{ - LIMITED_METHOD_CONTRACT; - - IGCHeap * pHeap = GCHeapUtilities::GetGCHeap(); - - if (m_gc_counts[2] != pHeap->CollectionCount(2)) - { - for (int i = 0; i < 3; i++) - { - m_gc_counts[i] = pHeap->CollectionCount(i); - } - - m_iteration++; - - UINT p = m_iteration % MEM_PRESSURE_COUNT; - - m_addPressure[p] = 0; // new pressure will be accumulated here - m_remPressure[p] = 0; - } -} - -// AddMemoryPressure implementation -// -// 1. Start budget - MIN_MEMORYPRESSURE_BUDGET -// 2. Focuses more on newly added memory pressure -// 3. Budget adjusted by effectiveness of last 3 triggered GC (add / remove ratio, max 10x) -// 4. Budget maxed with 30% of current managed GC size -// 5. If Gen2 GC is happening naturally, ignore past pressure -// -// Here's a brief description of the ideal algorithm for Add/Remove memory pressure: -// Do a GC when (HeapStart < X * MemPressureGrowth) where -// - HeapStart is GC Heap size after doing the last GC -// - MemPressureGrowth is the net of Add and Remove since the last GC -// - X is proportional to our guess of the ummanaged memory death rate per GC interval, -// and would be calculated based on historic data using standard exponential approximation: -// Xnew = UMDeath/UMTotal * 0.5 + Xprev -// +// Static wrapper methods to call managed AddMemoryPressure/RemoveMemoryPressure void GCInterface::AddMemoryPressure(UINT64 bytesAllocated) { CONTRACTL @@ -1310,103 +1234,15 @@ void GCInterface::AddMemoryPressure(UINT64 bytesAllocated) } CONTRACTL_END; - CheckCollectionCount(); - - UINT p = m_iteration % MEM_PRESSURE_COUNT; - - UINT64 newMemValue = InterlockedAdd(&m_addPressure[p], bytesAllocated); - - static_assert(MEM_PRESSURE_COUNT == 4, "AddMemoryPressure contains unrolled loops which depend on MEM_PRESSURE_COUNT"); - - UINT64 add = m_addPressure[0] + m_addPressure[1] + m_addPressure[2] + m_addPressure[3] - m_addPressure[p]; - UINT64 rem = m_remPressure[0] + m_remPressure[1] + m_remPressure[2] + m_remPressure[3] - m_remPressure[p]; - - STRESS_LOG4(LF_GCINFO, LL_INFO10000, "AMP Add: %llu => added=%llu total_added=%llu total_removed=%llu", - bytesAllocated, newMemValue, add, rem); - - SendEtwAddMemoryPressureEvent(bytesAllocated); - - if (newMemValue >= MIN_MEMORYPRESSURE_BUDGET) - { - UINT64 budget = MIN_MEMORYPRESSURE_BUDGET; - - if (m_iteration >= MEM_PRESSURE_COUNT) // wait until we have enough data points - { - // Adjust according to effectiveness of GC - // Scale budget according to past m_addPressure / m_remPressure ratio - if (add >= rem * MAX_MEMORYPRESSURE_RATIO) - { - budget = MIN_MEMORYPRESSURE_BUDGET * MAX_MEMORYPRESSURE_RATIO; - } - else if (add > rem) - { - CONSISTENCY_CHECK(rem != 0); - - // Avoid overflow by calculating addPressure / remPressure as fixed point (1 = 1024) - budget = (add * 1024 / rem) * budget / 1024; - } - } - - // If still over budget, check current managed heap size - if (newMemValue >= budget) - { - IGCHeap *pGCHeap = GCHeapUtilities::GetGCHeap(); - UINT64 heapOver3 = pGCHeap->GetCurrentObjSize() / 3; - - if (budget < heapOver3) // Max - { - budget = heapOver3; - } - - if (newMemValue >= budget) - { - // last check - if we would exceed 20% of GC "duty cycle", do not trigger GC at this time - if ((size_t)(pGCHeap->GetNow() - pGCHeap->GetLastGCStartTime(2)) > (pGCHeap->GetLastGCDuration(2) * 5)) - { - STRESS_LOG6(LF_GCINFO, LL_INFO10000, "AMP Budget: pressure=%llu ? budget=%llu (total_added=%llu, total_removed=%llu, mng_heap=%llu) pos=%d", - newMemValue, budget, add, rem, heapOver3 * 3, m_iteration); - - GarbageCollectModeAny(2); - - CheckCollectionCount(); - } - } - } - } -} - -extern "C" void QCALLTYPE GCInterface_RemoveMemoryPressure(UINT64 bytesAllocated) -{ - QCALL_CONTRACT; + GCX_COOP(); - BEGIN_QCALL; - GCInterface::RemoveMemoryPressure(bytesAllocated); - END_QCALL; + PREPARE_NONVIRTUAL_CALLSITE(METHOD__GC__ADD_MEMORY_PRESSURE); + DECLARE_ARGHOLDER_ARRAY(args, 1); + args[ARGNUM_0] = PTR_TO_ARGHOLDER((INT64)bytesAllocated); + CALL_MANAGED_METHOD_NORET(args); } void GCInterface::RemoveMemoryPressure(UINT64 bytesAllocated) -{ - CONTRACTL - { - NOTHROW; - GC_NOTRIGGER; - MODE_ANY; - } - CONTRACTL_END; - - CheckCollectionCount(); - - UINT p = m_iteration % MEM_PRESSURE_COUNT; - - SendEtwRemoveMemoryPressureEvent(bytesAllocated); - - InterlockedAdd(&m_remPressure[p], bytesAllocated); - - STRESS_LOG2(LF_GCINFO, LL_INFO10000, "AMP Remove: %llu => removed=%llu", - bytesAllocated, m_remPressure[p]); -} - -inline void GCInterface::SendEtwAddMemoryPressureEvent(UINT64 bytesAllocated) { CONTRACTL { @@ -1416,44 +1252,10 @@ inline void GCInterface::SendEtwAddMemoryPressureEvent(UINT64 bytesAllocated) } CONTRACTL_END; - FireEtwIncreaseMemoryPressure(bytesAllocated, GetClrInstanceId()); -} - -// Out-of-line helper to avoid EH prolog/epilog in functions that otherwise don't throw. -NOINLINE void GCInterface::SendEtwRemoveMemoryPressureEvent(UINT64 bytesAllocated) -{ - CONTRACTL - { - NOTHROW; - GC_NOTRIGGER; - MODE_ANY; - } - CONTRACTL_END; - - EX_TRY - { - FireEtwDecreaseMemoryPressure(bytesAllocated, GetClrInstanceId()); - } - EX_CATCH - { - // Ignore failures - } - EX_END_CATCH -} - -// Out-of-line helper to avoid EH prolog/epilog in functions that otherwise don't throw. -NOINLINE void GCInterface::GarbageCollectModeAny(int generation) -{ - CONTRACTL - { - THROWS; - GC_TRIGGERS; - MODE_ANY; - } - CONTRACTL_END; - - GCX_COOP(); - GCHeapUtilities::GetGCHeap()->GarbageCollect(generation, false, collection_non_blocking); + PREPARE_NONVIRTUAL_CALLSITE(METHOD__GC__REMOVE_MEMORY_PRESSURE); + DECLARE_ARGHOLDER_ARRAY(args, 1); + args[ARGNUM_0] = PTR_TO_ARGHOLDER((INT64)bytesAllocated); + CALL_MANAGED_METHOD_NORET(args); } // diff --git a/src/coreclr/vm/comutilnative.h b/src/coreclr/vm/comutilnative.h index 2d362b0fa0cc0c..7088aed3ebada7 100644 --- a/src/coreclr/vm/comutilnative.h +++ b/src/coreclr/vm/comutilnative.h @@ -154,7 +154,6 @@ class GCInterface { static UINT m_iteration; public: - static FORCEINLINE UINT64 InterlockedAdd(UINT64 *pAugend, UINT64 addend); static FORCEINLINE UINT64 InterlockedSub(UINT64 *pMinuend, UINT64 subtrahend); static FCDECL0(INT64, GetTotalPauseDuration); @@ -179,21 +178,14 @@ class GCInterface { static FCDECL0(INT64, GetAllocatedBytesForCurrentThread); static FCDECL0(INT64, GetTotalAllocatedBytesApproximate); - NOINLINE static void SendEtwRemoveMemoryPressureEvent(UINT64 bytesAllocated); - static void SendEtwAddMemoryPressureEvent(UINT64 bytesAllocated); - - static void CheckCollectionCount(); - static void RemoveMemoryPressure(UINT64 bytesAllocated); + // Wrapper methods that call managed implementation static void AddMemoryPressure(UINT64 bytesAllocated); + static void RemoveMemoryPressure(UINT64 bytesAllocated); static void EnumerateConfigurationValues(void* configurationContext, EnumerateConfigurationValuesCallback callback); static int RefreshMemoryLimit(); static enable_no_gc_region_callback_status EnableNoGCRegionCallback(NoGCRegionCallbackFinalizerWorkItem* callback, INT64 totalSize); static uint64_t GetGenerationBudget(int generation); - -private: - // Out-of-line helper to avoid EH prolog/epilog in functions that otherwise don't throw. - NOINLINE static void GarbageCollectModeAny(int generation); }; extern "C" INT64 QCALLTYPE GCInterface_GetTotalAllocatedBytesPrecise(); @@ -221,10 +213,6 @@ extern "C" int QCALLTYPE GCInterface_StartNoGCRegion(INT64 totalSize, BOOL lohSi extern "C" int QCALLTYPE GCInterface_EndNoGCRegion(); -extern "C" void QCALLTYPE GCInterface_AddMemoryPressure(UINT64 bytesAllocated); - -extern "C" void QCALLTYPE GCInterface_RemoveMemoryPressure(UINT64 bytesAllocated); - extern "C" void QCALLTYPE GCInterface_ReRegisterForFinalize(QCall::ObjectHandleOnStack pObj); extern "C" void QCALLTYPE GCInterface_EnumerateConfigurationValues(void* configurationContext, EnumerateConfigurationValuesCallback callback); diff --git a/src/coreclr/vm/corelib.h b/src/coreclr/vm/corelib.h index 9e2e71bb3a4bc3..9be013c6bf8599 100644 --- a/src/coreclr/vm/corelib.h +++ b/src/coreclr/vm/corelib.h @@ -1045,6 +1045,8 @@ DEFINE_METHOD(VALUE_TYPE, EQUALS, Equals, DEFINE_CLASS(GC, System, GC) DEFINE_METHOD(GC, KEEP_ALIVE, KeepAlive, SM_Obj_RetVoid) DEFINE_METHOD(GC, RUN_FINALIZERS, RunFinalizers, SM_RetUInt) +DEFINE_METHOD(GC, ADD_MEMORY_PRESSURE, AddMemoryPressure, SM_Long_RetVoid) +DEFINE_METHOD(GC, REMOVE_MEMORY_PRESSURE, RemoveMemoryPressure, SM_Long_RetVoid) DEFINE_CLASS_U(System, WeakReference, WeakReferenceObject) DEFINE_FIELD_U(_taggedHandle, WeakReferenceObject, m_taggedHandle) diff --git a/src/coreclr/vm/ecalllist.h b/src/coreclr/vm/ecalllist.h index 1776af6c065405..16e34bec5fed47 100644 --- a/src/coreclr/vm/ecalllist.h +++ b/src/coreclr/vm/ecalllist.h @@ -249,7 +249,6 @@ FCFuncStart(gMathFFuncs) FCFuncEnd() FCFuncStart(gThreadFuncs) - FCFuncElement("InternalFinalize", ThreadNative::Finalize) FCFuncElement("CatchAtSafePoint", ThreadNative::CatchAtSafePoint) FCFuncElement("CurrentThreadIsFinalizerThread", ThreadNative::CurrentThreadIsFinalizerThread) FCFuncElement("get_OptimalMaxSpinWaitsPerSpinIteration", ThreadNative::GetOptimalMaxSpinWaitsPerSpinIteration) diff --git a/src/coreclr/vm/loaderallocator.cpp b/src/coreclr/vm/loaderallocator.cpp index c2f5c813a305af..c6c61c82f99fba 100644 --- a/src/coreclr/vm/loaderallocator.cpp +++ b/src/coreclr/vm/loaderallocator.cpp @@ -1383,7 +1383,6 @@ void LoaderAllocator::Terminate() if (m_fGCPressure) { - GCX_PREEMP(); GCInterface::RemoveMemoryPressure(30000); m_fGCPressure = false; } diff --git a/src/coreclr/vm/metasig.h b/src/coreclr/vm/metasig.h index 3abae3a6fded9e..b461b5b257e21d 100644 --- a/src/coreclr/vm/metasig.h +++ b/src/coreclr/vm/metasig.h @@ -374,6 +374,7 @@ DEFINE_METASIG(SM(Int_Str_IntPtr_RetIntPtr, i s I, I)) DEFINE_METASIG(SM(Int_Str_IntPtr_Int_RetVoid, i s I i, v)) DEFINE_METASIG(SM(Str_IntPtr_RetIntPtr, s I, I)) DEFINE_METASIG(SM(Str_Bool_Int_RetV, s F i, v)) +DEFINE_METASIG(SM(Long_RetVoid, l, v)) DEFINE_METASIG_T(SM(Type_RetObj, C(TYPE), j)) DEFINE_METASIG_T(SM(Type_RetInt, C(TYPE), i)) diff --git a/src/coreclr/vm/object.cpp b/src/coreclr/vm/object.cpp index 877a565fda44b6..ae7534ea476645 100644 --- a/src/coreclr/vm/object.cpp +++ b/src/coreclr/vm/object.cpp @@ -208,14 +208,6 @@ TypeHandle Object::GetGCSafeTypeHandleIfPossible() const // MyValueType1's module's m_AssemblyRefByNameTable, which is garbage if its // AppDomain is unloading. // - // Another AV was encountered in a similar case, - // - // MyRefType1[] myArray - // - // where MyRefType2's module was unloaded by the time the GC occurred. In at least - // one case, the GC was caused by the AD unload itself (AppDomain::Unload -> - // AppDomain::Exit -> GCInterface::AddMemoryPressure -> WKS::GCHeapUtilities::GarbageCollect). - // // To protect against all scenarios, verify that // // * The MT of the object is not getting unloaded, OR diff --git a/src/coreclr/vm/qcallentrypoints.cpp b/src/coreclr/vm/qcallentrypoints.cpp index 7c3e125b375aa2..c5a89dc07a19b9 100644 --- a/src/coreclr/vm/qcallentrypoints.cpp +++ b/src/coreclr/vm/qcallentrypoints.cpp @@ -330,8 +330,6 @@ static const Entry s_QCall[] = DllImportEntry(GCInterface_ReRegisterForFinalize) DllImportEntry(GCInterface_GetNextFinalizableObject) DllImportEntry(GCInterface_WaitForPendingFinalizers) - DllImportEntry(GCInterface_AddMemoryPressure) - DllImportEntry(GCInterface_RemoveMemoryPressure) #ifdef FEATURE_BASICFREEZE DllImportEntry(GCInterface_RegisterFrozenSegment) DllImportEntry(GCInterface_UnregisterFrozenSegment) diff --git a/src/libraries/System.Private.CoreLib/src/System/GC.cs b/src/libraries/System.Private.CoreLib/src/System/GC.cs index de53aa1296c5a3..0386baebc3c1bd 100644 --- a/src/libraries/System.Private.CoreLib/src/System/GC.cs +++ b/src/libraries/System.Private.CoreLib/src/System/GC.cs @@ -1,6 +1,9 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. +using System.Diagnostics; +using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; using System.Threading; namespace System @@ -24,5 +27,209 @@ public static GCNotificationStatus WaitForFullGCApproach(TimeSpan timeout) /// The status of a registered full GC notification public static GCNotificationStatus WaitForFullGCComplete(TimeSpan timeout) => WaitForFullGCComplete(WaitHandle.ToTimeoutMilliseconds(timeout)); + +#if !MONO + // Support for AddMemoryPressure and RemoveMemoryPressure below. + private const uint PressureCount = 4; +#if TARGET_64BIT + private const uint MinGCMemoryPressureBudget = 4 * 1024 * 1024; +#else + private const uint MinGCMemoryPressureBudget = 3 * 1024 * 1024; +#endif + + private const uint MaxGCMemoryPressureRatio = 10; + + private static int[] s_gcCounts = new int[] { 0, 0, 0 }; + + private static long[] s_addPressure = new long[] { 0, 0, 0, 0 }; + private static long[] s_removePressure = new long[] { 0, 0, 0, 0 }; + private static uint s_iteration; + + /// + /// Resets the pressure accounting after a gen2 GC has occurred. + /// + private static void CheckCollectionCount() + { + if (s_gcCounts[2] != CollectionCount(2)) + { + for (int i = 0; i < 3; i++) + { + s_gcCounts[i] = CollectionCount(i); + } + + s_iteration++; + + uint p = s_iteration % PressureCount; + + s_addPressure[p] = 0; + s_removePressure[p] = 0; + } + } + + private static long InterlockedAddMemoryPressure(ref long pAugend, long addend) + { + long oldMemValue; + long newMemValue; + + do + { + oldMemValue = pAugend; + newMemValue = oldMemValue + addend; + + // check for overflow + if (newMemValue < oldMemValue) + { + newMemValue = long.MaxValue; + } + } while (Interlocked.CompareExchange(ref pAugend, newMemValue, oldMemValue) != oldMemValue); + + return newMemValue; + } + + /// + /// New AddMemoryPressure implementation (used by RCW and the CLRServicesImpl class) + /// 1. Less sensitive than the original implementation (start budget 3 MB) + /// 2. Focuses more on newly added memory pressure + /// 3. Budget adjusted by effectiveness of last 3 triggered GC (add / remove ratio, max 10x) + /// 4. Budget maxed with 30% of current managed GC size + /// 5. If Gen2 GC is happening naturally, ignore past pressure + /// + /// Here's a brief description of the ideal algorithm for Add/Remove memory pressure: + /// Do a GC when (HeapStart is less than X * MemPressureGrowth) where + /// - HeapStart is GC Heap size after doing the last GC + /// - MemPressureGrowth is the net of Add and Remove since the last GC + /// - X is proportional to our guess of the ummanaged memory death rate per GC interval, + /// and would be calculated based on historic data using standard exponential approximation: + /// Xnew = UMDeath/UMTotal * 0.5 + Xprev + /// + /// + public static void AddMemoryPressure(long bytesAllocated) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytesAllocated); +#if !TARGET_64BIT + ArgumentOutOfRangeException.ThrowIfGreaterThan(bytesAllocated, int.MaxValue); +#endif + + CheckCollectionCount(); + uint p = s_iteration % PressureCount; + long newMemValue = InterlockedAddMemoryPressure(ref s_addPressure[p], bytesAllocated); + + Debug.Assert(PressureCount == 4, "GC.AddMemoryPressure contains unrolled loops which depend on the PressureCount"); + + SendEtwAddMemoryPressureEvent((ulong)bytesAllocated); + + if (newMemValue >= MinGCMemoryPressureBudget) + { + long add = s_addPressure[0] + s_addPressure[1] + s_addPressure[2] + s_addPressure[3] - s_addPressure[p]; + long rem = s_removePressure[0] + s_removePressure[1] + s_removePressure[2] + s_removePressure[3] - s_removePressure[p]; + + long budget = MinGCMemoryPressureBudget; + + if (s_iteration >= PressureCount) // wait until we have enough data points + { + // Adjust according to effectiveness of GC + // Scale budget according to past m_addPressure / m_remPressure ratio + if (add >= rem * MaxGCMemoryPressureRatio) + { + budget = MinGCMemoryPressureBudget * MaxGCMemoryPressureRatio; + } + else if (add > rem) + { + Debug.Assert(rem != 0); + + // Avoid overflow by calculating addPressure / remPressure as fixed point (1 = 1024) + budget = (add * 1024 / rem) * budget / 1024; + } + } + + // If still over budget, check current managed heap size + if (newMemValue >= budget) + { + long heapOver3 = GetCurrentObjSize() / 3; + + if (budget < heapOver3) //Max + { + budget = heapOver3; + } + + if (newMemValue >= budget) + { + // last check - if we would exceed 20% of GC "duty cycle", do not trigger GC at this time + if ((GetNow() - GetLastGCStartTime(2)) > (GetLastGCDuration(2) * 5)) + { + TriggerCollection(2, InternalGCCollectionMode.NonBlocking); + CheckCollectionCount(); + } + } + } + } + } + + public static void RemoveMemoryPressure(long bytesAllocated) + { + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(bytesAllocated); +#if !TARGET_64BIT + ArgumentOutOfRangeException.ThrowIfGreaterThan(bytesAllocated, int.MaxValue); +#endif + + CheckCollectionCount(); + uint p = s_iteration % PressureCount; + SendEtwRemoveMemoryPressureEvent((ulong)bytesAllocated); + InterlockedAddMemoryPressure(ref s_removePressure[p], bytesAllocated); + } + +#if NATIVEAOT + [MethodImpl(MethodImplOptions.InternalCall)] + [System.Runtime.RuntimeImport("*", "RhGetCurrentObjSize")] + private static extern long GetCurrentObjSize(); + + [MethodImpl(MethodImplOptions.InternalCall)] + [System.Runtime.RuntimeImport("*", "RhGetGCNow")] + private static extern long GetNow(); + + [MethodImpl(MethodImplOptions.InternalCall)] + [System.Runtime.RuntimeImport("*", "RhGetLastGCStartTime")] + private static extern long GetLastGCStartTime(int generation); + + [MethodImpl(MethodImplOptions.InternalCall)] + [System.Runtime.RuntimeImport("*", "RhGetLastGCDuration")] + private static extern long GetLastGCDuration(int generation); + + [LibraryImport("*", EntryPoint = "RhCollect")] + private static partial void TriggerCollection(int generation, InternalGCCollectionMode mode); + + private static void SendEtwAddMemoryPressureEvent(ulong bytesAllocated) + { + // ETW events not currently sent for NativeAOT + _ = bytesAllocated; + } + + private static void SendEtwRemoveMemoryPressureEvent(ulong bytesAllocated) + { + // ETW events not currently sent for NativeAOT + _ = bytesAllocated; + } +#else + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GC_GetCurrentObjSize")] + private static partial long GetCurrentObjSize(); + + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GC_GetNow")] + private static partial long GetNow(); + + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GC_GetLastGCStartTime")] + private static partial long GetLastGCStartTime(int generation); + + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GC_GetLastGCDuration")] + private static partial long GetLastGCDuration(int generation); + + private static void TriggerCollection(int generation, InternalGCCollectionMode mode) => _Collect(generation, (int)mode, lowMemoryPressure: false); + + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GC_SendEtwAddMemoryPressureEvent")] + private static partial void SendEtwAddMemoryPressureEvent(ulong bytesAllocated); + + [LibraryImport(RuntimeHelpers.QCall, EntryPoint = "GC_SendEtwRemoveMemoryPressureEvent")] + private static partial void SendEtwRemoveMemoryPressureEvent(ulong bytesAllocated); +#endif +#endif // !MONO } }