Add randomized allocation sampling

src/coreclr/debug/daccess/dacdbiimpl.cpp

@@ -6543,10 +6543,10 @@ HRESULT DacHeapWalker::Init(CORDB_ADDRESS start, CORDB_ADDRESS end)
                 j++;
             }
         }
-        if ((&g_global_alloc_context)->alloc_ptr != nullptr)
+        if (g_global_alloc_context->alloc_ptr != nullptr)
         {
-            mAllocInfo[j].Ptr = (CORDB_ADDRESS)(&g_global_alloc_context)->alloc_ptr;
-            mAllocInfo[j].Limit = (CORDB_ADDRESS)(&g_global_alloc_context)->alloc_limit;
+            mAllocInfo[j].Ptr = (CORDB_ADDRESS)g_global_alloc_context->alloc_ptr;
+            mAllocInfo[j].Limit = (CORDB_ADDRESS)g_global_alloc_context->alloc_limit;
         }
 
         mThreadCount = j;

src/coreclr/debug/daccess/request.cpp

@@ -720,8 +720,8 @@ ClrDataAccess::GetThreadAllocData(CLRDATA_ADDRESS addr, struct DacpAllocData *da
 
     Thread* thread = PTR_Thread(TO_TADDR(addr));
 
-    data->allocBytes = TO_CDADDR(thread->m_alloc_context.alloc_bytes);
-    data->allocBytesLoh = TO_CDADDR(thread->m_alloc_context.alloc_bytes_uoh);
+    data->allocBytes = TO_CDADDR(thread->m_alloc_context.gc_alloc_context.alloc_bytes);
+    data->allocBytesLoh = TO_CDADDR(thread->m_alloc_context.gc_alloc_context.alloc_bytes_uoh);
 
     SOSDacLeave();
     return hr;
@@ -775,8 +775,8 @@ ClrDataAccess::GetThreadData(CLRDATA_ADDRESS threadAddr, struct DacpThreadData *
     threadData->osThreadId = (DWORD)thread->m_OSThreadId;
     threadData->state = thread->m_State;
     threadData->preemptiveGCDisabled = thread->m_fPreemptiveGCDisabled;
-    threadData->allocContextPtr = TO_CDADDR(thread->m_alloc_context.alloc_ptr);
-    threadData->allocContextLimit = TO_CDADDR(thread->m_alloc_context.alloc_limit);
+    threadData->allocContextPtr = TO_CDADDR(thread->m_alloc_context.gc_alloc_context.alloc_ptr);
+    threadData->allocContextLimit = TO_CDADDR(thread->m_alloc_context.gc_alloc_context.alloc_limit);
 
     threadData->fiberData = (CLRDATA_ADDRESS)NULL;
 
@@ -5347,8 +5347,8 @@ HRESULT ClrDataAccess::GetGlobalAllocationContext(
     }
 
     SOSDacEnter();
-    *allocPtr = (CLRDATA_ADDRESS)((&g_global_alloc_context)->alloc_ptr);
-    *allocLimit = (CLRDATA_ADDRESS)((&g_global_alloc_context)->alloc_limit);
+    *allocPtr = (CLRDATA_ADDRESS)(g_global_alloc_context->alloc_ptr);
+    *allocLimit = (CLRDATA_ADDRESS)(g_global_alloc_context->alloc_limit);
     SOSDacLeave();
     return hr;
 }

src/coreclr/gc/gc.cpp

@@ -44076,7 +44076,7 @@ size_t gc_heap::decommit_region (heap_segment* region, int bucket, int h_number)
     {
 #ifdef MULTIPLE_HEAPS
         // In return_free_region, we set heap_segment_heap (region) to nullptr so we cannot use it here.
-        // but since all heaps share the same mark array we simply pick the 0th heap to use. 
+        // but since all heaps share the same mark array we simply pick the 0th heap to use. 
         gc_heap* hp = g_heaps [0];
 #else
         gc_heap* hp = pGenGCHeap;
@@ -49293,7 +49293,6 @@ bool GCHeap::StressHeap(gc_alloc_context * context)
     }                                                                                       \
 } while (false)
 
-#ifdef FEATURE_64BIT_ALIGNMENT
 // Allocate small object with an alignment requirement of 8-bytes.
 Object* AllocAlign8(alloc_context* acontext, gc_heap* hp, size_t size, uint32_t flags)
 {
@@ -49359,7 +49358,6 @@ Object* AllocAlign8(alloc_context* acontext, gc_heap* hp, size_t size, uint32_t
 
     return newAlloc;
 }
-#endif // FEATURE_64BIT_ALIGNMENT
 
 Object*
 GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_DCL)
@@ -49420,15 +49418,11 @@ GCHeap::Alloc(gc_alloc_context* context, size_t size, uint32_t flags REQD_ALIGN_
     }
     else
     {
-#ifdef FEATURE_64BIT_ALIGNMENT
         if (flags & GC_ALLOC_ALIGN8)
         {
             newAlloc = AllocAlign8 (acontext, hp, size, flags);
         }
         else
-#else
-        assert ((flags & GC_ALLOC_ALIGN8) == 0);
-#endif
         {
             newAlloc = (Object*) hp->allocate (size + ComputeMaxStructAlignPad(requiredAlignment), acontext, flags);
         }

src/coreclr/gc/gcee.cpp

@@ -308,6 +308,12 @@ void gc_heap::fire_etw_allocation_event (size_t allocation_amount,
                                          uint8_t* object_address,
                                          size_t object_size)
 {
+    // do not emit AllocationTick event if AllocationSampled is enabled
+    if (EVENT_ENABLED(AllocationSampled))
+    {
+        return;
+    }
+
 #ifdef FEATURE_NATIVEAOT
     FIRE_EVENT(GCAllocationTick_V1, (uint32_t)allocation_amount, (uint32_t)gen_to_oh (gen_number));
 #else

src/coreclr/gc/gcevents.h

@@ -52,5 +52,10 @@ DYNAMIC_EVENT(CommittedUsage, GCEventLevel_Information, GCEventKeyword_GC, 1)
 DYNAMIC_EVENT(HeapCountTuning, GCEventLevel_Information, GCEventKeyword_GC, 1)
 DYNAMIC_EVENT(HeapCountSample, GCEventLevel_Information, GCEventKeyword_GC, 1)
 
+// non GC event that overrides AllocationTick
+// KNOWN_EVENT macro cannot be used because we don't want to emit the event; i.e. adding a method to IGCToCLREventSink
+// KNOWN_EVENT(AllocationSampled, GCEventProvider_Default, GCEventLevel_Information, EventKeyword_AllocationSampling)
+inline bool GCEventEnabledAllocationSampled() { return GCEventStatus::IsEnabled(GCEventProvider_Default, EventKeyword_AllocationSampling, GCEventLevel_Information); }
+
 #undef KNOWN_EVENT
 #undef DYNAMIC_EVENT

src/coreclr/gc/gcinterface.h

@@ -265,7 +265,7 @@ enum GCEventLevel
 // Event keywords corresponding to events that can be fired by the GC. These
 // numbers come from the ETW manifest itself - please make changes to this enum
 // if you add, remove, or change keyword sets that are used by the GC!
-enum GCEventKeyword
+enum GCEventKeyword : uint64_t
 {
     GCEventKeyword_None                          =       0x0,
     GCEventKeyword_GC                            =       0x1,
@@ -280,6 +280,12 @@ enum GCEventKeyword
     GCEventKeyword_ManagedHeapCollect            =  0x800000,
     GCEventKeyword_GCHeapAndTypeNames            = 0x1000000,
     GCEventKeyword_GCSampledObjectAllocationLow  = 0x2000000,
+
+    // Keyword outside of the GC where AllocationSampled event is defined.
+    // This is used to disable AllocationTick when AllocationSampling is enabled.
+    // Not counted in GCEventKeyword_All
+    EventKeyword_AllocationSampling          = 0x80000000000,
+
     GCEventKeyword_All = GCEventKeyword_GC
       | GCEventKeyword_GCPrivate
       | GCEventKeyword_GCHandle

src/coreclr/gc/gcpriv.h

@@ -1465,9 +1465,7 @@ class gc_heap
     friend struct ::alloc_context;
     friend void ProfScanRootsHelper(Object** object, ScanContext *pSC, uint32_t dwFlags);
     friend void GCProfileWalkHeapWorker(BOOL fProfilerPinned, BOOL fShouldWalkHeapRootsForEtw, BOOL fShouldWalkHeapObjectsForEtw);
-#ifdef FEATURE_64BIT_ALIGNMENT
     friend Object* AllocAlign8(alloc_context* acontext, gc_heap* hp, size_t size, uint32_t flags);
-#endif //FEATURE_64BIT_ALIGNMENT
     friend class t_join;
     friend class gc_mechanisms;
     friend class seg_free_spaces;

src/coreclr/inc/dacvars.h

@@ -140,7 +140,7 @@ DEFINE_DACVAR(ProfControlBlock, dac__g_profControlBlock, ::g_profControlBlock)
 DEFINE_DACVAR(PTR_DWORD, dac__g_card_table, ::g_card_table)
 DEFINE_DACVAR(PTR_BYTE, dac__g_lowest_address, ::g_lowest_address)
 DEFINE_DACVAR(PTR_BYTE, dac__g_highest_address, ::g_highest_address)
-DEFINE_DACVAR(gc_alloc_context, dac__g_global_alloc_context, ::g_global_alloc_context)
+DEFINE_DACVAR(UNKNOWN_POINTER_TYPE, dac__g_global_alloc_context, ::g_global_alloc_context)
 
 DEFINE_DACVAR(IGCHeap, dac__g_pGCHeap, ::g_pGCHeap)
 

src/coreclr/inc/eventtracebase.h

@@ -1323,17 +1323,19 @@ namespace ETW
 #define ETWLoaderStaticLoad 0 // Static reference load
 #define ETWLoaderDynamicLoad 1 // Dynamic assembly load
 
+#if defined(FEATURE_EVENT_TRACE)
+EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context;
+EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_PRIVATE_PROVIDER_DOTNET_Context;
+EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_RUNDOWN_PROVIDER_DOTNET_Context;
+EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_STRESS_PROVIDER_DOTNET_Context;
+#endif // FEATURE_EVENT_TRACE
+
 #if defined(FEATURE_EVENT_TRACE) && !defined(HOST_UNIX)
 //
 // The ONE and only ONE global instantiation of this class
 //
 extern ETW::CEtwTracer *  g_pEtwTracer;
 
-EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_PROVIDER_DOTNET_Context;
-EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_PRIVATE_PROVIDER_DOTNET_Context;
-EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_RUNDOWN_PROVIDER_DOTNET_Context;
-EXTERN_C DOTNET_TRACE_CONTEXT MICROSOFT_WINDOWS_DOTNETRUNTIME_STRESS_PROVIDER_DOTNET_Context;
-
 //
 // Special Handling of Startup events
 //

src/coreclr/nativeaot/Runtime/AsmOffsets.h

@@ -55,6 +55,7 @@ ASM_OFFSET(   44,    70, Thread, m_pvHijackedReturnAddress)
 ASM_OFFSET(   48,    78, Thread, m_uHijackedReturnValueFlags)
 ASM_OFFSET(   4c,    80, Thread, m_pExInfoStackHead)
 ASM_OFFSET(   50,    88, Thread, m_threadAbortException)
+ASM_OFFSET(   54,    90, Thread, m_combined_limit)
 
 ASM_SIZEOF(   14,    20, EHEnum)
 

src/coreclr/nativeaot/Runtime/GCHelpers.cpp

@@ -29,6 +29,16 @@
 
 #include "gcdesc.h"
 
+#ifdef FEATURE_EVENT_TRACE
+   #include "clretwallmain.h"
+#else // FEATURE_EVENT_TRACE
+   #include "etmdummy.h"
+#endif // FEATURE_EVENT_TRACE
+
+// TODO: used for dynamic allocation sampling
+// but generate duplicated symbols
+//#include "..\..\..\inc\sstring.h"
+
 #define RH_LARGE_OBJECT_SIZE 85000
 
 MethodTable g_FreeObjectEEType;
@@ -471,6 +481,43 @@ EXTERN_C int64_t QCALLTYPE RhGetTotalAllocatedBytesPrecise()
     return allocated;
 }
 
+inline void FireAllocationSampled(GC_ALLOC_FLAGS flags, size_t size, size_t samplingBudgetOffset, Object* orObject)
+{
+    // Note: this code is duplicated from GCToCLREventSink::FireGCAllocationTick_V4
+    void* typeId = nullptr;
+    const WCHAR* name = nullptr;
+    // TODO: this does not compile due to duplicated symbols when sstring.h is included
+    //InlineSString<MAX_CLASSNAME_LENGTH> strTypeName;
+    //EX_TRY
+    //{
+    //    TypeHandle th = GetThread()->GetTHAllocContextObj();
+
+    //    if (th != 0)
+    //    {
+    //        th.GetName(strTypeName);
+    //        name = strTypeName.GetUnicode();
+    //        typeId = th.GetMethodTable();
+    //    }
+    //}
+    //EX_CATCH{}
+    //EX_END_CATCH(SwallowAllExceptions)
+    // end of duplication
+
+    if (typeId != nullptr)
+    {
+        unsigned int allocKind =
+            (flags & GC_ALLOC_PINNED_OBJECT_HEAP) ? 2 :
+            (flags & GC_ALLOC_LARGE_OBJECT_HEAP) ? 1 :
+            0;  // SOH
+        unsigned int heapIndex = 0;
+#ifdef BACKGROUND_GC
+        gc_heap* hp = gc_heap::heap_of((BYTE*)orObject);
+        heapIndex = hp->heap_number;
+#endif
+        FireEtwAllocationSampled(allocKind, GetClrInstanceId(), typeId, name, heapIndex, (BYTE*)orObject, size, samplingBudgetOffset);
+    }
+}
+
 static Object* GcAllocInternal(MethodTable* pEEType, uint32_t uFlags, uintptr_t numElements, Thread* pThread)
 {
     ASSERT(!pThread->IsDoNotTriggerGcSet());
@@ -539,6 +586,9 @@ static Object* GcAllocInternal(MethodTable* pEEType, uint32_t uFlags, uintptr_t
     // Save the MethodTable for instrumentation purposes.
     tls_pLastAllocationEEType = pEEType;
 
+    // TODO: handle dynamic allocation sampling
+    //ee_alloc_context* acontext = pThread->GetEEAllocContext();
+
     Object* pObject = GCHeapUtilities::GetGCHeap()->Alloc(pThread->GetAllocContext(), cbSize, uFlags);
     if (pObject == NULL)
         return NULL;

src/coreclr/nativeaot/Runtime/amd64/AllocFast.S

@@ -28,7 +28,7 @@ NESTED_ENTRY RhpNewFast, _TEXT, NoHandler
 
         mov         rsi, [rax + OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         add         rdx, rsi
-        cmp         rdx, [rax + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rdx, [rax + OFFSETOF__Thread__m_combined_limit]
         ja          LOCAL_LABEL(RhpNewFast_RarePath)
 
         // set the new alloc pointer
@@ -143,7 +143,7 @@ NESTED_ENTRY RhNewString, _TEXT, NoHandler
         // rcx == Thread*
         // rdx == string size
         // r12 == element count
-        cmp         rax, [rcx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [rcx + OFFSETOF__Thread__m_combined_limit]
         ja          LOCAL_LABEL(RhNewString_RarePath)
 
         mov         [rcx + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax
@@ -226,7 +226,7 @@ NESTED_ENTRY RhpNewArray, _TEXT, NoHandler
         // rcx == Thread*
         // rdx == array size
         // r12 == element count
-        cmp         rax, [rcx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [rcx + OFFSETOF__Thread__m_combined_limit]
         ja          LOCAL_LABEL(RhpNewArray_RarePath)
 
         mov         [rcx + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax

src/coreclr/nativeaot/Runtime/amd64/AllocFast.asm

@@ -25,7 +25,7 @@ LEAF_ENTRY RhpNewFast, _TEXT
 
         mov         rax, [rdx + OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         add         r8, rax
-        cmp         r8, [rdx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         r8, [rdx + OFFSETOF__Thread__m_combined_limit]
         ja          RhpNewFast_RarePath
 
         ;; set the new alloc pointer
@@ -118,7 +118,7 @@ LEAF_ENTRY RhNewString, _TEXT
         ; rdx == element count
         ; r8 == array size
         ; r10 == thread
-        cmp         rax, [r10 + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [r10 + OFFSETOF__Thread__m_combined_limit]
         ja          RhpNewArrayRare
 
         mov         [r10 + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax
@@ -179,7 +179,7 @@ LEAF_ENTRY RhpNewArray, _TEXT
         ; rdx == element count
         ; r8 == array size
         ; r10 == thread
-        cmp         rax, [r10 + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [r10 + OFFSETOF__Thread__m_combined_limit]
         ja          RhpNewArrayRare
 
         mov         [r10 + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax

src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc

@@ -337,8 +337,6 @@ TSF_DoNotTriggerGc              equ 10h
 ;; Rename fields of nested structs
 ;;
 OFFSETOF__Thread__m_alloc_context__alloc_ptr        equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
-OFFSETOF__Thread__m_alloc_context__alloc_limit      equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
-
 
 
 ;; GC type flags

src/coreclr/nativeaot/Runtime/arm/AllocFast.S

@@ -26,7 +26,7 @@ LEAF_ENTRY RhpNewFast, _TEXT
 
         ldr         r3, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         add         r2, r3
-        ldr         r1, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r1, [r0, #OFFSETOF__Thread__m_combined_limit]
         cmp         r2, r1
         bhi         LOCAL_LABEL(RhpNewFast_RarePath)
 
@@ -132,7 +132,7 @@ LEAF_ENTRY RhNewString, _TEXT
         adds        r6, r12
         bcs         LOCAL_LABEL(RhNewString_RarePath) // if we get a carry here, the string is too large to fit below 4 GB
 
-        ldr         r12, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r12, [r0, #OFFSETOF__Thread__m_combined_limit]
         cmp         r6, r12
         bhi         LOCAL_LABEL(RhNewString_RarePath)
 
@@ -213,7 +213,7 @@ LOCAL_LABEL(ArrayAlignSize):
         adds        r6, r12
         bcs         LOCAL_LABEL(RhpNewArray_RarePath) // if we get a carry here, the array is too large to fit below 4 GB
 
-        ldr         r12, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r12, [r0, #OFFSETOF__Thread__m_combined_limit]
         cmp         r6, r12
         bhi         LOCAL_LABEL(RhpNewArray_RarePath)
 
@@ -349,7 +349,7 @@ LEAF_ENTRY RhpNewFastAlign8, _TEXT
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         r2, r3
-        ldr         r3, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r3, [r0, #OFFSETOF__Thread__m_combined_limit]
         cmp         r2, r3
         bhi         LOCAL_LABEL(Alloc8Failed)
 
@@ -412,7 +412,7 @@ LEAF_ENTRY RhpNewFastMisalign, _TEXT
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         r2, r3
-        ldr         r3, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r3, [r0, #OFFSETOF__Thread__m_combined_limit]
         cmp         r2, r3
         bhi         LOCAL_LABEL(BoxAlloc8Failed)