Add ee_alloc_context (NativeAOT)

This change is some preparatory refactoring for the randomized allocation sampling feature. We need to add more state onto allocation context but we don't want to do a breaking change of the GC interface. The new state only needs to be visible to the EE but we want it physically near the existing alloc context state for good cache locality. To accomplish this we created a new ee_alloc_context struct which contains an instance of gc_alloc_context within it.

In a future PR we will add a field called combined_limit which should be used by fast allocation helpers to determine when to go down the slow path. Most of the time combined_limit has the same value as alloc_limit, but periodically we need to emit an allocation sampling event on an object that is somewhere in the middle of an AC. Using combined_limit rather than alloc_limit as the slow path trigger allows us to keep all the sampling event logic in the slow path.

This PR introduces the abstraction for combined_limit and changes all the fast allocation helpers to use it, but it does not physically create the field yet. For now combined_limit is just an alias back to alloc_limit.

Overall this PR should not cause any change in runtime behavior and compiled code should be largely identical to before assuming modest inlining and optimization by the compiler.

src/coreclr/nativeaot/Runtime/AsmOffsets.h

@@ -46,7 +46,7 @@ ASM_OFFSET(    0,     0, MethodTable, m_uFlags)
 ASM_OFFSET(    4,     4, MethodTable, m_uBaseSize)
 ASM_OFFSET(   14,    18, MethodTable, m_VTable)
 
-ASM_OFFSET(    0,     0, Thread, m_rgbAllocContextBuffer)
+ASM_OFFSET(    0,     0, Thread, m_eeAllocContext)
 ASM_OFFSET(   28,    38, Thread, m_ThreadStateFlags)
 ASM_OFFSET(   2c,    40, Thread, m_pTransitionFrame)
 ASM_OFFSET(   30,    48, Thread, m_pDeferredTransitionFrame)
@@ -61,6 +61,8 @@ ASM_SIZEOF(   14,    20, EHEnum)
 ASM_OFFSET(    0,     0, gc_alloc_context, alloc_ptr)
 ASM_OFFSET(    4,     8, gc_alloc_context, alloc_limit)
 
+ASM_OFFSET(    0,     0, ee_alloc_context, m_rgbAllocContextBuffer)
+
 #ifdef FEATURE_CACHED_INTERFACE_DISPATCH
 ASM_OFFSET(    4,     8, InterfaceDispatchCell, m_pCache)
 #ifdef INTERFACE_DISPATCH_CACHE_HAS_CELL_BACKPOINTER

src/coreclr/nativeaot/Runtime/AsmOffsetsVerify.cpp

@@ -22,7 +22,7 @@
 
 class AsmOffsets
 {
-    static_assert(sizeof(Thread::m_rgbAllocContextBuffer) >= sizeof(gc_alloc_context), "Thread::m_rgbAllocContextBuffer is not big enough to hold a gc_alloc_context");
+    static_assert(sizeof(ee_alloc_context::m_rgbAllocContextBuffer) >= sizeof(gc_alloc_context), "ee_alloc_context::m_rgbAllocContextBuffer is not big enough to hold a gc_alloc_context");
 
     // Some assembly helpers for arrays and strings are shared and use the fact that arrays and strings have similar layouts)
     static_assert(offsetof(Array, m_Length) == offsetof(String, m_Length), "The length field of String and Array have different offsets");

src/coreclr/nativeaot/Runtime/DebugHeader.cpp

@@ -163,6 +163,9 @@ extern "C" void PopulateDebugHeaders()
     MAKE_DEBUG_FIELD_ENTRY(dac_gc_heap, finalize_queue);
     MAKE_DEBUG_FIELD_ENTRY(dac_gc_heap, generation_table);
 
+    MAKE_SIZE_ENTRY(ee_alloc_context);
+    MAKE_DEBUG_FIELD_ENTRY(ee_alloc_context, m_rgbAllocContextBuffer);
+
     MAKE_SIZE_ENTRY(gc_alloc_context);
     MAKE_DEBUG_FIELD_ENTRY(gc_alloc_context, alloc_ptr);
     MAKE_DEBUG_FIELD_ENTRY(gc_alloc_context, alloc_limit);
@@ -194,7 +197,7 @@ extern "C" void PopulateDebugHeaders()
 
     MAKE_SIZE_ENTRY(RuntimeThreadLocals);
     MAKE_DEBUG_FIELD_ENTRY(RuntimeThreadLocals, m_pNext);
-    MAKE_DEBUG_FIELD_ENTRY(RuntimeThreadLocals, m_rgbAllocContextBuffer);
+    MAKE_DEBUG_FIELD_ENTRY(RuntimeThreadLocals, m_eeAllocContext);
     MAKE_DEBUG_FIELD_ENTRY(RuntimeThreadLocals, m_threadId);
     MAKE_DEBUG_FIELD_ENTRY(RuntimeThreadLocals, m_pThreadStressLog);
     MAKE_DEBUG_FIELD_ENTRY(RuntimeThreadLocals, m_pExInfoStackHead);

src/coreclr/nativeaot/Runtime/amd64/AllocFast.S

@@ -28,7 +28,7 @@ NESTED_ENTRY RhpNewFast, _TEXT, NoHandler
 
         mov         rsi, [rax + OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         add         rdx, rsi
-        cmp         rdx, [rax + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rdx, [rax + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          LOCAL_LABEL(RhpNewFast_RarePath)
 
         // set the new alloc pointer
@@ -143,7 +143,7 @@ NESTED_ENTRY RhNewString, _TEXT, NoHandler
         // rcx == Thread*
         // rdx == string size
         // r12 == element count
-        cmp         rax, [rcx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [rcx + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          LOCAL_LABEL(RhNewString_RarePath)
 
         mov         [rcx + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax
@@ -226,7 +226,7 @@ NESTED_ENTRY RhpNewArray, _TEXT, NoHandler
         // rcx == Thread*
         // rdx == array size
         // r12 == element count
-        cmp         rax, [rcx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [rcx + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          LOCAL_LABEL(RhpNewArray_RarePath)
 
         mov         [rcx + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax

src/coreclr/nativeaot/Runtime/amd64/AllocFast.asm

@@ -25,7 +25,7 @@ LEAF_ENTRY RhpNewFast, _TEXT
 
         mov         rax, [rdx + OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         add         r8, rax
-        cmp         r8, [rdx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         r8, [rdx + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          RhpNewFast_RarePath
 
         ;; set the new alloc pointer
@@ -118,7 +118,7 @@ LEAF_ENTRY RhNewString, _TEXT
         ; rdx == element count
         ; r8 == array size
         ; r10 == thread
-        cmp         rax, [r10 + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [r10 + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          RhpNewArrayRare
 
         mov         [r10 + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax
@@ -179,7 +179,7 @@ LEAF_ENTRY RhpNewArray, _TEXT
         ; rdx == element count
         ; r8 == array size
         ; r10 == thread
-        cmp         rax, [r10 + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         rax, [r10 + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          RhpNewArrayRare
 
         mov         [r10 + OFFSETOF__Thread__m_alloc_context__alloc_ptr], rax

src/coreclr/nativeaot/Runtime/amd64/AsmMacros.inc

@@ -336,8 +336,11 @@ TSF_DoNotTriggerGc              equ 10h
 ;;
 ;; Rename fields of nested structs
 ;;
-OFFSETOF__Thread__m_alloc_context__alloc_ptr        equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
-OFFSETOF__Thread__m_alloc_context__alloc_limit      equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
+OFFSETOF__Thread__m_alloc_context__alloc_ptr        equ OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
+;;
+;; Combined_limit doesn't exist yet. It is planned to come as part of the randomized allocation sampling feature. For now this aliases alloc_limit
+;;
+OFFSETOF__Thread__m_eeAllocContext__combined_limit  equ OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
 
 
 

src/coreclr/nativeaot/Runtime/arm/AllocFast.S

@@ -26,7 +26,7 @@ LEAF_ENTRY RhpNewFast, _TEXT
 
         ldr         r3, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         add         r2, r3
-        ldr         r1, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r1, [r0, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         r2, r1
         bhi         LOCAL_LABEL(RhpNewFast_RarePath)
 
@@ -132,7 +132,7 @@ LEAF_ENTRY RhNewString, _TEXT
         adds        r6, r12
         bcs         LOCAL_LABEL(RhNewString_RarePath) // if we get a carry here, the string is too large to fit below 4 GB
 
-        ldr         r12, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r12, [r0, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         r6, r12
         bhi         LOCAL_LABEL(RhNewString_RarePath)
 
@@ -213,7 +213,7 @@ LOCAL_LABEL(ArrayAlignSize):
         adds        r6, r12
         bcs         LOCAL_LABEL(RhpNewArray_RarePath) // if we get a carry here, the array is too large to fit below 4 GB
 
-        ldr         r12, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r12, [r0, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         r6, r12
         bhi         LOCAL_LABEL(RhpNewArray_RarePath)
 
@@ -349,7 +349,7 @@ LEAF_ENTRY RhpNewFastAlign8, _TEXT
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         r2, r3
-        ldr         r3, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r3, [r0, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         r2, r3
         bhi         LOCAL_LABEL(Alloc8Failed)
 
@@ -412,7 +412,7 @@ LEAF_ENTRY RhpNewFastMisalign, _TEXT
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         r2, r3
-        ldr         r3, [r0, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         r3, [r0, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         r2, r3
         bhi         LOCAL_LABEL(BoxAlloc8Failed)
 

src/coreclr/nativeaot/Runtime/arm64/AllocFast.S

@@ -10,8 +10,11 @@ GC_ALLOC_FINALIZE               = 1
 //
 // Rename fields of nested structs
 //
-OFFSETOF__Thread__m_alloc_context__alloc_ptr        = OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
-OFFSETOF__Thread__m_alloc_context__alloc_limit      = OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
+OFFSETOF__Thread__m_alloc_context__alloc_ptr        = OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
+//
+// Combined_limit doesn't exist yet. It is planned to come as part of the randomized allocation sampling feature. For now this aliases alloc_limit
+//
+OFFSETOF__Thread__m_eeAllocContext__combined_limit  = OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
 
 
 
@@ -44,7 +47,7 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit      = OFFSETOF__Thread__m_rgbAll
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         x2, x2, x12
-        ldr         x13, [x1, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         x13, [x1, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         x2, x13
         bhi         LOCAL_LABEL(RhpNewFast_RarePath)
 
@@ -139,7 +142,7 @@ LOCAL_LABEL(NewOutOfMemory):
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         x2, x2, x12
-        ldr         x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         x12, [x3, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         x2, x12
         bhi         LOCAL_LABEL(RhNewString_Rare)
 
@@ -207,7 +210,7 @@ LOCAL_LABEL(RhNewString_Rare):
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         x2, x2, x12
-        ldr         x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         x12, [x3, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         x2, x12
         bhi         LOCAL_LABEL(RhpNewArray_Rare)
 

src/coreclr/nativeaot/Runtime/arm64/AllocFast.asm

@@ -30,7 +30,7 @@
         ;; Determine whether the end of the object would lie outside of the current allocation context. If so,
         ;; we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         x2, x2, x12
-        ldr         x13, [x1, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         x13, [x1, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         x2, x13
         bhi         RhpNewFast_RarePath
 
@@ -118,7 +118,7 @@ NewOutOfMemory
         ;; Determine whether the end of the object would lie outside of the current allocation context. If so,
         ;; we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         x2, x2, x12
-        ldr         x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         x12, [x3, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         x2, x12
         bhi         RhpNewArrayRare
 
@@ -179,7 +179,7 @@ StringSizeOverflow
         ;; Determine whether the end of the object would lie outside of the current allocation context. If so,
         ;; we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add         x2, x2, x12
-        ldr         x12, [x3, #OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        ldr         x12, [x3, #OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         cmp         x2, x12
         bhi         RhpNewArrayRare
 

src/coreclr/nativeaot/Runtime/arm64/AsmMacros.h

@@ -87,8 +87,11 @@ STATUS_REDHAWK_THREAD_ABORT      equ 0x43
 ;;
 ;; Rename fields of nested structs
 ;;
-OFFSETOF__Thread__m_alloc_context__alloc_ptr        equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
-OFFSETOF__Thread__m_alloc_context__alloc_limit      equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
+OFFSETOF__Thread__m_alloc_context__alloc_ptr        equ OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
+;;
+;; Combined_limit doesn't exist yet. It is planned to come as part of the randomized allocation sampling feature. For now this aliases alloc_limit
+;;
+OFFSETOF__Thread__m_eeAllocContext__combined_limit  equ OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
 
 ;;
 ;; IMPORTS

src/coreclr/nativeaot/Runtime/i386/AllocFast.asm

@@ -29,7 +29,7 @@ FASTCALL_FUNC   RhpNewFast, 4
         ;;
 
         add         eax, [edx + OFFSETOF__Thread__m_alloc_context__alloc_ptr]
-        cmp         eax, [edx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         eax, [edx + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          AllocFailed
 
         ;; set the new alloc pointer
@@ -165,7 +165,7 @@ FASTCALL_FUNC   RhNewString, 8
         mov         ecx, eax
         add         eax, [edx + OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         jc          StringAllocContextOverflow
-        cmp         eax, [edx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         eax, [edx + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          StringAllocContextOverflow
 
         ; ECX == allocation size
@@ -282,7 +282,7 @@ ArrayAlignSize:
         mov         ecx, eax
         add         eax, [edx + OFFSETOF__Thread__m_alloc_context__alloc_ptr]
         jc          ArrayAllocContextOverflow
-        cmp         eax, [edx + OFFSETOF__Thread__m_alloc_context__alloc_limit]
+        cmp         eax, [edx + OFFSETOF__Thread__m_eeAllocContext__combined_limit]
         ja          ArrayAllocContextOverflow
 
         ; ECX == array size

src/coreclr/nativeaot/Runtime/i386/AsmMacros.inc

@@ -140,8 +140,11 @@ STATUS_REDHAWK_THREAD_ABORT      equ 43h
 ;;
 ;; Rename fields of nested structs
 ;;
-OFFSETOF__Thread__m_alloc_context__alloc_ptr        equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
-OFFSETOF__Thread__m_alloc_context__alloc_limit      equ OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
+OFFSETOF__Thread__m_alloc_context__alloc_ptr        equ OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
+;;
+;; Combined_limit doesn't exist yet. It is planned to come as part of the randomized allocation sampling feature. For now this aliases alloc_limit
+;;
+OFFSETOF__Thread__m_eeAllocContext__combined_limit  equ OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
 
 ;;
 ;; CONSTANTS -- SYMBOLS

src/coreclr/nativeaot/Runtime/loongarch64/AllocFast.S

@@ -10,8 +10,11 @@ GC_ALLOC_FINALIZE               = 1
 //
 // Rename fields of nested structs
 //
-OFFSETOF__Thread__m_alloc_context__alloc_ptr        = OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
-OFFSETOF__Thread__m_alloc_context__alloc_limit      = OFFSETOF__Thread__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
+OFFSETOF__Thread__m_alloc_context__alloc_ptr        = OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_ptr
+//
+// Combined_limit doesn't exist yet. It is planned to come as part of the randomized allocation sampling feature. For now this aliases alloc_limit
+//
+OFFSETOF__Thread__m_eeAllocContext__combined_limit  = OFFSETOF__Thread__m_eeAllocContext + OFFSETOF__ee_alloc_context__m_rgbAllocContextBuffer + OFFSETOF__gc_alloc_context__alloc_limit
 
 
 
@@ -44,7 +47,7 @@ OFFSETOF__Thread__m_alloc_context__alloc_limit      = OFFSETOF__Thread__m_rgbAll
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add.d  $a2, $a2, $t3
-        ld.d   $t4, $a1, OFFSETOF__Thread__m_alloc_context__alloc_limit
+        ld.d   $t4, $a1, OFFSETOF__Thread__m_eeAllocContext__combined_limit
         bltu  $t4, $a2, RhpNewFast_RarePath
 
         // Update the alloc pointer to account for the allocation.
@@ -137,7 +140,7 @@ NewOutOfMemory:
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add.d  $a2, $a2, $t3
-        ld.d  $t3, $a3, OFFSETOF__Thread__m_alloc_context__alloc_limit
+        ld.d  $t3, $a3, OFFSETOF__Thread__m_eeAllocContext__combined_limit
         bltu  $t3, $a2, RhNewString_Rare
 
         // Reload new object address into r12.
@@ -199,7 +202,7 @@ RhNewString_Rare:
         // Determine whether the end of the object would lie outside of the current allocation context. If so,
         // we abandon the attempt to allocate the object directly and fall back to the slow helper.
         add.d  $a2, $a2, $t3
-        ld.d  $t3, $a3, OFFSETOF__Thread__m_alloc_context__alloc_limit
+        ld.d  $t3, $a3, OFFSETOF__Thread__m_eeAllocContext__combined_limit
         bltu  $t3, $a2, RhpNewArray_Rare
 
         // Reload new object address into t3.