Optimize AllocHeap: use new/delete with 4KB blocks and static allocation

src/coreclr/nativeaot/Runtime/CachedInterfaceDispatch_Aot.cpp

@@ -7,15 +7,11 @@
 #include "CachedInterfaceDispatch.h"
 
 // The base memory allocator.
-static AllocHeap * g_pAllocHeap = NULL;
+static AllocHeap g_allocHeap;
 
 bool InterfaceDispatch_InitializePal()
 {
-    g_pAllocHeap = new (nothrow) AllocHeap();
-    if (g_pAllocHeap == NULL)
-        return false;
-
-    if (!g_pAllocHeap->Init())
+    if (!g_allocHeap.Init())
         return false;
 
     return true;
@@ -24,14 +20,14 @@ bool InterfaceDispatch_InitializePal()
 // Allocate memory aligned at sizeof(void*)*2 boundaries
 void *InterfaceDispatch_AllocDoublePointerAligned(size_t size)
 {
-    return g_pAllocHeap->AllocAligned(size, sizeof(void*) * 2);
+    return g_allocHeap.AllocAligned(size, sizeof(void*) * 2);
 }
 
 // Allocate memory aligned at sizeof(void*) boundaries
 
 void *InterfaceDispatch_AllocPointerAligned(size_t size)
 {
-    return g_pAllocHeap->AllocAligned(size, sizeof(void*));
+    return g_allocHeap.AllocAligned(size, sizeof(void*));
 }
 
 FCIMPL4(PCODE, RhpUpdateDispatchCellCache, InterfaceDispatchCell * pCell, PCODE pTargetCode, MethodTable* pInstanceType, DispatchCellInfo *pNewCellInfo)

src/coreclr/nativeaot/Runtime/allocheap.cpp

@@ -19,12 +19,7 @@
 //-------------------------------------------------------------------------------------------------
 AllocHeap::AllocHeap()
     : m_blockList(),
-      m_pNextFree(NULL),
-      m_pFreeCommitEnd(NULL),
-      m_pFreeReserveEnd(NULL),
-      m_pbInitialMem(NULL),
-      m_fShouldFreeInitialMem(false),
-      m_lock(CrstAllocHeap)
+      m_cbCurBlockUsed(0)
       COMMA_INDEBUG(m_fIsInit(false))
 {
 }
@@ -33,62 +28,37 @@ AllocHeap::AllocHeap()
 bool AllocHeap::Init()
 {
     ASSERT(!m_fIsInit);
+    m_lock.Init(CrstAllocHeap);
     INDEBUG(m_fIsInit = true;)
 
     return true;
 }
 
 //-------------------------------------------------------------------------------------------------
-// This is for using pre-allocated memory on heap construction.
-// Should never use this more than once, and should always follow construction of heap.
-
-bool AllocHeap::Init(
-    uint8_t *    pbInitialMem,
-    uintptr_t cbInitialMemCommit,
-    uintptr_t cbInitialMemReserve,
-    bool       fShouldFreeInitialMem)
+void AllocHeap::Destroy()
 {
-    ASSERT(!m_fIsInit);
-
-    BlockListElem *pBlock = new (nothrow) BlockListElem(pbInitialMem, cbInitialMemReserve);
-    if (pBlock == NULL)
-        return false;
-    m_blockList.PushHead(pBlock);
-
-    if (!_UpdateMemPtrs(pbInitialMem,
-                        pbInitialMem + cbInitialMemCommit,
-                        pbInitialMem + cbInitialMemReserve))
+    while (!m_blockList.IsEmpty())
     {
-        return false;
+        BlockListElem *pCur = m_blockList.PopHead();
+        delete[] (uint8_t*)pCur;
     }
-
-    m_pbInitialMem = pbInitialMem;
-    m_fShouldFreeInitialMem = fShouldFreeInitialMem;
-
-    INDEBUG(m_fIsInit = true;)
-    return true;
+    m_lock.Destroy();
 }
 
 //-------------------------------------------------------------------------------------------------
-AllocHeap::~AllocHeap()
+uint8_t * AllocHeap::Alloc(
+    uintptr_t cbMem)
 {
-    while (!m_blockList.IsEmpty())
-    {
-        BlockListElem *pCur = m_blockList.PopHead();
-        if (pCur->GetStart() != m_pbInitialMem || m_fShouldFreeInitialMem)
-            PalVirtualFree(pCur->GetStart(), pCur->GetLength());
-        delete pCur;
-    }
+    return AllocAligned(cbMem, 1);
 }
 
 //-------------------------------------------------------------------------------------------------
-uint8_t * AllocHeap::_Alloc(
+uint8_t * AllocHeap::AllocAligned(
     uintptr_t cbMem,
-    uintptr_t alignment
-    )
+    uintptr_t alignment)
 {
     ASSERT((alignment & (alignment - 1)) == 0); // Power of 2 only.
-    ASSERT(alignment <= OS_PAGE_SIZE);          // Can't handle this right now.
+    ASSERT(alignment <= BLOCK_SIZE);            // Can't handle this right now.
 
     CrstHolder lock(&m_lock);
 
@@ -97,7 +67,7 @@ uint8_t * AllocHeap::_Alloc(
         return pbMem;
 
     // Must allocate new block
-    if (!_AllocNewBlock(cbMem))
+    if (!_AllocNewBlock(cbMem, alignment))
         return NULL;
 
     pbMem = _AllocFromCurBlock(cbMem, alignment);
@@ -107,105 +77,41 @@ uint8_t * AllocHeap::_Alloc(
 }
 
 //-------------------------------------------------------------------------------------------------
-uint8_t * AllocHeap::Alloc(
-    uintptr_t cbMem)
-{
-    return _Alloc(cbMem, 1);
-}
-
-//-------------------------------------------------------------------------------------------------
-uint8_t * AllocHeap::AllocAligned(
-    uintptr_t cbMem,
-    uintptr_t alignment)
-{
-    return _Alloc(cbMem, alignment);
-}
-
-//-------------------------------------------------------------------------------------------------
-bool AllocHeap::_UpdateMemPtrs(uint8_t* pNextFree, uint8_t* pFreeCommitEnd, uint8_t* pFreeReserveEnd)
-{
-    ASSERT(ALIGN_DOWN(pFreeCommitEnd, OS_PAGE_SIZE) == pFreeCommitEnd);
-    ASSERT(ALIGN_DOWN(pFreeReserveEnd, OS_PAGE_SIZE) == pFreeReserveEnd);
-
-    m_pNextFree = pNextFree;
-    m_pFreeCommitEnd = pFreeCommitEnd;
-    m_pFreeReserveEnd = pFreeReserveEnd;
-    return true;
-}
-
-//-------------------------------------------------------------------------------------------------
-bool AllocHeap::_UpdateMemPtrs(uint8_t* pNextFree, uint8_t* pFreeCommitEnd)
+bool AllocHeap::_AllocNewBlock(uintptr_t cbMem, uintptr_t alignment)
 {
-    return _UpdateMemPtrs(pNextFree, pFreeCommitEnd, m_pFreeReserveEnd);
-}
-
-//-------------------------------------------------------------------------------------------------
-bool AllocHeap::_UpdateMemPtrs(uint8_t* pNextFree)
-{
-    return _UpdateMemPtrs(pNextFree, m_pFreeCommitEnd);
-}
-
-//-------------------------------------------------------------------------------------------------
-bool AllocHeap::_AllocNewBlock(uintptr_t cbMem)
-{
-    cbMem = ALIGN_UP(cbMem, OS_PAGE_SIZE);
-
-    uint8_t * pbMem = reinterpret_cast<uint8_t*>
-        (PalVirtualAlloc(cbMem, PAGE_READWRITE));
+    uintptr_t cbBlockSize = ALIGN_UP(cbMem + sizeof(BlockListElem) + alignment, BLOCK_SIZE);
 
+    uint8_t * pbMem = new (nothrow) uint8_t[cbBlockSize];
     if (pbMem == NULL)
         return false;
 
-    BlockListElem *pBlockListElem = new (nothrow) BlockListElem(pbMem, cbMem);
-    if (pBlockListElem == NULL)
-    {
-        PalVirtualFree(pbMem, cbMem);
-        return false;
-    }
+    BlockListElem *pBlockListElem = reinterpret_cast<BlockListElem*>(pbMem);
+    pBlockListElem->m_cbMem = cbBlockSize;
 
-    // Add to the list. While there is no race for writers (we hold the lock) we have the
-    // possibility of simultaneous readers, and using the interlocked version creates a
-    // memory barrier to make sure any reader sees a consistent list.
-    m_blockList.PushHeadInterlocked(pBlockListElem);
+    m_blockList.PushHead(pBlockListElem);
+    m_cbCurBlockUsed = sizeof(BlockListElem);
 
-    return _UpdateMemPtrs(pbMem, pbMem + cbMem, pbMem + cbMem);
+    return true;
 }
 
 //-------------------------------------------------------------------------------------------------
 uint8_t * AllocHeap::_AllocFromCurBlock(
     uintptr_t cbMem,
     uintptr_t alignment)
 {
-    uint8_t * pbMem = NULL;
-
-    cbMem += (uint8_t *)ALIGN_UP(m_pNextFree, alignment) - m_pNextFree;
-
-    if (m_pNextFree + cbMem <= m_pFreeCommitEnd ||
-        _CommitFromCurBlock(cbMem))
-    {
-        ASSERT(cbMem + m_pNextFree <= m_pFreeCommitEnd);
-
-        pbMem = ALIGN_UP(m_pNextFree, alignment);
-
-        if (!_UpdateMemPtrs(m_pNextFree + cbMem))
-            return NULL;
-    }
-
-    return pbMem;
-}
+    BlockListElem *pCurBlock = m_blockList.GetHead();
+    if (pCurBlock == NULL)
+        return NULL;
 
-//-------------------------------------------------------------------------------------------------
-bool AllocHeap::_CommitFromCurBlock(uintptr_t cbMem)
-{
-    ASSERT(m_pFreeCommitEnd < m_pNextFree + cbMem);
+    uint8_t* pBlockStart = (uint8_t*)pCurBlock;
+    uint8_t* pAlloc = (uint8_t*)ALIGN_UP(pBlockStart + m_cbCurBlockUsed, alignment);
+    uintptr_t cbAllocEnd = pAlloc + cbMem - pBlockStart;
 
-    if (m_pNextFree + cbMem <= m_pFreeReserveEnd)
-    {
-        uintptr_t cbMemToCommit = ALIGN_UP(cbMem, OS_PAGE_SIZE);
-        return _UpdateMemPtrs(m_pNextFree, m_pFreeCommitEnd + cbMemToCommit);
-    }
+    if (cbAllocEnd > pCurBlock->m_cbMem)
+        return NULL;
 
-    return false;
+    m_cbCurBlockUsed = cbAllocEnd;
+    return pAlloc;
 }
 
 //-------------------------------------------------------------------------------------------------

src/coreclr/nativeaot/Runtime/allocheap.h

@@ -9,12 +9,7 @@ class AllocHeap
 
     bool Init();
 
-    bool Init(uint8_t *    pbInitialMem,
-              uintptr_t cbInitialMemCommit,
-              uintptr_t cbInitialMemReserve,
-              bool       fShouldFreeInitialMem);
-
-    ~AllocHeap();
+    void Destroy();
 
     // If AllocHeap was created with a MemAccessMgr, pRWAccessHolder must be non-NULL.
     // On return, the holder will permit R/W access to the allocated memory until it
@@ -29,41 +24,26 @@ class AllocHeap
 
   private:
     // Allocation Helpers
-    uint8_t* _Alloc(uintptr_t cbMem, uintptr_t alignment);
-    bool _AllocNewBlock(uintptr_t cbMem);
+    bool _AllocNewBlock(uintptr_t cbMem, uintptr_t alignment);
     uint8_t* _AllocFromCurBlock(uintptr_t cbMem, uintptr_t alignment);
-    bool _CommitFromCurBlock(uintptr_t cbMem);
 
-    // Access protection helpers
-    bool _UpdateMemPtrs(uint8_t* pNextFree, uint8_t* pFreeCommitEnd, uint8_t* pFreeReserveEnd);
-    bool _UpdateMemPtrs(uint8_t* pNextFree, uint8_t* pFreeCommitEnd);
-    bool _UpdateMemPtrs(uint8_t* pNextFree);
+    static const uintptr_t BLOCK_SIZE = 4096;
 
     struct BlockListElem
     {
-        BlockListElem(uint8_t * pbMem, uintptr_t  cbMem)
-            : m_pbMem(pbMem), m_cbMem(cbMem), m_pNext(NULL)
-            {}
+        BlockListElem* m_pNext;
+        uintptr_t m_cbMem;
 
-        uint8_t* GetStart() const { return m_pbMem; }
         uintptr_t GetLength() const { return m_cbMem; }
-
-        uint8_t *   m_pbMem;
-        uintptr_t   m_cbMem;
-        BlockListElem* m_pNext;
+        uint8_t* GetDataStart() const { return (uint8_t*)this + sizeof(BlockListElem); }
     };
 
     typedef SList<BlockListElem>    BlockList;
     BlockList                       m_blockList;
 
-    uint8_t *                         m_pNextFree;
-    uint8_t *                         m_pFreeCommitEnd;
-    uint8_t *                         m_pFreeReserveEnd;
-
-    uint8_t *                         m_pbInitialMem;
-    bool                            m_fShouldFreeInitialMem;
+    uintptr_t                       m_cbCurBlockUsed;
 
-    Crst                            m_lock;
+    CrstStatic                      m_lock;
 
     INDEBUG(bool                    m_fIsInit;)
 };