Make Large allocations naturally aligned

CMakeLists.txt

@@ -9,6 +9,12 @@ option(EXPOSE_EXTERNAL_RESERVE "Expose an interface to reserve memory using the
 option(SNMALLOC_RUST_SUPPORT "Build static library for rust" OFF)
 set(CACHE_FRIENDLY_OFFSET OFF CACHE STRING "Base offset to place linked-list nodes.")
 
+if ((CMAKE_BUILD_TYPE STREQUAL "Release") AND (NOT SNMALLOC_CI_BUILD))
+  option(USE_POSIX_COMMIT_CHECKS "Instrument Posix PAL to check for access to unused blocks of memory." Off)
+else ()
+  option(USE_POSIX_COMMIT_CHECKS "Instrument Posix PAL to check for access to unused blocks of memory." On)
+endif ()
+
 # Provide as macro so other projects can reuse
 macro(warnings_high)
   if(MSVC)
@@ -111,6 +117,11 @@ if(CACHE_FRIENDLY_OFFSET)
   target_compile_definitions(snmalloc_lib INTERFACE -DCACHE_FRIENDLY_OFFSET=${CACHE_FRIENDLY_OFFSET})
 endif()
 
+if(USE_POSIX_COMMIT_CHECKS)
+  target_compile_definitions(snmalloc_lib INTERFACE -DUSE_POSIX_COMMIT_CHECKS)
+endif()
+
+
 # To build with just the header library target define SNMALLOC_ONLY_HEADER_LIBRARY
 # in containing Cmake file.
 if(NOT DEFINED SNMALLOC_ONLY_HEADER_LIBRARY)

README.md

@@ -163,12 +163,12 @@ pages, rather than zeroing them synchronously in this call
 
 ```c++
 template<bool committed>
-void* reserve(size_t* size, size_t align);
+void* reserve(size_t size, size_t align);
 template<bool committed>
-void* reserve(const size_t* size) noexcept;
+void* reserve(size_t size) noexcept;
 ```
 Only one of these needs to be implemented, depending on whether the underlying
-system can provide strongly (e.g. 16MiB) aligned memory regions.
+system can provide strongly aligned memory regions.
 If the system guarantees only page alignment, implement the second and snmalloc
 will over-allocate and then trim the requested region.
 If the system provides strong alignment, implement the first to return memory

src/mem/alloc.h

@@ -1070,8 +1070,7 @@ namespace snmalloc
       bool was_full = super->is_full();
       SlabList* sl = &small_classes[sizeclass];
       Slab* slab = Metaslab::get_slab(p);
-      Superslab::Action a =
-        slab->dealloc_slow(sl, super, p);
+      Superslab::Action a = slab->dealloc_slow(sl, super, p);
       if (likely(a == Superslab::NoSlabReturn))
         return;
       stats().sizeclass_dealloc_slab(sizeclass);

src/mem/largealloc.h

@@ -63,16 +63,12 @@ namespace snmalloc
 
     void new_block()
     {
-      size_t size = SUPERSLAB_SIZE;
-      void* r = reserve<false>(&size, SUPERSLAB_SIZE);
-
-      if (size < SUPERSLAB_SIZE)
-        error("out of memory");
-
+      // Reserve the smallest large_class which is SUPERSLAB_SIZE
+      void* r = reserve<false>(0);
       PAL::template notify_using<NoZero>(r, OS_PAGE_SIZE);
 
       bump = r;
-      remaining = size;
+      remaining = SUPERSLAB_SIZE;
     }
 
     /**
@@ -126,6 +122,24 @@ namespace snmalloc
       lazy_decommit_guard.clear();
     }
 
+    void push_space(address_t start, size_t large_class)
+    {
+      void* p = pointer_cast<void>(start);
+      if (large_class > 0)
+        PAL::template notify_using<YesZero>(p, OS_PAGE_SIZE);
+      else
+      {
+        if (decommit_strategy == DecommitSuperLazy)
+        {
+          PAL::template notify_using<YesZero>(p, OS_PAGE_SIZE);
+          p = new (p) Decommittedslab();
+        }
+        else
+          PAL::template notify_using<YesZero>(p, SUPERSLAB_SIZE);
+      }
+      large_stack[large_class].push(reinterpret_cast<Largeslab*>(p));
+    }
+
   public:
     /**
      * Stack of large allocations that have been returned for reuse.
@@ -202,35 +216,68 @@ namespace snmalloc
     }
 
     template<bool committed>
-    void* reserve(size_t* size, size_t align) noexcept
+    void* reserve(size_t large_class) noexcept
     {
+      size_t size = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
+      size_t align = size;
+
       if constexpr (pal_supports<AlignedAllocation, PAL>)
       {
         return PAL::template reserve<committed>(size, align);
       }
       else
       {
-        size_t request = *size;
-        // Add align, so we can guarantee to provide at least size.
-        request += align;
-        // Alignment must be a power of 2.
-        assert(align == bits::next_pow2(align));
+        // Reserve 4 times the amount, and put aligned leftovers into the
+        // large_stack
+        size_t request = bits::max(size * 4, SUPERSLAB_SIZE * 8);
+        void* p = PAL::template reserve<false>(request);
 
-        void* p = PAL::template reserve<committed>(&request);
+        address_t p0 = address_cast(p);
+        address_t start = bits::align_up(p0, align);
+        address_t p1 = p0 + request;
+        address_t end = start + size;
 
-        *size = request;
-        auto p0 = address_cast(p);
-        auto start = bits::align_up(p0, align);
+        for (; end < bits::align_down(p1, align); end += size)
+        {
+          push_space(end, large_class);
+        }
+
+        // Put offcuts before alignment into the large stack
+        address_t offcut_end = start;
+        address_t offcut_start;
+        for (size_t i = large_class; i > 0;)
+        {
+          i--;
+          size_t offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i;
+          offcut_start = bits::align_up(p0, offcut_align);
+          if (offcut_start != offcut_end)
+          {
+            push_space(offcut_start, i);
+            offcut_end = offcut_start;
+          }
+        }
 
-        if (start > p0)
+        // Put offcuts after returned block into the large stack
+        offcut_start = end;
+        for (size_t i = large_class; i > 0;)
         {
-          uintptr_t end = bits::align_down(p0 + request, align);
-          *size = end - start;
-          PAL::notify_not_using(p, start - p0);
-          PAL::notify_not_using(pointer_cast<void>(end), (p0 + request) - end);
-          p = pointer_cast<void>(start);
+          i--;
+          auto offcut_align = bits::one_at_bit(SUPERSLAB_BITS) << i;
+          offcut_end = bits::align_down(p1, offcut_align);
+          if (offcut_start != offcut_end)
+          {
+            push_space(offcut_start, i);
+            offcut_start = offcut_end;
+          }
         }
-        return p;
+
+        // printf("Alloc %zx (size = %zx)\n", start, size);
+
+        void* result = pointer_cast<void>(start);
+        if (committed)
+          PAL::template notify_using<NoZero>(result, size);
+
+        return result;
       }
     }
 
@@ -281,9 +328,6 @@ namespace snmalloc
   template<class MemoryProvider>
   class LargeAlloc
   {
-    void* reserved_start = nullptr;
-    void* reserved_end = nullptr;
-
   public:
     // This will be a zero-size structure if stats are not enabled.
     Stats stats;
@@ -292,38 +336,6 @@ namespace snmalloc
 
     LargeAlloc(MemoryProvider& mp) : memory_provider(mp) {}
 
-    template<AllowReserve allow_reserve>
-    bool reserve_memory(size_t need, size_t add)
-    {
-      assert(reserved_start <= reserved_end);
-
-      /*
-       * Spell this comparison in terms of pointer subtraction like this,
-       * rather than "reserved_start + need < reserved_end" because the
-       * sum might not be representable on CHERI.
-       */
-      if (pointer_diff(reserved_start, reserved_end) < need)
-      {
-        if constexpr (allow_reserve == YesReserve)
-        {
-          stats.segment_create();
-          reserved_start =
-            memory_provider.template reserve<false>(&add, SUPERSLAB_SIZE);
-          reserved_end = pointer_offset(reserved_start, add);
-          reserved_start = pointer_align_up<SUPERSLAB_SIZE>(reserved_start);
-
-          if (add < need)
-            return false;
-        }
-        else
-        {
-          return false;
-        }
-      }
-
-      return true;
-    }
-
     template<ZeroMem zero_mem = NoZero, AllowReserve allow_reserve = YesReserve>
     void* alloc(size_t large_class, size_t size)
     {
@@ -337,23 +349,8 @@ namespace snmalloc
 
       if (p == nullptr)
       {
-        assert(reserved_start <= reserved_end);
-        size_t add;
-
-        if ((rsize + SUPERSLAB_SIZE) < RESERVE_SIZE)
-          add = RESERVE_SIZE;
-        else
-          add = rsize + SUPERSLAB_SIZE;
-
-        if (!reserve_memory<allow_reserve>(rsize, add))
-          return nullptr;
-
-        p = reserved_start;
-        reserved_start = pointer_offset(p, rsize);
-
-        stats.superslab_fresh();
-        // All memory is zeroed since it comes from reserved space.
-        memory_provider.template notify_using<NoZero>(p, size);
+        p = memory_provider.template reserve<false>(large_class);
+        memory_provider.template notify_using<zero_mem>(p, size);
       }
       else
       {
@@ -389,6 +386,7 @@ namespace snmalloc
         }
       }
 
+      assert(p == pointer_align_up(p, rsize));
       return p;
     }
 
@@ -403,13 +401,14 @@ namespace snmalloc
       }
 
       // Cross-reference largealloc's alloc() decommitted condition.
-      if ((decommit_strategy != DecommitNone) 
-          && (large_class != 0 || decommit_strategy == DecommitSuper))
+      if (
+        (decommit_strategy != DecommitNone) &&
+        (large_class != 0 || decommit_strategy == DecommitSuper))
       {
         size_t rsize = bits::one_at_bit(SUPERSLAB_BITS) << large_class;
 
         memory_provider.notify_not_using(
-        pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE);
+          pointer_offset(p, OS_PAGE_SIZE), rsize - OS_PAGE_SIZE);
       }
 
       stats.superslab_push();

src/mem/sizeclass.h

@@ -179,8 +179,6 @@ namespace snmalloc
   {
     // Client responsible for checking alignment is not zero
     assert(alignment != 0);
-    // Client responsible for checking alignment is not above SUPERSLAB_SIZE
-    assert(alignment <= SUPERSLAB_SIZE);
     // Client responsible for checking alignment is a power of two
     assert(bits::next_pow2(alignment) == alignment);
 

src/mem/slab.h

@@ -178,8 +178,8 @@ namespace snmalloc
     // This does not need to remove the "use" as done by the fast path.
     // Returns a complex return code for managing the superslab meta data.
     // i.e. This deallocation could make an entire superslab free.
-    SNMALLOC_SLOW_PATH typename Superslab::Action dealloc_slow(
-      SlabList* sl, Superslab* super, void* p)
+    SNMALLOC_SLOW_PATH typename Superslab::Action
+    dealloc_slow(SlabList* sl, Superslab* super, void* p)
     {
       Metaslab& meta = super->get_meta(this);
 

src/mem/superslab.h

@@ -84,7 +84,7 @@ namespace snmalloc
       {
         if (kind != Fresh)
         {
-        // If this wasn't previously Fresh, we need to zero some things.
+          // If this wasn't previously Fresh, we need to zero some things.
           used = 0;
           for (size_t i = 0; i < SLAB_COUNT; i++)
           {
@@ -105,7 +105,8 @@ namespace snmalloc
       {
         curr = (curr + meta[curr].next + 1) & (SLAB_COUNT - 1);
       }
-      if (curr != 0) abort();
+      if (curr != 0)
+        abort();
 
       for (size_t i = 0; i < SLAB_COUNT; i++)
       {

src/override/malloc.cc

@@ -107,9 +107,7 @@ extern "C"
   SNMALLOC_EXPORT void*
     SNMALLOC_NAME_MANGLE(memalign)(size_t alignment, size_t size)
   {
-    if (
-      (alignment == 0) || (alignment == size_t(-1)) ||
-      (alignment > SUPERSLAB_SIZE))
+    if ((alignment == 0) || (alignment == size_t(-1)))
     {
       errno = EINVAL;
       return nullptr;

src/pal/pal_apple.h

@@ -56,11 +56,11 @@ namespace snmalloc
      * See comment below.
      */
     template<bool committed>
-    void* reserve(const size_t* size)
+    void* reserve(size_t size)
     {
       void* p = mmap(
         nullptr,
-        *size,
+        size,
         PROT_READ | PROT_WRITE,
         MAP_PRIVATE | MAP_ANONYMOUS,
         pal_anon_id,

src/pal/pal_bsd_aligned.h

@@ -27,7 +27,7 @@ namespace snmalloc
      * Reserve memory at a specific alignment.
      */
     template<bool committed>
-    void* reserve(const size_t* size, size_t align) noexcept
+    void* reserve(size_t size, size_t align) noexcept
     {
       // Alignment must be a power of 2.
       assert(align == bits::next_pow2(align));
@@ -38,7 +38,7 @@ namespace snmalloc
 
       void* p = mmap(
         nullptr,
-        *size,
+        size,
         PROT_READ | PROT_WRITE,
         MAP_PRIVATE | MAP_ANONYMOUS | MAP_ALIGNED(log2align),
         -1,

src/pal/pal_freebsd_kernel.h

@@ -61,13 +61,12 @@ namespace snmalloc
     }
 
     template<bool committed>
-    void* reserve(size_t* size, size_t align)
+    void* reserve(size_t size, size_t align)
     {
-      size_t request = *size;
       vm_offset_t addr;
       if (vmem_xalloc(
             kernel_arena,
-            request,
+            size,
             align,
             0,
             0,
@@ -81,10 +80,10 @@ namespace snmalloc
       if (committed)
       {
         if (
-          kmem_back(kernel_object, addr, request, M_ZERO | M_WAITOK) !=
+          kmem_back(kernel_object, addr, size, M_ZERO | M_WAITOK) !=
           KERN_SUCCESS)
         {
-          vmem_xfree(kernel_arena, addr, request);
+          vmem_xfree(kernel_arena, addr, size);
           return nullptr;
         }
       }