Refill heuristic

src/snmalloc/backend/backend.h

@@ -97,42 +97,45 @@ namespace snmalloc
     static constexpr bool CONSOLIDATE_PAL_ALLOCS = true;
 #endif
 
-#if defined(OPEN_ENCLAVE)
-    // Single global buddy allocator is used on open enclave due to
-    // the limited address space.
-    using StatsR = StatsRange<SmallBuddyRange<
-      LargeBuddyRange<EmptyRange, bits::BITS - 1, bits::BITS - 1, Pagemap>>>;
-    using GlobalR = GlobalRange<StatsR>;
-    using ObjectRange = GlobalR;
-    using GlobalMetaRange = ObjectRange;
-#else
     // Set up source of memory
-    using P = PalRange<DefaultPal>;
+    using P = PalRange<PAL>;
     using Base = std::conditional_t<
       fixed_range,
       EmptyRange,
       PagemapRegisterRange<Pagemap, P, CONSOLIDATE_PAL_ALLOCS>>;
+
+    static constexpr size_t MinBaseSizeBits()
+    {
+      if constexpr (pal_supports<AlignedAllocation, PAL>)
+      {
+        return bits::next_pow2_bits_const(PAL::minimum_alloc_size);
+      }
+      else
+      {
+        return MIN_CHUNK_BITS;
+      }
+    }
+
     // Global range of memory
-    using StatsR =
-      StatsRange<LargeBuddyRange<Base, 24, bits::BITS - 1, Pagemap>>;
+    using StatsR = StatsRange<
+      LargeBuddyRange<Base, 24, bits::BITS - 1, Pagemap, MinBaseSizeBits()>>;
     using GlobalR = GlobalRange<StatsR>;
 
-#  ifdef SNMALLOC_META_PROTECTED
+#ifdef SNMALLOC_META_PROTECTED
     // Source for object allocations
     using ObjectRange =
-      LargeBuddyRange<CommitRange<GlobalR, DefaultPal>, 21, 21, Pagemap>;
+      LargeBuddyRange<CommitRange<GlobalR, PAL>, 21, 21, Pagemap>;
     // Set up protected range for metadata
-    using SubR = CommitRange<SubRange<GlobalR, DefaultPal, 6>, DefaultPal>;
+    using SubR = CommitRange<SubRange<GlobalR, PAL, 6>, PAL>;
     using MetaRange =
       SmallBuddyRange<LargeBuddyRange<SubR, 21 - 6, bits::BITS - 1, Pagemap>>;
     using GlobalMetaRange = GlobalRange<MetaRange>;
-#  else
+#else
     // Source for object allocations and metadata
     // No separation between the two
     using ObjectRange = SmallBuddyRange<
-      LargeBuddyRange<CommitRange<GlobalR, DefaultPal>, 21, 21, Pagemap>>;
+      LargeBuddyRange<CommitRange<GlobalR, PAL>, 21, 21, Pagemap>>;
     using GlobalMetaRange = GlobalRange<ObjectRange>;
-#  endif
 #endif
 
     struct LocalState

src/snmalloc/backend_helpers/largebuddyrange.h

@@ -165,19 +165,53 @@ namespace snmalloc
     }
   };
 
+  /**
+   * Used to represent a consolidating range of memory.  Uses a buddy allocator
+   * to consolidate adjacent blocks.
+   *
+   * ParentRange - Represents the range to get memory from to fill this range.
+   *
+   * REFILL_SIZE_BITS - Maximum size of a refill, may ask for less during warm
+   * up phase.
+   *
+   * MAX_SIZE_BITS - Maximum size that this range will store.
+   *
+   * Pagemap - How to access the pagemap, which is used to store the red black
+   * tree nodes for the buddy allocators.
+   *
+   * MIN_REFILL_SIZE_BITS - The minimum size that the ParentRange can be asked
+   * for
+   */
   template<
     typename ParentRange,
     size_t REFILL_SIZE_BITS,
     size_t MAX_SIZE_BITS,
-    SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap>
+    SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap,
+    size_t MIN_REFILL_SIZE_BITS = 0>
   class LargeBuddyRange
   {
     ParentRange parent{};
 
+    /**
+     * Maximum size of a refill
+     */
     static constexpr size_t REFILL_SIZE = bits::one_at_bit(REFILL_SIZE_BITS);
 
     /**
+     * Minimum size of a refill
+     */
+    static constexpr size_t MIN_REFILL_SIZE =
+      bits::one_at_bit(MIN_REFILL_SIZE_BITS);
+
+    /**
+     * The size of memory requested so far.
      *
+     * This is used to determine the refill size.
+     */
+    size_t requested_total = 0;
+
+    /**
+     * Buddy allocator used to represent this range of memory.
      */
     Buddy<BuddyChunkRep<Pagemap>, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large;
 
@@ -229,17 +263,36 @@ namespace snmalloc
     {
       if (ParentRange::Aligned)
       {
-        if (size >= REFILL_SIZE)
+        // Use amount currently requested to determine refill size.
+        // This will gradually increase the usage of the parent range.
+        // So small examples can grow local caches slowly, and larger
+        // examples will grow them by the refill size.
+        //
+        // The heuristic is designed to allocate the following sequence for
+        // 16KiB requests 16KiB, 16KiB, 32Kib, 64KiB, ..., REFILL_SIZE/2,
+        // REFILL_SIZE, REFILL_SIZE, ... Hence if this if they are coming from a
+        // contiguous aligned range, then they could be consolidated.  This
+        // depends on the ParentRange behaviour.
+        size_t refill_size = bits::min(REFILL_SIZE, requested_total);
+        refill_size = bits::max(refill_size, MIN_REFILL_SIZE);
+        refill_size = bits::max(refill_size, size);
+        refill_size = bits::next_pow2(refill_size);
+
+        auto refill_range = parent.alloc_range(refill_size);
+        if (refill_range != nullptr)
         {
-          return parent.alloc_range(size);
+          requested_total += refill_size;
+          add_range(pointer_offset(refill_range, size), refill_size - size);
         }
-
-        auto refill_range = parent.alloc_range(REFILL_SIZE);
-        if (refill_range != nullptr)
-          add_range(pointer_offset(refill_range, size), REFILL_SIZE - size);
         return refill_range;
       }
 
+      // Note the unaligned parent path does not use
+      // requested_total in the heuristic for the initial size
+      // this is because the request needs to introduce alignment.
+      // Currently the unaligned variant is not used as a local cache.
+      // So the gradual growing of refill_size is not needed.
+
       // Need to overallocate to get the alignment right.
       bool overflow = false;
       size_t needed_size = bits::umul(size, 2, overflow);
@@ -255,6 +308,7 @@ namespace snmalloc
 
         if (refill != nullptr)
         {
+          requested_total += refill_size;
           add_range(refill, refill_size);
 
           SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS));