diff --git a/src/snmalloc/backend/backend.h b/src/snmalloc/backend/backend.h index d9a2d2340..04a0fd02f 100644 --- a/src/snmalloc/backend/backend.h +++ b/src/snmalloc/backend/backend.h @@ -97,42 +97,45 @@ namespace snmalloc static constexpr bool CONSOLIDATE_PAL_ALLOCS = true; #endif -#if defined(OPEN_ENCLAVE) - // Single global buddy allocator is used on open enclave due to - // the limited address space. - using StatsR = StatsRange>>; - using GlobalR = GlobalRange; - using ObjectRange = GlobalR; - using GlobalMetaRange = ObjectRange; -#else // Set up source of memory - using P = PalRange; + using P = PalRange; using Base = std::conditional_t< fixed_range, EmptyRange, PagemapRegisterRange>; + + static constexpr size_t MinBaseSizeBits() + { + if constexpr (pal_supports) + { + return bits::next_pow2_bits_const(PAL::minimum_alloc_size); + } + else + { + return MIN_CHUNK_BITS; + } + } + // Global range of memory - using StatsR = - StatsRange>; + using StatsR = StatsRange< + LargeBuddyRange>; using GlobalR = GlobalRange; -# ifdef SNMALLOC_META_PROTECTED +#ifdef SNMALLOC_META_PROTECTED // Source for object allocations using ObjectRange = - LargeBuddyRange, 21, 21, Pagemap>; + LargeBuddyRange, 21, 21, Pagemap>; // Set up protected range for metadata - using SubR = CommitRange, DefaultPal>; + using SubR = CommitRange, PAL>; using MetaRange = SmallBuddyRange>; using GlobalMetaRange = GlobalRange; -# else +#else // Source for object allocations and metadata // No separation between the two using ObjectRange = SmallBuddyRange< - LargeBuddyRange, 21, 21, Pagemap>>; + LargeBuddyRange, 21, 21, Pagemap>>; using GlobalMetaRange = GlobalRange; -# endif #endif struct LocalState diff --git a/src/snmalloc/backend_helpers/largebuddyrange.h b/src/snmalloc/backend_helpers/largebuddyrange.h index 6a7bd2444..b81e392ef 100644 --- a/src/snmalloc/backend_helpers/largebuddyrange.h +++ b/src/snmalloc/backend_helpers/largebuddyrange.h @@ -165,19 +165,53 @@ namespace snmalloc } }; + /** + * Used to represent a consolidating range of memory. Uses a buddy allocator + * to consolidate adjacent blocks. + * + * ParentRange - Represents the range to get memory from to fill this range. + * + * REFILL_SIZE_BITS - Maximum size of a refill, may ask for less during warm + * up phase. + * + * MAX_SIZE_BITS - Maximum size that this range will store. + * + * Pagemap - How to access the pagemap, which is used to store the red black + * tree nodes for the buddy allocators. + * + * MIN_REFILL_SIZE_BITS - The minimum size that the ParentRange can be asked + * for + */ template< typename ParentRange, size_t REFILL_SIZE_BITS, size_t MAX_SIZE_BITS, - SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap> + SNMALLOC_CONCEPT(ConceptBuddyRangeMeta) Pagemap, + size_t MIN_REFILL_SIZE_BITS = 0> class LargeBuddyRange { ParentRange parent{}; + /** + * Maximum size of a refill + */ static constexpr size_t REFILL_SIZE = bits::one_at_bit(REFILL_SIZE_BITS); /** + * Minimum size of a refill + */ + static constexpr size_t MIN_REFILL_SIZE = + bits::one_at_bit(MIN_REFILL_SIZE_BITS); + + /** + * The size of memory requested so far. * + * This is used to determine the refill size. + */ + size_t requested_total = 0; + + /** + * Buddy allocator used to represent this range of memory. */ Buddy, MIN_CHUNK_BITS, MAX_SIZE_BITS> buddy_large; @@ -229,17 +263,36 @@ namespace snmalloc { if (ParentRange::Aligned) { - if (size >= REFILL_SIZE) + // Use amount currently requested to determine refill size. + // This will gradually increase the usage of the parent range. + // So small examples can grow local caches slowly, and larger + // examples will grow them by the refill size. + // + // The heuristic is designed to allocate the following sequence for + // 16KiB requests 16KiB, 16KiB, 32Kib, 64KiB, ..., REFILL_SIZE/2, + // REFILL_SIZE, REFILL_SIZE, ... Hence if this if they are coming from a + // contiguous aligned range, then they could be consolidated. This + // depends on the ParentRange behaviour. + size_t refill_size = bits::min(REFILL_SIZE, requested_total); + refill_size = bits::max(refill_size, MIN_REFILL_SIZE); + refill_size = bits::max(refill_size, size); + refill_size = bits::next_pow2(refill_size); + + auto refill_range = parent.alloc_range(refill_size); + if (refill_range != nullptr) { - return parent.alloc_range(size); + requested_total += refill_size; + add_range(pointer_offset(refill_range, size), refill_size - size); } - - auto refill_range = parent.alloc_range(REFILL_SIZE); - if (refill_range != nullptr) - add_range(pointer_offset(refill_range, size), REFILL_SIZE - size); return refill_range; } + // Note the unaligned parent path does not use + // requested_total in the heuristic for the initial size + // this is because the request needs to introduce alignment. + // Currently the unaligned variant is not used as a local cache. + // So the gradual growing of refill_size is not needed. + // Need to overallocate to get the alignment right. bool overflow = false; size_t needed_size = bits::umul(size, 2, overflow); @@ -255,6 +308,7 @@ namespace snmalloc if (refill != nullptr) { + requested_total += refill_size; add_range(refill, refill_size); SNMALLOC_ASSERT(refill_size < bits::one_at_bit(MAX_SIZE_BITS));