NFC: sizeclass: differentiate minimum step size and minimum allocation sizes

CMakeLists.txt

@@ -62,6 +62,9 @@ if (SNMALLOC_SANITIZER)
   message(STATUS "Using sanitizer=${SNMALLOC_SANITIZER}")
 endif()
 
+set(SNMALLOC_MIN_ALLOC_SIZE "" CACHE STRING "Minimum allocation bytes (power of 2)")
+set(SNMALLOC_MIN_ALLOC_STEP_SIZE "" CACHE STRING "Minimum allocation step (power of 2)")
+
 if(MSVC AND SNMALLOC_STATIC_LIBRARY AND (SNMALLOC_STATIC_LIBRARY_PREFIX STREQUAL ""))
   message(FATAL_ERROR "Empty static library prefix not supported on MSVC")
 endif()
@@ -226,6 +229,11 @@ endif()
 function(add_as_define FLAG)
   target_compile_definitions(snmalloc INTERFACE $<$<BOOL:${${FLAG}}>:${FLAG}>)
 endfunction()
+function(add_as_define_value KEY)
+  if (NOT ${${KEY}} STREQUAL "")
+    target_compile_definitions(snmalloc INTERFACE ${KEY}=${${KEY}})
+  endif ()
+endfunction()
 
 add_as_define(SNMALLOC_QEMU_WORKAROUND)
 add_as_define(SNMALLOC_TRACING)
@@ -238,6 +246,8 @@ endif()
 if (SNMALLOC_NO_REALLOCARR)
   add_as_define(SNMALLOC_NO_REALLOCARR)
 endif()
+add_as_define_value(SNMALLOC_MIN_ALLOC_SIZE)
+add_as_define_value(SNMALLOC_MIN_ALLOC_STEP_SIZE)
 
 target_compile_definitions(snmalloc INTERFACE $<$<BOOL:CONST_QUALIFIED_MALLOC_USABLE_SIZE>:MALLOC_USABLE_SIZE_QUALIFIER=const>)
 

src/snmalloc/ds/allocconfig.h

@@ -20,10 +20,31 @@ namespace snmalloc
   // Used to isolate values on cache lines to prevent false sharing.
   static constexpr size_t CACHELINE_SIZE = 64;
 
-  // Minimum allocation size is space for two pointers.
-  static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*));
-  static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*);
-  static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE);
+  /// The "machine epsilon" for the small sizeclass machinery.
+  static constexpr size_t MIN_ALLOC_STEP_SIZE =
+#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE)
+    SNMALLOC_MIN_ALLOC_STEP_SIZE;
+#else
+    2 * sizeof(void*);
+#endif
+
+  /// Derived from MIN_ALLOC_STEP_SIZE
+  static constexpr size_t MIN_ALLOC_STEP_BITS =
+    bits::ctz_const(MIN_ALLOC_STEP_SIZE);
+  static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE));
+
+  /**
+   * Minimum allocation size is space for two pointers.  If the small sizeclass
+   * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is
+   * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must
+   * be large or not a power of two, those smaller size classes will be unused.
+   */
+  static constexpr size_t MIN_ALLOC_SIZE =
+#if defined(SNMALLOC_MIN_ALLOC_SIZE)
+    SNMALLOC_MIN_ALLOC_SIZE;
+#else
+    2 * sizeof(void*);
+#endif
 
   // Minimum slab size.
 #if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64)
@@ -78,11 +99,18 @@ namespace snmalloc
   static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1;
 
   static_assert(
-    INTERMEDIATE_BITS < MIN_ALLOC_BITS,
+    INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS,
     "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS");
   static_assert(
     MIN_ALLOC_SIZE >= (sizeof(void*) * 2),
     "MIN_ALLOC_SIZE must be sufficient for two pointers");
+  static_assert(
+    1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >=
+      bits::next_pow2_const(MIN_ALLOC_SIZE),
+    "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE");
+  static_assert(
+    MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE,
+    "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE");
 
   // Return remote small allocs when the local cache reaches this size.
   static constexpr int64_t REMOTE_CACHE =

src/snmalloc/ds_core/bits.h

@@ -322,22 +322,6 @@ namespace snmalloc
      *
      * Does not work for value=0.
      ***********************************************/
-    template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
-    static size_t to_exp_mant(size_t value)
-    {
-      constexpr size_t LEADING_BIT = one_at_bit(MANTISSA_BITS + LOW_BITS) >> 1;
-      constexpr size_t MANTISSA_MASK = one_at_bit(MANTISSA_BITS) - 1;
-
-      value = value - 1;
-
-      size_t e =
-        bits::BITS - MANTISSA_BITS - LOW_BITS - clz(value | LEADING_BIT);
-      size_t b = (e == 0) ? 0 : 1;
-      size_t m = (value >> (LOW_BITS + e - b)) & MANTISSA_MASK;
-
-      return (e << MANTISSA_BITS) + m;
-    }
-
     template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
     constexpr size_t to_exp_mant_const(size_t value)
     {

src/snmalloc/mem/corealloc.h

@@ -597,23 +597,6 @@ namespace snmalloc
         init_message_queue();
         message_queue().invariant();
       }
-
-      if constexpr (DEBUG)
-      {
-        for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++)
-        {
-          size_t size = sizeclass_to_size(i);
-          smallsizeclass_t sc1 = size_to_sizeclass(size);
-          smallsizeclass_t sc2 = size_to_sizeclass_const(size);
-          size_t size1 = sizeclass_to_size(sc1);
-          size_t size2 = sizeclass_to_size(sc2);
-
-          SNMALLOC_CHECK(sc1 == i);
-          SNMALLOC_CHECK(sc1 == sc2);
-          SNMALLOC_CHECK(size1 == size);
-          SNMALLOC_CHECK(size1 == size2);
-        }
-      }
     }
 
   public:

src/snmalloc/mem/sizeclasstable.h

@@ -24,7 +24,7 @@ namespace snmalloc
     // For example, 24 byte allocations can be
     // problematic for some data due to alignment issues.
     auto sc = static_cast<smallsizeclass_t>(
-      bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
+      bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(size));
 
     SNMALLOC_ASSERT(sc == static_cast<uint8_t>(sc));
 
@@ -214,7 +214,8 @@ namespace snmalloc
         auto& meta = fast_small(sizeclass);
 
         size_t rsize =
-          bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
+          bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(
+            sizeclass);
         meta.size = rsize;
         size_t slab_bits = bits::max(
           bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS);
@@ -405,7 +406,7 @@ namespace snmalloc
   {
     // We subtract and shift to reduce the size of the table, i.e. we don't have
     // to store a value for every size.
-    return (s - 1) >> MIN_ALLOC_BITS;
+    return (s - 1) >> MIN_ALLOC_STEP_BITS;
   }
 
   constexpr size_t sizeclass_lookup_size =
@@ -421,13 +422,29 @@ namespace snmalloc
 
     constexpr SizeClassLookup()
     {
+      constexpr sizeclass_compress_t minimum_class =
+        static_cast<sizeclass_compress_t>(
+          size_to_sizeclass_const(MIN_ALLOC_SIZE));
+
+      /* Some unused sizeclasses is OK, but keep it within reason! */
+      static_assert(minimum_class < sizeclass_lookup_size);
+
       size_t curr = 1;
-      for (sizeclass_compress_t sizeclass = 0;
-           sizeclass < NUM_SMALL_SIZECLASSES;
-           sizeclass++)
+
+      sizeclass_compress_t sizeclass = 0;
+      for (; sizeclass < minimum_class; sizeclass++)
+      {
+        for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
+             curr += MIN_ALLOC_STEP_SIZE)
+        {
+          table[sizeclass_lookup_index(curr)] = minimum_class;
+        }
+      }
+
+      for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++)
       {
         for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
-             curr += 1 << MIN_ALLOC_BITS)
+             curr += MIN_ALLOC_STEP_SIZE)
         {
           auto i = sizeclass_lookup_index(curr);
           if (i == sizeclass_lookup_size)

src/test/func/memcpy/func-memcpy.cc

@@ -57,6 +57,9 @@ extern "C" void abort()
   {
     longjmp(jmp, 1);
   }
+#  if __has_builtin(__builtin_trap)
+  __builtin_trap();
+#  endif
   exit(-1);
 }
 
@@ -152,7 +155,11 @@ int main()
   // Some sizes to check for out-of-bounds access.  As we are only able to
   // catch overflows past the end of the sizeclass-padded allocation, make
   // sure we don't try to test on smaller allocations.
-  std::initializer_list<size_t> sizes = {MIN_ALLOC_SIZE, 1024, 2 * 1024 * 1024};
+
+  static constexpr size_t min_class_size =
+    sizeclass_to_size(size_to_sizeclass(MIN_ALLOC_SIZE));
+
+  std::initializer_list<size_t> sizes = {min_class_size, 1024, 2 * 1024 * 1024};
   static_assert(
     MIN_ALLOC_SIZE < 1024,
     "Can't detect overflow except at sizeclass boundaries");

src/test/func/memory/memory.cc

@@ -237,7 +237,9 @@ void test_external_pointer()
   // Malloc does not have an external pointer querying mechanism.
   auto& alloc = ThreadAlloc::get();
 
-  for (uint8_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++)
+  for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE);
+       sc < NUM_SMALL_SIZECLASSES;
+       sc++)
   {
     size_t size = sizeclass_to_size(sc);
     void* p1 = alloc.alloc(size);
@@ -470,7 +472,9 @@ void test_static_sized_allocs()
 void test_remaining_bytes()
 {
   auto& alloc = ThreadAlloc::get();
-  for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++)
+  for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE);
+       sc < NUM_SMALL_SIZECLASSES;
+       sc++)
   {
     auto size = sizeclass_to_size(sc);
     char* p = (char*)alloc.alloc(size);

src/test/func/sizeclass/sizeclass.cc

@@ -8,6 +8,9 @@ snmalloc::smallsizeclass_t size_to_sizeclass(size_t size)
   return snmalloc::size_to_sizeclass(size);
 }
 
+static constexpr snmalloc::smallsizeclass_t minimum_sizeclass =
+  snmalloc::size_to_sizeclass_const(snmalloc::MIN_ALLOC_SIZE);
+
 void test_align_size()
 {
   bool failed = false;
@@ -72,6 +75,10 @@ int main(int, char**)
   bool failed = false;
   size_t size_low = 0;
 
+  std::cout << "Configured with minimum allocation size "
+            << snmalloc::MIN_ALLOC_SIZE << " and step size "
+            << snmalloc::MIN_ALLOC_STEP_SIZE << std::endl;
+
   std::cout << "0 has sizeclass: " << (size_t)snmalloc::size_to_sizeclass(0)
             << std::endl;
 
@@ -86,12 +93,14 @@ int main(int, char**)
       slab_size != snmalloc::sizeclass_to_slab_size(sz))
     {
       slab_size = snmalloc::sizeclass_to_slab_size(sz);
-      std::cout << std::endl;
+      std::cout << std::endl << "slab size: " << slab_size << std::endl;
     }
 
     size_t size = snmalloc::sizeclass_to_size(sz);
     std::cout << (size_t)sz << " |-> "
-              << "[" << size_low + 1 << ", " << size << "]" << std::endl;
+              << "[" << size_low + 1 << ", " << size << "]"
+              << (sz == minimum_sizeclass ? " is minimum class" : "")
+              << std::endl;
 
     if (size < size_low)
     {
@@ -102,7 +111,30 @@ int main(int, char**)
 
     for (size_t i = size_low + 1; i <= size; i++)
     {
-      if (size_to_sizeclass(i) != sz)
+      /* All sizes should, via bit-math, come back to their class value */
+      if (snmalloc::size_to_sizeclass_const(i) != sz)
+      {
+        std::cout << "Size " << i << " has _const sizeclass "
+                  << (size_t)snmalloc::size_to_sizeclass_const(i)
+                  << " but expected sizeclass " << (size_t)sz << std::endl;
+        failed = true;
+      }
+
+      if (size < snmalloc::MIN_ALLOC_SIZE)
+      {
+        /*
+         * It is expected that these sizes have the "wrong" class from tabular
+         * lookup: they will have been clipped up to the minimum class.
+         */
+        if (size_to_sizeclass(i) != minimum_sizeclass)
+        {
+          std::cout << "Size " << i << " below minimum size; sizeclass "
+                    << (size_t)size_to_sizeclass(i) << " not expected minimum "
+                    << (size_t)minimum_sizeclass << std::endl;
+          failed = true;
+        }
+      }
+      else if (size_to_sizeclass(i) != sz)
       {
         std::cout << "Size " << i << " has sizeclass "
                   << (size_t)size_to_sizeclass(i) << " but expected sizeclass "