NFC: sizeclass: differentiate minimum step size and minimum allocatio…

…n sizes (#651)

* Move sizeclass debugging code to sizeclass test

The sizeclass was already testing most of this, so just add the missing bits.
Forgo some tests whose failure would have implied earlier failures.

This moves the last dynamic call of size_to_sizeclass_const into tests
(and so, too, to_exp_mant_const).  sizeclasstable.h still contains a static
call to compute NUM_SMALL_SIZECLASSES from MAX_SMALL_SIZECLASS_SIZE.

* Remove unused to_exp_mant

Only its _const sibling is used, and little at that, now that almost everything
to do with sizes and size classes is table-driven.

* test/memcpy: trap, if we can, before exiting

This just means I don't need to remember to set a breakpoint on exit

* test/memcpy: don't assume sizeclass 0 is allocable

* test/memory: don't assume sizeclass 0 is allocable

* test/sizeclass: handle nonzero minimum sizeclasses

* sizeclass: distinguish min alloc and step size

Add support for a minimum allocation size that isn't the minimum step of
the sizeclass table.

* Expose MIN_ALLOC_{,STEP}_SIZE through cmake

* test/sizeclass: report MIN_ALLOC_{STEP_,}SIZE

CMakeLists.txt

@@ -62,6 +62,9 @@ if (SNMALLOC_SANITIZER)
   message(STATUS "Using sanitizer=${SNMALLOC_SANITIZER}")
 endif()
 
+set(SNMALLOC_MIN_ALLOC_SIZE "" CACHE STRING "Minimum allocation bytes (power of 2)")
+set(SNMALLOC_MIN_ALLOC_STEP_SIZE "" CACHE STRING "Minimum allocation step (power of 2)")
+
 if(MSVC AND SNMALLOC_STATIC_LIBRARY AND (SNMALLOC_STATIC_LIBRARY_PREFIX STREQUAL ""))
   message(FATAL_ERROR "Empty static library prefix not supported on MSVC")
 endif()
@@ -226,6 +229,11 @@ endif()
 function(add_as_define FLAG)
   target_compile_definitions(snmalloc INTERFACE $<$<BOOL:${${FLAG}}>:${FLAG}>)
 endfunction()
+function(add_as_define_value KEY)
+  if (NOT ${${KEY}} STREQUAL "")
+    target_compile_definitions(snmalloc INTERFACE ${KEY}=${${KEY}})
+  endif ()
+endfunction()
 
 add_as_define(SNMALLOC_QEMU_WORKAROUND)
 add_as_define(SNMALLOC_TRACING)
@@ -238,6 +246,8 @@ endif()
 if (SNMALLOC_NO_REALLOCARR)
   add_as_define(SNMALLOC_NO_REALLOCARR)
 endif()
+add_as_define_value(SNMALLOC_MIN_ALLOC_SIZE)
+add_as_define_value(SNMALLOC_MIN_ALLOC_STEP_SIZE)
 
 target_compile_definitions(snmalloc INTERFACE $<$<BOOL:CONST_QUALIFIED_MALLOC_USABLE_SIZE>:MALLOC_USABLE_SIZE_QUALIFIER=const>)
 

src/snmalloc/ds/allocconfig.h

@@ -20,10 +20,31 @@ namespace snmalloc
   // Used to isolate values on cache lines to prevent false sharing.
   static constexpr size_t CACHELINE_SIZE = 64;
 
-  // Minimum allocation size is space for two pointers.
-  static_assert(bits::next_pow2_const(sizeof(void*)) == sizeof(void*));
-  static constexpr size_t MIN_ALLOC_SIZE = 2 * sizeof(void*);
-  static constexpr size_t MIN_ALLOC_BITS = bits::ctz_const(MIN_ALLOC_SIZE);
+  /// The "machine epsilon" for the small sizeclass machinery.
+  static constexpr size_t MIN_ALLOC_STEP_SIZE =
+#if defined(SNMALLOC_MIN_ALLOC_STEP_SIZE)
+    SNMALLOC_MIN_ALLOC_STEP_SIZE;
+#else
+    2 * sizeof(void*);
+#endif
+
+  /// Derived from MIN_ALLOC_STEP_SIZE
+  static constexpr size_t MIN_ALLOC_STEP_BITS =
+    bits::ctz_const(MIN_ALLOC_STEP_SIZE);
+  static_assert(bits::is_pow2(MIN_ALLOC_STEP_SIZE));
+
+  /**
+   * Minimum allocation size is space for two pointers.  If the small sizeclass
+   * machinery permits smaller values (that is, if MIN_ALLOC_STEP_SIZE is
+   * smaller than MIN_ALLOC_SIZE), which may be useful if MIN_ALLOC_SIZE must
+   * be large or not a power of two, those smaller size classes will be unused.
+   */
+  static constexpr size_t MIN_ALLOC_SIZE =
+#if defined(SNMALLOC_MIN_ALLOC_SIZE)
+    SNMALLOC_MIN_ALLOC_SIZE;
+#else
+    2 * sizeof(void*);
+#endif
 
   // Minimum slab size.
 #if defined(SNMALLOC_QEMU_WORKAROUND) && defined(SNMALLOC_VA_BITS_64)
@@ -78,11 +99,18 @@ namespace snmalloc
   static constexpr size_t REMOTE_MASK = REMOTE_SLOTS - 1;
 
   static_assert(
-    INTERMEDIATE_BITS < MIN_ALLOC_BITS,
+    INTERMEDIATE_BITS < MIN_ALLOC_STEP_BITS,
     "INTERMEDIATE_BITS must be less than MIN_ALLOC_BITS");
   static_assert(
     MIN_ALLOC_SIZE >= (sizeof(void*) * 2),
     "MIN_ALLOC_SIZE must be sufficient for two pointers");
+  static_assert(
+    1 << (INTERMEDIATE_BITS + MIN_ALLOC_STEP_BITS) >=
+      bits::next_pow2_const(MIN_ALLOC_SIZE),
+    "Entire sizeclass exponent is below MIN_ALLOC_SIZE; adjust STEP_SIZE");
+  static_assert(
+    MIN_ALLOC_SIZE >= MIN_ALLOC_STEP_SIZE,
+    "Minimum alloc sizes below minimum step size; raise MIN_ALLOC_SIZE");
 
   // Return remote small allocs when the local cache reaches this size.
   static constexpr int64_t REMOTE_CACHE =

src/snmalloc/ds_core/bits.h

@@ -322,22 +322,6 @@ namespace snmalloc
      *
      * Does not work for value=0.
      ***********************************************/
-    template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
-    static size_t to_exp_mant(size_t value)
-    {
-      constexpr size_t LEADING_BIT = one_at_bit(MANTISSA_BITS + LOW_BITS) >> 1;
-      constexpr size_t MANTISSA_MASK = one_at_bit(MANTISSA_BITS) - 1;
-
-      value = value - 1;
-
-      size_t e =
-        bits::BITS - MANTISSA_BITS - LOW_BITS - clz(value | LEADING_BIT);
-      size_t b = (e == 0) ? 0 : 1;
-      size_t m = (value >> (LOW_BITS + e - b)) & MANTISSA_MASK;
-
-      return (e << MANTISSA_BITS) + m;
-    }
-
     template<size_t MANTISSA_BITS, size_t LOW_BITS = 0>
     constexpr size_t to_exp_mant_const(size_t value)
     {

src/snmalloc/mem/corealloc.h

@@ -597,23 +597,6 @@ namespace snmalloc
         init_message_queue();
         message_queue().invariant();
       }
-
-      if constexpr (DEBUG)
-      {
-        for (smallsizeclass_t i = 0; i < NUM_SMALL_SIZECLASSES; i++)
-        {
-          size_t size = sizeclass_to_size(i);
-          smallsizeclass_t sc1 = size_to_sizeclass(size);
-          smallsizeclass_t sc2 = size_to_sizeclass_const(size);
-          size_t size1 = sizeclass_to_size(sc1);
-          size_t size2 = sizeclass_to_size(sc2);
-
-          SNMALLOC_CHECK(sc1 == i);
-          SNMALLOC_CHECK(sc1 == sc2);
-          SNMALLOC_CHECK(size1 == size);
-          SNMALLOC_CHECK(size1 == size2);
-        }
-      }
     }
 
   public:

src/snmalloc/mem/sizeclasstable.h

@@ -24,7 +24,7 @@ namespace snmalloc
     // For example, 24 byte allocations can be
     // problematic for some data due to alignment issues.
     auto sc = static_cast<smallsizeclass_t>(
-      bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(size));
+      bits::to_exp_mant_const<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(size));
 
     SNMALLOC_ASSERT(sc == static_cast<uint8_t>(sc));
 
@@ -214,7 +214,8 @@ namespace snmalloc
         auto& meta = fast_small(sizeclass);
 
         size_t rsize =
-          bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_BITS>(sizeclass);
+          bits::from_exp_mant<INTERMEDIATE_BITS, MIN_ALLOC_STEP_BITS>(
+            sizeclass);
         meta.size = rsize;
         size_t slab_bits = bits::max(
           bits::next_pow2_bits_const(MIN_OBJECT_COUNT * rsize), MIN_CHUNK_BITS);
@@ -405,7 +406,7 @@ namespace snmalloc
   {
     // We subtract and shift to reduce the size of the table, i.e. we don't have
     // to store a value for every size.
-    return (s - 1) >> MIN_ALLOC_BITS;
+    return (s - 1) >> MIN_ALLOC_STEP_BITS;
   }
 
   constexpr size_t sizeclass_lookup_size =
@@ -421,13 +422,29 @@ namespace snmalloc
 
     constexpr SizeClassLookup()
     {
+      constexpr sizeclass_compress_t minimum_class =
+        static_cast<sizeclass_compress_t>(
+          size_to_sizeclass_const(MIN_ALLOC_SIZE));
+
+      /* Some unused sizeclasses is OK, but keep it within reason! */
+      static_assert(minimum_class < sizeclass_lookup_size);
+
       size_t curr = 1;
-      for (sizeclass_compress_t sizeclass = 0;
-           sizeclass < NUM_SMALL_SIZECLASSES;
-           sizeclass++)
+
+      sizeclass_compress_t sizeclass = 0;
+      for (; sizeclass < minimum_class; sizeclass++)
+      {
+        for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
+             curr += MIN_ALLOC_STEP_SIZE)
+        {
+          table[sizeclass_lookup_index(curr)] = minimum_class;
+        }
+      }
+
+      for (; sizeclass < NUM_SMALL_SIZECLASSES; sizeclass++)
       {
         for (; curr <= sizeclass_metadata.fast_small(sizeclass).size;
-             curr += 1 << MIN_ALLOC_BITS)
+             curr += MIN_ALLOC_STEP_SIZE)
         {
           auto i = sizeclass_lookup_index(curr);
           if (i == sizeclass_lookup_size)

src/test/func/memcpy/func-memcpy.cc

@@ -57,6 +57,9 @@ extern "C" void abort()
   {
     longjmp(jmp, 1);
   }
+#  if __has_builtin(__builtin_trap)
+  __builtin_trap();
+#  endif
   exit(-1);
 }
 
@@ -152,7 +155,11 @@ int main()
   // Some sizes to check for out-of-bounds access.  As we are only able to
   // catch overflows past the end of the sizeclass-padded allocation, make
   // sure we don't try to test on smaller allocations.
-  std::initializer_list<size_t> sizes = {MIN_ALLOC_SIZE, 1024, 2 * 1024 * 1024};
+
+  static constexpr size_t min_class_size =
+    sizeclass_to_size(size_to_sizeclass(MIN_ALLOC_SIZE));
+
+  std::initializer_list<size_t> sizes = {min_class_size, 1024, 2 * 1024 * 1024};
   static_assert(
     MIN_ALLOC_SIZE < 1024,
     "Can't detect overflow except at sizeclass boundaries");

src/test/func/memory/memory.cc

@@ -237,7 +237,9 @@ void test_external_pointer()
   // Malloc does not have an external pointer querying mechanism.
   auto& alloc = ThreadAlloc::get();
 
-  for (uint8_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++)
+  for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE);
+       sc < NUM_SMALL_SIZECLASSES;
+       sc++)
   {
     size_t size = sizeclass_to_size(sc);
     void* p1 = alloc.alloc(size);
@@ -470,7 +472,9 @@ void test_static_sized_allocs()
 void test_remaining_bytes()
 {
   auto& alloc = ThreadAlloc::get();
-  for (size_t sc = 0; sc < NUM_SMALL_SIZECLASSES; sc++)
+  for (snmalloc::smallsizeclass_t sc = size_to_sizeclass(MIN_ALLOC_SIZE);
+       sc < NUM_SMALL_SIZECLASSES;
+       sc++)
   {
     auto size = sizeclass_to_size(sc);
     char* p = (char*)alloc.alloc(size);

src/test/func/sizeclass/sizeclass.cc

@@ -8,6 +8,9 @@ snmalloc::smallsizeclass_t size_to_sizeclass(size_t size)
   return snmalloc::size_to_sizeclass(size);
 }
 
+static constexpr snmalloc::smallsizeclass_t minimum_sizeclass =
+  snmalloc::size_to_sizeclass_const(snmalloc::MIN_ALLOC_SIZE);
+
 void test_align_size()
 {
   bool failed = false;
@@ -72,6 +75,10 @@ int main(int, char**)
   bool failed = false;
   size_t size_low = 0;
 
+  std::cout << "Configured with minimum allocation size "
+            << snmalloc::MIN_ALLOC_SIZE << " and step size "
+            << snmalloc::MIN_ALLOC_STEP_SIZE << std::endl;
+
   std::cout << "0 has sizeclass: " << (size_t)snmalloc::size_to_sizeclass(0)
             << std::endl;
 
@@ -86,12 +93,14 @@ int main(int, char**)
       slab_size != snmalloc::sizeclass_to_slab_size(sz))
     {
       slab_size = snmalloc::sizeclass_to_slab_size(sz);
-      std::cout << std::endl;
+      std::cout << std::endl << "slab size: " << slab_size << std::endl;
     }
 
     size_t size = snmalloc::sizeclass_to_size(sz);
     std::cout << (size_t)sz << " |-> "
-              << "[" << size_low + 1 << ", " << size << "]" << std::endl;
+              << "[" << size_low + 1 << ", " << size << "]"
+              << (sz == minimum_sizeclass ? " is minimum class" : "")
+              << std::endl;
 
     if (size < size_low)
     {
@@ -102,7 +111,30 @@ int main(int, char**)
 
     for (size_t i = size_low + 1; i <= size; i++)
     {
-      if (size_to_sizeclass(i) != sz)
+      /* All sizes should, via bit-math, come back to their class value */
+      if (snmalloc::size_to_sizeclass_const(i) != sz)
+      {
+        std::cout << "Size " << i << " has _const sizeclass "
+                  << (size_t)snmalloc::size_to_sizeclass_const(i)
+                  << " but expected sizeclass " << (size_t)sz << std::endl;
+        failed = true;
+      }
+
+      if (size < snmalloc::MIN_ALLOC_SIZE)
+      {
+        /*
+         * It is expected that these sizes have the "wrong" class from tabular
+         * lookup: they will have been clipped up to the minimum class.
+         */
+        if (size_to_sizeclass(i) != minimum_sizeclass)
+        {
+          std::cout << "Size " << i << " below minimum size; sizeclass "
+                    << (size_t)size_to_sizeclass(i) << " not expected minimum "
+                    << (size_t)minimum_sizeclass << std::endl;
+          failed = true;
+        }
+      }
+      else if (size_to_sizeclass(i) != sz)
       {
         std::cout << "Size " << i << " has sizeclass "
                   << (size_t)size_to_sizeclass(i) << " but expected sizeclass "