fix integer overflows in pool::ordered_malloc (#42)

Fixes trac #6701 (https://svn.boost.org/trac10/ticket/6701).

Originally-by: Jonathan Wakely <jwakely.boost@kayari.org>

.github/workflows/ci.yml

@@ -222,7 +222,7 @@ jobs:
             b2_defines: "define=BOOST_NO_STRESS_TEST=1"
             b2_variant: "variant=debug"
             b2_testflags: "testing.launcher=valgrind"
-            valgrind_opts: "--error-exitcode=1"
+            valgrind_opts: "--error-exitcode=1 --suppressions=libs/pool/test/suppressions.txt"
           - name: "COMMENT=Coverity Scan B2_TOOLSET=clang Job 17"
             buildtype: "b5847f804b-cce9827eb5"
             packages: "binutils-gold gdb libc6-dbg"

include/boost/pool/pool.hpp

@@ -26,6 +26,8 @@
 
 #include <boost/pool/poolfwd.hpp>
 
+// std::numeric_limits
+#include <boost/limits.hpp>
 // boost::integer::static_lcm
 #include <boost/integer/common_factor_ct.hpp>
 // boost::simple_segregated_storage
@@ -355,6 +357,12 @@ class pool: protected simple_segregated_storage < typename UserAllocator::size_t
       return s;
     }
 
+    size_type max_chunks() const
+    { //! Calculated maximum number of memory chunks that can be allocated in a single call by this Pool.
+      size_type POD_size = integer::static_lcm<sizeof(size_type), sizeof(void *)>::value + sizeof(size_type);
+      return (std::numeric_limits<size_type>::max() - POD_size) / alloc_size();
+    }
+
     static void * & nextof(void * const ptr)
     { //! \returns Pointer dereferenced.
       //! (Provided and used for the sake of code readability :)
@@ -375,6 +383,8 @@ class pool: protected simple_segregated_storage < typename UserAllocator::size_t
       //!   the first time that object needs to allocate system memory.
       //!   The default is 32. This parameter may not be 0.
       //! \param nmax_size is the maximum number of chunks to allocate in one block.
+      set_next_size(nnext_size);
+      set_max_size(nmax_size);
     }
 
     ~pool()
@@ -398,16 +408,17 @@ class pool: protected simple_segregated_storage < typename UserAllocator::size_t
     }
     void set_next_size(const size_type nnext_size)
     { //! Set number of chunks to request from the system the next time that object needs to allocate system memory. This value should never be set to 0.
-      //! \returns nnext_size.
-      next_size = start_size = nnext_size;
+      BOOST_USING_STD_MIN();
+      next_size = start_size = min BOOST_PREVENT_MACRO_SUBSTITUTION(nnext_size, max_chunks());
     }
     size_type get_max_size() const
     { //! \returns max_size.
       return max_size;
     }
     void set_max_size(const size_type nmax_size)
     { //! Set max_size.
-      max_size = nmax_size;
+      BOOST_USING_STD_MIN();
+      max_size = min BOOST_PREVENT_MACRO_SUBSTITUTION(nmax_size, max_chunks());
     }
     size_type get_requested_size() const
     { //!   \returns the requested size passed into the constructor.
@@ -708,9 +719,9 @@ void * pool<UserAllocator>::malloc_need_resize()
 
   BOOST_USING_STD_MIN();
   if(!max_size)
-    next_size <<= 1;
+    set_next_size(next_size << 1);
   else if( next_size*partition_size/requested_size < max_size)
-    next_size = min BOOST_PREVENT_MACRO_SUBSTITUTION(next_size << 1, max_size*requested_size/ partition_size);
+    set_next_size(min BOOST_PREVENT_MACRO_SUBSTITUTION(next_size << 1, max_size * requested_size / partition_size));
 
   //  initialize it,
   store().add_block(node.begin(), node.element_size(), partition_size);
@@ -748,9 +759,9 @@ void * pool<UserAllocator>::ordered_malloc_need_resize()
 
   BOOST_USING_STD_MIN();
   if(!max_size)
-    next_size <<= 1;
+    set_next_size(next_size << 1);
   else if( next_size*partition_size/requested_size < max_size)
-    next_size = min BOOST_PREVENT_MACRO_SUBSTITUTION(next_size << 1, max_size*requested_size/ partition_size);
+    set_next_size(min BOOST_PREVENT_MACRO_SUBSTITUTION(next_size << 1, max_size * requested_size / partition_size));
 
   //  initialize it,
   //  (we can use "add_block" here because we know that
@@ -792,6 +803,8 @@ void * pool<UserAllocator>::ordered_malloc(const size_type n)
 { //! Gets address of a chunk n, allocating new memory if not already available.
   //! \returns Address of chunk n if allocated ok.
   //! \returns 0 if not enough memory for n chunks.
+  if (n > max_chunks())
+    return 0;
 
   const size_type partition_size = alloc_size();
   const size_type total_req_size = n * requested_size;
@@ -840,9 +853,9 @@ void * pool<UserAllocator>::ordered_malloc(const size_type n)
 
   BOOST_USING_STD_MIN();
   if(!max_size)
-    next_size <<= 1;
+    set_next_size(next_size << 1);
   else if( next_size*partition_size/requested_size < max_size)
-    next_size = min BOOST_PREVENT_MACRO_SUBSTITUTION(next_size << 1, max_size*requested_size/ partition_size);
+    set_next_size(min BOOST_PREVENT_MACRO_SUBSTITUTION(next_size << 1, max_size * requested_size / partition_size));
 
   //  insert it into the list,
   //   handle border case.

test/Jamfile.v2

@@ -34,6 +34,7 @@ test-suite pool :
         <toolset>pathscale:<cxxflags>-Wno-long-long ]
     [ run test_bug_2696.cpp ]
     [ run test_bug_5526.cpp ]
+    [ run test_bug_6701.cpp ]
     [ run test_threading.cpp : : : <threading>multi <library>/boost/thread//boost_thread ]
     [ compile test_poisoned_macros.cpp ]
     ;

test/suppressions.txt

@@ -0,0 +1,7 @@
+{
+   no_fishy_value
+   Memcheck:FishyValue
+   __builtin_vec_new(size)
+   fun:_ZnamRKSt9nothrow_t
+   ...
+}

test/test_bug_6701.cpp

@@ -0,0 +1,27 @@
+/* Copyright (C) 2012 Étienne Dupuis
+*
+* Use, modification and distribution is subject to the
+* Boost Software License, Version 1.0. (See accompanying
+* file LICENSE_1_0.txt or http://www.boost.org/LICENSE_1_0.txt)
+*/
+
+// Test of bug #6701 (https://svn.boost.org/trac/boost/ticket/6701)
+
+#include <boost/pool/object_pool.hpp>
+#include <boost/limits.hpp>
+
+int main()
+{
+  boost::pool<> p(1024, std::numeric_limits<size_t>::max() / 768);
+
+  void *x = p.malloc();
+  BOOST_ASSERT(!x);
+
+  BOOST_ASSERT(std::numeric_limits<size_t>::max() / 1024 >= p.get_next_size());
+  BOOST_ASSERT(std::numeric_limits<size_t>::max() / 1024 >= p.get_max_size());
+
+  void *y = p.ordered_malloc(std::numeric_limits<size_t>::max() / 768);
+  BOOST_ASSERT(!y);
+
+  return 0;
+}