apache · sudheerv · Jun 19, 2020 · Jun 9, 2020 · Jun 17, 2020 · Jun 18, 2020
diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst
@@ -4407,6 +4407,31 @@ Sockets
    platforms.  (Currently only linux).  IO buffers are allocated with the MADV_DONTDUMP
    with madvise() on linux platforms that support MADV_DONTDUMP.  Enabled by default.
 
+.. ts:cv:: CONFIG proxy.config.ssl.misc.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for various SSL Operations
+   such as Handshake or Protocol Probe. Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.hostdb.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing HostDB records.
+   Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.payload.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing request payload buffer
+   for a POST request. Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.msg.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing miscellaneous transactional
+   buffers such as error response body. Default value is 8 which maps to a 32K buffer
+
+.. ts:cv:: CONFIG proxy.config.log.io.max_buffer_index INT 8
+
+   Configures the max IOBuffer Block index used for storing an access log entry.
+   Default value is 8 which maps to a 32K buffer
+
 .. ts:cv:: CONFIG proxy.config.http.enabled INT 1
 
    Turn on or off support for HTTP proxying. This is rarely used, the one

diff --git a/iocore/cache/CachePages.cc b/iocore/cache/CachePages.cc
@@ -427,7 +427,7 @@ ShowCache::handleCacheEvent(int event, Event *e)
 
   case VC_EVENT_READ_READY:
     if (!cvio) {
-      buffer         = new_empty_MIOBuffer();
+      buffer         = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K);
       buffer_reader  = buffer->alloc_reader();
       content_length = cache_vc->get_object_size();
       cvio           = cache_vc->do_io_read(this, content_length, buffer);

diff --git a/iocore/cache/CacheTest.cc b/iocore/cache/CacheTest.cc
@@ -95,7 +95,7 @@ CacheTestSM::event_handler(int event, void *data)
     cancel_timeout();
     cache_action  = nullptr;
     cache_vc      = static_cast<CacheVConnection *>(data);
-    buffer        = new_empty_MIOBuffer();
+    buffer        = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K);
     buffer_reader = buffer->alloc_reader();
     if (open_read_callout() < 0) {
       goto Lclose_error_next;
@@ -129,7 +129,7 @@ CacheTestSM::event_handler(int event, void *data)
     cancel_timeout();
     cache_action  = nullptr;
     cache_vc      = static_cast<CacheVConnection *>(data);
-    buffer        = new_empty_MIOBuffer();
+    buffer        = new_empty_MIOBuffer(BUFFER_SIZE_INDEX_32K);
     buffer_reader = buffer->alloc_reader();
     if (open_write_callout() < 0) {
       goto Lclose_error_next;

diff --git a/iocore/eventsystem/EventSystem.cc b/iocore/eventsystem/EventSystem.cc
@@ -34,25 +34,14 @@ void
 ink_event_system_init(ts::ModuleVersion v)
 {
   ink_release_assert(v.check(EVENT_SYSTEM_MODULE_INTERNAL_VERSION));
-  int config_max_iobuffer_size = DEFAULT_MAX_BUFFER_SIZE;
-  int iobuffer_advice          = 0;
+  int iobuffer_advice = 0;
 
   // For backwards compatibility make sure to allow thread_freelist_size
   // This needs to change in 6.0
   REC_EstablishStaticConfigInt32(thread_freelist_high_watermark, "proxy.config.allocator.thread_freelist_size");
 
   REC_EstablishStaticConfigInt32(thread_freelist_low_watermark, "proxy.config.allocator.thread_freelist_low_watermark");
 
-  REC_ReadConfigInteger(config_max_iobuffer_size, "proxy.config.io.max_buffer_size");
-
-  max_iobuffer_size = buffer_size_to_index(config_max_iobuffer_size, DEFAULT_BUFFER_SIZES - 1);
-  if (default_small_iobuffer_size > max_iobuffer_size) {
-    default_small_iobuffer_size = max_iobuffer_size;
-  }
-  if (default_large_iobuffer_size > max_iobuffer_size) {
-    default_large_iobuffer_size = max_iobuffer_size;
-  }
-
 #ifdef MADV_DONTDUMP // This should only exist on Linux 3.4 and higher.
   RecBool dont_dump_enabled = true;
   RecGetRecordBool("proxy.config.allocator.dontdump_iobuffers", &dont_dump_enabled, false);

diff --git a/iocore/eventsystem/I_IOBuffer.h b/iocore/eventsystem/I_IOBuffer.h
@@ -52,10 +52,6 @@ class MIOBuffer;
 class IOBufferReader;
 class VIO;
 
-inkcoreapi extern int64_t max_iobuffer_size;
-extern int64_t default_small_iobuffer_size;
-extern int64_t default_large_iobuffer_size; // matched to size of OS buffers
-
 enum AllocType {
   NO_ALLOC,
   MEMALIGNED,
@@ -417,7 +413,7 @@ class IOBufferBlock : public RefCountObj
     section in MIOBuffer.
 
   */
-  void alloc(int64_t i = default_large_iobuffer_size);
+  void alloc(int64_t i);
 
   /**
     Clear the IOBufferData this IOBufferBlock handles. Clears this
@@ -1148,7 +1144,7 @@ class MIOBuffer
   void dealloc_all_readers();
 
   void set(void *b, int64_t len);
-  void alloc(int64_t i = default_large_iobuffer_size);
+  void alloc(int64_t i);
   void append_block_internal(IOBufferBlock *b);
   int64_t write(IOBufferBlock const *b, int64_t len, int64_t offset);
 
@@ -1290,7 +1286,7 @@ struct MIOBufferAccessor {
   IOBufferReader *entry = nullptr;
 };
 
-extern MIOBuffer *new_MIOBuffer_internal(const char *loc, int64_t size_index = default_large_iobuffer_size);
+extern MIOBuffer *new_MIOBuffer_internal(const char *loc, int64_t size_index);
 
 class MIOBuffer_tracker
 {
@@ -1299,13 +1295,13 @@ class MIOBuffer_tracker
 public:
   explicit MIOBuffer_tracker(const char *_loc) : loc(_loc) {}
   MIOBuffer *
-  operator()(int64_t size_index = default_large_iobuffer_size)
+  operator()(int64_t size_index)
   {
     return new_MIOBuffer_internal(loc, size_index);
   }
 };
 
-extern MIOBuffer *new_empty_MIOBuffer_internal(const char *loc, int64_t size_index = default_large_iobuffer_size);
+extern MIOBuffer *new_empty_MIOBuffer_internal(const char *loc, int64_t size_index);
 
 class Empty_MIOBuffer_tracker
 {
@@ -1314,7 +1310,7 @@ class Empty_MIOBuffer_tracker
 public:
   explicit Empty_MIOBuffer_tracker(const char *_loc) : loc(_loc) {}
   MIOBuffer *
-  operator()(int64_t size_index = default_large_iobuffer_size)
+  operator()(int64_t size_index)
   {
     return new_empty_MIOBuffer_internal(loc, size_index);
   }
@@ -1352,8 +1348,7 @@ class IOBufferBlock_tracker
 #define new_IOBufferBlock IOBufferBlock_tracker(RES_PATH("memory/IOBuffer/"))
 ////////////////////////////////////////////////////////////
 
-extern IOBufferData *new_IOBufferData_internal(const char *location, int64_t size_index = default_large_iobuffer_size,
-                                               AllocType type = DEFAULT_ALLOC);
+extern IOBufferData *new_IOBufferData_internal(const char *location, int64_t size_index, AllocType type = DEFAULT_ALLOC);
 
 extern IOBufferData *new_xmalloc_IOBufferData_internal(const char *location, void *b, int64_t size);
 
@@ -1364,7 +1359,7 @@ class IOBufferData_tracker
 public:
   explicit IOBufferData_tracker(const char *_loc) : loc(_loc) {}
   IOBufferData *
-  operator()(int64_t size_index = default_large_iobuffer_size, AllocType type = DEFAULT_ALLOC)
+  operator()(int64_t size_index, AllocType type = DEFAULT_ALLOC)
   {
     return new_IOBufferData_internal(loc, size_index, type);
   }
@@ -1374,7 +1369,7 @@ class IOBufferData_tracker
 #define new_IOBufferData IOBufferData_tracker(RES_PATH("memory/IOBuffer/"))
 #define new_xmalloc_IOBufferData(b, size) new_xmalloc_IOBufferData_internal(RES_PATH("memory/IOBuffer/"), (b), (size))
 
-extern int64_t iobuffer_size_to_index(int64_t size, int64_t max = max_iobuffer_size);
+extern int64_t iobuffer_size_to_index(int64_t size, int64_t max);
 extern int64_t index_to_buffer_size(int64_t idx);
 /**
   Clone a IOBufferBlock chain. Used to snarf a IOBufferBlock chain

diff --git a/iocore/eventsystem/P_IOBuffer.h b/iocore/eventsystem/P_IOBuffer.h
@@ -36,7 +36,7 @@
 //
 //////////////////////////////////////////////////////////////
 TS_INLINE int64_t
-buffer_size_to_index(int64_t size, int64_t max = max_iobuffer_size)
+buffer_size_to_index(int64_t size, int64_t max)
 {
   int64_t r = max;
 
@@ -214,12 +214,6 @@ new_xmalloc_IOBufferData_internal(const char *location, void *b, int64_t size)
   return new_IOBufferData_internal(location, b, size, BUFFER_SIZE_INDEX_FOR_XMALLOC_SIZE(size));
 }
 
-TS_INLINE IOBufferData *
-new_IOBufferData_internal(const char *location, void *b, int64_t size)
-{
-  return new_IOBufferData_internal(location, b, size, iobuffer_size_to_index(size));
-}
-
 TS_INLINE IOBufferData *
 new_IOBufferData_internal(const char *loc, int64_t size_index, AllocType type)
 {

diff --git a/iocore/eventsystem/unit_tests/test_IOBuffer.cc b/iocore/eventsystem/unit_tests/test_IOBuffer.cc
@@ -35,26 +35,19 @@
 
 TEST_CASE("MIOBuffer", "[iocore]")
 {
-  // These value could be tweaked by `ink_event_system_init()` using `proxy.config.io.max_buffer_size`
-  REQUIRE(default_small_iobuffer_size == DEFAULT_SMALL_BUFFER_SIZE);
-  REQUIRE(default_large_iobuffer_size == DEFAULT_LARGE_BUFFER_SIZE);
-
-  REQUIRE(BUFFER_SIZE_FOR_INDEX(default_small_iobuffer_size) == 512);
-  REQUIRE(BUFFER_SIZE_FOR_INDEX(default_large_iobuffer_size) == 4096);
-
   SECTION("new_MIOBuffer 100 times")
   {
     int64_t read_avail_len1 = 0;
     int64_t read_avail_len2 = 0;
 
     for (unsigned i = 0; i < 100; ++i) {
-      MIOBuffer *b1            = new_MIOBuffer(default_small_iobuffer_size);
+      MIOBuffer *b1            = new_MIOBuffer(BUFFER_SIZE_INDEX_512);
       int64_t len1             = b1->write_avail();
       IOBufferReader *b1reader = b1->alloc_reader();
       b1->fill(len1);
       read_avail_len1 += b1reader->read_avail();
 
-      MIOBuffer *b2            = new_MIOBuffer(default_large_iobuffer_size);
+      MIOBuffer *b2            = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
       int64_t len2             = b2->write_avail();
       IOBufferReader *b2reader = b2->alloc_reader();
       b2->fill(len2);
@@ -64,19 +57,19 @@ TEST_CASE("MIOBuffer", "[iocore]")
       free_MIOBuffer(b1);
     }
 
-    CHECK(read_avail_len1 == 100 * BUFFER_SIZE_FOR_INDEX(default_small_iobuffer_size));
-    CHECK(read_avail_len2 == 100 * BUFFER_SIZE_FOR_INDEX(default_large_iobuffer_size));
+    CHECK(read_avail_len1 == 100 * BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX_512));
+    CHECK(read_avail_len2 == 100 * BUFFER_SIZE_FOR_INDEX(BUFFER_SIZE_INDEX_4K));
   }
 
   SECTION("write")
   {
-    MIOBuffer *miob            = new_MIOBuffer();
+    MIOBuffer *miob            = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
     IOBufferReader *miob_r     = miob->alloc_reader();
     const IOBufferBlock *block = miob->first_write_block();
 
     SECTION("initial state")
     {
-      CHECK(miob->size_index == default_large_iobuffer_size);
+      CHECK(miob->size_index == BUFFER_SIZE_INDEX_4K);
       CHECK(miob->water_mark == 0);
       CHECK(miob->first_write_block() != nullptr);
       CHECK(miob->block_size() == 4096);
@@ -176,7 +169,7 @@ TEST_CASE("MIOBuffer", "[iocore]")
 
   SECTION("write_avail")
   {
-    MIOBuffer *miob        = new_MIOBuffer();
+    MIOBuffer *miob        = new_MIOBuffer(BUFFER_SIZE_INDEX_4K);
     IOBufferReader *miob_r = miob->alloc_reader();
     uint8_t buf[8192];
     memset(buf, 0xAA, sizeof(buf));
@@ -347,7 +340,6 @@ struct EventProcessorListener : Catch::TestEventListenerBase {
     init_diags("", nullptr);
     RecProcessInit(RECM_STAND_ALONE);
 
-    // Initialize LibRecordsConfig for `proxy.config.io.max_buffer_size` (32K)
     LibRecordsConfigInit();
 
     ink_event_system_init(EVENT_SYSTEM_MODULE_PUBLIC_VERSION);

diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc
@@ -62,6 +62,7 @@ int hostdb_max_count                               = DEFAULT_HOST_DB_SIZE;
 char hostdb_hostfile_path[PATH_NAME_MAX]           = "";
 int hostdb_sync_frequency                          = 0;
 int hostdb_disable_reverse_lookup                  = 0;
+int hostdb_max_iobuf_index                         = BUFFER_SIZE_INDEX_32K;
 
 ClassAllocator<HostDBContinuation> hostDBContAllocator("hostDBContAllocator");
 
@@ -325,6 +326,8 @@ HostDBCache::start(int flags)
   // how often to sync hostdb to disk
   REC_EstablishStaticConfigInt32(hostdb_sync_frequency, "proxy.config.cache.hostdb.sync_frequency");
 
+  REC_EstablishStaticConfigInt32(hostdb_max_iobuf_index, "proxy.config.hostdb.io.max_buffer_index");
+
   if (hostdb_max_size == 0) {
     Fatal("proxy.config.hostdb.max_size must be a non-zero number");
   }

diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h
@@ -62,6 +62,8 @@ extern unsigned int hostdb_ip_fail_timeout_interval;
 extern unsigned int hostdb_serve_stale_but_revalidate;
 extern unsigned int hostdb_round_robin_max_count;
 
+extern int hostdb_max_iobuf_index;
+
 static inline unsigned int
 makeHostHash(const char *string)
 {
@@ -142,7 +144,7 @@ struct HostDBInfo : public RefCountObj {
   alloc(int size = 0)
   {
     size += sizeof(HostDBInfo);
-    int iobuffer_index = iobuffer_size_to_index(size);
+    int iobuffer_index = iobuffer_size_to_index(size, hostdb_max_iobuf_index);
     ink_release_assert(iobuffer_index >= 0);
     void *ptr = ioBufAllocator[iobuffer_index].alloc_void();
     memset(ptr, 0, size);

diff --git a/iocore/net/NetVCTest.cc b/iocore/net/NetVCTest.cc
@@ -138,8 +138,8 @@ NetVCTest::start_test()
   test_vc->set_inactivity_timeout(HRTIME_SECONDS(timeout));
   test_vc->set_active_timeout(HRTIME_SECONDS(timeout + 5));
 
-  read_buffer  = new_MIOBuffer();
-  write_buffer = new_MIOBuffer();
+  read_buffer  = new_MIOBuffer(BUFFER_SIZE_INDEX_32K);
+  write_buffer = new_MIOBuffer(BUFFER_SIZE_INDEX_32K);
 
   reader_for_rbuf = read_buffer->alloc_reader();
   reader_for_wbuf = write_buffer->alloc_reader();

diff --git a/iocore/net/P_SSLConfig.h b/iocore/net/P_SSLConfig.h
@@ -108,6 +108,7 @@ struct SSLConfigParams : public ConfigInfo {
   static bool server_allow_early_data_params;
 
   static int ssl_maxrecord;
+  static int ssl_misc_max_iobuffer_size_index;
   static bool ssl_allow_client_renegotiation;
 
   static bool ssl_ocsp_enabled;

diff --git a/iocore/net/P_SSLNetVConnection.h b/iocore/net/P_SSLNetVConnection.h
@@ -49,6 +49,7 @@
 #include "P_ALPNSupport.h"
 #include "TLSSessionResumptionSupport.h"
 #include "P_SSLUtils.h"
+#include "P_SSLConfig.h"
 
 // These are included here because older OpenSSL libraries don't have them.
 // Don't copy these defines, or use their values directly, they are merely
@@ -186,7 +187,7 @@ class SSLNetVConnection : public UnixNetVConnection, public ALPNSupport, public
   void
   initialize_handshake_buffers()
   {
-    this->handShakeBuffer    = new_MIOBuffer();
+    this->handShakeBuffer    = new_MIOBuffer(SSLConfigParams::ssl_misc_max_iobuffer_size_index);
     this->handShakeReader    = this->handShakeBuffer->alloc_reader();
     this->handShakeHolder    = this->handShakeReader->clone();
     this->handShakeBioStored = 0;

diff --git a/iocore/net/QUICNetVConnection.cc b/iocore/net/QUICNetVConnection.cc
@@ -1344,7 +1344,7 @@ QUICNetVConnection::_state_common_send_packet()
 
     Ptr<IOBufferBlock> udp_payload(new_IOBufferBlock());
     uint32_t udp_payload_len = std::min(window, this->_pmtu);
-    udp_payload->alloc(iobuffer_size_to_index(udp_payload_len));
+    udp_payload->alloc(iobuffer_size_to_index(udp_payload_len, BUFFER_SIZE_INDEX_32K));
 
     uint32_t written = 0;
     for (int i = static_cast<int>(this->_minimum_encryption_level); i <= static_cast<int>(QUICEncryptionLevel::ONE_RTT); ++i) {
@@ -1499,7 +1499,7 @@ QUICNetVConnection::_packetize_frames(QUICEncryptionLevel level, uint64_t max_pa
   size_t len                     = 0;
   Ptr<IOBufferBlock> first_block = make_ptr<IOBufferBlock>(new_IOBufferBlock());
   Ptr<IOBufferBlock> last_block  = first_block;
-  first_block->alloc(iobuffer_size_to_index(0));
+  first_block->alloc(iobuffer_size_to_index(0, BUFFER_SIZE_INDEX_32K));
   first_block->fill(0);
 
   uint32_t seq_num   = this->_seq_num++;

diff --git a/iocore/net/QUICPacketHandler.cc b/iocore/net/QUICPacketHandler.cc
@@ -76,7 +76,7 @@ QUICPacketHandler::_send_packet(const QUICPacket &packet, UDPConnection *udp_con
 {
   size_t udp_len;
   Ptr<IOBufferBlock> udp_payload(new_IOBufferBlock());
-  udp_payload->alloc(iobuffer_size_to_index(pmtu));
+  udp_payload->alloc(iobuffer_size_to_index(pmtu, BUFFER_SIZE_INDEX_32K));
   packet.store(reinterpret_cast<uint8_t *>(udp_payload->end()), &udp_len);
   udp_payload->fill(udp_len);
 

diff --git a/iocore/net/SSLConfig.cc b/iocore/net/SSLConfig.cc
@@ -52,6 +52,7 @@ int SSLConfig::configid                                     = 0;
 int SSLCertificateConfig::configid                          = 0;
 int SSLTicketKeyConfig::configid                            = 0;
 int SSLConfigParams::ssl_maxrecord                          = 0;
+int SSLConfigParams::ssl_misc_max_iobuffer_size_index       = 8;
 bool SSLConfigParams::ssl_allow_client_renegotiation        = false;
 bool SSLConfigParams::ssl_ocsp_enabled                      = false;
 int SSLConfigParams::ssl_ocsp_cache_timeout                 = 3600;
@@ -454,6 +455,8 @@ SSLConfigParams::initialize()
 
   REC_ReadConfigInt32(ssl_allow_client_renegotiation, "proxy.config.ssl.allow_client_renegotiation");
 
+  REC_ReadConfigInt32(ssl_misc_max_iobuffer_size_index, "proxy.config.ssl.misc.io.max_buffer_index");
+
   // Enable client regardless of config file settings as remap file
   // can cause HTTP layer to connect using SSL. But only if SSL
   // initialization hasn't failed already.

diff --git a/iocore/net/SSLNextProtocolAccept.cc b/iocore/net/SSLNextProtocolAccept.cc
@@ -167,7 +167,10 @@ SSLNextProtocolAccept::enableProtocols(const SessionProtocolSet &protos)
 }
 
 SSLNextProtocolAccept::SSLNextProtocolAccept(Continuation *ep, bool transparent_passthrough)
-  : SessionAccept(nullptr), buffer(new_empty_MIOBuffer()), endpoint(ep), transparent_passthrough(transparent_passthrough)
+  : SessionAccept(nullptr),
+    buffer(new_empty_MIOBuffer(SSLConfigParams::ssl_misc_max_iobuffer_size_index)),
+    endpoint(ep),
+    transparent_passthrough(transparent_passthrough)
 {
   SET_HANDLER(&SSLNextProtocolAccept::mainEvent);
 }