Issue#6850 - Customize Max IOBuffer Size for each usecase.

sudheerv · sudheerv · commit 3e88a958512f · 2020-06-08T18:22:44.000-07:00
Allow callers explicitly configure and pass in the max buffer sizes
when asking for unknown sized memory chunks depending on the usecase
instead of relying on a common/global default config.
diff --git a/iocore/hostdb/HostDB.cc b/iocore/hostdb/HostDB.cc
@@ -62,6 +62,7 @@ int hostdb_max_count                               = DEFAULT_HOST_DB_SIZE;
 char hostdb_hostfile_path[PATH_NAME_MAX]           = "";
 int hostdb_sync_frequency                          = 0;
 int hostdb_disable_reverse_lookup                  = 0;
+int hostdb_max_iobuf_index                         = BUFFER_SIZE_INDEX_32K;
 
 ClassAllocator<HostDBContinuation> hostDBContAllocator("hostDBContAllocator");
 
@@ -325,6 +326,8 @@ HostDBCache::start(int flags)
   // how often to sync hostdb to disk
   REC_EstablishStaticConfigInt32(hostdb_sync_frequency, "proxy.config.cache.hostdb.sync_frequency");
 
+  REC_EstablishStaticConfigInt32(hostdb_max_iobuf_index, "proxy.config.hostdb.io.max_buffer_index");
+
   if (hostdb_max_size == 0) {
     Fatal("proxy.config.hostdb.max_size must be a non-zero number");
   }
diff --git a/iocore/hostdb/I_HostDBProcessor.h b/iocore/hostdb/I_HostDBProcessor.h
@@ -62,6 +62,8 @@ extern unsigned int hostdb_ip_fail_timeout_interval;
 extern unsigned int hostdb_serve_stale_but_revalidate;
 extern unsigned int hostdb_round_robin_max_count;
 
+extern int hostdb_max_iobuf_index;
+
 static inline unsigned int
 makeHostHash(const char *string)
 {
@@ -142,7 +144,7 @@ struct HostDBInfo : public RefCountObj {
   alloc(int size = 0)
   {
     size += sizeof(HostDBInfo);
-    int iobuffer_index = iobuffer_size_to_index(size);
+    int iobuffer_index = iobuffer_size_to_index(size, hostdb_max_iobuf_index);
     ink_release_assert(iobuffer_index >= 0);
     void *ptr = ioBufAllocator[iobuffer_index].alloc_void();
     memset(ptr, 0, size);
diff --git a/iocore/net/QUICNetVConnection.cc b/iocore/net/QUICNetVConnection.cc
@@ -1344,7 +1344,7 @@ QUICNetVConnection::_state_common_send_packet()
 
     Ptr<IOBufferBlock> udp_payload(new_IOBufferBlock());
     uint32_t udp_payload_len = std::min(window, this->_pmtu);
-    udp_payload->alloc(iobuffer_size_to_index(udp_payload_len));
+    udp_payload->alloc(iobuffer_size_to_index(udp_payload_len, max_quic_iobuffer_index()));
 
     uint32_t written = 0;
     for (int i = static_cast<int>(this->_minimum_encryption_level); i <= static_cast<int>(QUICEncryptionLevel::ONE_RTT); ++i) {
@@ -1499,7 +1499,7 @@ QUICNetVConnection::_packetize_frames(QUICEncryptionLevel level, uint64_t max_pa
   size_t len                     = 0;
   Ptr<IOBufferBlock> first_block = make_ptr<IOBufferBlock>(new_IOBufferBlock());
   Ptr<IOBufferBlock> last_block  = first_block;
-  first_block->alloc(iobuffer_size_to_index(0));
+  first_block->alloc(iobuffer_size_to_index(0, BUFFER_SIZE_INDEX_32K));
   first_block->fill(0);
 
   uint32_t seq_num   = this->_seq_num++;
diff --git a/iocore/net/QUICPacketHandler.cc b/iocore/net/QUICPacketHandler.cc
@@ -76,7 +76,7 @@ QUICPacketHandler::_send_packet(const QUICPacket &packet, UDPConnection *udp_con
 {
   size_t udp_len;
   Ptr<IOBufferBlock> udp_payload(new_IOBufferBlock());
-  udp_payload->alloc(iobuffer_size_to_index(pmtu));
+  udp_payload->alloc(iobuffer_size_to_index(pmtu, BUFFER_SIZE_INDEX_32K));
   packet.store(reinterpret_cast<uint8_t *>(udp_payload->end()), &udp_len);
   udp_payload->fill(udp_len);
 
diff --git a/mgmt/RecordsConfig.cc b/mgmt/RecordsConfig.cc
@@ -720,6 +720,14 @@ static const RecordElement RecordsConfig[] =
   //##############################################################################
   {RECT_CONFIG, "proxy.config.io.max_buffer_size", RECD_INT, "32768", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.hostdb.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.payload.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.msg.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
+  {RECT_CONFIG, "proxy.config.log.io.max_buffer_index", RECD_INT, "8", RECU_NULL, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
+  ,
 
   //##############################################################################
   //#
diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc
@@ -1284,6 +1284,8 @@ HttpConfig::startup()
   HttpEstablishStaticConfigByte(c.enable_http_info, "proxy.config.http.enable_http_info");
 
   HttpEstablishStaticConfigLongLong(c.max_post_size, "proxy.config.http.max_post_size");
+  HttpEstablishStaticConfigLongLong(c.max_payload_iobuf_index, "proxy.config.payload.io.max_buffer_index");
+  HttpEstablishStaticConfigLongLong(c.max_msg_iobuf_index, "proxy.config.msg.io.max_buffer_index");
 
   //##############################################################################
   //#
@@ -1527,6 +1529,8 @@ HttpConfig::reconfigure()
 
   params->oride.cache_when_to_revalidate = m_master.oride.cache_when_to_revalidate;
   params->max_post_size                  = m_master.max_post_size;
+  params->max_payload_iobuf_index        = m_master.max_payload_iobuf_index;
+  params->max_msg_iobuf_index            = m_master.max_msg_iobuf_index;
 
   params->oride.cache_required_headers = m_master.oride.cache_required_headers;
   params->oride.cache_range_lookup     = INT_TO_BOOL(m_master.oride.cache_range_lookup);
diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h
@@ -747,6 +747,9 @@ struct HttpConfigParams : public ConfigInfo {
   MgmtInt post_copy_size = 2048;
   MgmtInt max_post_size  = 0;
 
+  MgmtInt max_payload_iobuf_index = BUFFER_SIZE_INDEX_32K;
+  MgmtInt max_msg_iobuf_index     = BUFFER_SIZE_INDEX_32K;
+
   char *redirect_actions_string                        = nullptr;
   IpMap *redirect_actions_map                          = nullptr;
   RedirectEnabled::Action redirect_actions_self_action = RedirectEnabled::Action::INVALID;
diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc
@@ -843,7 +843,7 @@ HttpSM::state_read_client_request_header(int event, void *data)
       // When receive an "Expect: 100-continue" request from client, ATS sends a "100 Continue" response to client
       // immediately, before receive the real response from original server.
       if ((len == HTTP_LEN_100_CONTINUE) && (strncasecmp(expect, HTTP_VALUE_100_CONTINUE, HTTP_LEN_100_CONTINUE) == 0)) {
-        int64_t alloc_index = buffer_size_to_index(len_100_continue_response);
+        int64_t alloc_index = buffer_size_to_index(len_100_continue_response, t_state.http_config_param->max_payload_iobuf_index);
         if (ua_entry->write_buffer) {
           free_MIOBuffer(ua_entry->write_buffer);
           ua_entry->write_buffer = nullptr;
@@ -898,7 +898,7 @@ HttpSM::wait_for_full_body()
       alloc_index = DEFAULT_REQUEST_BUFFER_SIZE_INDEX;
     }
   } else {
-    alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length);
+    alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length, t_state.http_config_param->max_payload_iobuf_index);
   }
   MIOBuffer *post_buffer    = new_MIOBuffer(alloc_index);
   IOBufferReader *buf_start = post_buffer->alloc_reader();
@@ -5676,7 +5676,7 @@ HttpSM::setup_transform_to_server_transfer()
   ink_assert(post_transform_info.entry->vc == post_transform_info.vc);
 
   int64_t nbytes            = t_state.hdr_info.transform_request_cl;
-  int64_t alloc_index       = buffer_size_to_index(nbytes);
+  int64_t alloc_index       = buffer_size_to_index(nbytes, t_state.http_config_param->max_payload_iobuf_index);
   MIOBuffer *post_buffer    = new_MIOBuffer(alloc_index);
   IOBufferReader *buf_start = post_buffer->alloc_reader();
 
@@ -5756,7 +5756,8 @@ HttpSM::do_setup_post_tunnel(HttpVC_t to_vc_type)
         alloc_index = DEFAULT_REQUEST_BUFFER_SIZE_INDEX;
       }
     } else {
-      alloc_index = buffer_size_to_index(t_state.hdr_info.request_content_length);
+      alloc_index =
+        buffer_size_to_index(t_state.hdr_info.request_content_length, t_state.http_config_param->max_payload_iobuf_index);
     }
     MIOBuffer *post_buffer    = new_MIOBuffer(alloc_index);
     IOBufferReader *buf_start = post_buffer->alloc_reader();
@@ -6177,7 +6178,8 @@ HttpSM::setup_cache_read_transfer()
   ink_assert(cache_sm.cache_read_vc != nullptr);
 
   doc_size    = t_state.cache_info.object_read->object_size_get();
-  alloc_index = buffer_size_to_index(doc_size + index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX));
+  alloc_index = buffer_size_to_index(doc_size + index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX),
+                                     t_state.http_config_param->max_payload_iobuf_index);
 
 #ifndef USE_NEW_EMPTY_MIOBUFFER
   MIOBuffer *buf = new_MIOBuffer(alloc_index);
@@ -6230,7 +6232,7 @@ HttpSM::setup_cache_transfer_to_transform()
   cache_response_hdr_bytes = t_state.hdr_info.cache_response.length_get();
 
   doc_size                  = t_state.cache_info.object_read->object_size_get();
-  alloc_index               = buffer_size_to_index(doc_size);
+  alloc_index               = buffer_size_to_index(doc_size, t_state.http_config_param->max_payload_iobuf_index);
   MIOBuffer *buf            = new_MIOBuffer(alloc_index);
   IOBufferReader *buf_start = buf->alloc_reader();
 
@@ -6375,7 +6377,7 @@ HttpSM::setup_internal_transfer(HttpSMHandler handler_arg)
   int64_t buf_size =
     index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX) + (is_msg_buf_present ? t_state.internal_msg_buffer_size : 0);
 
-  MIOBuffer *buf            = new_MIOBuffer(buffer_size_to_index(buf_size));
+  MIOBuffer *buf            = new_MIOBuffer(buffer_size_to_index(buf_size, t_state.http_config_param->max_payload_iobuf_index));
   IOBufferReader *buf_start = buf->alloc_reader();
 
   // First write the client response header into the buffer
@@ -6443,7 +6445,7 @@ HttpSM::find_http_resp_buffer_size(int64_t content_length)
     }
   } else {
     int64_t buf_size = index_to_buffer_size(HTTP_HEADER_BUFFER_SIZE_INDEX) + content_length;
-    alloc_index      = buffer_size_to_index(buf_size);
+    alloc_index      = buffer_size_to_index(buf_size, t_state.http_config_param->max_payload_iobuf_index);
   }
 
   return alloc_index;
diff --git a/proxy/http/HttpTransact.cc b/proxy/http/HttpTransact.cc
@@ -5560,8 +5560,9 @@ HttpTransact::handle_trace_and_options_requests(State *s, HTTPHdr *incoming_hdr)
       s->free_internal_msg_buffer();
       s->internal_msg_buffer_size = req_length * 2;
 
-      if (s->internal_msg_buffer_size <= max_iobuffer_size) {
-        s->internal_msg_buffer_fast_allocator_size = buffer_size_to_index(s->internal_msg_buffer_size);
+      if (s->internal_msg_buffer_size <= BUFFER_SIZE_FOR_INDEX(s->http_config_param->max_msg_iobuf_index)) {
+        s->internal_msg_buffer_fast_allocator_size =
+          buffer_size_to_index(s->internal_msg_buffer_size, s->http_config_param->max_msg_iobuf_index);
         s->internal_msg_buffer = static_cast<char *>(ioBufAllocator[s->internal_msg_buffer_fast_allocator_size].alloc_void());
       } else {
         s->internal_msg_buffer_fast_allocator_size = -1;
diff --git a/proxy/logging/LogBuffer.cc b/proxy/logging/LogBuffer.cc
@@ -111,8 +111,8 @@ LogBuffer::LogBuffer(LogObject *owner, size_t size, size_t buf_align, size_t wri
   //
   int64_t alloc_size = size + buf_align;
 
-  if (alloc_size <= max_iobuffer_size) {
-    m_buffer_fast_allocator_size = buffer_size_to_index(alloc_size);
+  if (alloc_size <= BUFFER_SIZE_FOR_INDEX(Log::config->logbuffer_max_iobuf_index)) {
+    m_buffer_fast_allocator_size = buffer_size_to_index(alloc_size, Log::config->logbuffer_max_iobuf_index);
     m_unaligned_buffer           = static_cast<char *>(ioBufAllocator[m_buffer_fast_allocator_size].alloc_void());
   } else {
     m_buffer_fast_allocator_size = -1;
diff --git a/proxy/logging/LogConfig.cc b/proxy/logging/LogConfig.cc
@@ -96,8 +96,9 @@ LogConfig::setup_default_values()
   file_stat_frequency  = 16;
   space_used_frequency = 900;
 
-  ascii_buffer_size = 4 * 9216;
-  max_line_size     = 9216; // size of pipe buffer for SunOS 5.6
+  ascii_buffer_size         = 4 * 9216;
+  max_line_size             = 9216; // size of pipe buffer for SunOS 5.6
+  logbuffer_max_iobuf_index = BUFFER_SIZE_INDEX_32K;
 }
 
 void LogConfig::reconfigure_mgmt_variables(ts::MemSpan<void>)
@@ -145,6 +146,11 @@ LogConfig::read_configuration_variables()
     max_space_mb_headroom = val;
   }
 
+  val = static_cast<int>(REC_ConfigReadInteger("proxy.config.log.io.max_buffer_index"));
+  if (val > 0) {
+    logbuffer_max_iobuf_index = val;
+  }
+
   ptr                     = REC_ConfigReadString("proxy.config.log.logfile_perm");
   int logfile_perm_parsed = ink_fileperm_parse(ptr);
   if (logfile_perm_parsed != -1) {
@@ -359,6 +365,7 @@ LogConfig::display(FILE *fd)
   fprintf(fd, "   sampling_frequency = %d\n", sampling_frequency);
   fprintf(fd, "   file_stat_frequency = %d\n", file_stat_frequency);
   fprintf(fd, "   space_used_frequency = %d\n", space_used_frequency);
+  fprintf(fd, "   logbuffer_max_iobuf_index = %d\n", logbuffer_max_iobuf_index);
 
   fprintf(fd, "\n");
   fprintf(fd, "************ Log Objects (%u objects) ************\n", log_object_manager.get_num_objects());
@@ -434,6 +441,7 @@ LogConfig::register_config_callbacks()
     "proxy.config.log.rolling_offset_hr",     "proxy.config.log.rolling_size_mb",     "proxy.config.log.auto_delete_rolled_files",
     "proxy.config.log.rolling_max_count",     "proxy.config.log.rolling_allow_empty", "proxy.config.log.config.filename",
     "proxy.config.log.sampling_frequency",    "proxy.config.log.file_stat_frequency", "proxy.config.log.space_used_frequency",
+    "proxy.config.log.io.max_buffer_size",
   };
 
   for (unsigned i = 0; i < countof(names); ++i) {
diff --git a/proxy/logging/LogConfig.h b/proxy/logging/LogConfig.h
@@ -205,6 +205,7 @@ class LogConfig : public ConfigInfo
 
   int ascii_buffer_size;
   int max_line_size;
+  int logbuffer_max_iobuf_index;
 
   char *hostname;
   char *logfile_dir;

Original file line number	Diff line number	Diff line change
`@@ -76,7 +76,7 @@ QUICPacketHandler::_send_packet(const QUICPacket &packet, UDPConnection *udp_con`
`76`	`76`	`{`
`77`	`77`	`size_t udp_len;`
`78`	`78`	`Ptr<IOBufferBlock> udp_payload(new_IOBufferBlock());`
`79`		`- udp_payload->alloc(iobuffer_size_to_index(pmtu));`
	`79`	`+ udp_payload->alloc(iobuffer_size_to_index(pmtu, BUFFER_SIZE_INDEX_32K));`
`80`	`80`	`packet.store(reinterpret_cast<uint8_t *>(udp_payload->end()), &udp_len);`
`81`	`81`	`udp_payload->fill(udp_len);`
`82`	`82`