Enforce Active Connection limits

1. Throttle connections when there's no room in active conn queue
2. Adjust manage_active_queue() to not fail when the conn is already in active queue
3. Metrics for throttling
4. metrics for timing out connections due to default inactivity timeout

Author: sudheerv

doc/admin-guide/monitoring/statistics/core/network-io.en.rst

@@ -60,7 +60,17 @@ Network I/O
 .. ts:stat:: global proxy.process.net.connections_currently_open integer
    :type: counter
 
+.. ts:stat:: global proxy.process.net.connections_throttled_in integer
+   :type: counter
+
+.. ts:stat:: global proxy.process.net.connections_throttled_out integer
+   :type: counter
+
+.. ts:stat:: global proxy.process.net.max.active.connections_throttled_in integer
+   :type: counter
+
 .. ts:stat:: global proxy.process.net.default_inactivity_timeout_applied integer
+.. ts:stat:: global proxy.process.net.default_inactivity_timeout_count integer
 .. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_count integer
 .. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_total integer
 .. ts:stat:: global proxy.process.net.inactivity_cop_lock_acquire_failure integer

iocore/net/Net.cc

@@ -102,7 +102,8 @@ register_net_stats()
   const std::pair<const char *, Net_Stats> non_persistent[] = {
     {"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat},
     {"proxy.process.net.connections_currently_open", net_connections_currently_open_stat},
-    {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_stat},
+    {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat},
+    {"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat},
     {"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat},
     {"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat},
     {"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat},
@@ -130,7 +131,8 @@ register_net_stats()
   NET_CLEAR_DYN_STAT(socks_connections_currently_open_stat);
   NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_total_stat);
   NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_count_stat);
-  NET_CLEAR_DYN_STAT(default_inactivity_timeout_stat);
+  NET_CLEAR_DYN_STAT(default_inactivity_timeout_count_stat);
+  NET_CLEAR_DYN_STAT(default_inactivity_timeout_applied_stat);
 
   RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.tcp.total_accepts", RECD_INT, RECP_NON_PERSISTENT,
                      static_cast<int>(net_tcp_accept_stat), RecRawStatSyncSum);
@@ -140,6 +142,8 @@ register_net_stats()
                      (int)net_connections_throttled_in_stat, RecRawStatSyncSum);
   RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_out", RECD_INT, RECP_PERSISTENT,
                      (int)net_connections_throttled_out_stat, RecRawStatSyncSum);
+  RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.max.active.connections_throttled_in", RECD_INT, RECP_PERSISTENT,
+                     (int)net_connections_max_active_throttled_in_stat, RecRawStatSyncSum);
 }
 
 void

iocore/net/P_Net.h

@@ -51,12 +51,14 @@ enum Net_Stats {
   inactivity_cop_lock_acquire_failure_stat,
   keep_alive_queue_timeout_total_stat,
   keep_alive_queue_timeout_count_stat,
-  default_inactivity_timeout_stat,
+  default_inactivity_timeout_applied_stat,
+  default_inactivity_timeout_count_stat,
   net_fastopen_attempts_stat,
   net_fastopen_successes_stat,
   net_tcp_accept_stat,
   net_connections_throttled_in_stat,
   net_connections_throttled_out_stat,
+  net_connections_max_active_throttled_in_stat,
   Net_Stat_Count
 };
 

iocore/net/UnixNet.cc

@@ -69,7 +69,25 @@ class InactivityCop : public Continuation
         continue;
       }
 
+      // set a default inactivity timeout if one is not set
+      bool use_default_inactivity_timeout = false;
+      if (ne->next_inactivity_timeout_at == 0 && nh.config.default_inactivity_timeout > 0) {
+        Debug("inactivity_cop", "vc: %p inactivity timeout not set, setting a default of %d", ne,
+              nh.config.default_inactivity_timeout);
+        use_default_inactivity_timeout = true;
+        ne->set_inactivity_timeout(HRTIME_SECONDS(nh.config.default_inactivity_timeout));
+        NET_INCREMENT_DYN_STAT(default_inactivity_timeout_applied_stat);
+      } else {
+        Debug("inactivity_cop_verbose", "vc: %p now: %" PRId64 " timeout at: %" PRId64 " timeout in: %" PRId64, ne,
+              ink_hrtime_to_sec(now), ink_hrtime_to_sec(ne->next_inactivity_timeout_at),
+              ink_hrtime_to_sec(ne->inactivity_timeout_in));
+      }
+
       if (ne->next_inactivity_timeout_at && ne->next_inactivity_timeout_at < now) {
+        if (use_default_inactivity_timeout) {
+          // track the connections that timed out due to default inactivity
+          NET_INCREMENT_DYN_STAT(default_inactivity_timeout_count_stat);
+        }
         if (nh.keep_alive_queue.in(ne)) {
           // only stat if the connection is in keep-alive, there can be other inactivity timeouts
           ink_hrtime diff = (now - (ne->next_inactivity_timeout_at - ne->inactivity_timeout_in)) / HRTIME_SECOND;
@@ -721,16 +739,22 @@ NetHandler::add_to_active_queue(NetEvent *ne)
         max_connections_per_thread_in, active_queue_size, keep_alive_queue_size);
   ink_assert(mutex->thread_holding == this_ethread());
 
+  bool active_queue_full = false;
+
   // if active queue is over size then close inactive connections
   if (manage_active_queue() == false) {
-    // there is no room left in the queue
-    return false;
+    active_queue_full = true;
   }
 
   if (active_queue.in(ne)) {
     // already in the active queue, move the head
     active_queue.remove(ne);
   } else {
+    if (active_queue_full) {
+      // there is no room left in the queue
+      NET_SUM_DYN_STAT(net_connections_max_active_throttled_in_stat, 1);
+      return false;
+    }
     // in the keep-alive queue or no queue, new to this queue
     remove_from_keep_alive_queue(ne);
     ++active_queue_size;

iocore/net/UnixNetVConnection.cc

@@ -1352,10 +1352,6 @@ TS_INLINE void
 UnixNetVConnection::set_inactivity_timeout(ink_hrtime timeout_in)
 {
   Debug("socket", "Set inactive timeout=%" PRId64 ", for NetVC=%p", timeout_in, this);
-  if (timeout_in == 0) {
-    // set default inactivity timeout
-    timeout_in = HRTIME_SECONDS(nh->config.default_inactivity_timeout);
-  }
   inactivity_timeout_in      = timeout_in;
   next_inactivity_timeout_at = Thread::get_hrtime() + inactivity_timeout_in;
 }

proxy/http/Http1ClientSession.cc

@@ -465,6 +465,12 @@ Http1ClientSession::new_transaction()
     return;
   }
 
+  if (!client_vc->add_to_active_queue()) {
+    // no room in the active queue close the connection
+    this->do_io_close();
+    return;
+  }
+
   // Defensive programming, make sure nothing persists across
   // connection re-use
   half_close = false;
@@ -474,7 +480,6 @@ Http1ClientSession::new_transaction()
   trans.set_proxy_ssn(this);
   transact_count++;
 
-  client_vc->add_to_active_queue();
   trans.new_transaction(read_from_early_data > 0 ? true : false);
 }
 

proxy/http2/Http2ConnectionState.cc

@@ -1153,6 +1153,13 @@ Http2ConnectionState::state_closed(int event, void *edata)
 Http2Stream *
 Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error)
 {
+  // first check if we've hit the active connection limit
+  if (!ua_session->get_netvc()->add_to_active_queue()) {
+    error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_INTERNAL_ERROR,
+                       "refused to create new stream, maxed out active connections");
+    return nullptr;
+  }
+
   // In half_close state, TS doesn't create new stream. Because GOAWAY frame is sent to client
   if (ua_session->get_half_close_local_flag()) {
     error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM,
@@ -1226,7 +1233,6 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error)
   new_stream->mutex                     = new_ProxyMutex();
   new_stream->is_first_transaction_flag = get_stream_requests() == 0;
   increment_stream_requests();
-  ua_session->get_netvc()->add_to_active_queue();
 
   return new_stream;
 }