Add metrics to track default inactivity timed out connections

doc/admin-guide/monitoring/statistics/core/network-io.en.rst

@@ -70,7 +70,14 @@ Network I/O
    :type: counter
 
 .. ts:stat:: global proxy.process.net.default_inactivity_timeout_applied integer
+   The total number of connections that had no transaction or connection level timer running on them and
+   had to fallback to the catch-all 'default_inactivity_timeout'
+   :type: counter
 .. ts:stat:: global proxy.process.net.default_inactivity_timeout_count integer
+   The total number of connections that were cleaned up due to 'default_inactivity_timeout'
+   :type: counter
+
+.. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_count integer
 .. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_total integer
 .. ts:stat:: global proxy.process.net.inactivity_cop_lock_acquire_failure integer
 .. ts:stat:: global proxy.process.net.net_handler_run integer

iocore/net/I_NetVConnection.h

@@ -539,7 +539,9 @@ class NetVConnection : public VConnection, public PluginUserArgs<TS_USER_ARGS_VC
     is currently active. See section on timeout semantics above.
 
    */
-  virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0;
+  virtual void set_inactivity_timeout(ink_hrtime timeout_in)         = 0;
+  virtual void set_default_inactivity_timeout(ink_hrtime timeout_in) = 0;
+  virtual bool is_default_inactivity_timeout()                       = 0;
 
   /**
     Clears the active timeout. No active timeouts will be sent until

iocore/net/Net.cc

@@ -102,7 +102,8 @@ register_net_stats()
   const std::pair<const char *, Net_Stats> non_persistent[] = {
     {"proxy.process.net.accepts_currently_open", net_accepts_currently_open_stat},
     {"proxy.process.net.connections_currently_open", net_connections_currently_open_stat},
-    {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_stat},
+    {"proxy.process.net.default_inactivity_timeout_applied", default_inactivity_timeout_applied_stat},
+    {"proxy.process.net.default_inactivity_timeout_count", default_inactivity_timeout_count_stat},
     {"proxy.process.net.dynamic_keep_alive_timeout_in_count", keep_alive_queue_timeout_count_stat},
     {"proxy.process.net.dynamic_keep_alive_timeout_in_total", keep_alive_queue_timeout_total_stat},
     {"proxy.process.socks.connections_currently_open", socks_connections_currently_open_stat},
@@ -130,7 +131,8 @@ register_net_stats()
   NET_CLEAR_DYN_STAT(socks_connections_currently_open_stat);
   NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_total_stat);
   NET_CLEAR_DYN_STAT(keep_alive_queue_timeout_count_stat);
-  NET_CLEAR_DYN_STAT(default_inactivity_timeout_stat);
+  NET_CLEAR_DYN_STAT(default_inactivity_timeout_count_stat);
+  NET_CLEAR_DYN_STAT(default_inactivity_timeout_applied_stat);
 
   RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.tcp.total_accepts", RECD_INT, RECP_NON_PERSISTENT,
                      static_cast<int>(net_tcp_accept_stat), RecRawStatSyncSum);

iocore/net/NetEvent.h

@@ -45,7 +45,9 @@ class NetEvent
 
   // Duplicate with `NetVConnection::set_inactivity_timeout`
   // TODO: more abstraction.
-  virtual void set_inactivity_timeout(ink_hrtime timeout_in) = 0;
+  virtual void set_inactivity_timeout(ink_hrtime timeout_in)         = 0;
+  virtual void set_default_inactivity_timeout(ink_hrtime timeout_in) = 0;
+  virtual bool is_default_inactivity_timeout()                       = 0;
 
   // get this vc's thread
   virtual EThread *get_thread() = 0;
@@ -71,6 +73,8 @@ class NetEvent
   ink_hrtime next_activity_timeout_at   = 0;
   ink_hrtime submit_time                = 0;
 
+  bool default_inactivity_timeout = false;
+
   LINK(NetEvent, open_link);
   LINK(NetEvent, cop_link);
   LINKM(NetEvent, read, ready_link)

iocore/net/P_Net.h

@@ -51,7 +51,8 @@ enum Net_Stats {
   inactivity_cop_lock_acquire_failure_stat,
   keep_alive_queue_timeout_total_stat,
   keep_alive_queue_timeout_count_stat,
-  default_inactivity_timeout_stat,
+  default_inactivity_timeout_applied_stat,
+  default_inactivity_timeout_count_stat,
   net_fastopen_attempts_stat,
   net_fastopen_successes_stat,
   net_tcp_accept_stat,

iocore/net/P_UnixNetVConnection.h

@@ -162,6 +162,8 @@ class UnixNetVConnection : public NetVConnection, public NetEvent
   ////////////////////////////////////////////////////////////
   virtual void set_active_timeout(ink_hrtime timeout_in) override;
   virtual void set_inactivity_timeout(ink_hrtime timeout_in) override;
+  virtual void set_default_inactivity_timeout(ink_hrtime timeout_in) override;
+  virtual bool is_default_inactivity_timeout() override;
   virtual void cancel_active_timeout() override;
   virtual void cancel_inactivity_timeout() override;
   void set_action(Continuation *c) override;

iocore/net/UnixNet.cc

@@ -69,7 +69,19 @@ class InactivityCop : public Continuation
         continue;
       }
 
+      // set a default inactivity timeout if one is not set
+      if (ne->next_inactivity_timeout_at == 0 && nh.config.default_inactivity_timeout > 0) {
+        Debug("inactivity_cop", "vc: %p inactivity timeout not set, setting a default of %d", ne,
+              nh.config.default_inactivity_timeout);
+        ne->set_default_inactivity_timeout(HRTIME_SECONDS(nh.config.default_inactivity_timeout));
+        NET_INCREMENT_DYN_STAT(default_inactivity_timeout_applied_stat);
+      }
+
       if (ne->next_inactivity_timeout_at && ne->next_inactivity_timeout_at < now) {
+        if (ne->is_default_inactivity_timeout()) {
+          // track the connections that timed out due to default inactivity
+          NET_INCREMENT_DYN_STAT(default_inactivity_timeout_count_stat);
+        }
         if (nh.keep_alive_queue.in(ne)) {
           // only stat if the connection is in keep-alive, there can be other inactivity timeouts
           ink_hrtime diff = (now - (ne->next_inactivity_timeout_at - ne->inactivity_timeout_in)) / HRTIME_SECOND;

iocore/net/UnixNetVConnection.cc

@@ -1352,14 +1352,25 @@ TS_INLINE void
 UnixNetVConnection::set_inactivity_timeout(ink_hrtime timeout_in)
 {
   Debug("socket", "Set inactive timeout=%" PRId64 ", for NetVC=%p", timeout_in, this);
-  if (timeout_in == 0) {
-    // set default inactivity timeout
-    timeout_in = HRTIME_SECONDS(nh->config.default_inactivity_timeout);
-  }
   inactivity_timeout_in      = timeout_in;
   next_inactivity_timeout_at = Thread::get_hrtime() + inactivity_timeout_in;
 }
 
+TS_INLINE void
+UnixNetVConnection::set_default_inactivity_timeout(ink_hrtime timeout_in)
+{
+  Debug("socket", "Set default inactive timeout=%" PRId64 ", for NetVC=%p", timeout_in, this);
+  inactivity_timeout_in      = 0;
+  default_inactivity_timeout = true;
+  next_inactivity_timeout_at = Thread::get_hrtime() + timeout_in;
+}
+
+TS_INLINE bool
+UnixNetVConnection::is_default_inactivity_timeout()
+{
+  return (default_inactivity_timeout && inactivity_timeout_in == 0);
+}
+
 /*
  * Close down the current netVC.  Save aside the socket and SSL information
  * and create new netVC in the current thread/netVC

proxy/PluginVC.cc

@@ -886,6 +886,18 @@ PluginVC::set_inactivity_timeout(ink_hrtime timeout_in)
   }
 }
 
+void
+PluginVC::set_default_inactivity_timeout(ink_hrtime timeout_in)
+{
+  set_inactivity_timeout(timeout_in);
+}
+
+bool
+PluginVC::is_default_inactivity_timeout()
+{
+  return false;
+}
+
 void
 PluginVC::cancel_active_timeout()
 {

proxy/PluginVC.h

@@ -89,6 +89,8 @@ class PluginVC : public NetVConnection, public PluginIdentity
   // Timeouts
   void set_active_timeout(ink_hrtime timeout_in) override;
   void set_inactivity_timeout(ink_hrtime timeout_in) override;
+  void set_default_inactivity_timeout(ink_hrtime timeout_in) override;
+  bool is_default_inactivity_timeout() override;
   void cancel_active_timeout() override;
   void cancel_inactivity_timeout() override;
   void add_to_keep_alive_queue() override;