Metrics for origin close (#6873)

(cherry picked from commit 613576f)

Author: shinrich

doc/admin-guide/monitoring/statistics/core/origin.en.rst

@@ -78,4 +78,63 @@ Origin Server
    :type: derivative
    :units: bytes
 
+.. ts:stat:: global proxy.process.http.origin_shutdown.pool_lock_contention integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.migration_failure integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_eos integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_transform_read integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_sharing integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_keep_alive integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_repsonse integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_request integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_modified integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.release_misc integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.cleanup_entry integer
+   :type counter
+   :units bytes
+
+.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_abort integer
+   :type counter
+   :units bytes
 

proxy/http/Http1ServerSession.cc

@@ -196,6 +196,7 @@ Http1ServerSession::release()
     //  due to lock contention
     // FIX:  should retry instead of closing
     this->do_io_close();
+    HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_pool_lock_contention);
   } else {
     // The session was successfully put into the session
     //    manager and it will manage it

proxy/http/HttpConfig.cc

@@ -350,6 +350,44 @@ register_stat_callbacks()
   RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.total_parent_marked_down_count", RECD_COUNTER, RECP_PERSISTENT,
                      (int)http_total_parent_marked_down_count, RecRawStatSyncCount);
 
+  // Stats to track causes of ATS initiated origin shutdowns
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.pool_lock_contention", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_pool_lock_contention, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.migration_failure", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_migration_failure, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_tunnel_server, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_no_keep_alive, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_eos", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_tunnel_server_eos, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_plugin_tunnel, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_detach", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_detach, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_client", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_tunnel_client, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_transform_read", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_transform_read, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_sharing", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_release_no_sharing, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_server", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_release_no_server, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_keep_alive", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_no_keep_alive, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_response", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_response, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_request", RECD_INT,
+                     RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_request, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_modified", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_release_modified, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_misc", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_release_misc, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.cleanup_entry", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_cleanup_entry, RecRawStatSyncCount);
+  RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_abort", RECD_INT, RECP_NON_PERSISTENT,
+                     (int)http_origin_shutdown_tunnel_abort, RecRawStatSyncCount);
+
   // Upstream current connections stats
   RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.current_parent_proxy_connections", RECD_INT, RECP_NON_PERSISTENT,
                      (int)http_current_parent_proxy_connections_stat, RecRawStatSyncSum);

proxy/http/HttpConfig.h

@@ -331,6 +331,25 @@ enum {
   http_origin_connect_adjust_thread_stat,
   http_cache_open_write_adjust_thread_stat,
 
+  http_origin_shutdown_pool_lock_contention,
+  http_origin_shutdown_migration_failure,
+  http_origin_shutdown_tunnel_server,
+  http_origin_shutdown_tunnel_server_no_keep_alive,
+  http_origin_shutdown_tunnel_server_eos,
+  http_origin_shutdown_tunnel_server_plugin_tunnel,
+  http_origin_shutdown_tunnel_server_detach,
+  http_origin_shutdown_tunnel_client,
+  http_origin_shutdown_tunnel_transform_read,
+  http_origin_shutdown_release_no_sharing,
+  http_origin_shutdown_release_no_server,
+  http_origin_shutdown_release_no_keep_alive,
+  http_origin_shutdown_release_invalid_response,
+  http_origin_shutdown_release_invalid_request,
+  http_origin_shutdown_release_modified,
+  http_origin_shutdown_release_misc,
+  http_origin_shutdown_cleanup_entry,
+  http_origin_shutdown_tunnel_abort,
+
   http_stat_count
 };
 

proxy/http/HttpSM.cc

@@ -229,6 +229,9 @@ HttpVCTable::cleanup_entry(HttpVCTableEntry *e)
       break;
     }
 
+    if (e->vc_type == HTTP_SERVER_VC) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_cleanup_entry);
+    }
     e->vc->do_io_close();
     e->vc = nullptr;
   }
@@ -2940,6 +2943,13 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p)
       plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL && t_state.txn_conf->keep_alive_enabled_out == 1) {
     close_connection = false;
   } else {
+    if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_no_keep_alive);
+    } else if (server_entry->eos == true) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_eos);
+    } else {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_plugin_tunnel);
+    }
     close_connection = true;
   }
 
@@ -2968,6 +2978,7 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p)
       break;
     }
 
+    HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server);
     close_connection = true;
 
     ink_assert(p->vc_type == HT_HTTP_SERVER);
@@ -3036,7 +3047,8 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p)
     p->read_success               = true;
     t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE;
     t_state.current.server->abort = HttpTransact::DIDNOT_ABORT;
-    close_connection              = true;
+    HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_detach);
+    close_connection = true;
     break;
 
   case VC_EVENT_READ_READY:
@@ -3899,6 +3911,7 @@ HttpSM::tunnel_handler_transform_read(int event, HttpTunnelProducer *p)
   //  transform hasn't detached yet.  If it is still alive,
   //  don't close the transform vc
   if (p->self_consumer->alive == false) {
+    HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_transform_read);
     p->vc->do_io_close();
   }
   p->handler_state = HTTP_SM_TRANSFORM_CLOSED;
@@ -5398,6 +5411,23 @@ HttpSM::release_server_session(bool serve_from_cache)
       ua_txn->attach_server_session(server_session, false);
     }
   } else {
+    if (TS_SERVER_SESSION_SHARING_MATCH_NONE == t_state.txn_conf->server_session_sharing_match) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_sharing);
+    } else if (t_state.current.server == nullptr) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_server);
+    } else if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_keep_alive);
+    } else if (!t_state.hdr_info.server_response.valid()) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_response);
+    } else if (!t_state.hdr_info.server_request.valid()) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_request);
+    } else if (t_state.hdr_info.server_response.status_get() != HTTP_STATUS_NOT_MODIFIED &&
+               (t_state.hdr_info.server_request.method_get_wksidx() != HTTP_WKSIDX_HEAD ||
+                t_state.www_auth_content == HttpTransact::CACHE_AUTH_NONE)) {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_modified);
+    } else {
+      HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_misc);
+    }
     server_session->do_io_close();
   }
 

proxy/http/HttpSessionManager.cc

@@ -401,6 +401,7 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad
               ink_assert(new_vc == nullptr || new_vc->nh != nullptr);
               if (!new_vc) {
                 // Close out to_return, we were't able to get a connection
+                HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_migration_failure);
                 to_return->do_io_close();
                 to_return = nullptr;
                 retval    = HSM_NOT_FOUND;

proxy/http/HttpTunnel.cc

@@ -1419,6 +1419,7 @@ HttpTunnel::chain_abort_all(HttpTunnelProducer *p)
     }
     p->read_vio = nullptr;
     p->vc->do_io_close(EHTTP_ERROR);
+    HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_abort);
     update_stats_after_abort(p->vc_type);
   }
 }