diff --git a/doc/admin-guide/monitoring/statistics/core/origin.en.rst b/doc/admin-guide/monitoring/statistics/core/origin.en.rst index 2b325c4ab6d..528d4e10658 100644 --- a/doc/admin-guide/monitoring/statistics/core/origin.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/origin.en.rst @@ -78,4 +78,63 @@ Origin Server :type: derivative :units: bytes +.. ts:stat:: global proxy.process.http.origin_shutdown.pool_lock_contention integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.migration_failure integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_eos integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_transform_read integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_sharing integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_no_keep_alive integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_repsonse integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_invalid_request integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_modified integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.release_misc integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.cleanup_entry integer + :type counter + :units bytes + +.. ts:stat:: global proxy.process.http.origin_shutdown.tunnel_abort integer + :type counter + :units bytes diff --git a/proxy/http/Http1ServerSession.cc b/proxy/http/Http1ServerSession.cc index 88c8f55ba5a..2adaf28ff41 100644 --- a/proxy/http/Http1ServerSession.cc +++ b/proxy/http/Http1ServerSession.cc @@ -196,6 +196,7 @@ Http1ServerSession::release() // due to lock contention // FIX: should retry instead of closing this->do_io_close(); + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_pool_lock_contention); } else { // The session was successfully put into the session // manager and it will manage it diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index 5877bc516b7..d3cc7b6bd7d 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -350,6 +350,44 @@ register_stat_callbacks() RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.total_parent_marked_down_count", RECD_COUNTER, RECP_PERSISTENT, (int)http_total_parent_marked_down_count, RecRawStatSyncCount); + // Stats to track causes of ATS initiated origin shutdowns + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.pool_lock_contention", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_pool_lock_contention, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.migration_failure", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_migration_failure, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_server, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_no_keep_alive", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_no_keep_alive, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_eos", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_server_eos, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_plugin_tunnel", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_plugin_tunnel, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_server_detach", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_server_detach, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_client", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_client, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_transform_read", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_tunnel_transform_read, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_sharing", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_no_sharing, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_server", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_no_server, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_no_keep_alive", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_no_keep_alive, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_response", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_response, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_invalid_request", RECD_INT, + RECP_NON_PERSISTENT, (int)http_origin_shutdown_release_invalid_request, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_modified", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_modified, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.release_misc", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_release_misc, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.cleanup_entry", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_cleanup_entry, RecRawStatSyncCount); + RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.origin_shutdown.tunnel_abort", RECD_INT, RECP_NON_PERSISTENT, + (int)http_origin_shutdown_tunnel_abort, RecRawStatSyncCount); + // Upstream current connections stats RecRegisterRawStat(http_rsb, RECT_PROCESS, "proxy.process.http.current_parent_proxy_connections", RECD_INT, RECP_NON_PERSISTENT, (int)http_current_parent_proxy_connections_stat, RecRawStatSyncSum); diff --git a/proxy/http/HttpConfig.h b/proxy/http/HttpConfig.h index c7ba480c62d..659e8e95034 100644 --- a/proxy/http/HttpConfig.h +++ b/proxy/http/HttpConfig.h @@ -331,6 +331,25 @@ enum { http_origin_connect_adjust_thread_stat, http_cache_open_write_adjust_thread_stat, + http_origin_shutdown_pool_lock_contention, + http_origin_shutdown_migration_failure, + http_origin_shutdown_tunnel_server, + http_origin_shutdown_tunnel_server_no_keep_alive, + http_origin_shutdown_tunnel_server_eos, + http_origin_shutdown_tunnel_server_plugin_tunnel, + http_origin_shutdown_tunnel_server_detach, + http_origin_shutdown_tunnel_client, + http_origin_shutdown_tunnel_transform_read, + http_origin_shutdown_release_no_sharing, + http_origin_shutdown_release_no_server, + http_origin_shutdown_release_no_keep_alive, + http_origin_shutdown_release_invalid_response, + http_origin_shutdown_release_invalid_request, + http_origin_shutdown_release_modified, + http_origin_shutdown_release_misc, + http_origin_shutdown_cleanup_entry, + http_origin_shutdown_tunnel_abort, + http_stat_count }; diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 96c6b325047..8019630a6b6 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -229,6 +229,9 @@ HttpVCTable::cleanup_entry(HttpVCTableEntry *e) break; } + if (e->vc_type == HTTP_SERVER_VC) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_cleanup_entry); + } e->vc->do_io_close(); e->vc = nullptr; } @@ -2983,6 +2986,13 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) plugin_tunnel_type == HTTP_NO_PLUGIN_TUNNEL && t_state.txn_conf->keep_alive_enabled_out == 1) { close_connection = false; } else { + if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_no_keep_alive); + } else if (server_entry->eos == true) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_eos); + } else { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_plugin_tunnel); + } close_connection = true; } @@ -3011,6 +3021,7 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) break; } + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server); close_connection = true; ink_assert(p->vc_type == HT_HTTP_SERVER); @@ -3079,7 +3090,8 @@ HttpSM::tunnel_handler_server(int event, HttpTunnelProducer *p) p->read_success = true; t_state.current.server->state = HttpTransact::TRANSACTION_COMPLETE; t_state.current.server->abort = HttpTransact::DIDNOT_ABORT; - close_connection = true; + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_server_detach); + close_connection = true; break; case VC_EVENT_READ_READY: @@ -3939,6 +3951,7 @@ HttpSM::tunnel_handler_transform_read(int event, HttpTunnelProducer *p) // transform hasn't detached yet. If it is still alive, // don't close the transform vc if (p->self_consumer->alive == false) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_transform_read); p->vc->do_io_close(); } p->handler_state = HTTP_SM_TRANSFORM_CLOSED; @@ -5438,6 +5451,23 @@ HttpSM::release_server_session(bool serve_from_cache) ua_txn->attach_server_session(server_session, false); } } else { + if (TS_SERVER_SESSION_SHARING_MATCH_NONE == t_state.txn_conf->server_session_sharing_match) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_sharing); + } else if (t_state.current.server == nullptr) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_server); + } else if (t_state.current.server->keep_alive != HTTP_KEEPALIVE) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_no_keep_alive); + } else if (!t_state.hdr_info.server_response.valid()) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_response); + } else if (!t_state.hdr_info.server_request.valid()) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_invalid_request); + } else if (t_state.hdr_info.server_response.status_get() != HTTP_STATUS_NOT_MODIFIED && + (t_state.hdr_info.server_request.method_get_wksidx() != HTTP_WKSIDX_HEAD || + t_state.www_auth_content == HttpTransact::CACHE_AUTH_NONE)) { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_modified); + } else { + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_release_misc); + } server_session->do_io_close(); } diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index a2a12644dd2..ac2fed477d1 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -401,6 +401,7 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad ink_assert(new_vc == nullptr || new_vc->nh != nullptr); if (!new_vc) { // Close out to_return, we were't able to get a connection + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_migration_failure); to_return->do_io_close(); to_return = nullptr; retval = HSM_NOT_FOUND; diff --git a/proxy/http/HttpTunnel.cc b/proxy/http/HttpTunnel.cc index 1bfc10b2836..0588acd311f 100644 --- a/proxy/http/HttpTunnel.cc +++ b/proxy/http/HttpTunnel.cc @@ -1419,6 +1419,7 @@ HttpTunnel::chain_abort_all(HttpTunnelProducer *p) } p->read_vio = nullptr; p->vc->do_io_close(EHTTP_ERROR); + HTTP_INCREMENT_DYN_STAT(http_origin_shutdown_tunnel_abort); update_stats_after_abort(p->vc_type); } }