diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 53bd09c8278..899f7a5ad55 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -409,6 +409,22 @@ Network handled. This should be tuned according to your memory size, and expected work load. If this is set to 0, the throttling logic is disabled. +.. ts:cv:: CONFIG proxy.config.net.max_connections_in INT 30000 + + The total number of client connections that the :program:`traffic_server` + can handle simultaneously. This should be tuned according to your memory size, + and expected work load (network, cpu etc). This limit includes both keepalive + and active client connections that :program:`traffic_server` can handle at + any given instant. + +.. ts:cv:: CONFIG proxy.config.net.max_active_connections_in INT 10000 + + The total number of active client connections that the |TS| can handle + simultaneously. This should be tuned according to your memory size, + and expected work load (network, cpu etc). If this is set to 0, active + connection tracking is disabled and active connections have no separate + limit and the total connections follow `proxy.config.net.connections_throttle` + .. ts:cv:: CONFIG proxy.config.net.default_inactivity_timeout INT 86400 :reloadable: diff --git a/doc/admin-guide/monitoring/statistics/core/network-io.en.rst b/doc/admin-guide/monitoring/statistics/core/network-io.en.rst index 5393086495a..8b832531d20 100644 --- a/doc/admin-guide/monitoring/statistics/core/network-io.en.rst +++ b/doc/admin-guide/monitoring/statistics/core/network-io.en.rst @@ -60,8 +60,17 @@ Network I/O .. ts:stat:: global proxy.process.net.connections_currently_open integer :type: counter +.. ts:stat:: global proxy.process.net.connections_throttled_in integer + :type: counter + +.. ts:stat:: global proxy.process.net.connections_throttled_out integer + :type: counter + +.. ts:stat:: global proxy.process.net.max.active.connections_throttled_in integer + :type: counter + .. ts:stat:: global proxy.process.net.default_inactivity_timeout_applied integer -.. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_count integer +.. ts:stat:: global proxy.process.net.default_inactivity_timeout_count integer .. ts:stat:: global proxy.process.net.dynamic_keep_alive_timeout_in_total integer .. ts:stat:: global proxy.process.net.inactivity_cop_lock_acquire_failure integer .. ts:stat:: global proxy.process.net.net_handler_run integer diff --git a/iocore/net/Net.cc b/iocore/net/Net.cc index f5bbb886af7..1813c0dc626 100644 --- a/iocore/net/Net.cc +++ b/iocore/net/Net.cc @@ -140,6 +140,8 @@ register_net_stats() (int)net_connections_throttled_in_stat, RecRawStatSyncSum); RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.connections_throttled_out", RECD_INT, RECP_PERSISTENT, (int)net_connections_throttled_out_stat, RecRawStatSyncSum); + RecRegisterRawStat(net_rsb, RECT_PROCESS, "proxy.process.net.max.active.connections_throttled_in", RECD_INT, RECP_PERSISTENT, + (int)net_connections_max_active_throttled_in_stat, RecRawStatSyncSum); } void diff --git a/iocore/net/P_Net.h b/iocore/net/P_Net.h index 15a55bb8bb6..9a749f9cce4 100644 --- a/iocore/net/P_Net.h +++ b/iocore/net/P_Net.h @@ -57,6 +57,7 @@ enum Net_Stats { net_tcp_accept_stat, net_connections_throttled_in_stat, net_connections_throttled_out_stat, + net_connections_max_active_throttled_in_stat, Net_Stat_Count }; diff --git a/iocore/net/UnixNet.cc b/iocore/net/UnixNet.cc index 788c2ad6f07..3f8f9960ef4 100644 --- a/iocore/net/UnixNet.cc +++ b/iocore/net/UnixNet.cc @@ -562,6 +562,11 @@ NetHandler::manage_active_queue(bool ignore_queue_size = false) max_connections_per_thread_in, max_connections_active_per_thread_in, total_connections_in, active_queue_size, keep_alive_queue_size); + if (!max_connections_active_per_thread_in) { + // active queue has no max + return true; + } + if (ignore_queue_size == false && max_connections_active_per_thread_in > active_queue_size) { return true; } @@ -721,16 +726,22 @@ NetHandler::add_to_active_queue(NetEvent *ne) max_connections_per_thread_in, active_queue_size, keep_alive_queue_size); ink_assert(mutex->thread_holding == this_ethread()); + bool active_queue_full = false; + // if active queue is over size then close inactive connections if (manage_active_queue() == false) { - // there is no room left in the queue - return false; + active_queue_full = true; } if (active_queue.in(ne)) { // already in the active queue, move the head active_queue.remove(ne); } else { + if (active_queue_full) { + // there is no room left in the queue + NET_SUM_DYN_STAT(net_connections_max_active_throttled_in_stat, 1); + return false; + } // in the keep-alive queue or no queue, new to this queue remove_from_keep_alive_queue(ne); ++active_queue_size; diff --git a/proxy/PluginVC.cc b/proxy/PluginVC.cc index f6cc4270592..126b4f0655e 100644 --- a/proxy/PluginVC.cc +++ b/proxy/PluginVC.cc @@ -926,7 +926,7 @@ bool PluginVC::add_to_active_queue() { // do nothing - return false; + return true; } SOCKET diff --git a/proxy/http/Http1ClientSession.cc b/proxy/http/Http1ClientSession.cc index 8f825dc32b1..0c402bf589d 100644 --- a/proxy/http/Http1ClientSession.cc +++ b/proxy/http/Http1ClientSession.cc @@ -465,6 +465,12 @@ Http1ClientSession::new_transaction() return; } + if (!client_vc->add_to_active_queue()) { + // no room in the active queue close the connection + this->do_io_close(); + return; + } + // Defensive programming, make sure nothing persists across // connection re-use half_close = false; @@ -474,7 +480,6 @@ Http1ClientSession::new_transaction() trans.set_proxy_ssn(this); transact_count++; - client_vc->add_to_active_queue(); trans.new_transaction(read_from_early_data > 0 ? true : false); } diff --git a/proxy/http2/Http2ConnectionState.cc b/proxy/http2/Http2ConnectionState.cc index 3711e5cb82f..8966c731941 100644 --- a/proxy/http2/Http2ConnectionState.cc +++ b/proxy/http2/Http2ConnectionState.cc @@ -1153,6 +1153,13 @@ Http2ConnectionState::state_closed(int event, void *edata) Http2Stream * Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) { + // first check if we've hit the active connection limit + if (!ua_session->get_netvc()->add_to_active_queue()) { + error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_CONNECTION, Http2ErrorCode::HTTP2_ERROR_NO_ERROR, + "refused to create new stream, maxed out active connections"); + return nullptr; + } + // In half_close state, TS doesn't create new stream. Because GOAWAY frame is sent to client if (ua_session->get_half_close_local_flag()) { error = Http2Error(Http2ErrorClass::HTTP2_ERROR_CLASS_STREAM, Http2ErrorCode::HTTP2_ERROR_REFUSED_STREAM, @@ -1226,7 +1233,6 @@ Http2ConnectionState::create_stream(Http2StreamId new_id, Http2Error &error) new_stream->mutex = new_ProxyMutex(); new_stream->is_first_transaction_flag = get_stream_requests() == 0; increment_stream_requests(); - ua_session->get_netvc()->add_to_active_queue(); return new_stream; }