diff --git a/doc/admin-guide/files/records.config.en.rst b/doc/admin-guide/files/records.config.en.rst index 395a5ad218a..8a1aa813b5e 100644 --- a/doc/admin-guide/files/records.config.en.rst +++ b/doc/admin-guide/files/records.config.en.rst @@ -994,8 +994,20 @@ mptcp ========== ================================================================= ``global`` Re-use sessions from a global pool of all server sessions. ``thread`` Re-use sessions from a per-thread pool. + ``hybrid`` Try to work as a global pool, but release server sessions to the + per-thread pool if there is lock contention on the global pool. ========== ================================================================= + + Setting :ts:cv:`proxy.config.http.server_session_sharing.pool` to global can reduce + the number of connections to origin for some traffic loads. However, if many + execute threads are active, the thread contention on the global pool can reduce the + lifetime of connections to origin and reduce effective origin connection reuse. + + For a hybrid pool, the operation starts as the global pool, but sessons are returned + to the local thread pool if the global pool lock is not acquired rather than just + closing the origin connection as is the case in standard global mode. + .. ts:cv:: CONFIG proxy.config.http.attach_server_session_to_client INT 0 :overridable: diff --git a/proxy/http/Http1ServerSession.cc b/proxy/http/Http1ServerSession.cc index 813a729d6aa..e794668d235 100644 --- a/proxy/http/Http1ServerSession.cc +++ b/proxy/http/Http1ServerSession.cc @@ -52,7 +52,7 @@ Http1ServerSession::destroy() } mutex.clear(); - if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sharing_pool) { + if (httpSessionManager.get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_THREAD) { THREAD_FREE(this, httpServerSessionAllocator, this_thread()); } else { httpServerSessionAllocator.free(this); diff --git a/proxy/http/HttpConfig.cc b/proxy/http/HttpConfig.cc index b32e39c9831..af647d308f1 100644 --- a/proxy/http/HttpConfig.cc +++ b/proxy/http/HttpConfig.cc @@ -33,6 +33,7 @@ #include "P_Net.h" #include "records/P_RecUtils.h" #include +#include "HttpSessionManager.h" #define HttpEstablishStaticConfigStringAlloc(_ix, _n) \ REC_EstablishStaticConfigStringAlloc(_ix, _n); \ @@ -164,7 +165,8 @@ http_config_enum_mask_read(const char *name, MgmtByte &value) static const ConfigEnumPair SessionSharingPoolStrings[] = { {TS_SERVER_SESSION_SHARING_POOL_GLOBAL, "global"}, - {TS_SERVER_SESSION_SHARING_POOL_THREAD, "thread"}}; + {TS_SERVER_SESSION_SHARING_POOL_THREAD, "thread"}, + {TS_SERVER_SESSION_SHARING_POOL_HYBRID, "hybrid"}}; int HttpConfig::m_id = 0; HttpConfigParams HttpConfig::m_master; @@ -1153,6 +1155,7 @@ HttpConfig::startup() http_config_enum_mask_read("proxy.config.http.server_session_sharing.match", c.oride.server_session_sharing_match); HttpEstablishStaticConfigStringAlloc(c.oride.server_session_sharing_match_str, "proxy.config.http.server_session_sharing.match"); http_config_enum_read("proxy.config.http.server_session_sharing.pool", SessionSharingPoolStrings, c.server_session_sharing_pool); + httpSessionManager.set_pool_type(c.server_session_sharing_pool); RecRegisterConfigUpdateCb("proxy.config.http.insert_forwarded", &http_insert_forwarded_cb, &c); { diff --git a/proxy/http/HttpProxyAPIEnums.h b/proxy/http/HttpProxyAPIEnums.h index f0d84557210..d27b1abd1fe 100644 --- a/proxy/http/HttpProxyAPIEnums.h +++ b/proxy/http/HttpProxyAPIEnums.h @@ -54,6 +54,7 @@ typedef enum { typedef enum { TS_SERVER_SESSION_SHARING_POOL_GLOBAL, TS_SERVER_SESSION_SHARING_POOL_THREAD, + TS_SERVER_SESSION_SHARING_POOL_HYBRID } TSServerSessionSharingPoolType; // This is use to signal apidefs.h to not define these again. diff --git a/proxy/http/HttpSM.cc b/proxy/http/HttpSM.cc index 546715eed03..b3c0363cd3e 100644 --- a/proxy/http/HttpSM.cc +++ b/proxy/http/HttpSM.cc @@ -1781,10 +1781,9 @@ HttpSM::state_http_server_open(int event, void *data) switch (event) { case NET_EVENT_OPEN: { - Http1ServerSession *session = - (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ? - THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : - httpServerSessionAllocator.alloc(); + Http1ServerSession *session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == httpSessionManager.get_pool_type()) ? + THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) : + httpServerSessionAllocator.alloc(); session->sharing_pool = static_cast(t_state.http_config_param->server_session_sharing_pool); session->sharing_match = static_cast(t_state.txn_conf->server_session_sharing_match); diff --git a/proxy/http/HttpSessionManager.cc b/proxy/http/HttpSessionManager.cc index bf5ea8d1fb0..f0564cd1b5e 100644 --- a/proxy/http/HttpSessionManager.cc +++ b/proxy/http/HttpSessionManager.cc @@ -330,6 +330,7 @@ HSMresult_t HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockaddr const *ip, const char *hostname, ProxyTransaction *ua_txn, HttpSM *sm) { + // First test for any session bound to the ua_txn Http1ServerSession *to_return = nullptr; TSServerSessionSharingMatchMask match_style = static_cast(sm->t_state.txn_conf->server_session_sharing_match); @@ -370,6 +371,27 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad to_return = nullptr; } + // Otherwise, check the thread pool first + if (this->get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_THREAD || + this->get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_HYBRID) { + retval = _acquire_session(ip, hostname_hash, sm, match_style, TS_SERVER_SESSION_SHARING_POOL_THREAD); + } + + // If you didn't get a match, and the global pool is an option go there. + if (retval != HSM_DONE && (TS_SERVER_SESSION_SHARING_POOL_GLOBAL == this->get_pool_type() || + TS_SERVER_SESSION_SHARING_POOL_HYBRID == this->get_pool_type())) { + retval = _acquire_session(ip, hostname_hash, sm, match_style, TS_SERVER_SESSION_SHARING_POOL_GLOBAL); + } + return retval; +} + +HSMresult_t +HttpSessionManager::_acquire_session(sockaddr const *ip, CryptoHash const &hostname_hash, HttpSM *sm, + TSServerSessionSharingMatchMask match_style, TSServerSessionSharingPoolType pool_type) +{ + Http1ServerSession *to_return = nullptr; + HSMresult_t retval = HSM_NOT_FOUND; + // Extend the mutex window until the acquired Server session is attached // to the SM. Releasing the mutex before that results in race conditions // due to a potential parallel network read on the VC with no mutex guarding @@ -377,12 +399,10 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad // Now check to see if we have a connection in our shared connection pool EThread *ethread = this_ethread(); Ptr pool_mutex = - (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) ? - ethread->server_session_pool->mutex : - m_g_pool->mutex; + (TS_SERVER_SESSION_SHARING_POOL_THREAD == pool_type) ? ethread->server_session_pool->mutex : m_g_pool->mutex; MUTEX_TRY_LOCK(lock, pool_mutex, ethread); if (lock.is_locked()) { - if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) { + if (TS_SERVER_SESSION_SHARING_POOL_THREAD == pool_type) { retval = ethread->server_session_pool->acquireSession(ip, hostname_hash, match_style, sm, to_return); Debug("http_ss", "[acquire session] thread pool search %s", to_return ? "successful" : "failed"); } else { @@ -446,6 +466,10 @@ HttpSessionManager::release_session(Http1ServerSession *to_release) MUTEX_TRY_LOCK(lock, pool->mutex, ethread); if (lock.is_locked()) { pool->releaseSession(to_release); + } else if (this->get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_HYBRID) { + // Try again with the thread pool + to_release->sharing_pool = TS_SERVER_SESSION_SHARING_POOL_THREAD; + return release_session(to_release); } else { Debug("http_ss", "[%" PRId64 "] [release session] could not release session due to lock contention", to_release->con_id); released_p = false; diff --git a/proxy/http/HttpSessionManager.h b/proxy/http/HttpSessionManager.h index dcb80c9bac4..66a98bded8d 100644 --- a/proxy/http/HttpSessionManager.h +++ b/proxy/http/HttpSessionManager.h @@ -67,6 +67,11 @@ class ServerSessionPool : public Continuation static bool validate_host_sni(HttpSM *sm, NetVConnection *netvc); static bool validate_sni(HttpSM *sm, NetVConnection *netvc); static bool validate_cert(HttpSM *sm, NetVConnection *netvc); + int + count() const + { + return m_ip_pool.count(); + } protected: using IPTable = IntrusiveHashMap; @@ -110,11 +115,24 @@ class HttpSessionManager void purge_keepalives(); void init(); int main_handler(int event, void *data); + void + set_pool_type(int pool_type) + { + m_pool_type = static_cast(pool_type); + } + TSServerSessionSharingPoolType + get_pool_type() const + { + return m_pool_type; + } private: /// Global pool, used if not per thread pools. /// @internal We delay creating this because the session manager is created during global statics init. ServerSessionPool *m_g_pool = nullptr; + HSMresult_t _acquire_session(sockaddr const *ip, CryptoHash const &hostname_hash, HttpSM *sm, + TSServerSessionSharingMatchMask match_style, TSServerSessionSharingPoolType pool_type); + TSServerSessionSharingPoolType m_pool_type = TS_SERVER_SESSION_SHARING_POOL_THREAD; }; extern HttpSessionManager httpSessionManager;