Add option for hybrid global and thread session pools

doc/admin-guide/files/records.config.en.rst

@@ -994,8 +994,20 @@ mptcp
    ========== =================================================================
    ``global`` Re-use sessions from a global pool of all server sessions.
    ``thread`` Re-use sessions from a per-thread pool.
+   ``hybrid`` Try to work as a global pool, but release server sessions to the
+              per-thread pool if there is lock contention on the global pool.
    ========== =================================================================
 
+
+   Setting :ts:cv:`proxy.config.http.server_session_sharing.pool` to global can reduce
+   the number of connections to origin for some traffic loads.  However, if many
+   execute threads are active, the thread contention on the global pool can reduce the
+   lifetime of connections to origin and reduce effective origin connection reuse.
+
+   For a hybrid pool, the operation starts as the global pool, but sessons are returned
+   to the local thread pool if the global pool lock is not acquired rather than just
+   closing the origin connection as is the case in standard global mode.
+
 .. ts:cv:: CONFIG proxy.config.http.attach_server_session_to_client INT 0
    :overridable:
 

proxy/http/Http1ServerSession.cc

@@ -52,7 +52,7 @@ Http1ServerSession::destroy()
   }
 
   mutex.clear();
-  if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sharing_pool) {
+  if (httpSessionManager.get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_THREAD) {
     THREAD_FREE(this, httpServerSessionAllocator, this_thread());
   } else {
     httpServerSessionAllocator.free(this);

proxy/http/HttpConfig.cc

@@ -33,6 +33,7 @@
 #include "P_Net.h"
 #include "records/P_RecUtils.h"
 #include <records/I_RecHttp.h>
+#include "HttpSessionManager.h"
 
 #define HttpEstablishStaticConfigStringAlloc(_ix, _n) \
   REC_EstablishStaticConfigStringAlloc(_ix, _n);      \
@@ -164,7 +165,8 @@ http_config_enum_mask_read(const char *name, MgmtByte &value)
 
 static const ConfigEnumPair<TSServerSessionSharingPoolType> SessionSharingPoolStrings[] = {
   {TS_SERVER_SESSION_SHARING_POOL_GLOBAL, "global"},
-  {TS_SERVER_SESSION_SHARING_POOL_THREAD, "thread"}};
+  {TS_SERVER_SESSION_SHARING_POOL_THREAD, "thread"},
+  {TS_SERVER_SESSION_SHARING_POOL_HYBRID, "hybrid"}};
 
 int HttpConfig::m_id = 0;
 HttpConfigParams HttpConfig::m_master;
@@ -1153,6 +1155,7 @@ HttpConfig::startup()
   http_config_enum_mask_read("proxy.config.http.server_session_sharing.match", c.oride.server_session_sharing_match);
   HttpEstablishStaticConfigStringAlloc(c.oride.server_session_sharing_match_str, "proxy.config.http.server_session_sharing.match");
   http_config_enum_read("proxy.config.http.server_session_sharing.pool", SessionSharingPoolStrings, c.server_session_sharing_pool);
+  httpSessionManager.set_pool_type(c.server_session_sharing_pool);
 
   RecRegisterConfigUpdateCb("proxy.config.http.insert_forwarded", &http_insert_forwarded_cb, &c);
   {

proxy/http/HttpProxyAPIEnums.h

@@ -54,6 +54,7 @@ typedef enum {
 typedef enum {
   TS_SERVER_SESSION_SHARING_POOL_GLOBAL,
   TS_SERVER_SESSION_SHARING_POOL_THREAD,
+  TS_SERVER_SESSION_SHARING_POOL_HYBRID
 } TSServerSessionSharingPoolType;
 
 // This is use to signal apidefs.h to not define these again.

proxy/http/HttpSM.cc

@@ -1781,10 +1781,9 @@ HttpSM::state_http_server_open(int event, void *data)
 
   switch (event) {
   case NET_EVENT_OPEN: {
-    Http1ServerSession *session =
-      (TS_SERVER_SESSION_SHARING_POOL_THREAD == t_state.http_config_param->server_session_sharing_pool) ?
-        THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) :
-        httpServerSessionAllocator.alloc();
+    Http1ServerSession *session = (TS_SERVER_SESSION_SHARING_POOL_THREAD == httpSessionManager.get_pool_type()) ?
+                                    THREAD_ALLOC_INIT(httpServerSessionAllocator, mutex->thread_holding) :
+                                    httpServerSessionAllocator.alloc();
     session->sharing_pool  = static_cast<TSServerSessionSharingPoolType>(t_state.http_config_param->server_session_sharing_pool);
     session->sharing_match = static_cast<TSServerSessionSharingMatchMask>(t_state.txn_conf->server_session_sharing_match);
 

proxy/http/HttpSessionManager.cc

@@ -330,6 +330,7 @@ HSMresult_t
 HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockaddr const *ip, const char *hostname,
                                     ProxyTransaction *ua_txn, HttpSM *sm)
 {
+  // First test for any session bound to the ua_txn
   Http1ServerSession *to_return = nullptr;
   TSServerSessionSharingMatchMask match_style =
     static_cast<TSServerSessionSharingMatchMask>(sm->t_state.txn_conf->server_session_sharing_match);
@@ -370,19 +371,38 @@ HttpSessionManager::acquire_session(Continuation * /* cont ATS_UNUSED */, sockad
     to_return = nullptr;
   }
 
+  // Otherwise, check the thread pool first
+  if (this->get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_THREAD ||
+      this->get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_HYBRID) {
+    retval = _acquire_session(ip, hostname_hash, sm, match_style, TS_SERVER_SESSION_SHARING_POOL_THREAD);
+  }
+
+  //  If you didn't get a match, and the global pool is an option go there.
+  if (retval != HSM_DONE && (TS_SERVER_SESSION_SHARING_POOL_GLOBAL == this->get_pool_type() ||
+                             TS_SERVER_SESSION_SHARING_POOL_HYBRID == this->get_pool_type())) {
+    retval = _acquire_session(ip, hostname_hash, sm, match_style, TS_SERVER_SESSION_SHARING_POOL_GLOBAL);
+  }
+  return retval;
+}
+
+HSMresult_t
+HttpSessionManager::_acquire_session(sockaddr const *ip, CryptoHash const &hostname_hash, HttpSM *sm,
+                                     TSServerSessionSharingMatchMask match_style, TSServerSessionSharingPoolType pool_type)
+{
+  Http1ServerSession *to_return = nullptr;
+  HSMresult_t retval            = HSM_NOT_FOUND;
+
   // Extend the mutex window until the acquired Server session is attached
   // to the SM. Releasing the mutex before that results in race conditions
   // due to a potential parallel network read on the VC with no mutex guarding
   {
     // Now check to see if we have a connection in our shared connection pool
     EThread *ethread = this_ethread();
     Ptr<ProxyMutex> pool_mutex =
-      (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) ?
-        ethread->server_session_pool->mutex :
-        m_g_pool->mutex;
+      (TS_SERVER_SESSION_SHARING_POOL_THREAD == pool_type) ? ethread->server_session_pool->mutex : m_g_pool->mutex;
     MUTEX_TRY_LOCK(lock, pool_mutex, ethread);
     if (lock.is_locked()) {
-      if (TS_SERVER_SESSION_SHARING_POOL_THREAD == sm->t_state.http_config_param->server_session_sharing_pool) {
+      if (TS_SERVER_SESSION_SHARING_POOL_THREAD == pool_type) {
         retval = ethread->server_session_pool->acquireSession(ip, hostname_hash, match_style, sm, to_return);
         Debug("http_ss", "[acquire session] thread pool search %s", to_return ? "successful" : "failed");
       } else {
@@ -446,6 +466,10 @@ HttpSessionManager::release_session(Http1ServerSession *to_release)
   MUTEX_TRY_LOCK(lock, pool->mutex, ethread);
   if (lock.is_locked()) {
     pool->releaseSession(to_release);
+  } else if (this->get_pool_type() == TS_SERVER_SESSION_SHARING_POOL_HYBRID) {
+    // Try again with the thread pool
+    to_release->sharing_pool = TS_SERVER_SESSION_SHARING_POOL_THREAD;
+    return release_session(to_release);
   } else {
     Debug("http_ss", "[%" PRId64 "] [release session] could not release session due to lock contention", to_release->con_id);
     released_p = false;

proxy/http/HttpSessionManager.h

@@ -67,6 +67,11 @@ class ServerSessionPool : public Continuation
   static bool validate_host_sni(HttpSM *sm, NetVConnection *netvc);
   static bool validate_sni(HttpSM *sm, NetVConnection *netvc);
   static bool validate_cert(HttpSM *sm, NetVConnection *netvc);
+  int
+  count() const
+  {
+    return m_ip_pool.count();
+  }
 
 protected:
   using IPTable   = IntrusiveHashMap<Http1ServerSession::IPLinkage>;
@@ -110,11 +115,24 @@ class HttpSessionManager
   void purge_keepalives();
   void init();
   int main_handler(int event, void *data);
+  void
+  set_pool_type(int pool_type)
+  {
+    m_pool_type = static_cast<TSServerSessionSharingPoolType>(pool_type);
+  }
+  TSServerSessionSharingPoolType
+  get_pool_type() const
+  {
+    return m_pool_type;
+  }
 
 private:
   /// Global pool, used if not per thread pools.
   /// @internal We delay creating this because the session manager is created during global statics init.
   ServerSessionPool *m_g_pool = nullptr;
+  HSMresult_t _acquire_session(sockaddr const *ip, CryptoHash const &hostname_hash, HttpSM *sm,
+                               TSServerSessionSharingMatchMask match_style, TSServerSessionSharingPoolType pool_type);
+  TSServerSessionSharingPoolType m_pool_type = TS_SERVER_SESSION_SHARING_POOL_THREAD;
 };
 
 extern HttpSessionManager httpSessionManager;