HTTP2 drain

Author: Zizhong Zhang

mgmt/RecordsConfig.cc

@@ -1111,6 +1111,8 @@ static const RecordElement RecordsConfig[] =
   //        ###########
   {RECT_CONFIG, "proxy.config.http.connect_ports", RECD_STRING, "443", RECU_DYNAMIC, RR_NULL, RECC_STR, "^(\\*|[[:digit:][:space:]]+)$", RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.http.http2_drain_timeout", RECD_INT, "0", RECU_RESTART_TS, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
+  ,
   //        ##########################
   //        # Various update periods #
   //        ##########################

proxy/Main.cc

@@ -173,6 +173,8 @@ static int cmd_line_dprintf_level = 0;  // default debug output level from ink_d
 static int poll_timeout           = -1; // No value set.
 static int cmd_disable_freelist   = 0;
 
+static int http2_drain_timeout = 0;
+
 static volatile bool sigusr1_received = false;
 static volatile bool sigusr2_received = false;
 
@@ -460,6 +462,11 @@ proxy_signal_handler(int signo, siginfo_t *info, void *ctx)
     signal_crash_handler(signo, info, ctx);
   }
 
+  if (http2_drain_timeout) {
+    http2_drain = true;
+    sleep(http2_drain_timeout);
+  }
+
   shutdown_event_system = true;
   sleep(1);
 }
@@ -1688,6 +1695,8 @@ main(int /* argc ATS_UNUSED */, const char **argv)
     ::exit(0);
   }
 
+  REC_ReadConfigInteger(http2_drain_timeout, "proxy.config.http.http2_drain_timeout");
+
   // We need to do this early so we can initialize the Machine
   // singleton, which depends on configuration values loaded in this.
   // We want to initialize Machine as early as possible because it

proxy/http2/HTTP2.cc

@@ -28,6 +28,8 @@
 #include "P_RecCore.h"
 #include "P_RecProcess.h"
 
+volatile bool http2_drain = false;
+
 const char *const HTTP2_CONNECTION_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n";
 
 // Constant strings for pseudo headers

proxy/http2/HTTP2.h

@@ -38,6 +38,8 @@ typedef unsigned Http2StreamId;
 // the flow control window can be come negative so we need to track it with a signed type.
 typedef int32_t Http2WindowSize;
 
+extern volatile bool http2_drain;
+
 extern const char *const HTTP2_CONNECTION_PREFACE;
 const size_t HTTP2_CONNECTION_PREFACE_LEN = 24;
 

proxy/http2/Http2ClientSession.cc

@@ -297,6 +297,10 @@ Http2ClientSession::main_event_handler(int event, void *edata)
     schedule_event = nullptr;
   }
 
+  if (http2_drain && this->connection_state.get_shutdown_state() == NOT_INITATED) {
+    send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_SHUTDOWN_INIT, this);
+  }
+
   switch (event) {
   case VC_EVENT_READ_COMPLETE:
   case VC_EVENT_READ_READY:

proxy/http2/Http2ClientSession.h

@@ -41,6 +41,8 @@
 #define HTTP2_SESSION_EVENT_FINI (HTTP2_SESSION_EVENTS_START + 2)
 #define HTTP2_SESSION_EVENT_RECV (HTTP2_SESSION_EVENTS_START + 3)
 #define HTTP2_SESSION_EVENT_XMIT (HTTP2_SESSION_EVENTS_START + 4)
+#define HTTP2_SESSION_EVENT_SHUTDOWN_INIT (HTTP2_SESSION_EVENTS_START + 5)
+#define HTTP2_SESSION_EVENT_SHUTDOWN_CONT (HTTP2_SESSION_EVENTS_START + 6)
 
 size_t const HTTP2_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX;
 

proxy/http2/Http2ConnectionState.cc

@@ -913,8 +913,33 @@ Http2ConnectionState::main_event_handler(int event, void *edata)
       }
     }
 
-    break;
-  }
+  } break;
+
+  // Initiate a gracefull shutdown
+  case HTTP2_SESSION_EVENT_SHUTDOWN_INIT: {
+    SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread());
+    if (shutdown_state != NOT_INITATED)
+      break;
+    shutdown_state = INITIATED;
+    // [RFC 7540] 6.8.  GOAWAY
+    // A server that is attempting to gracefully shut down a
+    // connection SHOULD send an initial GOAWAY frame with the last stream
+    // identifier set to 2^31-1 and a NO_ERROR code.
+    send_goaway_frame(INT32_MAX, Http2ErrorCode::HTTP2_ERROR_NO_ERROR);
+    // After allowing time for any in-flight stream creation (at least one round-trip time),
+    this_ethread()->schedule_in((Continuation *)this, HRTIME_SECONDS(2), HTTP2_SESSION_EVENT_SHUTDOWN_CONT);
+  } break;
+
+  // Continue a gracefull shutdown
+  case HTTP2_SESSION_EVENT_SHUTDOWN_CONT: {
+    SCOPED_MUTEX_LOCK(lock, this->mutex, this_ethread());
+    if (shutdown_state != INITIATED)
+      break;
+    shutdown_state = IN_PROGREASS;
+    // [RFC 7540] 6.8.  GOAWAY
+    // ..., the server can send another GOAWAY frame with an updated last stream identifier
+    send_goaway_frame(latest_streamid_in, Http2ErrorCode::HTTP2_ERROR_NO_ERROR);
+  } break;
 
   default:
     DebugHttp2Con(ua_session, "unexpected event=%d edata=%p", event, edata);
@@ -1120,6 +1145,9 @@ Http2ConnectionState::release_stream(Http2Stream *stream)
     // We were shutting down, go ahead and terminate the session
     ua_session->destroy();
     ua_session = nullptr;
+  } else if (total_client_streams_count == 0 && http2_drain && ua_session && stream) {
+    send_goaway_frame(stream->get_id(), Http2ErrorCode::HTTP2_ERROR_NO_ERROR);
+    Note("[Http2ConnectionState::release_stream]: draining http2 connection, GOAWAY is sent");
   }
 }
 

proxy/http2/Http2ConnectionState.h

@@ -38,6 +38,8 @@ enum Http2SendADataFrameResult {
   HTTP2_SEND_A_DATA_FRAME_DONE       = 3,
 };
 
+enum Http2ShutdownState { NOT_INITATED, INITIATED, IN_PROGREASS };
+
 class Http2ConnectionSettings
 {
 public:
@@ -126,7 +128,8 @@ class Http2ConnectionState : public Continuation
       continued_stream_id(0),
       _scheduled(false),
       fini_received(false),
-      recursion(0)
+      recursion(0),
+      shutdown_state(NOT_INITATED)
   {
     SET_HANDLER(&Http2ConnectionState::main_event_handler);
   }
@@ -265,6 +268,18 @@ class Http2ConnectionState : public Continuation
     }
   }
 
+  Http2ShutdownState
+  get_shutdown_state() const
+  {
+    return shutdown_state;
+  }
+
+  void
+  set_shutdown_state(Http2ShutdownState state)
+  {
+    shutdown_state = state;
+  }
+
 private:
   Http2ConnectionState(const Http2ConnectionState &);            // noncopyable
   Http2ConnectionState &operator=(const Http2ConnectionState &); // noncopyable
@@ -303,6 +318,7 @@ class Http2ConnectionState : public Continuation
   bool _scheduled;
   bool fini_received;
   int recursion;
+  Http2ShutdownState shutdown_state;
 };
 
 #endif // __HTTP2_CONNECTION_STATE_H__