HTTP2 drain

doc/admin-guide/files/records.config.en.rst

@@ -416,6 +416,14 @@ Thread Variables
    This setting specifies the number of active client connections
    for use by :option:`traffic_ctl server restart --drain`.
 
+.. ts:cv:: CONFIG proxy.config.stop.shutdown_timeout INT 0
+   :reloadable:
+
+   The shutdown timeout(in seconds) to apply when stopping Traffic
+   Server, in which ATS can initiate graceful shutdowns. It only supports
+   HTTP/2 graceful shutdown for now. Stopping Traffic Server here means sending
+   `traffic_server` a signal either by `bin/trafficserver stop` or `kill`.
+
 Network
 =======
 

mgmt/RecordsConfig.cc

@@ -138,6 +138,8 @@ static const RecordElement RecordsConfig[] =
   ,
   {RECT_CONFIG, "proxy.config.restart.active_client_threshold", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_NULL, nullptr, RECA_NULL}
   ,
+  {RECT_CONFIG, "proxy.config.stop.shutdown_timeout", RECD_INT, "0", RECU_DYNAMIC, RR_NULL, RECC_STR, "^[0-9]+$", RECA_NULL}
+  ,
 
   //##############################################################################
   //#

proxy/Main.cc

@@ -173,6 +173,8 @@ static int cmd_line_dprintf_level = 0;  // default debug output level from ink_d
 static int poll_timeout           = -1; // No value set.
 static int cmd_disable_freelist   = 0;
 
+static int shutdown_timeout = 0;
+
 static volatile bool sigusr1_received = false;
 static volatile bool sigusr2_received = false;
 
@@ -460,6 +462,11 @@ proxy_signal_handler(int signo, siginfo_t *info, void *ctx)
     signal_crash_handler(signo, info, ctx);
   }
 
+  if (shutdown_timeout) {
+    http2_drain = true;
+    sleep(shutdown_timeout);
+  }
+
   shutdown_event_system = true;
   sleep(1);
 }
@@ -1688,6 +1695,8 @@ main(int /* argc ATS_UNUSED */, const char **argv)
     ::exit(0);
   }
 
+  REC_ReadConfigInteger(shutdown_timeout, "proxy.config.stop.shutdown_timeout");
+
   // We need to do this early so we can initialize the Machine
   // singleton, which depends on configuration values loaded in this.
   // We want to initialize Machine as early as possible because it

proxy/http2/HTTP2.cc

@@ -28,6 +28,8 @@
 #include "P_RecCore.h"
 #include "P_RecProcess.h"
 
+volatile bool http2_drain = false;
+
 const char *const HTTP2_CONNECTION_PREFACE = "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n";
 
 // Constant strings for pseudo headers

proxy/http2/HTTP2.h

@@ -38,6 +38,8 @@ typedef unsigned Http2StreamId;
 // the flow control window can be come negative so we need to track it with a signed type.
 typedef int32_t Http2WindowSize;
 
+extern volatile bool http2_drain;
+
 extern const char *const HTTP2_CONNECTION_PREFACE;
 const size_t HTTP2_CONNECTION_PREFACE_LEN = 24;
 

proxy/http2/Http2ClientSession.cc

@@ -307,6 +307,10 @@ Http2ClientSession::main_event_handler(int event, void *edata)
     schedule_event = nullptr;
   }
 
+  if (http2_drain && this->connection_state.get_shutdown_state() == NOT_INITIATED) {
+    send_connection_event(&this->connection_state, HTTP2_SESSION_EVENT_SHUTDOWN_INIT, this);
+  }
+
   switch (event) {
   case VC_EVENT_READ_COMPLETE:
   case VC_EVENT_READ_READY:

proxy/http2/Http2ClientSession.h

@@ -41,6 +41,8 @@
 #define HTTP2_SESSION_EVENT_FINI (HTTP2_SESSION_EVENTS_START + 2)
 #define HTTP2_SESSION_EVENT_RECV (HTTP2_SESSION_EVENTS_START + 3)
 #define HTTP2_SESSION_EVENT_XMIT (HTTP2_SESSION_EVENTS_START + 4)
+#define HTTP2_SESSION_EVENT_SHUTDOWN_INIT (HTTP2_SESSION_EVENTS_START + 5)
+#define HTTP2_SESSION_EVENT_SHUTDOWN_CONT (HTTP2_SESSION_EVENTS_START + 6)
 
 size_t const HTTP2_HEADER_BUFFER_SIZE_INDEX = CLIENT_CONNECTION_FIRST_READ_BUFFER_SIZE_INDEX;
 

proxy/http2/Http2ConnectionState.cc

@@ -919,8 +919,32 @@ Http2ConnectionState::main_event_handler(int event, void *edata)
       }
     }
 
-    break;
-  }
+  } break;
+
+  // Initiate a gracefull shutdown
+  case HTTP2_SESSION_EVENT_SHUTDOWN_INIT: {
+    ink_assert(shutdown_state == NOT_INITIATED);
+    shutdown_state = INITIATED;
+    // [RFC 7540] 6.8.  GOAWAY
+    // A server that is attempting to gracefully shut down a
+    // connection SHOULD send an initial GOAWAY frame with the last stream
+    // identifier set to 2^31-1 and a NO_ERROR code.
+    send_goaway_frame(INT32_MAX, Http2ErrorCode::HTTP2_ERROR_NO_ERROR);
+    // After allowing time for any in-flight stream creation (at least one round-trip time),
+    this_ethread()->schedule_in((Continuation *)this, HRTIME_SECONDS(2), HTTP2_SESSION_EVENT_SHUTDOWN_CONT);
+  } break;
+
+  // Continue a gracefull shutdown
+  case HTTP2_SESSION_EVENT_SHUTDOWN_CONT: {
+    ink_assert(shutdown_state == INITIATED);
+    shutdown_state = IN_PROGRESS;
+    // [RFC 7540] 6.8.  GOAWAY
+    // ..., the server can send another GOAWAY frame with an updated last stream identifier
+    send_goaway_frame(latest_streamid_in, Http2ErrorCode::HTTP2_ERROR_NO_ERROR);
+    // Stop creating new streams
+    SCOPED_MUTEX_LOCK(lock, this->ua_session->mutex, this_ethread());
+    this->ua_session->set_half_close_flag(true);
+  } break;
 
   default:
     DebugHttp2Con(ua_session, "unexpected event=%d edata=%p", event, edata);
@@ -1134,7 +1158,7 @@ Http2ConnectionState::release_stream(Http2Stream *stream)
       }
     }
 
-    if (fini_received && total_client_streams_count == 0) {
+    if ((fini_received || shutdown_state == IN_PROGRESS) && total_client_streams_count == 0) {
       // We were shutting down, go ahead and terminate the session
       ua_session->destroy();
       ua_session = nullptr;

proxy/http2/Http2ConnectionState.h

@@ -38,6 +38,8 @@ enum Http2SendADataFrameResult {
   HTTP2_SEND_A_DATA_FRAME_DONE       = 3,
 };
 
+enum Http2ShutdownState { NOT_INITIATED, INITIATED, IN_PROGRESS };
+
 class Http2ConnectionSettings
 {
 public:
@@ -126,7 +128,8 @@ class Http2ConnectionState : public Continuation
       continued_stream_id(0),
       _scheduled(false),
       fini_received(false),
-      recursion(0)
+      recursion(0),
+      shutdown_state(NOT_INITIATED)
   {
     SET_HANDLER(&Http2ConnectionState::main_event_handler);
   }
@@ -265,6 +268,18 @@ class Http2ConnectionState : public Continuation
     }
   }
 
+  Http2ShutdownState
+  get_shutdown_state() const
+  {
+    return shutdown_state;
+  }
+
+  void
+  set_shutdown_state(Http2ShutdownState state)
+  {
+    shutdown_state = state;
+  }
+
 private:
   Http2ConnectionState(const Http2ConnectionState &);            // noncopyable
   Http2ConnectionState &operator=(const Http2ConnectionState &); // noncopyable
@@ -303,6 +318,7 @@ class Http2ConnectionState : public Continuation
   bool _scheduled;
   bool fini_received;
   int recursion;
+  Http2ShutdownState shutdown_state;
 };
 
 #endif // __HTTP2_CONNECTION_STATE_H__

proxy/http2/Http2SessionAccept.cc

@@ -45,6 +45,11 @@ Http2SessionAccept::accept(NetVConnection *netvc, MIOBuffer *iobuf, IOBufferRead
     Warning("HTTP/2 client '%s' prohibited by ip-allow policy", ats_ip_ntop(client_ip, ipb, sizeof(ipb)));
     return false;
   }
+
+  if (http2_drain) {
+    return false;
+  }
+
   netvc->attributes = this->options.transport_type;
 
   if (is_debug_tag_set("http2_seq")) {