Initial support for upstream HTTP/1.1 tunneling (envoyproxy#13293)

Commit Message:
Additional Description:
Risk Level: Low
Testing: unit test, integration, manual testing
Docs Changes: Added documentation on how to configure Envoy for tunneling TCP over HTTP/1
Release Notes: n/a (still hidden)
Part of envoyproxy#11308

Signed-off-by: Iacopo Rozzo <iacopo@kubermatic.com>
Signed-off-by: Qin Qin <qqin@google.com>

configs/configgen.py

@@ -139,5 +139,7 @@ def generate_config(template_path, template, output_file, **context):
                 mongos_servers=mongos_servers)
 
 shutil.copy(os.path.join(SCRIPT_DIR, 'envoyproxy_io_proxy.yaml'), OUT_DIR)
-shutil.copy(os.path.join(SCRIPT_DIR, 'encapsulate_in_connect.yaml'), OUT_DIR)
-shutil.copy(os.path.join(SCRIPT_DIR, 'terminate_connect.yaml'), OUT_DIR)
+shutil.copy(os.path.join(SCRIPT_DIR, 'encapsulate_in_http1_connect.yaml'), OUT_DIR)
+shutil.copy(os.path.join(SCRIPT_DIR, 'encapsulate_in_http2_connect.yaml'), OUT_DIR)
+shutil.copy(os.path.join(SCRIPT_DIR, 'terminate_http1_connect.yaml'), OUT_DIR)
+shutil.copy(os.path.join(SCRIPT_DIR, 'terminate_http2_connect.yaml'), OUT_DIR)

configs/encapsulate_in_http1_connect.yaml

@@ -0,0 +1,44 @@
+# This configuration takes incoming data on port 10000 and encapsulates it in a CONNECT
+# request which is sent upstream port 10001.
+# It can be used to test TCP tunneling as described in docs/root/intro/arch_overview/http/upgrades.rst
+# and running `curl --x 127.0.0.1:10000 https://www.google.com`
+
+admin:
+  access_log_path: /tmp/admin_access.log
+  address:
+    socket_address:
+      protocol: TCP
+      address: 127.0.0.1
+      port_value: 9903
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address:
+        protocol: TCP
+        address: 127.0.0.1
+        port_value: 10000
+    filter_chains:
+    - filters:
+      - name: tcp
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.tcp_proxy.v3.TcpProxy
+          stat_prefix: tcp_stats
+          cluster: "cluster_0"
+          tunneling_config:
+            hostname: host.com:443
+  clusters:
+    - name: cluster_0
+      connect_timeout: 5s
+      # This ensures HTTP/1.1 CONNECT is used for establishing the tunnel.
+      http_protocol_options:
+        {}
+      load_assignment:
+        cluster_name: cluster_0
+        endpoints:
+          - lb_endpoints:
+              - endpoint:
+                  address:
+                    socket_address:
+                      address: 127.0.0.1
+                      port_value: 10001

configs/encapsulate_in_connect.yaml → configs/encapsulate_in_http2_connect.yaml

@@ -26,10 +26,11 @@ static_resources:
           stat_prefix: tcp_stats
           cluster: "cluster_0"
           tunneling_config:
-            hostname: host.com
+            hostname: host.com:443
   clusters:
     - name: cluster_0
       connect_timeout: 5s
+      # This ensures HTTP/2 CONNECT is used for establishing the tunnel.
       http2_protocol_options:
         {}
       load_assignment:

configs/terminate_http1_connect.yaml

@@ -0,0 +1,61 @@
+# This configuration terminates a CONNECT request and sends the CONNECT payload upstream.
+# It can be used to test TCP tunneling as described in docs/root/intro/arch_overview/http/upgrades.rst
+# or used to test CONNECT directly, by running `curl -k -v -x 127.0.0.1:10001 https://www.google.com`
+admin:
+  access_log_path: /tmp/admin_access.log
+  address:
+    socket_address:
+      protocol: TCP
+      address: 127.0.0.1
+      port_value: 9902
+static_resources:
+  listeners:
+  - name: listener_0
+    address:
+      socket_address:
+        protocol: TCP
+        address: 127.0.0.1
+        port_value: 10001
+    filter_chains:
+    - filters:
+      - name: envoy.filters.network.http_connection_manager
+        typed_config:
+          "@type": type.googleapis.com/envoy.extensions.filters.network.http_connection_manager.v3.HttpConnectionManager
+          stat_prefix: ingress_http
+          route_config:
+            name: local_route
+            virtual_hosts:
+            - name: local_service
+              domains:
+                - "*"
+              routes:
+                - match:
+                    connect_matcher:
+                      {}
+                  route:
+                    cluster: service_google
+                    upgrade_configs:
+                      - upgrade_type: CONNECT
+                        connect_config:
+                          {}
+          http_filters:
+          - name: envoy.filters.http.router
+          http_protocol_options: {}
+          upgrade_configs:
+            - upgrade_type: CONNECT
+  clusters:
+  - name: service_google
+    connect_timeout: 0.25s
+    type: LOGICAL_DNS
+    # Comment out the following line to test on v6 networks
+    dns_lookup_family: V4_ONLY
+    lb_policy: ROUND_ROBIN
+    load_assignment:
+      cluster_name: service_google
+      endpoints:
+      - lb_endpoints:
+        - endpoint:
+            address:
+              socket_address:
+                address: www.google.com
+                port_value: 443

docs/root/intro/arch_overview/http/upgrades.rst

@@ -94,17 +94,32 @@ will synthesize 200 response headers, and then forward the TCP data as the HTTP
 For an example of proxying connect, please see :repo:`configs/proxy_connect.yaml <configs/proxy_connect.yaml>`
 For an example of terminating connect, please see :repo:`configs/terminate_connect.yaml <configs/terminate_connect.yaml>`
 
-Tunneling TCP over HTTP/2
-^^^^^^^^^^^^^^^^^^^^^^^^^
-Envoy also has support for transforming raw TCP into HTTP/2 CONNECT requests. This can be used to
-proxy multiplexed TCP over pre-warmed secure connections and amortize the cost of any TLS handshake.
-An example set up proxying SMTP would look something like this
+Tunneling TCP over HTTP
+^^^^^^^^^^^^^^^^^^^^^^^
+Envoy also has support for tunneling raw TCP over HTTP CONNECT requests. Find
+below some usage scenarios.
+
+HTTP/2 CONNECT can be used to proxy multiplexed TCP over pre-warmed secure connections and amortize the cost of any TLS
+handshake.
+An example set up proxying SMTP would look something like this:
 
 [SMTP Upstream] --- raw SMTP --- [L2 Envoy]  --- SMTP tunneled over HTTP/2  --- [L1 Envoy]  --- raw SMTP  --- [Client]
 
+HTTP/1.1 CONNECT can be used to have TCP client connecting to its own
+destination passing through an HTTP proxy server (e.g. corporate proxy not
+supporting HTTP/2):
+
+[HTTP Server] --- raw HTTP --- [L2 Envoy]  --- HTTP tunneled over HTTP/1.1  --- [L1 Envoy]  --- raw HTTP  --- [HTTP Client]
+
+Note that when using HTTP/1 CONNECT you will end up having a TCP connection
+between L1 and L2 Envoy for each TCP client connection, it is preferable to use
+HTTP/2 when you have the choice.
+
 Examples of such a set up can be found in the Envoy example config :repo:`directory <configs/>`
-If you run `bazel-bin/source/exe/envoy-static --config-path configs/encapsulate_in_connect.yaml --base-id 1`
-and `bazel-bin/source/exe/envoy-static --config-path  configs/terminate_connect.yaml`
-you will be running two Envoys, the first listening for TCP traffic on port 10000 and encapsulating it in an HTTP/2
-CONNECT request, and the second listening for HTTP/2 on 10001, stripping the CONNECT headers, and forwarding the
+For HTTP/1.1 run `bazel-bin/source/exe/envoy-static --config-path configs/encapsulate_in_http1_connect.yaml --base-id 1`
+and `bazel-bin/source/exe/envoy-static --config-path configs/terminate_http1_connect.yaml`.
+For HTTP/2 run `bazel-bin/source/exe/envoy-static --config-path configs/encapsulate_in_http2_connect.yaml --base-id 1`
+and `bazel-bin/source/exe/envoy-static --config-path configs/terminate_http2_connect.yaml`.
+In both cases you will be running a first Envoy listening for TCP traffic on port 10000 and encapsulating it in an HTTP
+CONNECT request, and a second one listening on 10001, stripping the CONNECT headers, and forwarding the 
 original TCP upstream, in this case to google.com.

include/envoy/network/connection.h

@@ -118,6 +118,11 @@ class Connection : public Event::DeferredDeletable, public FilterManager {
    */
   virtual void enableHalfClose(bool enabled) PURE;
 
+  /**
+   * @return true if half-close semantics are enabled, false otherwise.
+   */
+  virtual bool isHalfCloseEnabled() PURE;
+
   /**
    * Close the connection.
    */

source/common/http/codec_client.h

@@ -66,6 +66,11 @@ class CodecClient : Logger::Loggable<Logger::Id::client>,
     connection_->addConnectionCallbacks(cb);
   }
 
+  /**
+   * Return if half-close semantics are enabled on the underlying connection.
+   */
+  bool isHalfCloseEnabled() { return connection_->isHalfCloseEnabled(); }
+
   /**
    * Close the underlying network connection. This is immediate and will not attempt to flush any
    * pending write data.
@@ -176,8 +181,14 @@ class CodecClient : Logger::Loggable<Logger::Id::client>,
     CodecReadFilter(CodecClient& parent) : parent_(parent) {}
 
     // Network::ReadFilter
-    Network::FilterStatus onData(Buffer::Instance& data, bool) override {
+    Network::FilterStatus onData(Buffer::Instance& data, bool end_stream) override {
       parent_.onData(data);
+      if (end_stream && parent_.isHalfCloseEnabled()) {
+        // Note that this results in the connection closed as if it was closed
+        // locally, it would be more correct to convey the end stream to the
+        // response decoder, but it would require some refactoring.
+        parent_.close();
+      }
       return Network::FilterStatus::StopIteration;
     }
 

source/common/http/http1/codec_impl.cc

@@ -77,8 +77,8 @@ const std::string StreamEncoderImpl::LAST_CHUNK = "0\r\n";
 StreamEncoderImpl::StreamEncoderImpl(ConnectionImpl& connection,
                                      HeaderKeyFormatter* header_key_formatter)
     : connection_(connection), disable_chunk_encoding_(false), chunk_encoding_(true),
-      is_response_to_head_request_(false), is_response_to_connect_request_(false),
-      header_key_formatter_(header_key_formatter) {
+      connect_request_(false), is_response_to_head_request_(false),
+      is_response_to_connect_request_(false), header_key_formatter_(header_key_formatter) {
   if (connection_.connection().aboveHighWatermark()) {
     runHighWatermarkCallbacks();
   }
@@ -261,6 +261,10 @@ void StreamEncoderImpl::endEncode() {
 
   connection_.flushOutput(true);
   connection_.onEncodeComplete();
+  // With CONNECT, half-closing the connection is used to signal end stream.
+  if (connect_request_) {
+    connection_.connection().close(Network::ConnectionCloseType::FlushWriteAndDelay);
+  }
 }
 
 void ServerConnectionImpl::maybeAddSentinelBufferFragment(Buffer::WatermarkBuffer& output_buffer) {
@@ -380,6 +384,7 @@ Status RequestEncoderImpl::encodeHeaders(const RequestHeaderMap& headers, bool e
     head_request_ = true;
   } else if (method->value() == Headers::get().MethodValues.Connect) {
     disableChunkEncoding();
+    connection_.connection().enableHalfClose(true);
     connect_request_ = true;
   }
   if (Utility::isUpgrade(headers)) {

source/common/http/http1/codec_impl.h

@@ -87,6 +87,7 @@ class StreamEncoderImpl : public virtual StreamEncoder,
   uint32_t read_disable_calls_{};
   bool disable_chunk_encoding_ : 1;
   bool chunk_encoding_ : 1;
+  bool connect_request_ : 1;
   bool is_response_to_head_request_ : 1;
   bool is_response_to_connect_request_ : 1;
 
@@ -162,7 +163,6 @@ class RequestEncoderImpl : public StreamEncoderImpl, public RequestEncoder {
 private:
   bool upgrade_request_{};
   bool head_request_{};
-  bool connect_request_{};
 };
 
 /**

source/common/http/http1/codec_impl_legacy.cc

@@ -78,8 +78,8 @@ const std::string StreamEncoderImpl::LAST_CHUNK = "0\r\n";
 StreamEncoderImpl::StreamEncoderImpl(ConnectionImpl& connection,
                                      HeaderKeyFormatter* header_key_formatter)
     : connection_(connection), disable_chunk_encoding_(false), chunk_encoding_(true),
-      is_response_to_head_request_(false), is_response_to_connect_request_(false),
-      header_key_formatter_(header_key_formatter) {
+      connect_request_(false), is_response_to_head_request_(false),
+      is_response_to_connect_request_(false), header_key_formatter_(header_key_formatter) {
   if (connection_.connection().aboveHighWatermark()) {
     runHighWatermarkCallbacks();
   }
@@ -262,6 +262,10 @@ void StreamEncoderImpl::endEncode() {
 
   connection_.flushOutput(true);
   connection_.onEncodeComplete();
+  // With CONNECT half-closing the connection is used to signal end stream.
+  if (connect_request_) {
+    connection_.connection().close(Network::ConnectionCloseType::FlushWriteAndDelay);
+  }
 }
 
 void ServerConnectionImpl::maybeAddSentinelBufferFragment(Buffer::WatermarkBuffer& output_buffer) {
@@ -381,6 +385,7 @@ Status RequestEncoderImpl::encodeHeaders(const RequestHeaderMap& headers, bool e
     head_request_ = true;
   } else if (method->value() == Headers::get().MethodValues.Connect) {
     disableChunkEncoding();
+    connection_.connection().enableHalfClose(true);
     connect_request_ = true;
   }
   if (Utility::isUpgrade(headers)) {

source/common/http/http1/codec_impl_legacy.h

@@ -89,6 +89,7 @@ class StreamEncoderImpl : public virtual StreamEncoder,
   uint32_t read_disable_calls_{};
   bool disable_chunk_encoding_ : 1;
   bool chunk_encoding_ : 1;
+  bool connect_request_ : 1;
   bool is_response_to_head_request_ : 1;
   bool is_response_to_connect_request_ : 1;
 
@@ -166,7 +167,6 @@ class RequestEncoderImpl : public StreamEncoderImpl, public RequestEncoder {
 private:
   bool upgrade_request_{};
   bool head_request_{};
-  bool connect_request_{};
 };
 
 /**

source/common/network/connection_impl.cc

@@ -375,6 +375,7 @@ void ConnectionImpl::readDisable(bool disable) {
 }
 
 void ConnectionImpl::raiseEvent(ConnectionEvent event) {
+  ENVOY_CONN_LOG(trace, "raising connection event {}", *this, event);
   ConnectionImplBase::raiseConnectionEvent(event);
   // We may have pending data in the write buffer on transport handshake
   // completion, which may also have completed in the context of onReadReady(),

source/common/network/connection_impl.h

@@ -61,6 +61,7 @@ class ConnectionImpl : public ConnectionImplBase, public TransportSocketCallback
   // Network::Connection
   void addBytesSentCallback(BytesSentCb cb) override;
   void enableHalfClose(bool enabled) override;
+  bool isHalfCloseEnabled() override { return enable_half_close_; }
   void close(ConnectionCloseType type) final;
   std::string nextProtocol() const override { return transport_socket_->protocol(); }
   void noDelay(bool enable) override;

source/common/tcp_proxy/BUILD

@@ -21,6 +21,7 @@ envoy_cc_library(
         "//include/envoy/tcp:upstream_interface",
         "//include/envoy/upstream:cluster_manager_interface",
         "//include/envoy/upstream:load_balancer_interface",
+        "//source/common/http:codec_client_lib",
         "//source/common/http:header_map_lib",
         "//source/common/http:headers_lib",
         "//source/common/http:utility_lib",
@@ -58,6 +59,7 @@ envoy_cc_library(
         "//source/common/common:empty_string",
         "//source/common/common:macros",
         "//source/common/common:minimal_logger_lib",
+        "//source/common/http:codec_client_lib",
         "//source/common/network:application_protocol_lib",
         "//source/common/network:cidr_range_lib",
         "//source/common/network:filter_lib",