Add allow-plain server ports attribute

doc/admin-guide/files/records.yaml.en.rst

@@ -697,6 +697,7 @@ HTTP Engine
    tr-out                      Outbound transparent.
    tr-pass                     Pass through enabled.
    mptcp                       Multipath TCP.
+   allow-plain                 Allow failback to non-TLS for TLS ports
    =========== =============== ========================================
 
 *number*
@@ -721,6 +722,8 @@ ssl
 
    Not compatible with: ``blind`` and ``quic``.
 
+   ``allow-plain`` allows a failback to non SSL for such ports.
+
 quic
    Require QUIC termination for inbound connections. SSL :ref:`must be configured <admin-ssl-termination>` for this option to provide a functional server port.
    **THIS IS EXPERIMENTAL SUPPORT AND NOT READY FOR PRODUCTION USE.**
@@ -779,6 +782,10 @@ mptcp
 
    Requires custom Linux kernel available at https://multipath-tcp.org.
 
+allow-plain
+   For TLS ports, will fall back to non-TLS processing if the TLS handshake fails. Incompatible with
+   quic ports.
+
 .. topic:: Example
 
    Listen on port 80 on any address for IPv4 and IPv6.::

include/records/I_RecHttp.h

@@ -271,6 +271,8 @@ struct HttpProxyPort {
   bool m_outbound_transparent_p = false;
   // True if transparent pass-through is enabled on this port.
   bool m_transparent_passthrough = false;
+  // True if allow-plain is enabled on this port.
+  bool m_allow_plain = false;
   /// True if MPTCP is enabled on this port.
   bool m_mptcp = false;
   /// Local address for inbound connections (listen address).
@@ -420,6 +422,7 @@ struct HttpProxyPort {
   static const char *const OPT_TRANSPARENT_OUTBOUND;    ///< Outbound transparent.
   static const char *const OPT_TRANSPARENT_FULL;        ///< Full transparency.
   static const char *const OPT_TRANSPARENT_PASSTHROUGH; ///< Pass-through non-HTTP.
+  static const char *const OPT_ALLOW_PLAIN;             ///< Backup to plain HTTP.
   static const char *const OPT_SSL;                     ///< SSL (experimental)
   static const char *const OPT_QUIC;                    ///< QUIC (experimental)
   static const char *const OPT_PROXY_PROTO;             ///< Proxy Protocol

iocore/net/P_SSLNetVConnection.h

@@ -181,6 +181,18 @@ class SSLNetVConnection : public UnixNetVConnection,
     transparentPassThrough = val;
   }
 
+  bool
+  getAllowPlain() const
+  {
+    return allowPlain;
+  }
+
+  void
+  setAllowPlain(bool val)
+  {
+    allowPlain = val;
+  }
+
   // Copy up here so we overload but don't override
   using super::reenable;
 
@@ -438,6 +450,7 @@ class SSLNetVConnection : public UnixNetVConnection,
   int handShakeBioStored                     = 0;
 
   bool transparentPassThrough = false;
+  bool allowPlain             = false;
 
   int sent_cert = 0;
 
@@ -484,6 +497,8 @@ class SSLNetVConnection : public UnixNetVConnection,
   void _make_ssl_connection(SSL_CTX *ctx);
   void _bindSSLObject();
   void _unbindSSLObject();
+  UnixNetVConnection *_migrateFromSSL();
+  void _propagateHandShakeBuffer(UnixNetVConnection *target, EThread *t);
 
   int _ssl_read_from_net(EThread *lthread, int64_t &ret);
   ssl_error_t _ssl_read_buffer(void *buf, int64_t nbytes, int64_t &nread);

iocore/net/P_SSLNextProtocolAccept.h

@@ -33,7 +33,7 @@
 class SSLNextProtocolAccept : public SessionAccept
 {
 public:
-  SSLNextProtocolAccept(Continuation *, bool);
+  SSLNextProtocolAccept(Continuation *, bool, bool);
   ~SSLNextProtocolAccept() override;
 
   bool accept(NetVConnection *, MIOBuffer *, IOBufferReader *) override;
@@ -60,6 +60,7 @@ class SSLNextProtocolAccept : public SessionAccept
   SSLNextProtocolSet protoset;
   SessionProtocolSet protoenabled;
   bool transparent_passthrough;
+  bool allow_plain;
 
   friend struct SSLNextProtocolTrampoline;
 };

iocore/net/SSLNetVConnection.cc

@@ -71,6 +71,7 @@ using namespace std::literals;
 #define SSL_WRITE_WOULD_BLOCK      10
 #define SSL_WAIT_FOR_HOOK          11
 #define SSL_WAIT_FOR_ASYNC         12
+#define SSL_RESTART                13
 
 ClassAllocator<SSLNetVConnection> sslNetVCAllocator("sslNetVCAllocator");
 
@@ -608,6 +609,12 @@ SSLNetVConnection::net_read_io(NetHandler *nh, EThread *lthread)
     } else {
       ret = sslStartHandShake(SSL_EVENT_SERVER, err);
     }
+    if (ret == SSL_RESTART) {
+      // VC migrated into a new object
+      // Just give up and go home. Events should trigger on the new vc
+      Dbg(dbg_ctl_ssl, "Restart for allow plain");
+      return;
+    }
     // If we have flipped to blind tunnel, don't read ahead
     if (this->handShakeReader) {
       if (this->attributes == HttpProxyPort::TRANSPORT_BLIND_TUNNEL) {
@@ -1360,13 +1367,24 @@ SSLNetVConnection::sslServerHandShakeEvent(int &err)
     err = errno;
     SSLVCDebug(this, "SSL handshake error: %s (%d), errno=%d", SSLErrorName(ssl_error), ssl_error, err);
 
-    // start a blind tunnel if tr-pass is set and data does not look like ClientHello
     char *buf = handShakeBuffer ? handShakeBuffer->buf() : nullptr;
-    if (getTransparentPassThrough() && buf && *buf != SSL_OP_HANDSHAKE) {
-      SSLVCDebug(this, "Data does not look like SSL handshake, starting blind tunnel");
-      this->attributes   = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
-      sslHandshakeStatus = SSLHandshakeStatus::SSL_HANDSHAKE_ONGOING;
-      return EVENT_CONT;
+    if (buf && *buf != SSL_OP_HANDSHAKE) {
+      SSLVCDebug(this, "SSL hanshake error with bad HS buffer");
+      if (getAllowPlain()) {
+        SSLVCDebug(this, "Try plain");
+        // If this doesn't look like a ClientHello, convert this connection to a UnixNetVC and send the
+        // packet for Http Processing
+        this->_migrateFromSSL();
+        return SSL_RESTART;
+      } else if (getTransparentPassThrough()) {
+        // start a blind tunnel if tr-pass is set and data does not look like ClientHello
+        SSLVCDebug(this, "Data does not look like SSL handshake, starting blind tunnel");
+        this->attributes   = HttpProxyPort::TRANSPORT_BLIND_TUNNEL;
+        sslHandshakeStatus = SSLHandshakeStatus::SSL_HANDSHAKE_ONGOING;
+        return EVENT_CONT;
+      } else {
+        SSLVCDebug(this, "Give up");
+      }
     }
   }
 
@@ -2141,6 +2159,76 @@ SSLNetVConnection::_getNetProcessor()
   return &sslNetProcessor;
 }
 
+void
+SSLNetVConnection::_propagateHandShakeBuffer(UnixNetVConnection *target, EThread *t)
+{
+  Debug("ssl", "allow-plain, handshake buffer ready to read=%" PRId64, this->handShakeHolder->read_avail());
+  // Take ownership of the handShake buffer
+  this->sslHandshakeStatus = SSLHandshakeStatus::SSL_HANDSHAKE_DONE;
+  NetState *s              = &target->read;
+  s->vio.buffer.writer_for(this->handShakeBuffer);
+  s->vio.set_reader(this->handShakeHolder);
+  this->handShakeHolder = nullptr;
+  this->handShakeBuffer = nullptr;
+  s->vio.vc_server      = target;
+  s->vio.cont           = this->read.vio.cont;
+  s->vio.mutex          = this->read.vio.cont->mutex;
+  // Passing along the buffer, don't keep a reading holding early in the buffer
+  this->handShakeReader->dealloc();
+  this->handShakeReader = nullptr;
+
+  // Kick things again, so the data that was copied into the
+  // vio.read buffer gets processed
+  target->readSignalDone(VC_EVENT_READ_COMPLETE, get_NetHandler(t));
+}
+
+/*
+ * Replaces the current SSLNetVConnection with a UnixNetVConnection
+ * Propagates any data in the SSL handShakeBuffer to be processed
+ * by the UnixNetVConnection logic
+ */
+UnixNetVConnection *
+SSLNetVConnection::_migrateFromSSL()
+{
+  EThread *t            = this_ethread();
+  NetHandler *client_nh = get_NetHandler(t);
+  ink_assert(client_nh);
+
+  Connection hold_con;
+  hold_con.move(this->con);
+
+  // We will leave the SSL object with the original SSLNetVC to be
+  // cleaned up.  Only moving the socket and handShakeBuffer
+  // So no need to call _prepareMigration
+
+  // Do_io_close will signal the VC to be freed on the original thread
+  // Since we moved the con context, the fd will not be closed
+  // Go ahead and remove the fd from the original thread's epoll structure, so it is not
+  // processed on two threads simultaneously
+  this->ep.stop();
+
+  // Create new VC:
+  UnixNetVConnection *newvc = static_cast<UnixNetVConnection *>(unix_netProcessor.allocate_vc(t));
+  ink_assert(newvc != nullptr);
+  if (newvc != nullptr && newvc->populate(hold_con, this->read.vio.cont, nullptr) != EVENT_DONE) {
+    newvc->do_io_close();
+    Debug("ssl", "Failed to populate unixvc for allow-plain");
+    newvc = nullptr;
+  }
+  if (newvc != nullptr) {
+    newvc->attributes = HttpProxyPort::TRANSPORT_DEFAULT;
+    newvc->set_is_transparent(this->is_transparent);
+    newvc->set_context(get_context());
+    newvc->options = this->options;
+    Debug("ssl", "Move to unixvc for allow-plain");
+    this->_propagateHandShakeBuffer(newvc, t);
+  }
+
+  // Do not mark this closed until the end so it does not get freed by the other thread too soon
+  this->do_io_close();
+  return newvc;
+}
+
 ssl_curve_id
 SSLNetVConnection::_get_tls_curve() const
 {

iocore/net/SSLNextProtocolAccept.cc

@@ -75,16 +75,16 @@ struct SSLNextProtocolTrampoline : public Continuation {
 
     SSLNetVConnection *netvc;
 
-    vio   = static_cast<VIO *>(edata);
-    netvc = dynamic_cast<SSLNetVConnection *>(vio->vc_server);
-    ink_assert(netvc != nullptr);
+    vio = static_cast<VIO *>(edata);
 
     switch (event) {
     case VC_EVENT_EOS:
     case VC_EVENT_ERROR:
     case VC_EVENT_ACTIVE_TIMEOUT:
     case VC_EVENT_INACTIVITY_TIMEOUT:
-      netvc->do_io_close();
+      if (vio->vc_server != nullptr) {
+        vio->vc_server->do_io_close();
+      }
       delete this;
       return EVENT_ERROR;
     case VC_EVENT_READ_COMPLETE:
@@ -93,25 +93,32 @@ struct SSLNextProtocolTrampoline : public Continuation {
       return EVENT_ERROR;
     }
 
-    // Cancel the action, so later timeouts and errors don't try to
-    // send the event to the Accept object.  After this point, the accept
-    // object does not care.
-    netvc->set_action(nullptr);
-
-    Continuation *endpoint_cont = netvc->endpoint();
-    if (!endpoint_cont) {
-      // Route to the default endpoint
-      endpoint_cont = npnParent->endpoint;
-    }
-
-    if (endpoint_cont) {
-      // disable read io, send events to endpoint
-      netvc->do_io_read(endpoint_cont, 0, nullptr);
-
-      send_plugin_event(endpoint_cont, NET_EVENT_ACCEPT, netvc);
+    // This wasn't really a TLS connection
+    // Trying to process it as a TCP connection
+    netvc = dynamic_cast<SSLNetVConnection *>(vio->vc_server);
+    if (netvc == nullptr) {
+      send_plugin_event(npnParent->endpoint, NET_EVENT_ACCEPT, vio->vc_server);
     } else {
-      // No handler, what should we do? Best to just kill the VC while we can.
-      netvc->do_io_close();
+      // Cancel the action, so later timeouts and errors don't try to
+      // send the event to the Accept object.  After this point, the accept
+      // object does not care.
+      netvc->set_action(nullptr);
+
+      Continuation *endpoint_cont = netvc->endpoint();
+      if (!endpoint_cont) {
+        // Route to the default endpoint
+        endpoint_cont = npnParent->endpoint;
+      }
+
+      if (endpoint_cont) {
+        // disable read io, send events to endpoint
+        netvc->do_io_read(endpoint_cont, 0, nullptr);
+
+        send_plugin_event(endpoint_cont, NET_EVENT_ACCEPT, netvc);
+      } else {
+        // No handler, what should we do? Best to just kill the VC while we can.
+        netvc->do_io_close();
+      }
     }
 
     delete this;
@@ -132,6 +139,7 @@ SSLNextProtocolAccept::mainEvent(int event, void *edata)
     ink_release_assert(netvc != nullptr);
 
     netvc->setTransparentPassThrough(transparent_passthrough);
+    netvc->setAllowPlain(allow_plain);
 
     // Register our protocol set with the VC and kick off a zero-length read to
     // force the SSLNetVConnection to complete the SSL handshake. Don't tell
@@ -167,11 +175,12 @@ SSLNextProtocolAccept::enableProtocols(const SessionProtocolSet &protos)
   this->protoenabled = protos;
 }
 
-SSLNextProtocolAccept::SSLNextProtocolAccept(Continuation *ep, bool transparent_passthrough)
+SSLNextProtocolAccept::SSLNextProtocolAccept(Continuation *ep, bool transparent_passthrough, bool allow_plain)
   : SessionAccept(nullptr),
     buffer(new_empty_MIOBuffer(SSLConfigParams::ssl_misc_max_iobuffer_size_index)),
     endpoint(ep),
-    transparent_passthrough(transparent_passthrough)
+    transparent_passthrough(transparent_passthrough),
+    allow_plain(allow_plain)
 {
   SET_HANDLER(&SSLNextProtocolAccept::mainEvent);
 }

proxy/ProtocolProbeSessionAccept.cc

@@ -170,8 +170,14 @@ ProtocolProbeSessionAccept::mainEvent(int event, void *data)
     ink_assert(data);
 
     VIO *vio;
-    NetVConnection *netvc          = static_cast<NetVConnection *>(data);
-    ProtocolProbeTrampoline *probe = new ProtocolProbeTrampoline(this, netvc->mutex, nullptr, nullptr);
+    NetVConnection *netvc = static_cast<NetVConnection *>(data);
+    ProtocolProbeTrampoline *probe;
+    UnixNetVConnection *unix_netvc = dynamic_cast<UnixNetVConnection *>(netvc);
+    if (unix_netvc != nullptr && unix_netvc->read.vio.get_writer() != nullptr) {
+      probe = new ProtocolProbeTrampoline(this, netvc->mutex, unix_netvc->read.vio.get_writer(), unix_netvc->read.vio.get_reader());
+    } else {
+      probe = new ProtocolProbeTrampoline(this, netvc->mutex, nullptr, nullptr);
+    }
 
     // The connection has completed, set the accept inactivity timeout here to watch over the difference between the
     // connection set up and the first transaction..

proxy/http/HttpProxyServerMain.cc

@@ -206,7 +206,7 @@ MakeHttpProxyAcceptor(HttpProxyAcceptor &acceptor, HttpProxyPort &port, unsigned
   ProtocolSessionCreateMap.insert({TS_ALPN_PROTOCOL_INDEX_HTTP_2_0, create_h2_server_session});
 
   if (port.isSSL()) {
-    SSLNextProtocolAccept *ssl = new SSLNextProtocolAccept(probe, port.m_transparent_passthrough);
+    SSLNextProtocolAccept *ssl = new SSLNextProtocolAccept(probe, port.m_transparent_passthrough, port.m_allow_plain);
 
     // ALPN selects the first server-offered protocol,
     // so make sure that we offer the newest protocol first.

src/records/RecHttp.cc

@@ -191,6 +191,7 @@ const char *const HttpProxyPort::OPT_TRANSPARENT_INBOUND     = "tr-in";
 const char *const HttpProxyPort::OPT_TRANSPARENT_OUTBOUND    = "tr-out";
 const char *const HttpProxyPort::OPT_TRANSPARENT_FULL        = "tr-full";
 const char *const HttpProxyPort::OPT_TRANSPARENT_PASSTHROUGH = "tr-pass";
+const char *const HttpProxyPort::OPT_ALLOW_PLAIN             = "allow-plain";
 const char *const HttpProxyPort::OPT_SSL                     = "ssl";
 const char *const HttpProxyPort::OPT_PROXY_PROTO             = "pp";
 const char *const HttpProxyPort::OPT_PLUGIN                  = "plugin";
@@ -450,6 +451,8 @@ HttpProxyPort::processOptions(const char *opts)
 #else
       Warning("Transparent pass-through requested [%s] in port descriptor '%s' but TPROXY was not configured.", item, opts);
 #endif
+    } else if (0 == strcasecmp(OPT_ALLOW_PLAIN, item)) {
+      m_allow_plain = true;
     } else if (0 == strcasecmp(OPT_MPTCP, item)) {
       if (mptcp_supported()) {
         m_mptcp = true;
@@ -500,6 +503,17 @@ HttpProxyPort::processOptions(const char *opts)
     m_transparent_passthrough = false;
   }
 
+  // Make sure QUIC is not enabled with incompatible options
+  if (this->isQUIC()) {
+    if (this->m_allow_plain) {
+      Warning("allow_plain incompatible with QUIC");
+      zret = false;
+    } else if (this->m_inbound_transparent_p || this->m_outbound_transparent_p) {
+      Warning("transparent mode not supported with QUIC");
+      zret = false;
+    }
+  }
+
   // Set the default session protocols.
   if (!sp_set_p) {
     if (this->isSSL()) {
@@ -655,6 +669,10 @@ HttpProxyPort::print(char *out, size_t n)
     zret += snprintf(out + zret, n - zret, ":%s", OPT_TRANSPARENT_PASSTHROUGH);
   }
 
+  if (m_allow_plain) {
+    zret += snprintf(out + zret, n - zret, ":%s", OPT_ALLOW_PLAIN);
+  }
+
   /* Don't print the IP resolution preferences if the port is outbound
    * transparent (which means the preference order is forced) or if
    * the order is the same as the default.