SNI: Add support to match/replace captured group from fqdn into tunnel_route.

Author: Damian Meden

doc/admin-guide/files/sni.yaml.en.rst

@@ -110,6 +110,8 @@ disable_h2                Deprecated for the more general h2 setting.  Setting d
                           to :code:`true` is the same as setting http2 to :code:`on`.
 
 tunnel_route              Destination as an FQDN and port, separated by a colon ``:``.
+                          Match group number can be specified by ``$N`` where N should refer to a specified group in
+                          the FQDN, ``tunnel_route: $1.domain``.
 
                           This will forward all traffic to the specified destination without first terminating
                           the incoming TLS connection.
@@ -212,6 +214,21 @@ client certificate.
      verify_server_policy: DISABLED
      verify_client: STRICT
 
+Use FQDN captured group to match in ``tunnel_route``.
+
+.. code-block:: yaml
+
+   sni:
+   - fqdn: '*.foo.com'
+     tunnel_route: '$1.myfoo'
+   - fqdn: '*.bar.*.com'
+     tunnel_route: '$2.some.$1.yahoo'
+
+FQDN ``some.foo.com`` will match and the captured string will be replaced in the ``tunnel_route`` which will end up being
+``some.myfoo``.
+Second part is using multiple groups, having ``bob.bar.example.com`` as FQDN, ``tunnel_route`` will end up being
+``bar.some.example.yahoo``.
+
 See Also
 ========
 

iocore/net/P_SNIActionPerformer.h

@@ -38,7 +38,18 @@
 class ActionItem
 {
 public:
-  virtual int SNIAction(Continuation *cont) const = 0;
+  /**
+   * Context should contain extra data needed to be passed to the actual SNIAction.
+   */
+  struct Context {
+    /**
+     * if any, fqdn_wildcard_captured_groups will hold the captured groups from the `fqdn`
+     * match which will be used to construct the tunnel destination.
+     */
+    std::optional<std::vector<std::string>> _fqdn_wildcard_captured_groups;
+  };
+
+  virtual int SNIAction(Continuation *cont, const Context &ctx) const = 0;
 
   /**
     This method tests whether this action would have been triggered by a
@@ -61,7 +72,7 @@ class ControlH2 : public ActionItem
   ~ControlH2() override {}
 
   int
-  SNIAction(Continuation *cont) const override
+  SNIAction(Continuation *cont, const Context &ctx) const override
   {
     auto ssl_vc = dynamic_cast<SSLNetVConnection *>(cont);
     if (ssl_vc) {
@@ -81,21 +92,98 @@ class ControlH2 : public ActionItem
 class TunnelDestination : public ActionItem
 {
 public:
-  TunnelDestination(const std::string_view &dest, bool decrypt) : destination(dest), tunnel_decrypt(decrypt) {}
+  TunnelDestination(const std::string_view &dest, bool decrypt) : destination(dest), tunnel_decrypt(decrypt)
+  {
+    need_fix = (destination.find_first_of('$') != std::string::npos);
+  }
   ~TunnelDestination() override {}
 
   int
-  SNIAction(Continuation *cont) const override
+  SNIAction(Continuation *cont, const Context &ctx) const override
   {
     // Set the netvc option?
     SSLNetVConnection *ssl_netvc = dynamic_cast<SSLNetVConnection *>(cont);
     if (ssl_netvc) {
-      ssl_netvc->set_tunnel_destination(destination, tunnel_decrypt);
+      // If needed, we will try to amend the tunnel destination.
+      if (ctx._fqdn_wildcard_captured_groups && need_fix) {
+        const auto &fixed_dst = replace_match_groups(destination, *ctx._fqdn_wildcard_captured_groups);
+        ssl_netvc->set_tunnel_destination(fixed_dst, tunnel_decrypt);
+        Debug("TunnelDestination", "Destination now is [%s], configured [%s]", fixed_dst.c_str(), destination.c_str());
+      } else {
+        ssl_netvc->set_tunnel_destination(destination, tunnel_decrypt);
+      }
     }
     return SSL_TLSEXT_ERR_OK;
   }
   std::string destination;
   bool tunnel_decrypt = false;
+
+private:
+  bool
+  is_number(const std::string &s) const
+  {
+    return !s.empty() &&
+           std::find_if(std::begin(s), std::end(s), [](std::string::value_type c) { return !std::isdigit(c); }) == std::end(s);
+  }
+
+  /**
+   * `tunnel_route` may contain matching groups ie: `$1` which needs to be replaced by the corresponding
+   * captured group from the `fqdn`, this function will replace them using proper group string. Matching
+   * groups could be at any order.
+   */
+  std::string
+  replace_match_groups(const std::string &dst, const std::vector<std::string> &groups) const
+  {
+    if (dst.empty() || groups.empty()) {
+      return dst;
+    }
+    std::string real_dst;
+    std::string::size_type pos{0};
+
+    const auto end = std::end(dst);
+    // We need to split the tunnel string and place each corresponding match on the
+    // configured one, so we need to first, get the match, then get the match number
+    // making sure that it does exist in the captured group.
+    for (auto c = std::begin(dst); c != end; c++, pos++) {
+      if (*c == '$') {
+        // find the next '.' so we can get the group number.
+        const auto dot            = dst.find('.', pos);
+        std::string::size_type to = std::string::npos;
+        if (dot != std::string::npos) {
+          to = dot - (pos + 1);
+        } else {
+          // It may not have a dot, which could be because it's the last part. In that case
+          // we should check for the port separator.
+          if (const auto port = dst.find(':', pos); port != std::string::npos) {
+            to = (port - pos) - 1;
+          }
+        }
+        const auto &number_str = dst.substr(pos + 1, to);
+        if (!is_number(number_str)) {
+          // it may be some issue on the configured string, place the char and keep going.
+          real_dst += *c;
+          continue;
+        }
+        const std::size_t group_index = std::stoi(number_str);
+        if ((group_index - 1) < groups.size()) {
+          // place the captured group.
+          real_dst += groups[group_index - 1];
+          // if it was the last match, then ...
+          if (dot == std::string::npos && to == std::string::npos) {
+            // that's it.
+            break;
+          }
+          pos += number_str.size() + 1;
+          std::advance(c, number_str.size() + 1);
+        }
+      }
+      real_dst += *c;
+    }
+
+    return real_dst;
+  }
+
+  bool need_fix;
 };
 
 class VerifyClient : public ActionItem
@@ -107,7 +195,7 @@ class VerifyClient : public ActionItem
   VerifyClient(uint8_t param) : mode(param) {}
   ~VerifyClient() override {}
   int
-  SNIAction(Continuation *cont) const override
+  SNIAction(Continuation *cont, const Context &ctx) const override
   {
     auto ssl_vc = dynamic_cast<SSLNetVConnection *>(cont);
     Debug("ssl_sni", "action verify param %d", this->mode);
@@ -131,7 +219,7 @@ class HostSniPolicy : public ActionItem
   HostSniPolicy(uint8_t param) : policy(param) {}
   ~HostSniPolicy() override {}
   int
-  SNIAction(Continuation *cont) const override
+  SNIAction(Continuation *cont, const Context &ctx) const override
   {
     // On action this doesn't do anything
     return SSL_TLSEXT_ERR_OK;
@@ -160,7 +248,7 @@ class TLSValidProtocols : public ActionItem
   TLSValidProtocols() : protocol_mask(max_mask) {}
   TLSValidProtocols(unsigned long protocols) : unset(false), protocol_mask(protocols) {}
   int
-  SNIAction(Continuation *cont) const override
+  SNIAction(Continuation *cont, const Context &ctx) const override
   {
     if (!unset) {
       auto ssl_vc = dynamic_cast<SSLNetVConnection *>(cont);
@@ -205,7 +293,7 @@ class SNI_IpAllow : public ActionItem
   } // end function SNI_IpAllow
 
   int
-  SNIAction(Continuation *cont) const override
+  SNIAction(Continuation *cont, const Context &ctx) const override
   {
     // i.e, ip filtering is not required
     if (ip_map.count() == 0) {

iocore/net/P_SSLSNI.h

@@ -66,7 +66,7 @@ struct namedElement {
     }
     pos = 0;
     while ((pos = name.find('*', pos)) != std::string::npos) {
-      name.replace(pos, 1, ".{0,}");
+      name.replace(pos, 1, "(.{0,})");
     }
     Debug("ssl_sni", "Regexed fqdn=%s", name.c_str());
     setRegexName(name);
@@ -111,7 +111,7 @@ struct SNIConfigParams : public ConfigInfo {
   void cleanup();
   int Initialize();
   void loadSNIConfig();
-  const actionVector *get(const std::string &servername) const;
+  std::pair<const actionVector *, ActionItem::Context> get(const std::string &servername) const;
 };
 
 struct SNIConfig {

iocore/net/SSLSNIConfig.cc

@@ -40,6 +40,7 @@
 #include <pcre.h>
 
 static ConfigUpdateHandler<SNIConfig> *sniConfigUpdate;
+static constexpr int OVECSIZE{30};
 
 const NextHopProperty *
 SNIConfigParams::getPropertyConfig(const std::string &servername) const
@@ -108,20 +109,42 @@ int SNIConfig::configid = 0;
 /*definition of member functions of SNIConfigParams*/
 SNIConfigParams::SNIConfigParams() {}
 
-const actionVector *
+std::pair<const actionVector *, ActionItem::Context>
 SNIConfigParams::get(const std::string &servername) const
 {
-  int ovector[2];
+  int ovector[OVECSIZE];
+  ActionItem::Context context;
+
   for (const auto &retval : sni_action_list) {
     int length = servername.length();
     if (retval.match == nullptr && length == 0) {
-      return &retval.actions;
-    } else if (pcre_exec(retval.match, nullptr, servername.c_str(), length, 0, 0, ovector, 2) == 1 && ovector[0] == 0 &&
-               ovector[1] == length) {
-      return &retval.actions;
+      return {&retval.actions, context};
+    } else if (auto offset = pcre_exec(retval.match, nullptr, servername.c_str(), length, 0, 0, ovector, OVECSIZE); offset >= 0) {
+      if (offset == 1 && ovector[1] == length) {
+        // full match
+        return {&retval.actions, context};
+      }
+      // If contains groups
+      if (offset == 0) {
+        // reset to max if too many.
+        offset = OVECSIZE / 3;
+      }
+
+      const char *psubStrMatchStr = nullptr;
+      std::vector<std::string> groups;
+      for (int strnum = 1; strnum < offset; strnum++) {
+        pcre_get_substring(servername.c_str(), ovector, offset, strnum, &(psubStrMatchStr));
+        groups.emplace_back(psubStrMatchStr);
+      }
+      context._fqdn_wildcard_captured_groups = std::move(groups);
+      if (psubStrMatchStr) {
+        pcre_free_substring(psubStrMatchStr);
+      }
+
+      return {&retval.actions, context};
     }
   }
-  return nullptr;
+  return {nullptr, context};
 }
 
 int
@@ -197,9 +220,10 @@ SNIConfig::TestClientAction(const char *servername, const IpEndpoint &ep, int &h
 {
   bool retval = false;
   SNIConfig::scoped_config params;
-  const actionVector *actionvec = params->get(servername);
-  if (actionvec) {
-    for (auto &&item : *actionvec) {
+
+  const auto &actions = params->get(servername);
+  if (actions.first) {
+    for (auto &&item : *actions.first) {
       retval |= item->TestClientSNIAction(servername, ep, host_sni_policy);
     }
   }

iocore/net/SSLUtils.cc

@@ -376,12 +376,11 @@ static int
 PerformAction(Continuation *cont, const char *servername)
 {
   SNIConfig::scoped_config params;
-  const actionVector *actionvec = params->get(servername);
-  if (!actionvec) {
+  if (const auto &actions = params->get(servername); !actions.first) {
     Debug("ssl_sni", "%s not available in the map", servername);
   } else {
-    for (auto &&item : *actionvec) {
-      auto ret = item->SNIAction(cont);
+    for (auto &&item : *actions.first) {
+      auto ret = item->SNIAction(cont, actions.second);
       if (ret != SSL_TLSEXT_ERR_OK) {
         return ret;
       }

tests/gold_tests/tls/tls_tunnel.test.py

@@ -26,6 +26,8 @@
 server_foo = Test.MakeOriginServer("server_foo", ssl=True)
 server_bar = Test.MakeOriginServer("server_bar", ssl=True)
 server2 = Test.MakeOriginServer("server2")
+#dns = Test.MakeDNServer("dns", default=['127.0.0.1'])
+dns = Test.MakeDNServer("dns")
 
 request_foo_header = {"headers": "GET / HTTP/1.1\r\nHost: foo.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
 request_bar_header = {"headers": "GET / HTTP/1.1\r\nHost: bar.com\r\n\r\n", "timestamp": "1469733493.993", "body": ""}
@@ -44,6 +46,9 @@
 ts.addSSLfile("ssl/signer.pem")
 ts.addSSLfile("ssl/signer.key")
 
+dns.addRecords(records={"localhost": ["127.0.0.1"]})
+dns.addRecords(records={"one.testmatch": ["127.0.0.1"]})
+dns.addRecords(records={"two.example.one": ["127.0.0.1"]})
 # Need no remap rules.  Everything should be proccessed by sni
 
 # Make sure the TS server certs are different from the origin certs
@@ -61,7 +66,9 @@
     'proxy.config.ssl.client.CA.cert.path': '{0}'.format(ts.Variables.SSLDir),
     'proxy.config.ssl.client.CA.cert.filename': 'signer.pem',
     'proxy.config.exec_thread.autoconfig.scale': 1.0,
-    'proxy.config.url_remap.pristine_host_hdr': 1
+    'proxy.config.url_remap.pristine_host_hdr': 1,
+    'proxy.config.dns.nameservers': '127.0.0.1:{0}'.format(dns.Variables.Port),
+    'proxy.config.dns.resolv_conf': 'NULL'
 })
 
 # foo.com should not terminate.  Just tunnel to server_foo
@@ -73,6 +80,10 @@
   "  tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port),
   "- fqdn: bob.*.com",
   "  tunnel_route: localhost:{0}".format(server_foo.Variables.SSL_Port),
+  "- fqdn: '*.match.com'",
+  "  tunnel_route: $1.testmatch:{0}".format(server_foo.Variables.SSL_Port),
+  "- fqdn: '*.ok.*.com'",
+  "  tunnel_route: $2.example.$1:{0}".format(server_foo.Variables.SSL_Port),
   "- fqdn: ''", # No SNI sent
   "  tunnel_route: localhost:{0}".format(server_bar.Variables.SSL_Port)
 ])
@@ -82,6 +93,7 @@
 tr.ReturnCode = 0
 tr.Processes.Default.StartBefore(server_foo)
 tr.Processes.Default.StartBefore(server_bar)
+tr.Processes.Default.StartBefore(dns)
 tr.Processes.Default.StartBefore(Test.Processes.ts)
 tr.StillRunningAfter = ts
 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded")
@@ -120,6 +132,31 @@
 tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server")
 tr.Processes.Default.Streams.All += Testers.ContainsExpression("bar ok", "Should get a response from bar")
 
+
+tr = Test.AddTestRun("one.match.com Tunnel-test")
+tr.Processes.Default.Command = "curl -vvv --resolve 'one.match.com:{0}:127.0.0.1' -k  https://one.match.com:{0}".format(ts.Variables.ssl_port)
+tr.ReturnCode = 0
+tr.StillRunningAfter = ts
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded")
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server")
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server")
+tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response")
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server")
+tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from tm")
+
+
+tr = Test.AddTestRun("one.ok.two.com Tunnel-test")
+tr.Processes.Default.Command = "curl -vvv --resolve 'one.ok.two.com:{0}:127.0.0.1' -k  https:/one.ok.two.com:{0}".format(ts.Variables.ssl_port)
+tr.ReturnCode = 0
+tr.StillRunningAfter = ts
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Could Not Connect", "Curl attempt should have succeeded")
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("Not Found on Accelerato", "Should not try to remap on Traffic Server")
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("CN=foo.com", "Should not TLS terminate on Traffic Server")
+tr.Processes.Default.Streams.All += Testers.ContainsExpression("HTTP/1.1 200 OK", "Should get a successful response")
+tr.Processes.Default.Streams.All += Testers.ExcludesExpression("ATS", "Do not terminate on Traffic Server")
+tr.Processes.Default.Streams.All += Testers.ContainsExpression("foo ok", "Should get a response from tm")
+
+
 # Update sni file and reload
 tr = Test.AddTestRun("Update config files")
 # Update the SNI config