Enable tproxy to individual upstream endpoints

agent/proxycfg/snapshot.go

@@ -47,6 +47,10 @@ type ConfigSnapshotUpstreams struct {
 
 	// UpstreamConfig is a map to an upstream's configuration.
 	UpstreamConfig map[string]*structs.Upstream
+
+	// PassthroughEndpoints is a set of upstream addresses that transparent
+	// proxies can dial directly.
+	PassthroughEndpoints map[string]struct{}
 }
 
 type configSnapshotConnectProxy struct {
@@ -80,6 +84,7 @@ func (c *configSnapshotConnectProxy) IsEmpty() bool {
 		len(c.WatchedServiceChecks) == 0 &&
 		len(c.PreparedQueryEndpoints) == 0 &&
 		len(c.UpstreamConfig) == 0 &&
+		len(c.PassthroughEndpoints) == 0 &&
 		!c.MeshConfigSet
 }
 

agent/proxycfg/state.go

@@ -582,6 +582,7 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot {
 		snap.ConnectProxy.WatchedServiceChecks = make(map[structs.ServiceID][]structs.CheckType)
 		snap.ConnectProxy.PreparedQueryEndpoints = make(map[string]structs.CheckServiceNodes)
 		snap.ConnectProxy.UpstreamConfig = make(map[string]*structs.Upstream)
+		snap.ConnectProxy.PassthroughEndpoints = make(map[string]struct{})
 	case structs.ServiceKindTerminatingGateway:
 		snap.TerminatingGateway.WatchedServices = make(map[structs.ServiceName]context.CancelFunc)
 		snap.TerminatingGateway.WatchedIntentions = make(map[structs.ServiceName]context.CancelFunc)
@@ -931,6 +932,13 @@ func (s *state) handleUpdateUpstreams(u cache.UpdateEvent, snap *ConfigSnapshotU
 		}
 		snap.WatchedUpstreamEndpoints[svc][targetID] = resp.Nodes
 
+		for _, node := range resp.Nodes {
+			if node.Service.Proxy.TransparentProxy.DialedDirectly {
+				addr, _ := node.Service.BestAddress(false)
+				snap.PassthroughEndpoints[addr] = struct{}{}
+			}
+		}
+
 	case strings.HasPrefix(u.CorrelationID, "mesh-gateway:"):
 		resp, ok := u.Result.(*structs.IndexedNodesWithGateways)
 		if !ok {

agent/xds/clusters.go

@@ -96,20 +96,12 @@ func (s *ResourceGenerator) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.C
 	}
 	clusters = append(clusters, appCluster)
 
-	// In transparent proxy mode there needs to be a passthrough cluster for traffic going to destinations
-	// that aren't in Consul's catalog.
-	if cfgSnap.Proxy.Mode == structs.ProxyModeTransparent &&
-		(cfgSnap.ConnectProxy.MeshConfig == nil ||
-			!cfgSnap.ConnectProxy.MeshConfig.TransparentProxy.CatalogDestinationsOnly) {
-
-		clusters = append(clusters, &envoy_cluster_v3.Cluster{
-			Name: OriginalDestinationClusterName,
-			ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{
-				Type: envoy_cluster_v3.Cluster_ORIGINAL_DST,
-			},
-			LbPolicy:       envoy_cluster_v3.Cluster_CLUSTER_PROVIDED,
-			ConnectTimeout: ptypes.DurationProto(5 * time.Second),
-		})
+	if cfgSnap.Proxy.Mode == structs.ProxyModeTransparent {
+		passthroughs, err := makePassthroughClusters(cfgSnap)
+		if err != nil {
+			return nil, fmt.Errorf("failed to make passthrough clusters for transparent proxy: %v", err)
+		}
+		clusters = append(clusters, passthroughs...)
 	}
 
 	for id, chain := range cfgSnap.ConnectProxy.DiscoveryChain {
@@ -176,6 +168,52 @@ func makeExposeClusterName(destinationPort int) string {
 	return fmt.Sprintf("exposed_cluster_%d", destinationPort)
 }
 
+// In transparent proxy mode there are potentially two passthrough clusters added.
+// The first is for destinations inside the mesh, which require certificates for mTLS.
+// The second is for destinations outside of Consul's catalog. This is for a plain TCP proxy.
+// Both use Envoy's ORIGINAL_DST listener filter, which forwards to the original destination address
+// (before the iptables redirection).
+func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) {
+	clusters := make([]proto.Message, 0)
+
+	if len(cfgSnap.ConnectProxy.PassthroughEndpoints) > 0 {
+		c := envoy_cluster_v3.Cluster{
+			Name: MeshPassthroughClusterName,
+			ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{
+				Type: envoy_cluster_v3.Cluster_ORIGINAL_DST,
+			},
+			LbPolicy:       envoy_cluster_v3.Cluster_CLUSTER_PROVIDED,
+			ConnectTimeout: ptypes.DurationProto(5 * time.Second),
+		}
+
+		tlsContext := envoy_tls_v3.UpstreamTlsContext{
+			CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()),
+		}
+		transportSocket, err := makeUpstreamTLSTransportSocket(&tlsContext)
+		if err != nil {
+			return nil, err
+		}
+		c.TransportSocket = transportSocket
+		clusters = append(clusters, &c)
+	}
+
+	if cfgSnap.ConnectProxy.MeshConfigSet ||
+		(cfgSnap.ConnectProxy.MeshConfig == nil ||
+			!cfgSnap.ConnectProxy.MeshConfig.TransparentProxy.CatalogDestinationsOnly) {
+
+		clusters = append(clusters, &envoy_cluster_v3.Cluster{
+			Name: OriginalDestinationClusterName,
+			ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{
+				Type: envoy_cluster_v3.Cluster_ORIGINAL_DST,
+			},
+			LbPolicy:       envoy_cluster_v3.Cluster_CLUSTER_PROVIDED,
+			ConnectTimeout: ptypes.DurationProto(5 * time.Second),
+		})
+	}
+
+	return clusters, nil
+}
+
 // clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters"
 // for a mesh gateway. This will include 1 cluster per remote datacenter as well as
 // 1 cluster for each service subset.

agent/xds/listeners.go

@@ -156,35 +156,10 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
 		// For every potential address we collected, create the appropriate address prefix to match on.
 		// In this case we are matching on exact addresses, so the prefix is the address itself,
 		// and the prefix length is based on whether it's IPv4 or IPv6.
-		ranges := make([]*envoy_core_v3.CidrRange, 0)
-
-		for addr := range uniqueAddrs {
-			ip := net.ParseIP(addr)
-			if ip == nil {
-				continue
-			}
-
-			pfxLen := uint32(32)
-			if ip.To4() == nil {
-				pfxLen = 128
-			}
-			ranges = append(ranges, &envoy_core_v3.CidrRange{
-				AddressPrefix: addr,
-				PrefixLen:     &wrappers.UInt32Value{Value: pfxLen},
-			})
-		}
-
-		// The match rules are stable sorted to avoid draining if the list is provided out of order
-		sort.SliceStable(ranges, func(i, j int) bool {
-			return ranges[i].AddressPrefix < ranges[j].AddressPrefix
-		})
-
-		filterChain.FilterChainMatch = &envoy_listener_v3.FilterChainMatch{
-			PrefixRanges: ranges,
-		}
+		filterChain.FilterChainMatch = makeFilterChainMatchFromAddrs(uniqueAddrs)
 
 		// Only attach the filter chain if there are addresses to match on
-		if len(ranges) > 0 {
+		if filterChain.FilterChainMatch != nil && len(filterChain.FilterChainMatch.PrefixRanges) > 0 {
 			outboundListener.FilterChains = append(outboundListener.FilterChains, filterChain)
 		}
 		hasFilterChains = true
@@ -197,6 +172,26 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
 			return outboundListener.FilterChains[i].Name < outboundListener.FilterChains[j].Name
 		})
 
+		// Add a passthrough for every mesh endpoint that can be dialed directly,
+		// as opposed to via a virtual IP.
+		if len(cfgSnap.ConnectProxy.PassthroughEndpoints) > 0 {
+			filterChain, err := s.makeUpstreamFilterChainForDiscoveryChain(
+				MeshPassthroughClusterName,
+				MeshPassthroughClusterName,
+				"tcp",
+				nil,
+				nil,
+				cfgSnap,
+				nil,
+			)
+			if err != nil {
+				return nil, err
+			}
+			filterChain.FilterChainMatch = makeFilterChainMatchFromAddrs(cfgSnap.ConnectProxy.PassthroughEndpoints)
+
+			outboundListener.FilterChains = append(outboundListener.FilterChains, filterChain)
+		}
+
 		// Add a catch-all filter chain that acts as a TCP proxy to non-catalog destinations
 		if cfgSnap.ConnectProxy.MeshConfig == nil ||
 			!cfgSnap.ConnectProxy.MeshConfig.TransparentProxy.CatalogDestinationsOnly {
@@ -209,7 +204,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
 			}
 
 			filterChain, err := s.makeUpstreamFilterChainForDiscoveryChain(
-				"passthrough",
+				OriginalDestinationClusterName,
 				OriginalDestinationClusterName,
 				"tcp",
 				nil,
@@ -291,6 +286,35 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg.
 	return resources, nil
 }
 
+func makeFilterChainMatchFromAddrs(addrs map[string]struct{}) *envoy_listener_v3.FilterChainMatch {
+	ranges := make([]*envoy_core_v3.CidrRange, 0)
+
+	for addr := range addrs {
+		ip := net.ParseIP(addr)
+		if ip == nil {
+			continue
+		}
+
+		pfxLen := uint32(32)
+		if ip.To4() == nil {
+			pfxLen = 128
+		}
+		ranges = append(ranges, &envoy_core_v3.CidrRange{
+			AddressPrefix: addr,
+			PrefixLen:     &wrappers.UInt32Value{Value: pfxLen},
+		})
+	}
+
+	// The match rules are stable sorted to avoid draining if the list is provided out of order
+	sort.SliceStable(ranges, func(i, j int) bool {
+		return ranges[i].AddressPrefix < ranges[j].AddressPrefix
+	})
+
+	return &envoy_listener_v3.FilterChainMatch{
+		PrefixRanges: ranges,
+	}
+}
+
 func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) {
 	var path structs.ExposePath
 

agent/xds/server.go

@@ -96,10 +96,16 @@ const (
 
 	// OriginalDestinationClusterName is the name we give to the passthrough
 	// cluster which redirects transparently-proxied requests to their original
-	// destination. This cluster prevents Consul from blocking connections to
-	// destinations outside of the catalog when in transparent proxy mode.
+	// destination outside the mesh. This cluster prevents Consul from blocking
+	// connections to destinations outside of the catalog when in transparent
+	// proxy mode.
 	OriginalDestinationClusterName = "original-destination"
 
+	// MeshPassthroughClusterName is the name we give to the passthrough
+	// cluster which redirects transparently-proxied requests to endpoints
+	// in the mesh. For this cluster we present a client certificate.
+	MeshPassthroughClusterName = "mesh-passthrough"
+
 	// DefaultAuthCheckFrequency is the default value for
 	// Server.AuthCheckFrequency to use when the zero value is provided.
 	DefaultAuthCheckFrequency = 5 * time.Minute