From 2477b731b391badeb4a4f0614a8a4d37c8b2379f Mon Sep 17 00:00:00 2001 From: freddygv Date: Tue, 1 Jun 2021 11:07:25 -0400 Subject: [PATCH] Enable tproxy to individual upstream endpoints --- agent/proxycfg/snapshot.go | 5 +++ agent/proxycfg/state.go | 8 ++++ agent/xds/clusters.go | 66 ++++++++++++++++++++++++------- agent/xds/listeners.go | 80 +++++++++++++++++++++++++------------- agent/xds/server.go | 10 ++++- 5 files changed, 125 insertions(+), 44 deletions(-) diff --git a/agent/proxycfg/snapshot.go b/agent/proxycfg/snapshot.go index c7674186a4a2..9b437a5d8244 100644 --- a/agent/proxycfg/snapshot.go +++ b/agent/proxycfg/snapshot.go @@ -47,6 +47,10 @@ type ConfigSnapshotUpstreams struct { // UpstreamConfig is a map to an upstream's configuration. UpstreamConfig map[string]*structs.Upstream + + // PassthroughEndpoints is a set of upstream addresses that transparent + // proxies can dial directly. + PassthroughEndpoints map[string]struct{} } type configSnapshotConnectProxy struct { @@ -80,6 +84,7 @@ func (c *configSnapshotConnectProxy) IsEmpty() bool { len(c.WatchedServiceChecks) == 0 && len(c.PreparedQueryEndpoints) == 0 && len(c.UpstreamConfig) == 0 && + len(c.PassthroughEndpoints) == 0 && !c.MeshConfigSet } diff --git a/agent/proxycfg/state.go b/agent/proxycfg/state.go index f53f46086d46..ae50b14018e6 100644 --- a/agent/proxycfg/state.go +++ b/agent/proxycfg/state.go @@ -582,6 +582,7 @@ func (s *state) initialConfigSnapshot() ConfigSnapshot { snap.ConnectProxy.WatchedServiceChecks = make(map[structs.ServiceID][]structs.CheckType) snap.ConnectProxy.PreparedQueryEndpoints = make(map[string]structs.CheckServiceNodes) snap.ConnectProxy.UpstreamConfig = make(map[string]*structs.Upstream) + snap.ConnectProxy.PassthroughEndpoints = make(map[string]struct{}) case structs.ServiceKindTerminatingGateway: snap.TerminatingGateway.WatchedServices = make(map[structs.ServiceName]context.CancelFunc) snap.TerminatingGateway.WatchedIntentions = make(map[structs.ServiceName]context.CancelFunc) @@ -931,6 +932,13 @@ func (s *state) handleUpdateUpstreams(u cache.UpdateEvent, snap *ConfigSnapshotU } snap.WatchedUpstreamEndpoints[svc][targetID] = resp.Nodes + for _, node := range resp.Nodes { + if node.Service.Proxy.TransparentProxy.DialedDirectly { + addr, _ := node.Service.BestAddress(false) + snap.PassthroughEndpoints[addr] = struct{}{} + } + } + case strings.HasPrefix(u.CorrelationID, "mesh-gateway:"): resp, ok := u.Result.(*structs.IndexedNodesWithGateways) if !ok { diff --git a/agent/xds/clusters.go b/agent/xds/clusters.go index d32fc57a473e..f487e4c6388b 100644 --- a/agent/xds/clusters.go +++ b/agent/xds/clusters.go @@ -96,20 +96,12 @@ func (s *ResourceGenerator) clustersFromSnapshotConnectProxy(cfgSnap *proxycfg.C } clusters = append(clusters, appCluster) - // In transparent proxy mode there needs to be a passthrough cluster for traffic going to destinations - // that aren't in Consul's catalog. - if cfgSnap.Proxy.Mode == structs.ProxyModeTransparent && - (cfgSnap.ConnectProxy.MeshConfig == nil || - !cfgSnap.ConnectProxy.MeshConfig.TransparentProxy.CatalogDestinationsOnly) { - - clusters = append(clusters, &envoy_cluster_v3.Cluster{ - Name: OriginalDestinationClusterName, - ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{ - Type: envoy_cluster_v3.Cluster_ORIGINAL_DST, - }, - LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, - ConnectTimeout: ptypes.DurationProto(5 * time.Second), - }) + if cfgSnap.Proxy.Mode == structs.ProxyModeTransparent { + passthroughs, err := makePassthroughClusters(cfgSnap) + if err != nil { + return nil, fmt.Errorf("failed to make passthrough clusters for transparent proxy: %v", err) + } + clusters = append(clusters, passthroughs...) } for id, chain := range cfgSnap.ConnectProxy.DiscoveryChain { @@ -176,6 +168,52 @@ func makeExposeClusterName(destinationPort int) string { return fmt.Sprintf("exposed_cluster_%d", destinationPort) } +// In transparent proxy mode there are potentially two passthrough clusters added. +// The first is for destinations inside the mesh, which require certificates for mTLS. +// The second is for destinations outside of Consul's catalog. This is for a plain TCP proxy. +// Both use Envoy's ORIGINAL_DST listener filter, which forwards to the original destination address +// (before the iptables redirection). +func makePassthroughClusters(cfgSnap *proxycfg.ConfigSnapshot) ([]proto.Message, error) { + clusters := make([]proto.Message, 0) + + if len(cfgSnap.ConnectProxy.PassthroughEndpoints) > 0 { + c := envoy_cluster_v3.Cluster{ + Name: MeshPassthroughClusterName, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{ + Type: envoy_cluster_v3.Cluster_ORIGINAL_DST, + }, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ConnectTimeout: ptypes.DurationProto(5 * time.Second), + } + + tlsContext := envoy_tls_v3.UpstreamTlsContext{ + CommonTlsContext: makeCommonTLSContextFromLeaf(cfgSnap, cfgSnap.Leaf()), + } + transportSocket, err := makeUpstreamTLSTransportSocket(&tlsContext) + if err != nil { + return nil, err + } + c.TransportSocket = transportSocket + clusters = append(clusters, &c) + } + + if cfgSnap.ConnectProxy.MeshConfigSet || + (cfgSnap.ConnectProxy.MeshConfig == nil || + !cfgSnap.ConnectProxy.MeshConfig.TransparentProxy.CatalogDestinationsOnly) { + + clusters = append(clusters, &envoy_cluster_v3.Cluster{ + Name: OriginalDestinationClusterName, + ClusterDiscoveryType: &envoy_cluster_v3.Cluster_Type{ + Type: envoy_cluster_v3.Cluster_ORIGINAL_DST, + }, + LbPolicy: envoy_cluster_v3.Cluster_CLUSTER_PROVIDED, + ConnectTimeout: ptypes.DurationProto(5 * time.Second), + }) + } + + return clusters, nil +} + // clustersFromSnapshotMeshGateway returns the xDS API representation of the "clusters" // for a mesh gateway. This will include 1 cluster per remote datacenter as well as // 1 cluster for each service subset. diff --git a/agent/xds/listeners.go b/agent/xds/listeners.go index c31f0fd6c59c..d84af71425d3 100644 --- a/agent/xds/listeners.go +++ b/agent/xds/listeners.go @@ -156,35 +156,10 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. // For every potential address we collected, create the appropriate address prefix to match on. // In this case we are matching on exact addresses, so the prefix is the address itself, // and the prefix length is based on whether it's IPv4 or IPv6. - ranges := make([]*envoy_core_v3.CidrRange, 0) - - for addr := range uniqueAddrs { - ip := net.ParseIP(addr) - if ip == nil { - continue - } - - pfxLen := uint32(32) - if ip.To4() == nil { - pfxLen = 128 - } - ranges = append(ranges, &envoy_core_v3.CidrRange{ - AddressPrefix: addr, - PrefixLen: &wrappers.UInt32Value{Value: pfxLen}, - }) - } - - // The match rules are stable sorted to avoid draining if the list is provided out of order - sort.SliceStable(ranges, func(i, j int) bool { - return ranges[i].AddressPrefix < ranges[j].AddressPrefix - }) - - filterChain.FilterChainMatch = &envoy_listener_v3.FilterChainMatch{ - PrefixRanges: ranges, - } + filterChain.FilterChainMatch = makeFilterChainMatchFromAddrs(uniqueAddrs) // Only attach the filter chain if there are addresses to match on - if len(ranges) > 0 { + if filterChain.FilterChainMatch != nil && len(filterChain.FilterChainMatch.PrefixRanges) > 0 { outboundListener.FilterChains = append(outboundListener.FilterChains, filterChain) } hasFilterChains = true @@ -197,6 +172,26 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. return outboundListener.FilterChains[i].Name < outboundListener.FilterChains[j].Name }) + // Add a passthrough for every mesh endpoint that can be dialed directly, + // as opposed to via a virtual IP. + if len(cfgSnap.ConnectProxy.PassthroughEndpoints) > 0 { + filterChain, err := s.makeUpstreamFilterChainForDiscoveryChain( + MeshPassthroughClusterName, + MeshPassthroughClusterName, + "tcp", + nil, + nil, + cfgSnap, + nil, + ) + if err != nil { + return nil, err + } + filterChain.FilterChainMatch = makeFilterChainMatchFromAddrs(cfgSnap.ConnectProxy.PassthroughEndpoints) + + outboundListener.FilterChains = append(outboundListener.FilterChains, filterChain) + } + // Add a catch-all filter chain that acts as a TCP proxy to non-catalog destinations if cfgSnap.ConnectProxy.MeshConfig == nil || !cfgSnap.ConnectProxy.MeshConfig.TransparentProxy.CatalogDestinationsOnly { @@ -209,7 +204,7 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. } filterChain, err := s.makeUpstreamFilterChainForDiscoveryChain( - "passthrough", + OriginalDestinationClusterName, OriginalDestinationClusterName, "tcp", nil, @@ -291,6 +286,35 @@ func (s *ResourceGenerator) listenersFromSnapshotConnectProxy(cfgSnap *proxycfg. return resources, nil } +func makeFilterChainMatchFromAddrs(addrs map[string]struct{}) *envoy_listener_v3.FilterChainMatch { + ranges := make([]*envoy_core_v3.CidrRange, 0) + + for addr := range addrs { + ip := net.ParseIP(addr) + if ip == nil { + continue + } + + pfxLen := uint32(32) + if ip.To4() == nil { + pfxLen = 128 + } + ranges = append(ranges, &envoy_core_v3.CidrRange{ + AddressPrefix: addr, + PrefixLen: &wrappers.UInt32Value{Value: pfxLen}, + }) + } + + // The match rules are stable sorted to avoid draining if the list is provided out of order + sort.SliceStable(ranges, func(i, j int) bool { + return ranges[i].AddressPrefix < ranges[j].AddressPrefix + }) + + return &envoy_listener_v3.FilterChainMatch{ + PrefixRanges: ranges, + } +} + func parseCheckPath(check structs.CheckType) (structs.ExposePath, error) { var path structs.ExposePath diff --git a/agent/xds/server.go b/agent/xds/server.go index 259b01437052..19472c1371ca 100644 --- a/agent/xds/server.go +++ b/agent/xds/server.go @@ -96,10 +96,16 @@ const ( // OriginalDestinationClusterName is the name we give to the passthrough // cluster which redirects transparently-proxied requests to their original - // destination. This cluster prevents Consul from blocking connections to - // destinations outside of the catalog when in transparent proxy mode. + // destination outside the mesh. This cluster prevents Consul from blocking + // connections to destinations outside of the catalog when in transparent + // proxy mode. OriginalDestinationClusterName = "original-destination" + // MeshPassthroughClusterName is the name we give to the passthrough + // cluster which redirects transparently-proxied requests to endpoints + // in the mesh. For this cluster we present a client certificate. + MeshPassthroughClusterName = "mesh-passthrough" + // DefaultAuthCheckFrequency is the default value for // Server.AuthCheckFrequency to use when the zero value is provided. DefaultAuthCheckFrequency = 5 * time.Minute