[1.10] Generate tunneling cluster once per upstream (#7425)

* cherry-pick multiple routes fix

* remove unneeded cherry pick artifact
TLS from Envoy to proxy not supported in this version

* move changelog

changelog/v1.10.41/tunneling-upstream-tls.yaml

@@ -0,0 +1,7 @@
+changelog:
+- type: FIX
+  issueLink: https://github.com/solo-io/gloo/issues/7313
+  resolvesIssue: true
+  description: >-
+    Fix for issue where multiple routes defined for the same upstream would remove the UpstreamTlsContext associated with the tunneling cluster
+    and all requests would be sent through the tunnel as http.

projects/gloo/pkg/plugins/tunneling/plugin.go

@@ -11,6 +11,7 @@ import (
 	"github.com/solo-io/gloo/projects/gloo/pkg/plugins"
 	"github.com/solo-io/gloo/projects/gloo/pkg/translator"
 	"github.com/solo-io/gloo/projects/gloo/pkg/utils"
+	"k8s.io/apimachinery/pkg/util/sets"
 )
 
 var (
@@ -48,6 +49,9 @@ func (p *plugin) GeneratedResources(params plugins.Params,
 
 	upstreams := params.Snapshot.Upstreams
 
+	// keep track of clusters we've seen in case of multiple routes to same cluster
+	processedClusters := sets.NewString()
+
 	// find all the route config that points to upstreams with tunneling
 	for _, rtConfig := range inRouteConfigurations {
 		for _, vh := range rtConfig.GetVirtualHosts() {
@@ -70,6 +74,7 @@ func (p *plugin) GeneratedResources(params plugins.Params,
 						return generatedClusters, nil, nil, generatedListeners, nil
 					}
 
+					// the existence of this value is our indicator that this is a tunneling upstream
 					tunnelingHostname := us.GetHttpProxyHostname().GetValue()
 					if tunnelingHostname == "" {
 						continue
@@ -81,6 +86,10 @@ func (p *plugin) GeneratedResources(params plugins.Params,
 					// update the old cluster to route to ourselves first
 					rtAction.ClusterSpecifier = &envoy_config_route_v3.RouteAction_Cluster{Cluster: selfCluster}
 
+					// we only want to generate a new encapsulating cluster and pipe to ourselves if we have not done so already
+					if processedClusters.Has(cluster) {
+						continue
+					}
 					var originalTransportSocket *envoy_config_core_v3.TransportSocket
 					for _, inCluster := range inClusters {
 						if inCluster.GetName() == cluster {
@@ -97,6 +106,7 @@ func (p *plugin) GeneratedResources(params plugins.Params,
 
 					generatedClusters = append(generatedClusters, generateSelfCluster(selfCluster, selfPipe, originalTransportSocket))
 					generatedListeners = append(generatedListeners, generateForwardingTcpListener(cluster, selfPipe, tunnelingHostname))
+					processedClusters.Insert(cluster)
 				}
 			}
 		}

projects/gloo/pkg/plugins/tunneling/plugin_test.go

@@ -6,6 +6,7 @@ import (
 	envoy_config_endpoint_v3 "github.com/envoyproxy/go-control-plane/envoy/config/endpoint/v3"
 	envoy_config_route_v3 "github.com/envoyproxy/go-control-plane/envoy/config/route/v3"
 	envoytcp "github.com/envoyproxy/go-control-plane/envoy/extensions/filters/network/tcp_proxy/v3"
+	envoyauth "github.com/envoyproxy/go-control-plane/envoy/extensions/transport_sockets/tls/v3"
 	"github.com/golang/protobuf/ptypes/wrappers"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -19,6 +20,10 @@ import (
 	"google.golang.org/protobuf/proto"
 )
 
+const (
+	httpProxyHostname string = "host.com:443"
+)
+
 var _ = Describe("Plugin", func() {
 
 	var (
@@ -32,7 +37,7 @@ var _ = Describe("Plugin", func() {
 				Namespace: "gloo-system",
 			},
 			SslConfig:         nil,
-			HttpProxyHostname: &wrappers.StringValue{Value: "host.com:443"},
+			HttpProxyHostname: &wrappers.StringValue{Value: httpProxyHostname},
 		}
 	)
 
@@ -53,6 +58,7 @@ var _ = Describe("Plugin", func() {
 						Domains: []string{"*"},
 						Routes: []*envoy_config_route_v3.Route{
 							{
+								Name: "testroute",
 								Match: &envoy_config_route_v3.RouteMatch{
 									PathSpecifier: &envoy_config_route_v3.RouteMatch_Prefix{
 										Prefix: "/",
@@ -72,11 +78,13 @@ var _ = Describe("Plugin", func() {
 			},
 		}
 
+		// use UpstreamToClusterName to emulate a real translation loop.
+		clusterName := translator.UpstreamToClusterName(us.Metadata.Ref())
 		inClusters = []*envoy_config_cluster_v3.Cluster{
 			{
-				Name: "http_proxy",
+				Name: clusterName,
 				LoadAssignment: &envoy_config_endpoint_v3.ClusterLoadAssignment{
-					ClusterName: "http_proxy",
+					ClusterName: clusterName,
 					Endpoints: []*envoy_config_endpoint_v3.LocalityLbEndpoints{
 						{
 							LbEndpoints: []*envoy_config_endpoint_v3.LbEndpoint{
@@ -128,6 +136,44 @@ var _ = Describe("Plugin", func() {
 		Expect(typedTcpConfig.GetCluster()).To(Equal(originalCluster), "should forward to original destination")
 	})
 
+	Context("UpstreamTlsContext", func() {
+		BeforeEach(func() {
+			// add an UpstreamTlsContext
+			cfg, err := utils.MessageToAny(&envoyauth.UpstreamTlsContext{
+				CommonTlsContext: &envoyauth.CommonTlsContext{},
+				Sni:              httpProxyHostname,
+			})
+			Expect(err).ToNot(HaveOccurred())
+			inClusters[0].TransportSocket = &envoy_config_core_v3.TransportSocket{
+				Name: "",
+				ConfigType: &envoy_config_core_v3.TransportSocket_TypedConfig{
+					TypedConfig: cfg,
+				},
+			}
+
+			inRoute := inRouteConfigurations[0].VirtualHosts[0].Routes[0]
+
+			// update route input with duplicate route, the duplicate points to the same upstream as existing
+			dupRoute := proto.Clone(inRoute).(*envoy_config_route_v3.Route)
+			dupRoute.Name = dupRoute.Name + "-duplicate"
+			dupRoute.Action = &envoy_config_route_v3.Route_Route{
+				Route: &envoy_config_route_v3.RouteAction{
+					ClusterSpecifier: &envoy_config_route_v3.RouteAction_Cluster{
+						Cluster: translator.UpstreamToClusterName(us.Metadata.Ref()),
+					},
+				},
+			}
+			inRouteConfigurations[0].VirtualHosts[0].Routes = append(inRouteConfigurations[0].VirtualHosts[0].Routes, dupRoute)
+		})
+
+		It("should allow multiple routes to same upstream", func() {
+			p := tunneling.NewPlugin()
+			generatedClusters, _, _, _, err := p.GeneratedResources(params, inClusters, nil, inRouteConfigurations, nil)
+			Expect(err).ToNot(HaveOccurred())
+			Expect(generatedClusters).To(HaveLen(1), "should generate a single cluster for the upstream")
+			Expect(generatedClusters[0].GetTransportSocket()).ToNot(BeNil())
+		})
+	})
 	Context("multiple routes and clusters", func() {
 
 		BeforeEach(func() {
@@ -183,7 +229,6 @@ var _ = Describe("Plugin", func() {
 			generatedClusters[1].LoadAssignment.Endpoints[0].LbEndpoints[0] = nil
 
 			Expect(generatedClusters[0]).To(matchers.MatchProto(generatedClusters[1]), "generated clusters should be identical, barring name, clustername, endpoints")
-
 		})
 
 		It("should namespace generated listeners, avoiding duplicates", func() {
@@ -215,9 +260,7 @@ var _ = Describe("Plugin", func() {
 			generatedListeners[1].FilterChains[0].Filters[0].ConfigType = nil
 
 			Expect(generatedListeners[0]).To(matchers.MatchProto(generatedListeners[1]), "generated listeners should be identical, barring name, address, and tcp stats prefix")
-
 		})
-
 	})
 
 })