feat: support attaching EnvoyProxy resource to Gateways (envoyproxy#3532

) * refactor: rename EnvoyProxy field Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * feat(controller): process gateway.infrastructure.parametersref Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * feat(translator): support attaching EnvoyProxy to Gateways Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * chore: fix lint Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * chore: update generated code Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * chore: fix test Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * test: gateway-with-infrastructure-parametersref Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * test: add more cases Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * fix: EnvoyExtensionPolicy status Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * trigger reconciliation for envoyproxies Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * chore: rename processGCParamsRef as suggested Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * chore: add comments to Resources Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * refactor: update as suggested Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * refactor: extract resources.GetEnvoyProxy Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * refactor: move attachEnvoyProxy into GatewayContext.ResetListeners Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * refactor: change ListenerContext.gateway to GatewayContext Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * refactor: move getIRKey out of inner loop Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * fix: reset *ir.ExtAuth for each gateway Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * refactor: continue early to reduce nesting Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * refactor: rename envoyProxies and ClassEnvoyProxy Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * rename json/yaml name for envoyproxy Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * fix: pass the right envoyProxy to t.IsEnvoyServiceRouting Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * test: add more cases Signed-off-by: haoqixu <hq.xu0o0@gmail.com> * chore: regenerate Signed-off-by: haoqixu <hq.xu0o0@gmail.com> --------- Signed-off-by: haoqixu <hq.xu0o0@gmail.com> Signed-off-by: “bjlhlin” <“lihonglin1@jd.com”> Signed-off-by: bjlhlin <lihonglin1@jd.com>
bjlhlin · Jun 24, 2024 · cc9ad54 · cc9ad54
1 parent 477dba6
commit cc9ad54
Show file tree

Hide file tree

Showing 61 changed files with 1,466 additions and 560 deletions.
diff --git a/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml b/internal/cmd/egctl/testdata/translate/out/default-resources.all.yaml
@@ -1,4 +1,4 @@
-envoyProxy:
+envoyProxyForGatewayClass:
   metadata:
     creationTimestamp: null
     name: default-envoy-proxy

diff --git a/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml b/internal/cmd/egctl/testdata/translate/out/invalid-envoyproxy.all.yaml
@@ -1,4 +1,4 @@
-envoyProxy:
+envoyProxyForGatewayClass:
   metadata:
     creationTimestamp: null
     name: example

diff --git a/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml b/internal/cmd/egctl/testdata/translate/out/valid-envoyproxy.all.yaml
@@ -1,4 +1,4 @@
-envoyProxy:
+envoyProxyForGatewayClass:
   metadata:
     creationTimestamp: null
     name: example

diff --git a/internal/cmd/egctl/translate.go b/internal/cmd/egctl/translate.go
@@ -315,13 +315,13 @@ func translateGatewayAPIToGatewayAPI(resources *gatewayapi.Resources) (gatewayap
 	gRes, _ := gTranslator.Translate(resources)
 	// Update the status of the GatewayClass based on EnvoyProxy validation
 	epInvalid := false
-	if resources.EnvoyProxy != nil {
-		if err := validation.ValidateEnvoyProxy(resources.EnvoyProxy); err != nil {
+	if resources.EnvoyProxyForGatewayClass != nil {
+		if err := validation.ValidateEnvoyProxy(resources.EnvoyProxyForGatewayClass); err != nil {
 			epInvalid = true
 			msg := fmt.Sprintf("%s: %v", status.MsgGatewayClassInvalidParams, err)
 			status.SetGatewayClassAccepted(resources.GatewayClass, false, string(gwapiv1.GatewayClassReasonInvalidParameters), msg)
 		}
-		gRes.EnvoyProxy = resources.EnvoyProxy
+		gRes.EnvoyProxyForGatewayClass = resources.EnvoyProxyForGatewayClass
 	}
 	if !epInvalid {
 		status.SetGatewayClassAccepted(resources.GatewayClass, true, string(gwapiv1.GatewayClassReasonAccepted), status.MsgValidGatewayClass)
@@ -364,8 +364,8 @@ func translateGatewayAPIToXds(dnsDomain string, resourceType string, resources *
 				ServiceURL: ratelimit.GetServiceURL("envoy-gateway", dnsDomain),
 			},
 		}
-		if resources.EnvoyProxy != nil {
-			xTranslator.FilterOrder = resources.EnvoyProxy.Spec.FilterOrder
+		if resources.EnvoyProxyForGatewayClass != nil {
+			xTranslator.FilterOrder = resources.EnvoyProxyForGatewayClass.Spec.FilterOrder
 		}
 		xRes, err := xTranslator.Translate(val)
 		if err != nil {
@@ -445,8 +445,8 @@ func constructConfigDump(resources *gatewayapi.Resources, tCtx *xds_types.Resour
 
 	// Apply Bootstrap from EnvoyProxy API if set by the user
 	// The config should have been validated already
-	if resources.EnvoyProxy != nil && resources.EnvoyProxy.Spec.Bootstrap != nil {
-		bootstrapConfigurations, err = bootstrap.ApplyBootstrapConfig(resources.EnvoyProxy.Spec.Bootstrap, bootstrapConfigurations)
+	if resources.EnvoyProxyForGatewayClass != nil && resources.EnvoyProxyForGatewayClass.Spec.Bootstrap != nil {
+		bootstrapConfigurations, err = bootstrap.ApplyBootstrapConfig(resources.EnvoyProxyForGatewayClass.Spec.Bootstrap, bootstrapConfigurations)
 		if err != nil {
 			return nil, err
 		}
@@ -685,7 +685,7 @@ func kubernetesYAMLToResources(str string, addMissingResources bool) (*gatewayap
 				},
 				Spec: typedSpec.(egv1a1.EnvoyProxySpec),
 			}
-			resources.EnvoyProxy = envoyProxy
+			resources.EnvoyProxyForGatewayClass = envoyProxy
 		case gatewayapi.KindGatewayClass:
 			typedSpec := spec.Interface()
 			gatewayClass := &gwapiv1.GatewayClass{
@@ -923,7 +923,7 @@ func kubernetesYAMLToResources(str string, addMissingResources bool) (*gatewayap
 		}
 
 		// Add EnvoyProxy if it does not exist
-		if resources.EnvoyProxy == nil {
+		if resources.EnvoyProxyForGatewayClass == nil {
 			if err := addDefaultEnvoyProxy(resources); err != nil {
 				return nil, err
 			}
@@ -955,7 +955,7 @@ func addDefaultEnvoyProxy(resources *gatewayapi.Resources) error {
 			},
 		},
 	}
-	resources.EnvoyProxy = ep
+	resources.EnvoyProxyForGatewayClass = ep
 	ns := gwapiv1.Namespace(namespace)
 	resources.GatewayClass.Spec.ParametersRef = &gwapiv1.ParametersReference{
 		Group:     gwapiv1.Group(egv1a1.GroupVersion.Group),

diff --git a/internal/gatewayapi/backendtlspolicy.go b/internal/gatewayapi/backendtlspolicy.go
@@ -13,20 +13,22 @@ import (
 	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
 	gwapiv1a3 "sigs.k8s.io/gateway-api/apis/v1alpha3"
 
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 	"github.com/envoyproxy/gateway/internal/gatewayapi/status"
 	"github.com/envoyproxy/gateway/internal/ir"
 )
 
-func (t *Translator) applyBackendTLSSetting(backendRef gwapiv1.BackendObjectReference, backendNamespace string, parent gwapiv1a2.ParentReference, resources *Resources) *ir.TLSUpstreamConfig {
-	upstreamConfig, policy := t.processBackendTLSPolicy(backendRef, backendNamespace, parent, resources)
-	return t.applyEnvoyProxyBackendTLSSetting(policy, upstreamConfig, resources, parent)
+func (t *Translator) applyBackendTLSSetting(backendRef gwapiv1.BackendObjectReference, backendNamespace string, parent gwapiv1a2.ParentReference, resources *Resources, envoyProxy *egv1a1.EnvoyProxy) *ir.TLSUpstreamConfig {
+	upstreamConfig, policy := t.processBackendTLSPolicy(backendRef, backendNamespace, parent, resources, envoyProxy)
+	return t.applyEnvoyProxyBackendTLSSetting(policy, upstreamConfig, resources, parent, envoyProxy)
 }
 
 func (t *Translator) processBackendTLSPolicy(
 	backendRef gwapiv1.BackendObjectReference,
 	backendNamespace string,
 	parent gwapiv1a2.ParentReference,
 	resources *Resources,
+	envoyProxy *egv1a1.EnvoyProxy,
 ) (*ir.TLSUpstreamConfig, *gwapiv1a3.BackendTLSPolicy) {
 	policy := getBackendTLSPolicy(resources.BackendTLSPolicies, backendRef, backendNamespace)
 	if policy == nil {
@@ -53,37 +55,35 @@ func (t *Translator) processBackendTLSPolicy(
 
 	status.SetAcceptedForPolicyAncestors(&policy.Status, ancestorRefs, t.GatewayControllerName)
 	// apply defaults as per envoyproxy
-	if resources.EnvoyProxy != nil {
-		if resources.EnvoyProxy.Spec.BackendTLS != nil {
-			if len(resources.EnvoyProxy.Spec.BackendTLS.Ciphers) > 0 {
-				tlsBundle.Ciphers = resources.EnvoyProxy.Spec.BackendTLS.Ciphers
+	if envoyProxy != nil {
+		if envoyProxy.Spec.BackendTLS != nil {
+			if len(envoyProxy.Spec.BackendTLS.Ciphers) > 0 {
+				tlsBundle.Ciphers = envoyProxy.Spec.BackendTLS.Ciphers
 			}
-			if len(resources.EnvoyProxy.Spec.BackendTLS.ECDHCurves) > 0 {
-				tlsBundle.ECDHCurves = resources.EnvoyProxy.Spec.BackendTLS.ECDHCurves
+			if len(envoyProxy.Spec.BackendTLS.ECDHCurves) > 0 {
+				tlsBundle.ECDHCurves = envoyProxy.Spec.BackendTLS.ECDHCurves
 			}
-			if len(resources.EnvoyProxy.Spec.BackendTLS.SignatureAlgorithms) > 0 {
-				tlsBundle.SignatureAlgorithms = resources.EnvoyProxy.Spec.BackendTLS.SignatureAlgorithms
+			if len(envoyProxy.Spec.BackendTLS.SignatureAlgorithms) > 0 {
+				tlsBundle.SignatureAlgorithms = envoyProxy.Spec.BackendTLS.SignatureAlgorithms
 			}
-			if resources.EnvoyProxy.Spec.BackendTLS.MinVersion != nil {
-				tlsBundle.MinVersion = ptr.To(ir.TLSVersion(*resources.EnvoyProxy.Spec.BackendTLS.MinVersion))
+			if envoyProxy.Spec.BackendTLS.MinVersion != nil {
+				tlsBundle.MinVersion = ptr.To(ir.TLSVersion(*envoyProxy.Spec.BackendTLS.MinVersion))
 			}
-			if resources.EnvoyProxy.Spec.BackendTLS.MinVersion != nil {
-				tlsBundle.MaxVersion = ptr.To(ir.TLSVersion(*resources.EnvoyProxy.Spec.BackendTLS.MaxVersion))
+			if envoyProxy.Spec.BackendTLS.MinVersion != nil {
+				tlsBundle.MaxVersion = ptr.To(ir.TLSVersion(*envoyProxy.Spec.BackendTLS.MaxVersion))
 			}
-			if len(resources.EnvoyProxy.Spec.BackendTLS.ALPNProtocols) > 0 {
-				tlsBundle.ALPNProtocols = make([]string, len(resources.EnvoyProxy.Spec.BackendTLS.ALPNProtocols))
-				for i := range resources.EnvoyProxy.Spec.BackendTLS.ALPNProtocols {
-					tlsBundle.ALPNProtocols[i] = string(resources.EnvoyProxy.Spec.BackendTLS.ALPNProtocols[i])
+			if len(envoyProxy.Spec.BackendTLS.ALPNProtocols) > 0 {
+				tlsBundle.ALPNProtocols = make([]string, len(envoyProxy.Spec.BackendTLS.ALPNProtocols))
+				for i := range envoyProxy.Spec.BackendTLS.ALPNProtocols {
+					tlsBundle.ALPNProtocols[i] = string(envoyProxy.Spec.BackendTLS.ALPNProtocols[i])
 				}
 			}
 		}
 	}
 	return tlsBundle, policy
 }
 
-func (t *Translator) applyEnvoyProxyBackendTLSSetting(policy *gwapiv1a3.BackendTLSPolicy, tlsConfig *ir.TLSUpstreamConfig, resources *Resources, parent gwapiv1a2.ParentReference) *ir.TLSUpstreamConfig {
-	ep := resources.EnvoyProxy
-
+func (t *Translator) applyEnvoyProxyBackendTLSSetting(policy *gwapiv1a3.BackendTLSPolicy, tlsConfig *ir.TLSUpstreamConfig, resources *Resources, parent gwapiv1a2.ParentReference, ep *egv1a1.EnvoyProxy) *ir.TLSUpstreamConfig {
 	if ep == nil || ep.Spec.BackendTLS == nil || tlsConfig == nil {
 		return tlsConfig
 	}

diff --git a/internal/gatewayapi/clienttrafficpolicy.go b/internal/gatewayapi/clienttrafficpolicy.go
@@ -39,9 +39,11 @@ func hasSectionName(target *gwapiv1a2.LocalPolicyTargetReferenceWithSectionName)
 	return target.SectionName != nil
 }
 
-func (t *Translator) ProcessClientTrafficPolicies(resources *Resources,
+func (t *Translator) ProcessClientTrafficPolicies(
+	resources *Resources,
 	gateways []*GatewayContext,
-	xdsIR XdsIRMap, infraIR InfraIRMap,
+	xdsIR XdsIRMap,
+	infraIR InfraIRMap,
 ) []*egv1a1.ClientTrafficPolicy {
 	var res []*egv1a1.ClientTrafficPolicy
 
@@ -132,7 +134,7 @@ func (t *Translator) ProcessClientTrafficPolicies(resources *Resources,
 				var err error
 				for _, l := range gateway.listeners {
 					// Find IR
-					irKey := t.getIRKey(l.gateway)
+					irKey := t.getIRKey(l.gateway.Gateway)
 					// It must exist since we've already finished processing the gateways
 					gwXdsIR := xdsIR[irKey]
 					if string(l.Name) == section {
@@ -251,7 +253,7 @@ func (t *Translator) ProcessClientTrafficPolicies(resources *Resources,
 					}
 
 					// Find IR
-					irKey := t.getIRKey(l.gateway)
+					irKey := t.getIRKey(l.gateway.Gateway)
 					// It must exist since we've already finished processing the gateways
 					gwXdsIR := xdsIR[irKey]
 					if err := validatePortOverlapForClientTrafficPolicy(l, gwXdsIR, true); err != nil {
@@ -378,7 +380,7 @@ func (t *Translator) translateClientTrafficPolicyForListener(policy *egv1a1.Clie
 	xdsIR XdsIRMap, infraIR InfraIRMap, resources *Resources,
 ) error {
 	// Find IR
-	irKey := t.getIRKey(l.gateway)
+	irKey := t.getIRKey(l.gateway.Gateway)
 	// It must exist since we've already finished processing the gateways
 	gwXdsIR := xdsIR[irKey]
 

diff --git a/internal/gatewayapi/contexts.go b/internal/gatewayapi/contexts.go
@@ -14,19 +14,22 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	gwapiv1 "sigs.k8s.io/gateway-api/apis/v1"
 	gwapiv1a2 "sigs.k8s.io/gateway-api/apis/v1alpha2"
+
+	egv1a1 "github.com/envoyproxy/gateway/api/v1alpha1"
 )
 
 // GatewayContext wraps a Gateway and provides helper methods for
 // setting conditions, accessing Listeners, etc.
 type GatewayContext struct {
 	*gwapiv1.Gateway
 
-	listeners []*ListenerContext
+	listeners  []*ListenerContext
+	envoyProxy *egv1a1.EnvoyProxy
 }
 
 // ResetListeners resets the listener statuses and re-generates the GatewayContext
 // ListenerContexts from the Gateway spec.
-func (g *GatewayContext) ResetListeners() {
+func (g *GatewayContext) ResetListeners(resource *Resources) {
 	numListeners := len(g.Spec.Listeners)
 	g.Status.Listeners = make([]gwapiv1.ListenerStatus, numListeners)
 	g.listeners = make([]*ListenerContext, numListeners)
@@ -35,10 +38,28 @@ func (g *GatewayContext) ResetListeners() {
 		g.Status.Listeners[i] = gwapiv1.ListenerStatus{Name: listener.Name}
 		g.listeners[i] = &ListenerContext{
 			Listener:          listener,
-			gateway:           g.Gateway,
+			gateway:           g,
 			listenerStatusIdx: i,
 		}
 	}
+
+	g.attachEnvoyProxy(resource)
+}
+
+func (g *GatewayContext) attachEnvoyProxy(resources *Resources) {
+	if g.Spec.Infrastructure != nil && g.Spec.Infrastructure.ParametersRef != nil && !IsMergeGatewaysEnabled(resources) {
+		ref := g.Spec.Infrastructure.ParametersRef
+		if string(ref.Group) == egv1a1.GroupVersion.Group && ref.Kind == egv1a1.KindEnvoyProxy {
+			ep := resources.GetEnvoyProxy(g.Namespace, ref.Name)
+			if ep != nil {
+				g.envoyProxy = ep
+				return
+			}
+		}
+		// not found, fallthrough to use envoyProxy attached to gatewayclass
+	}
+
+	g.envoyProxy = resources.EnvoyProxyForGatewayClass
 }
 
 // ListenerContext wraps a Listener and provides helper methods for
@@ -47,7 +68,7 @@ func (g *GatewayContext) ResetListeners() {
 type ListenerContext struct {
 	*gwapiv1.Listener
 
-	gateway           *gwapiv1.Gateway
+	gateway           *GatewayContext
 	listenerStatusIdx int
 	namespaceSelector labels.Selector
 	tlsSecrets        []*corev1.Secret
@@ -209,18 +230,8 @@ func GetHostnames(route RouteContext) []string {
 // GetParentReferences returns the ParentReference of the Route object.
 func GetParentReferences(route RouteContext) []gwapiv1.ParentReference {
 	rv := reflect.ValueOf(route).Elem()
-	kind := rv.FieldByName("Kind").String()
 	pr := rv.FieldByName("Spec").FieldByName("ParentRefs")
-	if kind == KindHTTPRoute || kind == KindGRPCRoute {
-		return pr.Interface().([]gwapiv1.ParentReference)
-	}
-
-	parentReferences := make([]gwapiv1.ParentReference, pr.Len())
-	for i := 0; i < len(parentReferences); i++ {
-		p := pr.Index(i).Interface().(gwapiv1.ParentReference)
-		parentReferences[i] = UpgradeParentReference(p)
-	}
-	return parentReferences
+	return pr.Interface().([]gwapiv1.ParentReference)
 }
 
 // GetRouteStatus returns the RouteStatus object associated with the Route.
@@ -254,17 +265,8 @@ func GetRouteParentContext(route RouteContext, forParentRef gwapiv1.ParentRefere
 	specParentRefs := rv.FieldByName("Spec").FieldByName("ParentRefs")
 	for i := 0; i < specParentRefs.Len(); i++ {
 		p := specParentRefs.Index(i).Interface().(gwapiv1.ParentReference)
-		up := p
-		if !isHTTPRoute {
-			up = UpgradeParentReference(p)
-		}
-		if reflect.DeepEqual(up, forParentRef) {
-			if isHTTPRoute {
-				parentRef = &p
-			} else {
-				upgraded := UpgradeParentReference(p)
-				parentRef = &upgraded
-			}
+		if reflect.DeepEqual(p, forParentRef) {
+			parentRef = &p
 			break
 		}
 	}
@@ -332,6 +334,14 @@ type RouteParentContext struct {
 	listeners            []*ListenerContext
 }
 
+// GetGateway returns the GatewayContext if parent resource is a gateway.
+func (r *RouteParentContext) GetGateway() *GatewayContext {
+	if r == nil || len(r.listeners) == 0 {
+		return nil
+	}
+	return r.listeners[0].gateway
+}
+
 func (r *RouteParentContext) SetListeners(listeners ...*ListenerContext) {
 	r.listeners = append(r.listeners, listeners...)
 }