From 4a520c3c1422d9144ad1d0989cd16ab3d4e42fff Mon Sep 17 00:00:00 2001 From: Andrew Stucki Date: Fri, 19 May 2023 11:12:20 -0400 Subject: [PATCH] [API Gateway] API Gateway Binding Logic (#2142) * initial commit * Add additional TODO * Add some basic lifecycle unit tests * split up implementation * Add more tests and fix some bugs * remove one parallel call in a loop * Fix binding * Add resolvedRefs statuses for routes * Fix issue with empty parent ref that k8s doesn't like * Fix up updates/status ordering * Add basic gateway status setting * Finish up first pass on gateway statuses * Re-organize and begin adding comments * More comments * More comments * More comments * More comments * More comments * Add file that wasn't saved * Add utils unit tests * Add more tests * Final tests * Fix tests * Fix up gateway annotation with binding logic * Update doc comments for linter * Add forgotten file * Fix block in tests due to buffered channel size and better handle context cancelation --- .../templates/connect-inject-clusterrole.yaml | 6 + .../api-gateway/binding/annotations.go | 38 + .../api-gateway/binding/annotations_test.go | 203 ++ control-plane/api-gateway/binding/binder.go | 275 ++ .../api-gateway/binding/binder_test.go | 2532 +++++++++++++++++ .../api-gateway/binding/references.go | 132 + control-plane/api-gateway/binding/result.go | 439 +++ .../api-gateway/binding/route_binding.go | 478 ++++ control-plane/api-gateway/binding/setter.go | 177 ++ .../api-gateway/binding/setter_test.go | 39 + control-plane/api-gateway/binding/snapshot.go | 43 + control-plane/api-gateway/binding/utils.go | 102 + .../api-gateway/binding/utils_test.go | 236 ++ .../api-gateway/binding/validation.go | 298 ++ .../api-gateway/binding/validation_test.go | 644 +++++ .../gateway_class_config_controller_test.go | 9 +- .../controllers/gateway_class_serializer.go | 91 - .../gateway_class_serializer_test.go | 486 ---- .../controllers/gateway_controller.go | 303 +- .../controllers/gateway_controller_test.go | 15 +- .../gatewayclass_controller_test.go | 6 +- .../api-gateway/controllers/index.go | 110 +- .../api-gateway/gatekeeper/gatekeeper_test.go | 5 + .../translation/config_entry_translation.go | 203 +- .../config_entry_translation_test.go | 113 +- control-plane/cache/consul.go | 269 +- control-plane/cache/consul_test.go | 7 +- .../subcommand/inject-connect/command.go | 2 + 28 files changed, 6428 insertions(+), 833 deletions(-) create mode 100644 control-plane/api-gateway/binding/annotations.go create mode 100644 control-plane/api-gateway/binding/annotations_test.go create mode 100644 control-plane/api-gateway/binding/binder.go create mode 100644 control-plane/api-gateway/binding/binder_test.go create mode 100644 control-plane/api-gateway/binding/references.go create mode 100644 control-plane/api-gateway/binding/result.go create mode 100644 control-plane/api-gateway/binding/route_binding.go create mode 100644 control-plane/api-gateway/binding/setter.go create mode 100644 control-plane/api-gateway/binding/setter_test.go create mode 100644 control-plane/api-gateway/binding/snapshot.go create mode 100644 control-plane/api-gateway/binding/utils.go create mode 100644 control-plane/api-gateway/binding/utils_test.go create mode 100644 control-plane/api-gateway/binding/validation.go create mode 100644 control-plane/api-gateway/binding/validation_test.go delete mode 100644 control-plane/api-gateway/controllers/gateway_class_serializer.go delete mode 100644 control-plane/api-gateway/controllers/gateway_class_serializer_test.go diff --git a/charts/consul/templates/connect-inject-clusterrole.yaml b/charts/consul/templates/connect-inject-clusterrole.yaml index f44fde949a..e414048755 100644 --- a/charts/consul/templates/connect-inject-clusterrole.yaml +++ b/charts/consul/templates/connect-inject-clusterrole.yaml @@ -137,12 +137,18 @@ rules: - gateway.networking.k8s.io resources: - gatewayclasses/finalizers + - gateways/finalizers + - httproutes/finalizers + - tcproutes/finalizers verbs: - update - apiGroups: - gateway.networking.k8s.io resources: - gatewayclasses/status + - gateways/status + - httproutes/status + - tcproutes/status verbs: - get - patch diff --git a/control-plane/api-gateway/binding/annotations.go b/control-plane/api-gateway/binding/annotations.go new file mode 100644 index 0000000000..a596ede73a --- /dev/null +++ b/control-plane/api-gateway/binding/annotations.go @@ -0,0 +1,38 @@ +package binding + +import ( + "encoding/json" + + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" +) + +const ( + group = "api-gateway.consul.hashicorp.com" + annotationConfigKey = "api-gateway.consul.hashicorp.com/config" +) + +func serializeGatewayClassConfig(gw *gwv1beta1.Gateway, gwcc *v1alpha1.GatewayClassConfig) (*v1alpha1.GatewayClassConfig, bool) { + if gwcc == nil { + return nil, false + } + + if gw.Annotations == nil { + gw.Annotations = make(map[string]string) + } + + if annotatedConfig, ok := gw.Annotations[annotationConfigKey]; ok { + var config v1alpha1.GatewayClassConfig + if err := json.Unmarshal([]byte(annotatedConfig), &config.Spec); err == nil { + // if we can unmarshal the gateway, return it + return &config, false + } + } + + // otherwise if we failed to unmarshal or there was no annotation, marshal it onto + // the gateway + marshaled, _ := json.Marshal(gwcc.Spec) + gw.Annotations[annotationConfigKey] = string(marshaled) + return gwcc, true +} diff --git a/control-plane/api-gateway/binding/annotations_test.go b/control-plane/api-gateway/binding/annotations_test.go new file mode 100644 index 0000000000..2f4a413f07 --- /dev/null +++ b/control-plane/api-gateway/binding/annotations_test.go @@ -0,0 +1,203 @@ +package binding + +import ( + "encoding/json" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" +) + +func TestSerializeGatewayClassConfig_HappyPath(t *testing.T) { + t.Parallel() + + type args struct { + gw *gwv1beta1.Gateway + gwcc *v1alpha1.GatewayClassConfig + } + tests := []struct { + name string + args args + expectedDidUpdate bool + }{ + { + name: "when gateway has not been annotated yet and annotations are nil", + args: args{ + gw: &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-gw", + }, + Spec: gwv1beta1.GatewaySpec{}, + Status: gwv1beta1.GatewayStatus{}, + }, + gwcc: &v1alpha1.GatewayClassConfig{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "the config", + }, + Spec: v1alpha1.GatewayClassConfigSpec{ + ServiceType: pointerTo(corev1.ServiceType("serviceType")), + NodeSelector: map[string]string{ + "selector": "of node", + }, + Tolerations: []v1.Toleration{ + { + Key: "key", + Operator: "op", + Value: "120", + Effect: "to the moon", + TolerationSeconds: new(int64), + }, + }, + CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ + Service: []string{"service"}, + }, + }, + }, + }, + expectedDidUpdate: true, + }, + { + name: "when gateway has not been annotated yet but annotations are empty", + args: args{ + gw: &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-gw", + Annotations: make(map[string]string), + }, + Spec: gwv1beta1.GatewaySpec{}, + Status: gwv1beta1.GatewayStatus{}, + }, + gwcc: &v1alpha1.GatewayClassConfig{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "the config", + }, + Spec: v1alpha1.GatewayClassConfigSpec{ + ServiceType: pointerTo(corev1.ServiceType("serviceType")), + NodeSelector: map[string]string{ + "selector": "of node", + }, + Tolerations: []v1.Toleration{ + { + Key: "key", + Operator: "op", + Value: "120", + Effect: "to the moon", + TolerationSeconds: new(int64), + }, + }, + CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ + Service: []string{"service"}, + }, + }, + }, + }, + expectedDidUpdate: true, + }, + { + name: "when gateway has been annotated", + args: args{ + gw: &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-gw", + Annotations: map[string]string{ + annotationConfigKey: `{"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"copyAnnotations":{"service":["service"]}}`, + }, + }, + Spec: gwv1beta1.GatewaySpec{}, + Status: gwv1beta1.GatewayStatus{}, + }, + gwcc: &v1alpha1.GatewayClassConfig{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "the config", + }, + Spec: v1alpha1.GatewayClassConfigSpec{ + ServiceType: pointerTo(corev1.ServiceType("serviceType")), + NodeSelector: map[string]string{ + "selector": "of node", + }, + Tolerations: []v1.Toleration{ + { + Key: "key", + Operator: "op", + Value: "120", + Effect: "to the moon", + TolerationSeconds: new(int64), + }, + }, + CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ + Service: []string{"service"}, + }, + }, + }, + }, + expectedDidUpdate: false, + }, + { + name: "when gateway has been annotated but the serialization was invalid", + args: args{ + gw: &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "my-gw", + Annotations: map[string]string{ + // we remove the opening brace to make unmarshalling fail + annotationConfigKey: `"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"copyAnnotations":{"service":["service"]}}`, + }, + }, + Spec: gwv1beta1.GatewaySpec{}, + Status: gwv1beta1.GatewayStatus{}, + }, + gwcc: &v1alpha1.GatewayClassConfig{ + TypeMeta: metav1.TypeMeta{}, + ObjectMeta: metav1.ObjectMeta{ + Name: "the config", + }, + Spec: v1alpha1.GatewayClassConfigSpec{ + ServiceType: pointerTo(corev1.ServiceType("serviceType")), + NodeSelector: map[string]string{ + "selector": "of node", + }, + Tolerations: []v1.Toleration{ + { + Key: "key", + Operator: "op", + Value: "120", + Effect: "to the moon", + TolerationSeconds: new(int64), + }, + }, + CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ + Service: []string{"service"}, + }, + }, + }, + }, + expectedDidUpdate: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + _, actualDidUpdate := serializeGatewayClassConfig(tt.args.gw, tt.args.gwcc) + + if actualDidUpdate != tt.expectedDidUpdate { + t.Errorf("SerializeGatewayClassConfig() = %v, want %v", actualDidUpdate, tt.expectedDidUpdate) + } + + var config v1alpha1.GatewayClassConfig + err := json.Unmarshal([]byte(tt.args.gw.Annotations[annotationConfigKey]), &config.Spec) + require.NoError(t, err) + + if diff := cmp.Diff(config.Spec, tt.args.gwcc.Spec); diff != "" { + t.Errorf("Expected gwconfig spec to match serialized version (-want,+got):\n%s", diff) + } + }) + } +} diff --git a/control-plane/api-gateway/binding/binder.go b/control-plane/api-gateway/binding/binder.go new file mode 100644 index 0000000000..bc02b1fab1 --- /dev/null +++ b/control-plane/api-gateway/binding/binder.go @@ -0,0 +1,275 @@ +package binding + +import ( + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/consul-k8s/control-plane/api-gateway/translation" + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul/api" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +const ( + // gatewayFinalizer is the finalizer we add to any gateway object. + gatewayFinalizer = "gateway-finalizer.consul.hashicorp.com" + + // namespaceNameLabel represents that label added automatically to namespaces in newer Kubernetes clusters. + namespaceNameLabel = "kubernetes.io/metadata.name" +) + +var ( + // constants extracted for ease of use. + kindGateway = "Gateway" + kindSecret = "Secret" + betaGroup = gwv1beta1.GroupVersion.Group + + // the list of kinds we can support by listener protocol. + supportedKindsForProtocol = map[gwv1beta1.ProtocolType][]gwv1beta1.RouteGroupKind{ + gwv1beta1.HTTPProtocolType: {{ + Group: (*gwv1beta1.Group)(&gwv1beta1.GroupVersion.Group), + Kind: "HTTPRoute", + }}, + gwv1beta1.HTTPSProtocolType: {{ + Group: (*gwv1beta1.Group)(&gwv1beta1.GroupVersion.Group), + Kind: "HTTPRoute", + }}, + gwv1beta1.TCPProtocolType: {{ + Group: (*gwv1alpha2.Group)(&gwv1alpha2.GroupVersion.Group), + Kind: "TCPRoute", + }}, + } +) + +// BinderConfig configures a binder instance with all of the information +// that it needs to know to generate a snapshot of bound state. +type BinderConfig struct { + // Translator instance initialized with proper name/namespace translation + // configuration from helm. + Translator translation.K8sToConsulTranslator + // ControllerName is the name of the controller used in determining which + // gateways we control, also leveraged for setting route statuses. + ControllerName string + + // GatewayClassConfig is the configuration corresponding to the given + // GatewayClass -- if it is nil we should treat the gateway as deleted + // since the gateway is now pointing to an invalid gateway class + GatewayClassConfig *v1alpha1.GatewayClassConfig + // GatewayClass is the GatewayClass corresponding to the Gateway we want to + // bind routes to. It is passed as a pointer because it could be nil. If no + // GatewayClass corresponds to a Gateway, we ought to clean up any sort of + // state that we may have set on the Gateway, its corresponding Routes or in + // Consul, because we should no longer be managing the Gateway (its association + // to our controller is through a parameter on the GatewayClass). + GatewayClass *gwv1beta1.GatewayClass + // Gateway is the Gateway being reconciled that we want to bind routes to. + Gateway gwv1beta1.Gateway + // HTTPRoutes is a list of HTTPRoute objects that ought to be bound to the Gateway. + HTTPRoutes []gwv1beta1.HTTPRoute + // TCPRoutes is a list of TCPRoute objects that ought to be bound to the Gateway. + TCPRoutes []gwv1alpha2.TCPRoute + // Secrets is a list of Secret objects that a Gateway references. + Secrets []corev1.Secret + + // TODO: Do we need to pass in Routes that have references to a Gateway in their statuses + // for cleanup purposes or is the below enough for record keeping? + + // ConsulHTTPRoutes are a list of HTTPRouteConfigEntry objects that currently reference the + // Gateway we've created in Consul. + ConsulHTTPRoutes []api.HTTPRouteConfigEntry + // ConsulTCPRoutes are a list of TCPRouteConfigEntry objects that currently reference the + // Gateway we've created in Consul. + ConsulTCPRoutes []api.TCPRouteConfigEntry + // ConsulInlineCertificates is a list of certificates that have been created in Consul. + ConsulInlineCertificates []api.InlineCertificateConfigEntry + // ConnectInjectedServices is a list of all services that have been injected by our connect-injector + // and that we can, therefore reference on the mesh. + ConnectInjectedServices []api.CatalogService + + // Namespaces is a map of all namespaces in Kubernetes indexed by their names for looking up labels + // for AllowedRoutes matching purposes. + Namespaces map[string]corev1.Namespace + // ControlledGateways is a map of all Gateway objects that we currently should be interested in. This + // is used to determine whether we should garbage collect Certificate or Route objects when they become + // disassociated with a particular Gateway. + ControlledGateways map[types.NamespacedName]gwv1beta1.Gateway +} + +// Binder is used for generating a Snapshot of all operations that should occur both +// in Kubernetes and Consul as a result of binding routes to a Gateway. +type Binder struct { + statusSetter *setter + config BinderConfig +} + +// NewBinder creates a Binder object with the given configuration. +func NewBinder(config BinderConfig) *Binder { + return &Binder{config: config, statusSetter: newSetter(config.ControllerName)} +} + +// gatewayRef returns a Consul-based reference for the given Kubernetes gateway to +// be used for marking a deletion that is needed in Consul. +func (b *Binder) gatewayRef() api.ResourceReference { + return b.config.Translator.ReferenceForGateway(&b.config.Gateway) +} + +// isGatewayDeleted returns whether we should treat the given gateway as a deleted object. +// This is true if the gateway has a deleted timestamp, if its GatewayClass does not match +// our controller name, or if the GatewayClass it references doesn't exist. +func (b *Binder) isGatewayDeleted() bool { + gatewayClassMismatch := b.config.GatewayClass == nil || b.config.ControllerName != string(b.config.GatewayClass.Spec.ControllerName) + isGatewayDeleted := isDeleted(&b.config.Gateway) || gatewayClassMismatch || b.config.GatewayClassConfig == nil + return isGatewayDeleted +} + +// Snapshot generates a snapshot of operations that need to occur in Kubernetes and Consul +// in order for a Gateway to be reconciled. +func (b *Binder) Snapshot() Snapshot { + // at this point we assume all tcp routes and http routes + // actually reference this gateway + tracker := b.references() + serviceMap := serviceMap(b.config.ConnectInjectedServices) + seenRoutes := map[api.ResourceReference]struct{}{} + snapshot := Snapshot{} + gwcc := b.config.GatewayClassConfig + + isGatewayDeleted := b.isGatewayDeleted() + if !isGatewayDeleted { + var updated bool + gwcc, updated = serializeGatewayClassConfig(&b.config.Gateway, gwcc) + + // we don't have a deletion but if we add a finalizer for the gateway, then just add it and return + // otherwise try and resolve as much as possible + if ensureFinalizer(&b.config.Gateway) || updated { + // if we've added the finalizer or serialized the class config, then update + snapshot.Kubernetes.Updates = append(snapshot.Kubernetes.Updates, &b.config.Gateway) + return snapshot + } + } + + httpRouteBinder := b.newHTTPRouteBinder(tracker, serviceMap) + tcpRouteBinder := b.newTCPRouteBinder(tracker, serviceMap) + + // used for tracking how many routes have successfully bound to which listeners + // on a gateway for reporting the number of bound routes in a gateway listener's + // status + boundCounts := make(map[gwv1beta1.SectionName]int) + + // attempt to bind all routes + + for _, r := range b.config.HTTPRoutes { + snapshot = httpRouteBinder.bind(pointerTo(r), boundCounts, seenRoutes, snapshot) + } + + for _, r := range b.config.TCPRoutes { + snapshot = tcpRouteBinder.bind(pointerTo(r), boundCounts, seenRoutes, snapshot) + } + + // now cleanup any routes that we haven't already processed + + for _, r := range b.config.ConsulHTTPRoutes { + snapshot = b.cleanHTTPRoute(pointerTo(r), seenRoutes, snapshot) + } + + for _, r := range b.config.ConsulTCPRoutes { + snapshot = b.cleanTCPRoute(pointerTo(r), seenRoutes, snapshot) + } + + // process certificates + + seenCerts := make(map[types.NamespacedName]api.ResourceReference) + for _, secret := range b.config.Secrets { + if isGatewayDeleted { + // we bypass the secret creation since we want to be able to GC if necessary + continue + } + + certificate := b.config.Translator.SecretToInlineCertificate(secret) + certificateRef := translation.EntryToReference(&certificate) + + // mark the certificate as processed + seenCerts[objectToMeta(&secret)] = certificateRef + // add the certificate to the set of upsert operations needed in Consul + snapshot.Consul.Updates = append(snapshot.Consul.Updates, &certificate) + } + + // clean up any inline certs that are now stale and can be GC'd + for _, cert := range b.config.ConsulInlineCertificates { + certRef := translation.EntryToNamespacedName(&cert) + if _, ok := seenCerts[certRef]; !ok { + // check to see if nothing is now referencing the certificate + if tracker.canGCSecret(certRef) { + ref := translation.EntryToReference(&cert) + // we can GC this now since it's not referenced by any Gateway + snapshot.Consul.Deletions = append(snapshot.Consul.Deletions, ref) + } + } + } + + // we only want to upsert the gateway into Consul or update its status + // if the gateway hasn't been marked for deletion + if !isGatewayDeleted { + snapshot.GatewayClassConfig = gwcc + + entry := b.config.Translator.GatewayToAPIGateway(b.config.Gateway, seenCerts) + snapshot.Consul.Updates = append(snapshot.Consul.Updates, &entry) + + // calculate the status for the gateway + var status gwv1beta1.GatewayStatus + gatewayValidation := validateGateway(b.config.Gateway) + listenerValidation := validateListeners(b.config.Gateway.Namespace, b.config.Gateway.Spec.Listeners, b.config.Secrets) + for i, listener := range b.config.Gateway.Spec.Listeners { + status.Listeners = append(status.Listeners, gwv1beta1.ListenerStatus{ + Name: listener.Name, + SupportedKinds: supportedKindsForProtocol[listener.Protocol], + AttachedRoutes: int32(boundCounts[listener.Name]), + Conditions: listenerValidation.Conditions(b.config.Gateway.Generation, i), + }) + } + // TODO: addresses + status.Conditions = gatewayValidation.Conditions(b.config.Gateway.Generation, listenerValidation.Invalid()) + status.Addresses = []gwv1beta1.GatewayAddress{} + + // only mark the gateway as needing a status update if there's a diff with its old + // status, this keeps the controller from infinitely reconciling + if !cmp.Equal(status, b.config.Gateway.Status, cmp.FilterPath(func(p cmp.Path) bool { + path := p.String() + return path == "Listeners.Conditions.LastTransitionTime" || path == "Conditions.LastTransitionTime" + }, cmp.Ignore())) { + b.config.Gateway.Status = status + snapshot.Kubernetes.StatusUpdates = append(snapshot.Kubernetes.StatusUpdates, &b.config.Gateway) + } + } else { + // if the gateway has been deleted, unset whatever we've set on it + snapshot.Consul.Deletions = append(snapshot.Consul.Deletions, b.gatewayRef()) + if removeFinalizer(&b.config.Gateway) { + snapshot.Kubernetes.Updates = append(snapshot.Kubernetes.Updates, &b.config.Gateway) + } + } + + return snapshot +} + +// serviceMap constructs a map of services indexed by their Kubernetes namespace and name +// from the annotations that are set on the service. +func serviceMap(services []api.CatalogService) map[types.NamespacedName]api.CatalogService { + smap := make(map[types.NamespacedName]api.CatalogService) + for _, service := range services { + smap[serviceToNamespacedName(&service)] = service + } + return smap +} + +// serviceToNamespacedName returns the Kubernetes namespace and name of a Consul catalog service +// based on the Metadata annotations written on the service. +func serviceToNamespacedName(s *api.CatalogService) types.NamespacedName { + var ( + metaKeyKubeNS = "k8s-namespace" + metaKeyKubeServiceName = "k8s-service-name" + ) + return types.NamespacedName{ + Namespace: s.ServiceMeta[metaKeyKubeNS], + Name: s.ServiceMeta[metaKeyKubeServiceName], + } +} diff --git a/control-plane/api-gateway/binding/binder_test.go b/control-plane/api-gateway/binding/binder_test.go new file mode 100644 index 0000000000..dbff0072b7 --- /dev/null +++ b/control-plane/api-gateway/binding/binder_test.go @@ -0,0 +1,2532 @@ +package binding + +import ( + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +func TestBinder_Lifecycle(t *testing.T) { + t.Parallel() + + className := "gateway-class" + gatewayClassName := gwv1beta1.ObjectName(className) + controllerName := "test-controller" + deletionTimestamp := pointerTo(metav1.Now()) + gatewayClass := &gwv1beta1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: className, + }, + Spec: gwv1beta1.GatewayClassSpec{ + ControllerName: gwv1beta1.GatewayController(controllerName), + }, + } + + for name, tt := range map[string]struct { + config BinderConfig + expected Snapshot + }{ + "no gateway class and empty routes": { + config: BinderConfig{ + Gateway: gwv1beta1.Gateway{}, + }, + expected: Snapshot{ + Consul: ConsulSnapshot{ + Deletions: []api.ResourceReference{{ + Kind: api.APIGateway, + }}, + }, + }, + }, + "no gateway class and empty routes remove finalizer": { + config: BinderConfig{ + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{gatewayFinalizer}, + }, + }, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + Updates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{}, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Deletions: []api.ResourceReference{{ + Kind: api.APIGateway, + }}, + }, + }, + }, + "deleting gateway empty routes": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + Updates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Deletions: []api.ResourceReference{{ + Kind: api.APIGateway, + }}, + }, + }, + }, + "basic gateway no finalizer": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + Updates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + }, + }, + Consul: ConsulSnapshot{}, + }, + }, + "basic gateway": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + Listeners: []gwv1beta1.Listener{{ + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }}, + }, + }}, + }, + }, + Secrets: []corev1.Secret{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-one", + }, + }}, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + StatusUpdates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + Listeners: []gwv1beta1.Listener{{ + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }}, + }, + }}, + }, + Status: gwv1beta1.GatewayStatus{ + Addresses: []gwv1beta1.GatewayAddress{}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "ListenersNotValid", + Message: "one or more listeners are invalid", + }}, + Listeners: []gwv1beta1.ListenerStatus{{ + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "UnsupportedProtocol", + Message: "listener protocol is unsupported", + }, { + Type: "Conflicted", + Status: metav1.ConditionFalse, + Reason: "NoConflicts", + Message: "listener has no conflicts", + }, { + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved certificate references", + }}, + }}, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Updates: []api.ConfigEntry{ + &api.InlineCertificateConfigEntry{ + Kind: api.InlineCertificate, + Name: "secret-one", + Meta: map[string]string{ + "k8s-name": "secret-one", + "k8s-namespace": "", + "k8s-service-name": "secret-one", + "managed-by": "consul-k8s-gateway-controller", + }, + }, + &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Meta: map[string]string{ + "k8s-name": "", + "k8s-namespace": "", + "k8s-service-name": "", + "managed-by": "consul-k8s-gateway-controller", + }, + Listeners: []api.APIGatewayListener{{ + TLS: api.APIGatewayTLSConfiguration{ + Certificates: []api.ResourceReference{{ + Kind: api.InlineCertificate, + Name: "secret-one", + }}, + }, + }}, + }, + }, + }, + }, + }, + "gateway http route no finalizer": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + HTTPRoutes: []gwv1beta1.HTTPRoute{{ + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }}, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + Updates: []client.Object{ + &gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + }, + StatusUpdates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + Status: gwv1beta1.GatewayStatus{ + Addresses: []gwv1beta1.GatewayAddress{}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "gateway accepted", + }}, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Updates: []api.ConfigEntry{ + &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Name: "gateway", + Meta: map[string]string{ + "k8s-name": "gateway", + "k8s-namespace": "", + "k8s-service-name": "gateway", + "managed-by": "consul-k8s-gateway-controller", + }, + Listeners: []api.APIGatewayListener{}, + }, + }, + }, + }, + }, + "gateway http route deleting": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + HTTPRoutes: []gwv1beta1.HTTPRoute{{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }}, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + Updates: []client.Object{ + &gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + }, + StatusUpdates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + Status: gwv1beta1.GatewayStatus{ + Addresses: []gwv1beta1.GatewayAddress{}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "gateway accepted", + }}, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Updates: []api.ConfigEntry{ + &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Name: "gateway", + Meta: map[string]string{ + "k8s-name": "gateway", + "k8s-namespace": "", + "k8s-service-name": "gateway", + "managed-by": "consul-k8s-gateway-controller", + }, + Listeners: []api.APIGatewayListener{}, + }, + }, + Deletions: []api.ResourceReference{{ + Kind: api.HTTPRoute, + }}, + }, + }, + }, + "gateway tcp route no finalizer": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + TCPRoutes: []gwv1alpha2.TCPRoute{{ + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }}, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + Updates: []client.Object{ + &gwv1alpha2.TCPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + }, + StatusUpdates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + Status: gwv1beta1.GatewayStatus{ + Addresses: []gwv1beta1.GatewayAddress{}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "gateway accepted", + }}, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Updates: []api.ConfigEntry{ + &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Name: "gateway", + Meta: map[string]string{ + "k8s-name": "gateway", + "k8s-namespace": "", + "k8s-service-name": "gateway", + "managed-by": "consul-k8s-gateway-controller", + }, + Listeners: []api.APIGatewayListener{}, + }, + }, + }, + }, + }, + "gateway tcp route deleting": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + }, + TCPRoutes: []gwv1alpha2.TCPRoute{{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }}, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + Updates: []client.Object{ + &gwv1alpha2.TCPRoute{ + ObjectMeta: metav1.ObjectMeta{ + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + }, + StatusUpdates: []client.Object{ + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + }, + Status: gwv1beta1.GatewayStatus{ + Addresses: []gwv1beta1.GatewayAddress{}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "gateway accepted", + }}, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Updates: []api.ConfigEntry{ + &api.APIGatewayConfigEntry{ + Kind: api.APIGateway, + Name: "gateway", + Meta: map[string]string{ + "k8s-name": "gateway", + "k8s-namespace": "", + "k8s-service-name": "gateway", + "managed-by": "consul-k8s-gateway-controller", + }, + Listeners: []api.APIGatewayListener{}, + }, + }, + Deletions: []api.ResourceReference{{ + Kind: api.TCPRoute, + }}, + }, + }, + }, + "gateway deletion routes and secrets": { + config: BinderConfig{ + ControllerName: controllerName, + GatewayClass: gatewayClass, + Gateway: gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + Listeners: []gwv1beta1.Listener{{ + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }, { + Name: "secret-two", + }}, + }, + }}, + }, + }, + ControlledGateways: map[types.NamespacedName]gwv1beta1.Gateway{ + {Name: "gateway"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + Listeners: []gwv1beta1.Listener{{ + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }, { + Name: "secret-two", + }}, + }, + }}, + }, + }, + {Name: "gateway-two"}: { + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway-two", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + Listeners: []gwv1beta1.Listener{{ + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }, { + Name: "secret-three", + }}, + }, + }}, + }, + }, + }, + Secrets: []corev1.Secret{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-one", + }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-two", + }, + }}, + HTTPRoutes: []gwv1beta1.HTTPRoute{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "http-route-one", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "http-route-two", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }, { + Name: "gateway-two", + }}, + }, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gwv1beta1.GatewayController(controllerName), + ParentRef: gwv1beta1.ParentReference{Name: "gateway"}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + }}, + }, { + ControllerName: gwv1beta1.GatewayController(controllerName), + ParentRef: gwv1beta1.ParentReference{Name: "gateway-two"}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + }}, + }}, + }, + }, + }}, + TCPRoutes: []gwv1alpha2.TCPRoute{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-route-one", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, { + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-route-two", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }, { + Name: "gateway-two", + }}, + }, + }, + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gwv1beta1.GatewayController(controllerName), + ParentRef: gwv1beta1.ParentReference{Name: "gateway"}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + }}, + }, { + ControllerName: gwv1beta1.GatewayController(controllerName), + ParentRef: gwv1beta1.ParentReference{Name: "gateway-two"}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + }}, + }}, + }, + }, + }}, + ConsulHTTPRoutes: []api.HTTPRouteConfigEntry{{ + Kind: api.HTTPRoute, + Name: "http-route-two", + Meta: map[string]string{ + "k8s-name": "http-route-two", + "k8s-namespace": "", + "k8s-service-name": "http-route-two", + "managed-by": "consul-k8s-gateway-controller", + }, + Parents: []api.ResourceReference{{ + Kind: api.APIGateway, + Name: "gateway", + }, { + Kind: api.APIGateway, + Name: "gateway-two", + }}, + }}, + ConsulTCPRoutes: []api.TCPRouteConfigEntry{{ + Kind: api.TCPRoute, + Name: "tcp-route-two", + Meta: map[string]string{ + "k8s-name": "tcp-route-two", + "k8s-namespace": "", + "k8s-service-name": "tcp-route-two", + "managed-by": "consul-k8s-gateway-controller", + }, + Parents: []api.ResourceReference{{ + Kind: api.APIGateway, + Name: "gateway", + }, { + Kind: api.APIGateway, + Name: "gateway-two", + }}, + }}, + ConsulInlineCertificates: []api.InlineCertificateConfigEntry{{ + Kind: api.InlineCertificate, + Name: "secret-one", + Meta: map[string]string{ + "k8s-name": "secret-one", + "k8s-namespace": "", + "k8s-service-name": "secret-one", + "managed-by": "consul-k8s-gateway-controller", + }, + }, { + Kind: api.InlineCertificate, + Name: "secret-two", + Meta: map[string]string{ + "k8s-name": "secret-two", + "k8s-namespace": "", + "k8s-service-name": "secret-two", + "managed-by": "consul-k8s-gateway-controller", + }, + }}, + }, + expected: Snapshot{ + Kubernetes: KubernetesSnapshot{ + StatusUpdates: []client.Object{ + &gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "http-route-two", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }, { + Name: "gateway-two", + }}, + }, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + // removed gateway status + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gwv1beta1.GatewayController(controllerName), + ParentRef: gwv1beta1.ParentReference{Name: "gateway-two"}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + }}, + }}, + }, + }, + }, + &gwv1alpha2.TCPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-route-two", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }, { + Name: "gateway-two", + }}, + }, + }, + // removed gateway status + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gwv1beta1.GatewayController(controllerName), + ParentRef: gwv1beta1.ParentReference{Name: "gateway-two"}, + Conditions: []metav1.Condition{{ + Type: "Accepted", + Status: metav1.ConditionTrue, + }}, + }}, + }, + }, + }, + }, + Updates: []client.Object{ + &gwv1beta1.HTTPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "http-route-one", + Finalizers: []string{}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + &gwv1alpha2.TCPRoute{ + ObjectMeta: metav1.ObjectMeta{ + Name: "tcp-route-one", + Finalizers: []string{}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + &gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + DeletionTimestamp: deletionTimestamp, + Finalizers: []string{}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + Listeners: []gwv1beta1.Listener{{ + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }, { + Name: "secret-two", + }}, + }, + }}, + }, + }, + }, + }, + Consul: ConsulSnapshot{ + Updates: []api.ConfigEntry{ + &api.HTTPRouteConfigEntry{ + Kind: api.HTTPRoute, + Name: "http-route-two", + Meta: map[string]string{ + "k8s-name": "http-route-two", + "k8s-namespace": "", + "k8s-service-name": "http-route-two", + "managed-by": "consul-k8s-gateway-controller", + }, + // dropped ref to gateway + Parents: []api.ResourceReference{{ + Kind: api.APIGateway, + Name: "gateway-two", + }}, + }, + &api.TCPRouteConfigEntry{ + Kind: api.TCPRoute, + Name: "tcp-route-two", + Meta: map[string]string{ + "k8s-name": "tcp-route-two", + "k8s-namespace": "", + "k8s-service-name": "tcp-route-two", + "managed-by": "consul-k8s-gateway-controller", + }, + // dropped ref to gateway + Parents: []api.ResourceReference{{ + Kind: api.APIGateway, + Name: "gateway-two", + }}, + }, + }, + Deletions: []api.ResourceReference{{ + Kind: api.HTTPRoute, + Name: "http-route-one", + }, { + Kind: api.TCPRoute, + Name: "tcp-route-one", + }, { + Kind: api.InlineCertificate, + Name: "secret-two", + }, { + Kind: api.APIGateway, + Name: "gateway", + }}, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + tt.config.ControllerName = controllerName + tt.config.GatewayClassConfig = &v1alpha1.GatewayClassConfig{} + serializeGatewayClassConfig(&tt.config.Gateway, tt.config.GatewayClassConfig) + + binder := NewBinder(tt.config) + actual := binder.Snapshot() + + diff := cmp.Diff(tt.expected, actual, cmp.FilterPath(func(p cmp.Path) bool { + return p.String() == "GatewayClassConfig" || strings.HasSuffix(p.String(), "LastTransitionTime") || strings.HasSuffix(p.String(), "Annotations") + }, cmp.Ignore())) + if diff != "" { + t.Error("undexpected diff", diff) + } + }) + } +} + +func TestBinder_BindingRulesKitchenSink(t *testing.T) { + t.Parallel() + + className := "gateway-class" + gatewayClassName := gwv1beta1.ObjectName(className) + controllerName := "test-controller" + gatewayClass := &gwv1beta1.GatewayClass{ + ObjectMeta: metav1.ObjectMeta{ + Name: className, + }, + Spec: gwv1beta1.GatewayClassSpec{ + ControllerName: gwv1beta1.GatewayController(controllerName), + }, + } + + gateway := gwv1beta1.Gateway{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.GatewaySpec{ + GatewayClassName: gatewayClassName, + Listeners: []gwv1beta1.Listener{{ + Name: "http-listener-default-same", + Protocol: gwv1beta1.HTTPProtocolType, + }, { + Name: "http-listener-hostname", + Protocol: gwv1beta1.HTTPProtocolType, + Hostname: pointerTo[gwv1beta1.Hostname]("host.name"), + }, { + Name: "http-listener-mismatched-kind-allowed", + Protocol: gwv1beta1.HTTPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Kinds: []gwv1beta1.RouteGroupKind{{ + Kind: "Foo", + }}, + }, + }, { + Name: "http-listener-explicit-all-allowed", + Protocol: gwv1beta1.HTTPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromAll), + }, + }, + }, { + Name: "http-listener-explicit-allowed-same", + Protocol: gwv1beta1.HTTPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromSame), + }, + }, + }, { + Name: "http-listener-allowed-selector", + Protocol: gwv1beta1.HTTPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromSelector), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "test": "foo", + }, + }, + }, + }, + }, { + Name: "http-listener-tls", + Protocol: gwv1beta1.HTTPSProtocolType, + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }}, + }, + }, { + Name: "tcp-listener-default-same", + Protocol: gwv1beta1.TCPProtocolType, + }, { + Name: "tcp-listener-mismatched-kind-allowed", + Protocol: gwv1beta1.TCPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Kinds: []gwv1beta1.RouteGroupKind{{ + Kind: "Foo", + }}, + }, + }, { + Name: "tcp-listener-explicit-all-allowed", + Protocol: gwv1beta1.TCPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromAll), + }, + }, + }, { + Name: "tcp-listener-explicit-allowed-same", + Protocol: gwv1beta1.TCPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromSame), + }, + }, + }, { + Name: "tcp-listener-allowed-selector", + Protocol: gwv1beta1.TCPProtocolType, + AllowedRoutes: &gwv1beta1.AllowedRoutes{ + Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromSelector), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "test": "foo", + }, + }, + }, + }, + }, { + Name: "tcp-listener-tls", + Protocol: gwv1beta1.TCPProtocolType, + TLS: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{{ + Name: "secret-one", + }}, + }, + }}, + }, + } + + namespaces := map[string]corev1.Namespace{ + "": { + ObjectMeta: metav1.ObjectMeta{ + Name: "", + }, + }, + "test": { + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{ + "test": "foo", + }, + }, + }, + } + + defaultNamespacePointer := pointerTo[gwv1beta1.Namespace]("") + + httpTypeMeta := metav1.TypeMeta{} + httpTypeMeta.SetGroupVersionKind(gwv1beta1.SchemeGroupVersion.WithKind("HTTPRoute")) + tcpTypeMeta := metav1.TypeMeta{} + tcpTypeMeta.SetGroupVersionKind(gwv1beta1.SchemeGroupVersion.WithKind("TCPRoute")) + + for name, tt := range map[string]struct { + httpRoute *gwv1beta1.HTTPRoute + expectedHTTPRouteUpdate *gwv1beta1.HTTPRoute + expectedHTTPRouteUpdateStatus *gwv1beta1.HTTPRoute + expectedHTTPConsulRouteUpdate *api.HTTPRouteConfigEntry + expectedHTTPConsulRouteDelete *api.ResourceReference + + tcpRoute *gwv1alpha2.TCPRoute + expectedTCPRouteUpdate *gwv1alpha2.TCPRoute + expectedTCPRouteUpdateStatus *gwv1alpha2.TCPRoute + expectedTCPConsulRouteUpdate *api.TCPRouteConfigEntry + expectedTCPConsulRouteDelete *api.ResourceReference + }{ + "untargeted http route same namespace": { + httpRoute: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + expectedHTTPRouteUpdateStatus: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "untargeted http route same namespace missing backend": { + httpRoute: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1beta1.HTTPRouteRule{{ + BackendRefs: []gwv1beta1.HTTPBackendRef{{ + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + }, + }, + }}, + }}, + }, + }, + expectedHTTPRouteUpdateStatus: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1beta1.HTTPRouteRule{{ + BackendRefs: []gwv1beta1.HTTPBackendRef{{ + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + }, + }, + }}, + }}, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "BackendNotFound", + Message: "/backend: backend not found", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "untargeted http route same namespace invalid backend type": { + httpRoute: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1beta1.HTTPRouteRule{{ + BackendRefs: []gwv1beta1.HTTPBackendRef{{ + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + Group: pointerTo[gwv1beta1.Group]("invalid.foo.com"), + }, + }, + }}, + }}, + }, + }, + expectedHTTPRouteUpdateStatus: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1beta1.HTTPRouteRule{{ + BackendRefs: []gwv1beta1.HTTPBackendRef{{ + BackendRef: gwv1beta1.BackendRef{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + Group: pointerTo[gwv1beta1.Group]("invalid.foo.com"), + }, + }, + }}, + }}, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "InvalidKind", + Message: "/backend [Service.invalid.foo.com]: invalid backend kind", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "untargeted http route different namespace": { + httpRoute: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + }}, + }, + }, + }, + expectedHTTPRouteUpdateStatus: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + }}, + }, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "targeted http route same namespace": { + httpRoute: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-default-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-hostname"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-allowed-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-allowed-selector"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-tls"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }}, + }, + }, + }, + expectedHTTPRouteUpdateStatus: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-default-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-hostname"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-allowed-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-allowed-selector"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-tls"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }}, + }, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-default-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-hostname"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-mismatched-kind-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-mismatched-kind-allowed: listener does not support route protocol", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-allowed-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-allowed-selector"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-allowed-selector: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-tls"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-explicit-all-allowed: listener does not support route protocol", + }}, + }}, + }, + }, + }, + }, + "targeted http route different namespace": { + httpRoute: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-default-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-hostname"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-allowed-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-allowed-selector"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-tls"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }}, + }, + }, + }, + expectedHTTPRouteUpdateStatus: &gwv1beta1.HTTPRoute{ + TypeMeta: httpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-default-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-hostname"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-allowed-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-allowed-selector"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-tls"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }}, + }, + }, + Status: gwv1beta1.HTTPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-default-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-default-same: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-hostname"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-hostname: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-mismatched-kind-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-mismatched-kind-allowed: listener does not support route protocol", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-allowed-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-explicit-allowed-same: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-allowed-selector"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-tls"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-tls: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-explicit-all-allowed: listener does not support route protocol", + }}, + }}, + }, + }, + }, + }, + "untargeted tcp route same namespace": { + tcpRoute: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + }, + expectedTCPRouteUpdateStatus: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + }, + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "untargeted tcp route same namespace missing backend": { + tcpRoute: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1alpha2.TCPRouteRule{{ + BackendRefs: []gwv1beta1.BackendRef{{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + }, + }}, + }}, + }, + }, + expectedTCPRouteUpdateStatus: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1alpha2.TCPRouteRule{{ + BackendRefs: []gwv1beta1.BackendRef{{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + }, + }}, + }}, + }, + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "BackendNotFound", + Message: "/backend: backend not found", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "untargeted tcp route same namespace invalid backend type": { + tcpRoute: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1alpha2.TCPRouteRule{{ + BackendRefs: []gwv1beta1.BackendRef{{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + Group: pointerTo[gwv1beta1.Group]("invalid.foo.com"), + }, + }}, + }}, + }, + }, + expectedTCPRouteUpdateStatus: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + }}, + }, + Rules: []gwv1alpha2.TCPRouteRule{{ + BackendRefs: []gwv1beta1.BackendRef{{ + BackendObjectReference: gwv1beta1.BackendObjectReference{ + Name: gwv1beta1.ObjectName("backend"), + Group: pointerTo[gwv1beta1.Group]("invalid.foo.com"), + }, + }}, + }}, + }, + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "InvalidKind", + Message: "/backend [Service.invalid.foo.com]: invalid backend kind", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "untargeted tcp route different namespace": { + tcpRoute: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + }}, + }, + }, + }, + expectedTCPRouteUpdateStatus: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + }}, + }, + }, + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }}, + }, + }, + }, + }, + "targeted tcp route same namespace": { + tcpRoute: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-default-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-allowed-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-allowed-selector"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-tls"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }}, + }, + }, + }, + expectedTCPRouteUpdateStatus: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-default-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-allowed-same"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-allowed-selector"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-tls"), + }, { + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }}, + }, + }, + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-default-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-mismatched-kind-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-mismatched-kind-allowed: listener does not support route protocol", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-allowed-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-allowed-selector"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-allowed-selector: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-tls"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-explicit-all-allowed: listener does not support route protocol", + }}, + }}, + }, + }, + }, + }, + "targeted tcp route different namespace": { + tcpRoute: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-default-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-allowed-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-allowed-selector"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-tls"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }}, + }, + }, + }, + expectedTCPRouteUpdateStatus: &gwv1alpha2.TCPRoute{ + TypeMeta: tcpTypeMeta, + ObjectMeta: metav1.ObjectMeta{ + Name: "route", + Namespace: "test", + Finalizers: []string{gatewayFinalizer}, + }, + Spec: gwv1alpha2.TCPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-default-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-mismatched-kind-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-allowed-same"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-allowed-selector"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-tls"), + }, { + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }}, + }, + }, + Status: gwv1alpha2.TCPRouteStatus{ + RouteStatus: gwv1beta1.RouteStatus{ + Parents: []gwv1beta1.RouteParentStatus{{ + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-default-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-default-same: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-mismatched-kind-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-mismatched-kind-allowed: listener does not support route protocol", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-explicit-allowed-same"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-explicit-allowed-same: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-allowed-selector"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("tcp-listener-tls"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "tcp-listener-tls: listener does not allow binding routes from the given namespace", + }}, + }, { + ControllerName: gatewayClass.Spec.ControllerName, + ParentRef: gwv1beta1.ParentReference{ + Name: "gateway", + Namespace: defaultNamespacePointer, + SectionName: pointerTo[gwv1beta1.SectionName]("http-listener-explicit-all-allowed"), + }, + Conditions: []metav1.Condition{{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + }, { + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "NotAllowedByListeners", + Message: "http-listener-explicit-all-allowed: listener does not support route protocol", + }}, + }}, + }, + }, + }, + }, + } { + t.Run(name, func(t *testing.T) { + config := BinderConfig{ + ControllerName: controllerName, + GatewayClassConfig: &v1alpha1.GatewayClassConfig{}, + GatewayClass: gatewayClass, + Gateway: gateway, + Namespaces: namespaces, + ControlledGateways: map[types.NamespacedName]gwv1beta1.Gateway{ + {Name: "gateway"}: gateway, + }, + Secrets: []corev1.Secret{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "secret-one", + }, + }}, + } + serializeGatewayClassConfig(&config.Gateway, config.GatewayClassConfig) + + if tt.httpRoute != nil { + config.HTTPRoutes = append(config.HTTPRoutes, *tt.httpRoute) + } + if tt.tcpRoute != nil { + config.TCPRoutes = append(config.TCPRoutes, *tt.tcpRoute) + } + + binder := NewBinder(config) + actual := binder.Snapshot() + + compareUpdates(t, tt.expectedHTTPRouteUpdate, actual.Kubernetes.Updates) + compareUpdates(t, tt.expectedTCPRouteUpdate, actual.Kubernetes.Updates) + compareUpdates(t, tt.expectedHTTPRouteUpdateStatus, actual.Kubernetes.StatusUpdates) + compareUpdates(t, tt.expectedTCPRouteUpdateStatus, actual.Kubernetes.StatusUpdates) + }) + } +} + +func compareUpdates[T client.Object](t *testing.T, expected T, updates []client.Object) { + t.Helper() + + if isNil(expected) { + for _, update := range updates { + if u, ok := update.(T); ok { + t.Error("found unexpected update", u) + } + } + } else { + found := false + for _, update := range updates { + if u, ok := update.(T); ok { + diff := cmp.Diff(expected, u, cmp.FilterPath(func(p cmp.Path) bool { + return p.String() == "Status.RouteStatus.Parents.Conditions.LastTransitionTime" + }, cmp.Ignore())) + if diff != "" { + t.Error("diff between actual and expected", diff) + } + found = true + } + } + if !found { + t.Error("expected route update not found in", updates) + } + } +} diff --git a/control-plane/api-gateway/binding/references.go b/control-plane/api-gateway/binding/references.go new file mode 100644 index 0000000000..640e59cdfb --- /dev/null +++ b/control-plane/api-gateway/binding/references.go @@ -0,0 +1,132 @@ +package binding + +import ( + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// referenceTracker acts as a reference counting object for: +// 1. the number of controlled gateways that are referenced by an HTTPRoute +// 2. the number of controlled gateways that are referenced by a TCPRoute +// 3. the number of gateways that reference a certificate Secret +// +// These are used for determining when dissasociating from a gateway +// should cause us to cleanup a route or certificate both in Consul and +// whatever state we have set on the object in Kubernetes. +type referenceTracker struct { + httpRouteReferencesGateways map[types.NamespacedName]int + tcpRouteReferencesGateways map[types.NamespacedName]int + certificatesReferencedByGateways map[types.NamespacedName]int +} + +// isLastReference checks if the given gateway is the last controlled gateway +// that a route references. If it is and the gateway has been deleted, we +// should clean up all state created for the route. +func (r referenceTracker) isLastReference(object client.Object) bool { + key := types.NamespacedName{ + Namespace: object.GetNamespace(), + Name: object.GetName(), + } + + switch object.(type) { + case *gwv1alpha2.TCPRoute: + return r.tcpRouteReferencesGateways[key] == 1 + case *gwv1beta1.HTTPRoute: + return r.httpRouteReferencesGateways[key] == 1 + default: + return false + } +} + +// canGCSecret checks if we can garbage collect a secret that has +// not been upserted. +func (r referenceTracker) canGCSecret(key types.NamespacedName) bool { + // should this be 1 or 0? + return r.certificatesReferencedByGateways[key] == 1 +} + +// references initializes a referenceTracker based on the HTTPRoutes, TCPRoutes, +// and ControlledGateways associated with this Binder. +func (b *Binder) references() referenceTracker { + tracker := referenceTracker{ + httpRouteReferencesGateways: make(map[types.NamespacedName]int), + tcpRouteReferencesGateways: make(map[types.NamespacedName]int), + certificatesReferencedByGateways: make(map[types.NamespacedName]int), + } + + for _, route := range b.config.HTTPRoutes { + references := map[types.NamespacedName]struct{}{} + for _, ref := range route.Spec.ParentRefs { + for _, gateway := range b.config.ControlledGateways { + parentName := string(ref.Name) + parentNamespace := valueOr(ref.Namespace, route.Namespace) + if nilOrEqual(ref.Group, betaGroup) && + nilOrEqual(ref.Kind, kindGateway) && + gateway.Namespace == parentNamespace && + gateway.Name == parentName { + // the route references a gateway we control, store the ref to this gateway + references[types.NamespacedName{ + Namespace: parentNamespace, + Name: parentName, + }] = struct{}{} + } + } + } + tracker.httpRouteReferencesGateways[types.NamespacedName{ + Namespace: route.Namespace, + Name: route.Name, + }] = len(references) + } + + for _, route := range b.config.TCPRoutes { + references := map[types.NamespacedName]struct{}{} + for _, ref := range route.Spec.ParentRefs { + for _, gateway := range b.config.ControlledGateways { + parentName := string(ref.Name) + parentNamespace := valueOr(ref.Namespace, route.Namespace) + if nilOrEqual(ref.Group, betaGroup) && + nilOrEqual(ref.Kind, kindGateway) && + gateway.Namespace == parentNamespace && + gateway.Name == parentName { + // the route references a gateway we control, store the ref to this gateway + references[types.NamespacedName{ + Namespace: parentNamespace, + Name: parentName, + }] = struct{}{} + } + } + } + tracker.tcpRouteReferencesGateways[types.NamespacedName{ + Namespace: route.Namespace, + Name: route.Name, + }] = len(references) + } + + for _, gateway := range b.config.ControlledGateways { + references := map[types.NamespacedName]struct{}{} + for _, listener := range gateway.Spec.Listeners { + if listener.TLS == nil { + continue + } + for _, ref := range listener.TLS.CertificateRefs { + if nilOrEqual(ref.Group, "") && + nilOrEqual(ref.Kind, kindSecret) { + // the gateway references a secret, store it + references[types.NamespacedName{ + Namespace: valueOr(ref.Namespace, gateway.Namespace), + Name: string(ref.Name), + }] = struct{}{} + } + } + } + + for ref := range references { + count := tracker.certificatesReferencedByGateways[ref] + tracker.certificatesReferencedByGateways[ref] = count + 1 + } + } + + return tracker +} diff --git a/control-plane/api-gateway/binding/result.go b/control-plane/api-gateway/binding/result.go new file mode 100644 index 0000000000..eeeb029c40 --- /dev/null +++ b/control-plane/api-gateway/binding/result.go @@ -0,0 +1,439 @@ +package binding + +import ( + "errors" + "fmt" + "sort" + "strings" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +var ( + // Each of the below are specified in the Gateway spec under RouteConditionReason + // the general usage is that each error is specified as errRoute* where * corresponds + // to the RouteConditionReason given in the spec. If a reason is overloaded and can + // be used with two different types of things (i.e. something is not found or it's not supported) + // then we distinguish those two usages with errRoute*_Usage. + errRouteNotAllowedByListeners_Namespace = errors.New("listener does not allow binding routes from the given namespace") + errRouteNotAllowedByListeners_Protocol = errors.New("listener does not support route protocol") + errRouteNoMatchingListenerHostname = errors.New("listener cannot bind route with a non-aligned hostname") + errRouteInvalidKind = errors.New("invalid backend kind") + errRouteBackendNotFound = errors.New("backend not found") + errRouteRefNotPermitted = errors.New("reference not permitted due to lack of ReferenceGrant") +) + +// routeValidationResult holds the result of validating a route globally, in other +// words, for a particular backend reference without consideration to its particular +// gateway. Unfortunately, due to the fact that the spec requires a route status be +// associated with a parent reference, what it means is that anything that is global +// in nature, like this status will need to be duplicated for every parent reference +// on a given route status. +type routeValidationResult struct { + namespace string + backend gwv1beta1.BackendRef + err error +} + +// Type is used for error printing a backend reference type that we don't support on +// a validation error. +func (v routeValidationResult) Type() string { + return (&metav1.GroupKind{ + Group: valueOr(v.backend.Group, ""), + Kind: valueOr(v.backend.Kind, "Service"), + }).String() +} + +// String is the namespace/name of the reference that has an error. +func (v routeValidationResult) String() string { + return (types.NamespacedName{Namespace: v.namespace, Name: string(v.backend.Name)}).String() +} + +// routeValidationResults contains a list of validation results for the backend references +// on a route. +type routeValidationResults []routeValidationResult + +// Condition returns the ResolvedRefs condition that gets duplicated across every relevant +// parent on a route's status. +func (e routeValidationResults) Condition() metav1.Condition { + // we only use the first error due to the way the spec is structured + // where you can only have a single condition + for _, v := range e { + err := v.err + if err != nil { + switch err { + case errRouteInvalidKind: + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "InvalidKind", + Message: fmt.Sprintf("%s [%s]: %s", v.String(), v.Type(), err.Error()), + } + case errRouteBackendNotFound: + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "BackendNotFound", + Message: fmt.Sprintf("%s: %s", v.String(), err.Error()), + } + case errRouteRefNotPermitted: + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "RefNotPermitted", + Message: fmt.Sprintf("%s: %s", v.String(), err.Error()), + } + default: + // this should never happen + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "UnhandledValidationError", + Message: err.Error(), + } + } + } + } + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + Message: "resolved backend references", + } +} + +// bindResult holds the result of attempting to bind a route to a particular gateway listener +// an error value here means that the route did not bind successfully, no error means that +// the route should be considered bound. +type bindResult struct { + section gwv1beta1.SectionName + err error +} + +// bindResults holds the results of attempting to bind a route to a gateway, having a separate +// bindResult for each listener on the gateway. +type bindResults []bindResult + +// Error constructs a human readable error for bindResults, containing any errors that a route +// had in binding to a gateway, note that this is only used if a route failed to bind to every +// listener it attempted to bind to. +func (b bindResults) Error() string { + messages := []string{} + for _, result := range b { + if result.err != nil { + messages = append(messages, fmt.Sprintf("%s: %s", result.section, result.err.Error())) + } + } + + sort.Strings(messages) + return strings.Join(messages, "; ") +} + +// DidBind returns whether a route successfully bound to any listener on a gateway. +func (b bindResults) DidBind() bool { + for _, result := range b { + if result.err == nil { + return true + } + } + return false +} + +// Condition constructs an Accepted condition for a route that will be scoped +// to the particular parent reference it's using to attempt binding. +func (b bindResults) Condition() metav1.Condition { + // if we bound to any listeners, say we're accepted + if b.DidBind() { + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + } + } + + // default to the most generic reason in the spec "NotAllowedByListeners" + reason := "NotAllowedByListeners" + + // if we only have a single binding error, we can get more specific + if len(b) == 1 { + for _, result := range b { + // if we have a hostname mismatch error, then use the more specific reason + if result.err == errRouteNoMatchingListenerHostname { + reason = "NoMatchingListenerHostname" + } + } + } + + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: reason, + Message: b.Error(), + } +} + +// parentBindResult associates a binding result with the given parent reference. +type parentBindResult struct { + parent gwv1beta1.ParentReference + results bindResults +} + +// parentBindResults contains the list of all results that occurred when this route +// attempted to bind to a gateway using its parent references. +type parentBindResults []parentBindResult + +var ( + // Each of the below are specified in the Gateway spec under ListenerConditionReason + // the general usage is that each error is specified as errListener* where * corresponds + // to the ListenerConditionReason given in the spec. If a reason is overloaded and can + // be used with two different types of things (i.e. something is not found or it's not supported) + // then we distinguish those two usages with errListener*_Usage. + errListenerUnsupportedProtocol = errors.New("listener protocol is unsupported") + errListenerPortUnavailable = errors.New("listener port is unavailable") + errListenerHostnameConflict = errors.New("listener hostname conflicts with another listener") + errListenerProtocolConflict = errors.New("listener protocol conflicts with another listener") + errListenerInvalidCertificateRef_NotFound = errors.New("certificate not found") + errListenerInvalidCertificateRef_NotSupported = errors.New("certificate type is not supported") + + // Below is where any custom generic listener validation errors should go. + // We map anything under here to a custom ListenerConditionReason of Invalid on + // an Accepted status type. + errListenerNoTLSPassthrough = errors.New("TLS passthrough is not supported") +) + +// listenerValidationResult contains the result of internally validating a single listener +// as well as the result of validating it in relation to all its peers (via conflictedErr). +// an error set on any of its members corresponds to an error condition on the corresponding +// status type. +type listenerValidationResult struct { + // status type: Accepted + acceptedErr error + // status type: Conflicted + conflictedErr error + // status type: ResolvedRefs + refErr error + // TODO: programmed +} + +// acceptedCondition constructs the condition for the Accepted status type. +func (l listenerValidationResult) acceptedCondition(generation int64) metav1.Condition { + now := metav1.Now() + switch l.acceptedErr { + case errListenerPortUnavailable: + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "PortUnavailable", + ObservedGeneration: generation, + Message: l.acceptedErr.Error(), + LastTransitionTime: now, + } + case errListenerUnsupportedProtocol: + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "UnsupportedProtocol", + ObservedGeneration: generation, + Message: l.acceptedErr.Error(), + LastTransitionTime: now, + } + case nil: + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + ObservedGeneration: generation, + Message: "listener accepted", + LastTransitionTime: now, + } + default: + // falback to invalid + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "Invalid", + ObservedGeneration: generation, + Message: l.acceptedErr.Error(), + LastTransitionTime: now, + } + } +} + +// conflictedCondition constructs the condition for the Conflicted status type. +func (l listenerValidationResult) conflictedCondition(generation int64) metav1.Condition { + now := metav1.Now() + + switch l.conflictedErr { + case errListenerProtocolConflict: + return metav1.Condition{ + Type: "Conflicted", + Status: metav1.ConditionTrue, + Reason: "ProtocolConflict", + ObservedGeneration: generation, + Message: l.conflictedErr.Error(), + LastTransitionTime: now, + } + case errListenerHostnameConflict: + return metav1.Condition{ + Type: "Conflicted", + Status: metav1.ConditionTrue, + Reason: "HostnameConflict", + ObservedGeneration: generation, + Message: l.conflictedErr.Error(), + LastTransitionTime: now, + } + default: + return metav1.Condition{ + Type: "Conflicted", + Status: metav1.ConditionFalse, + Reason: "NoConflicts", + ObservedGeneration: generation, + Message: "listener has no conflicts", + LastTransitionTime: now, + } + } +} + +// acceptedCondition constructs the condition for the ResolvedRefs status type. +func (l listenerValidationResult) resolvedRefsCondition(generation int64) metav1.Condition { + now := metav1.Now() + + switch l.refErr { + case errListenerInvalidCertificateRef_NotFound: + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "InvalidCertificateRef", + ObservedGeneration: generation, + Message: l.refErr.Error(), + LastTransitionTime: now, + } + case errListenerInvalidCertificateRef_NotSupported: + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionFalse, + Reason: "InvalidCertificateRef", + ObservedGeneration: generation, + Message: l.refErr.Error(), + LastTransitionTime: now, + } + default: + return metav1.Condition{ + Type: "ResolvedRefs", + Status: metav1.ConditionTrue, + Reason: "ResolvedRefs", + ObservedGeneration: generation, + Message: "resolved certificate references", + LastTransitionTime: now, + } + } +} + +// Conditions constructs the entire set of conditions for a given gateway listener. +func (l listenerValidationResult) Conditions(generation int64) []metav1.Condition { + return []metav1.Condition{ + l.acceptedCondition(generation), + l.conflictedCondition(generation), + l.resolvedRefsCondition(generation), + } +} + +// listenerValidationResults holds all of the results for a gateway's listeners +// the index of each result needs to correspond exactly to the index of the listener +// on the gateway spec for which it is describing. +type listenerValidationResults []listenerValidationResult + +// Invalid returns whether or not there is any listener that is not "Accepted" +// this is used in constructing a gateway's status where the Accepted status +// at the top-level can have a GatewayConditionReason of ListenersNotValid. +func (l listenerValidationResults) Invalid() bool { + for _, r := range l { + if r.acceptedErr != nil { + return true + } + } + return false +} + +// Conditions returns the listener conditions at a given index. +func (l listenerValidationResults) Conditions(generation int64, index int) []metav1.Condition { + result := l[index] + return result.Conditions(generation) +} + +var ( + // Each of the below are specified in the Gateway spec under GatewayConditionReason + // the general usage is that each error is specified as errGateway* where * corresponds + // to the GatewayConditionReason given in the spec. + errGatewayUnsupportedAddress = errors.New("gateway does not support specifying addresses") + errGatewayListenersNotValid = errors.New("one or more listeners are invalid") +) + +// gatewayValidationResult contains the result of internally validating a gateway. +// An error set on any of its members corresponds to an error condition on the corresponding +// status type. +type gatewayValidationResult struct { + acceptedErr error + // TODO: programmed +} + +// acceptedCondition returns a condition for the Accepted status type. It takes a boolean argument +// for whether or not any of the gateway's listeners are invalid, if they are, it overrides whatever +// Reason is set as an error on the result and instead uses the ListenersNotValid reason. +func (l gatewayValidationResult) acceptedCondition(generation int64, listenersInvalid bool) metav1.Condition { + now := metav1.Now() + + if l.acceptedErr == nil { + if listenersInvalid { + return metav1.Condition{ + Type: "Accepted", + // should one invalid listener cause the entire gateway to become invalid? + Status: metav1.ConditionFalse, + Reason: "ListenersNotValid", + ObservedGeneration: generation, + Message: errGatewayListenersNotValid.Error(), + LastTransitionTime: now, + } + } + + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + ObservedGeneration: generation, + Message: "gateway accepted", + LastTransitionTime: now, + } + } + + if l.acceptedErr == errGatewayUnsupportedAddress { + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "UnsupportedAddress", + ObservedGeneration: generation, + Message: l.acceptedErr.Error(), + LastTransitionTime: now, + } + } + + // fallback to Invalid reason + return metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionFalse, + Reason: "Invalid", + ObservedGeneration: generation, + Message: l.acceptedErr.Error(), + LastTransitionTime: now, + } +} + +// Conditions constructs the gateway conditions given whether its listeners are valid. +func (l gatewayValidationResult) Conditions(generation int64, listenersInvalid bool) []metav1.Condition { + return []metav1.Condition{ + l.acceptedCondition(generation, listenersInvalid), + } +} diff --git a/control-plane/api-gateway/binding/route_binding.go b/control-plane/api-gateway/binding/route_binding.go new file mode 100644 index 0000000000..a1f6a578ac --- /dev/null +++ b/control-plane/api-gateway/binding/route_binding.go @@ -0,0 +1,478 @@ +package binding + +import ( + "github.com/hashicorp/consul-k8s/control-plane/api-gateway/translation" + "github.com/hashicorp/consul/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// consulHTTPRouteFor returns the Consul HTTPRouteConfigEntry for the given reference. +func (b *Binder) consulHTTPRouteFor(ref api.ResourceReference) *api.HTTPRouteConfigEntry { + for _, route := range b.config.ConsulHTTPRoutes { + if route.Namespace == ref.Namespace && route.Partition == ref.Partition && route.Name == ref.Name { + return &route + } + } + return nil +} + +// consulTCPRouteFor returns the Consul TCPRouteConfigEntry for the given reference. +func (b *Binder) consulTCPRouteFor(ref api.ResourceReference) *api.TCPRouteConfigEntry { + for _, route := range b.config.ConsulTCPRoutes { + if route.Namespace == ref.Namespace && route.Partition == ref.Partition && route.Name == ref.Name { + return &route + } + } + return nil +} + +// routeBinder encapsulates the binding logic for binding a route to the given Gateway. +// The logic for route binding is almost identical between different route types, but +// due to the strong typing in the Spec and Go's inability to deal with fields via generics +// we have to pull in a bunch of accessors (which ideally should be in the upstream spec) +// for each route type. +// +// From the generic signature -- T: the type of Kubernetes route, U: the type of Consul config entry +// +// TODO: consider moving the function closures to something like an interface that we can +// implement the accessors on for each route type. +type routeBinder[T client.Object, U api.ConfigEntry] struct { + // isGatewayDeleted is used to determine whether we should just ignore + // attempting to bind the route (since we no longer know whether we + // should manage the route we only want to remove any state we've + // set on it). + isGatewayDeleted bool + // gateway is the gateway that we want to use for binding + gateway *gwv1beta1.Gateway + // gatewayRef is a Consul reference used to prune no-longer bound + // parents from a Consul resource we've created. + gatewayRef api.ResourceReference + // tracker is the referenceTracker used to determine when we want to cleanup + // routes based on a deleted gateway. + tracker referenceTracker + // namespaces is the set of namespaces in Consul that use for determining + // whether a route in a given namespace can bind to a gateway with AllowedRoutes set + namespaces map[string]corev1.Namespace + // services is a catalog of all connect-injected services to check a route against + // for resolving its backend refs + services map[types.NamespacedName]api.CatalogService + + // translationReferenceFunc is a function used to translate a Kubernetes object into + // a Consul object reference + translationReferenceFunc func(route T) api.ResourceReference + // lookupFunc is a function used for finding an existing Consul object based on + // its object reference + lookupFunc func(api.ResourceReference) U + // getParentsFunc is a function used for getting the parent references of a Consul route object + getParentsFunc func(U) []api.ResourceReference + // setParentsFunc is a function used for setting the parent references of a route object + setParentsFunc func(U, []api.ResourceReference) + // removeStatusRefsFunc is a function used for removing the statuses for the given parent + // references from a route + removeStatusRefsFunc func(T, []gwv1beta1.ParentReference) bool + // getHostnamesFunc is a function used for getting the hostnames associated with a route + getHostnamesFunc func(T) []gwv1beta1.Hostname + // getParentRefsFunc is used for getting the parent references of a Kubernetes route object + getParentRefsFunc func(T) []gwv1beta1.ParentReference + // translationFunc is used for translating a Kubernetes route into the corresponding Consul config entry + translationFunc func(T, map[types.NamespacedName]api.ResourceReference) U + // setRouteConditionFunc is used for adding or overwriting a condition on a route at the given + // parent + setRouteConditionFunc func(T, *gwv1beta1.ParentReference, metav1.Condition) bool + // getBackendRefsFunc returns a list of all backend references that we need to validate against the + // list of known connect-injected services + getBackendRefsFunc func(T) []gwv1beta1.BackendRef + // removeControllerStatusFunc is used to remove all of the statuses set by our controller when GC'ing + // a route + removeControllerStatusFunc func(T) bool +} + +// newRouteBinder creates a new route binder for the given Kubernetes and Consul route types +// generally this is lightly wrapped by other constructors that pass in the various closures +// needed for accessing fields on the objects. +func newRouteBinder[T client.Object, U api.ConfigEntry]( + isGatewayDeleted bool, + gateway *gwv1beta1.Gateway, + gatewayRef api.ResourceReference, + namespaces map[string]corev1.Namespace, + services map[types.NamespacedName]api.CatalogService, + tracker referenceTracker, + translationReferenceFunc func(route T) api.ResourceReference, + lookupFunc func(api.ResourceReference) U, + getParentsFunc func(U) []api.ResourceReference, + setParentsFunc func(U, []api.ResourceReference), + removeStatusRefsFunc func(T, []gwv1beta1.ParentReference) bool, + getHostnamesFunc func(T) []gwv1beta1.Hostname, + getParentRefsFunc func(T) []gwv1beta1.ParentReference, + translationFunc func(T, map[types.NamespacedName]api.ResourceReference) U, + setRouteConditionFunc func(T, *gwv1beta1.ParentReference, metav1.Condition) bool, + getBackendRefsFunc func(T) []gwv1beta1.BackendRef, + removeControllerStatusFunc func(T) bool, +) *routeBinder[T, U] { + return &routeBinder[T, U]{ + isGatewayDeleted: isGatewayDeleted, + gateway: gateway, + gatewayRef: gatewayRef, + namespaces: namespaces, + services: services, + tracker: tracker, + translationReferenceFunc: translationReferenceFunc, + lookupFunc: lookupFunc, + getParentsFunc: getParentsFunc, + setParentsFunc: setParentsFunc, + removeStatusRefsFunc: removeStatusRefsFunc, + getHostnamesFunc: getHostnamesFunc, + getParentRefsFunc: getParentRefsFunc, + translationFunc: translationFunc, + setRouteConditionFunc: setRouteConditionFunc, + getBackendRefsFunc: getBackendRefsFunc, + removeControllerStatusFunc: removeControllerStatusFunc, + } +} + +// bind contains the main logic for binding a route to a given gateway. +func (r *routeBinder[T, U]) bind(route T, boundCount map[gwv1beta1.SectionName]int, seenRoutes map[api.ResourceReference]struct{}, snapshot Snapshot) (updatedSnapshot Snapshot) { + routeRef := r.translationReferenceFunc(route) + existing := r.lookupFunc(routeRef) + gatewayRefs := filterParentRefs(objectToMeta(r.gateway), route.GetNamespace(), r.getParentRefsFunc(route)) + + // mark this route as having been processed + seenRoutes[routeRef] = struct{}{} + + // flags to mark that some operation needs to occur + consulNeedsDelete := false + kubernetesNeedsUpdate := false + kubernetesNeedsStatusUpdate := false + // Since the update can either be for an existing resource (in the case + // of a deleted gateway) or for a resource generated by translating a + // bound gateway we just set the resource that we want to push out an + // update for. If this is not nil, we push it into the snapshot. + var consulUpdate U + + // we do this in a closure at the end to make sure we don't accidentally + // add something multiple times into the list of update/delete operations + // instead we just set a flag indicating that an update is needed and then + // append to the snapshot right before returning + defer func() { + if !isNil(consulUpdate) { + snapshot.Consul.Updates = append(snapshot.Consul.Updates, consulUpdate) + } + if consulNeedsDelete { + snapshot.Consul.Deletions = append(snapshot.Consul.Deletions, routeRef) + } + if kubernetesNeedsUpdate { + snapshot.Kubernetes.Updates = append(snapshot.Kubernetes.Updates, route) + } + if kubernetesNeedsStatusUpdate { + snapshot.Kubernetes.StatusUpdates = append(snapshot.Kubernetes.StatusUpdates, route) + } + + updatedSnapshot = snapshot + }() + + if isDeleted(route) { + // mark the route as needing to get cleaned up if we detect that it's being deleted + consulNeedsDelete = true + if removeFinalizer(route) { + kubernetesNeedsUpdate = true + } + return + } + + if r.isGatewayDeleted { + // first check if this is our only ref for the route + if r.tracker.isLastReference(route) { + // if it is, then mark everything for deletion + consulNeedsDelete = true + if r.removeControllerStatusFunc(route) { + kubernetesNeedsStatusUpdate = true + } + if removeFinalizer(route) { + kubernetesNeedsUpdate = true + } + return + } + + // otherwise remove the condition since we no longer know if we should + // control the route and drop any references for the Consul route + if !isNil(existing) { + // this drops all the parent refs + r.setParentsFunc(existing, parentsForRoute(r.gatewayRef, r.getParentsFunc(existing), nil)) + // and then we mark the route as needing updated + consulUpdate = existing + // drop the status conditions + if r.removeStatusRefsFunc(route, gatewayRefs) { + kubernetesNeedsStatusUpdate = true + } + } + return + } + + if ensureFinalizer(route) { + kubernetesNeedsUpdate = true + return + } + + // TODO: scrub route refs from statuses that no longer exist + + validation := validateRefs(route.GetNamespace(), r.getBackendRefsFunc(route), r.services) + // the spec is dumb and makes you set a parent for any status, even when the + // status is not with respect to a parent, as is the case of resolved refs + // so we need to set the status on all parents + for _, ref := range gatewayRefs { + if r.setRouteConditionFunc(route, &ref, validation.Condition()) { + kubernetesNeedsStatusUpdate = true + } + } + + results := make(parentBindResults, 0) + namespace := r.namespaces[route.GetNamespace()] + gk := route.GetObjectKind().GroupVersionKind().GroupKind() + for _, ref := range gatewayRefs { + result := make(bindResults, 0) + for _, listener := range listenersFor(r.gateway, ref.SectionName) { + if !routeKindIsAllowedForListener(supportedKindsForProtocol[listener.Protocol], gk) { + result = append(result, bindResult{ + section: listener.Name, + err: errRouteNotAllowedByListeners_Protocol, + }) + continue + } + + if !routeKindIsAllowedForListenerExplicit(listener.AllowedRoutes, gk) { + result = append(result, bindResult{ + section: listener.Name, + err: errRouteNotAllowedByListeners_Protocol, + }) + continue + } + + if !routeAllowedForListenerNamespaces(r.gateway.Namespace, listener.AllowedRoutes, namespace) { + result = append(result, bindResult{ + section: listener.Name, + err: errRouteNotAllowedByListeners_Namespace, + }) + continue + } + + if !routeAllowedForListenerHostname(listener.Hostname, r.getHostnamesFunc(route)) { + result = append(result, bindResult{ + section: listener.Name, + err: errRouteNoMatchingListenerHostname, + }) + continue + } + + result = append(result, bindResult{ + section: listener.Name, + }) + + boundCount[listener.Name] = boundCount[listener.Name] + 1 + } + + results = append(results, parentBindResult{ + parent: ref, + results: result, + }) + } + + updated := false + for _, result := range results { + if r.setRouteConditionFunc(route, &result.parent, result.results.Condition()) { + updated = true + } + } + + if updated { + kubernetesNeedsStatusUpdate = true + } + + entry := r.translationFunc(route, nil) + // make all parent refs explicit based on what actually bound + if isNil(existing) { + r.setParentsFunc(entry, parentsForRoute(r.gatewayRef, nil, results)) + } else { + r.setParentsFunc(entry, parentsForRoute(r.gatewayRef, r.getParentsFunc(existing), results)) + } + consulUpdate = entry + + return +} + +// newTCPRouteBinder wraps newRouteBinder with the proper closures needed for accessing TCPRoutes and their config entries. +func (b *Binder) newTCPRouteBinder(tracker referenceTracker, services map[types.NamespacedName]api.CatalogService) *routeBinder[*gwv1alpha2.TCPRoute, *api.TCPRouteConfigEntry] { + return newRouteBinder( + b.isGatewayDeleted(), + &b.config.Gateway, + b.gatewayRef(), + b.config.Namespaces, + services, + tracker, + b.config.Translator.ReferenceForTCPRoute, + b.consulTCPRouteFor, + func(t *api.TCPRouteConfigEntry) []api.ResourceReference { return t.Parents }, + func(t *api.TCPRouteConfigEntry, parents []api.ResourceReference) { t.Parents = parents }, + b.statusSetter.removeTCPRouteReferences, + func(t *gwv1alpha2.TCPRoute) []gwv1beta1.Hostname { return nil }, + func(t *gwv1alpha2.TCPRoute) []gwv1beta1.ParentReference { return t.Spec.ParentRefs }, + b.config.Translator.TCPRouteToTCPRoute, + b.statusSetter.setTCPRouteCondition, + func(t *gwv1alpha2.TCPRoute) []gwv1beta1.BackendRef { + refs := []gwv1beta1.BackendRef{} + for _, rule := range t.Spec.Rules { + refs = append(refs, rule.BackendRefs...) + } + return refs + }, + b.statusSetter.removeTCPStatuses, + ) +} + +// newHTTPRouteBinder wraps newRouteBinder with the proper closures needed for accessing HTTPRoutes and their config entries. +func (b *Binder) newHTTPRouteBinder(tracker referenceTracker, services map[types.NamespacedName]api.CatalogService) *routeBinder[*gwv1beta1.HTTPRoute, *api.HTTPRouteConfigEntry] { + return newRouteBinder( + b.isGatewayDeleted(), + &b.config.Gateway, + b.gatewayRef(), + b.config.Namespaces, + services, + tracker, + b.config.Translator.ReferenceForHTTPRoute, + b.consulHTTPRouteFor, + func(t *api.HTTPRouteConfigEntry) []api.ResourceReference { return t.Parents }, + func(t *api.HTTPRouteConfigEntry, parents []api.ResourceReference) { t.Parents = parents }, + b.statusSetter.removeHTTPRouteReferences, + func(t *gwv1beta1.HTTPRoute) []gwv1beta1.Hostname { return t.Spec.Hostnames }, + func(t *gwv1beta1.HTTPRoute) []gwv1beta1.ParentReference { return t.Spec.ParentRefs }, + b.config.Translator.HTTPRouteToHTTPRoute, + b.statusSetter.setHTTPRouteCondition, + func(t *gwv1beta1.HTTPRoute) []gwv1beta1.BackendRef { + refs := []gwv1beta1.BackendRef{} + for _, rule := range t.Spec.Rules { + for _, ref := range rule.BackendRefs { + refs = append(refs, ref.BackendRef) + } + } + return refs + }, + b.statusSetter.removeHTTPStatuses, + ) +} + +// cleanRoute removes a gateway reference from the given route config entry +// and marks adds it to the snapshot if its mutated the entry at all. +func cleanRoute[T api.ConfigEntry]( + route T, + seenRoutes map[api.ResourceReference]struct{}, + snapshot Snapshot, + gatewayRef api.ResourceReference, + getParentsFunc func(T) []api.ResourceReference, + setParentsFunc func(T, []api.ResourceReference), +) Snapshot { + routeRef := translation.EntryToReference(route) + if _, ok := seenRoutes[routeRef]; !ok { + existingParents := getParentsFunc(route) + parents := parentsForRoute(gatewayRef, existingParents, nil) + if len(parents) == 0 { + // we can GC this now since we've dropped all refs from it + snapshot.Consul.Deletions = append(snapshot.Consul.Deletions, routeRef) + } else if len(existingParents) != len(parents) { + // we've mutated the length, which means this route needs an update + setParentsFunc(route, parents) + snapshot.Consul.Updates = append(snapshot.Consul.Updates, route) + } + } + return snapshot +} + +// cleanHTTPRoute wraps cleanRoute with the proper closures for HTTPRoute config entries. +func (b *Binder) cleanHTTPRoute(route *api.HTTPRouteConfigEntry, seenRoutes map[api.ResourceReference]struct{}, snapshot Snapshot) Snapshot { + return cleanRoute(route, seenRoutes, snapshot, b.gatewayRef(), + func(route *api.HTTPRouteConfigEntry) []api.ResourceReference { return route.Parents }, + func(route *api.HTTPRouteConfigEntry, parents []api.ResourceReference) { route.Parents = parents }, + ) +} + +// cleanTCPRoute wraps cleanRoute with the proper closures for TCPRoute config entries. +func (b *Binder) cleanTCPRoute(route *api.TCPRouteConfigEntry, seenRoutes map[api.ResourceReference]struct{}, snapshot Snapshot) Snapshot { + return cleanRoute(route, seenRoutes, snapshot, b.gatewayRef(), + func(route *api.TCPRouteConfigEntry) []api.ResourceReference { return route.Parents }, + func(route *api.TCPRouteConfigEntry, parents []api.ResourceReference) { route.Parents = parents }, + ) +} + +// parentsForRoute constructs a list of Consul route parent references based on what parents actually bound +// on a given route. This is necessary due to the fact that some additional validation in Kubernetes might +// require a route not to actually be accepted by a gateway, whereas we may have laxer logic inside of Consul +// itself. In these cases we want to just drop the parent reference in the Consul config entry we are going +// to write in order for it not to succeed in binding where Kubernetes failed to bind. +func parentsForRoute(ref api.ResourceReference, existing []api.ResourceReference, results parentBindResults) []api.ResourceReference { + // store all section names that bound + parentSet := map[string]struct{}{} + for _, result := range results { + for _, r := range result.results { + if r.err != nil { + parentSet[string(r.section)] = struct{}{} + } + } + } + + // first, filter out all of the parent refs that don't correspond to this gateway + parents := []api.ResourceReference{} + for _, parent := range existing { + if parent.Kind == api.APIGateway && + parent.Name == ref.Name && + parent.Namespace == ref.Namespace { + continue + } + parents = append(parents, parent) + } + + // now construct the bound set + for parent := range parentSet { + parents = append(parents, api.ResourceReference{ + Kind: api.APIGateway, + Name: ref.Name, + Namespace: ref.Namespace, + SectionName: parent, + }) + } + return parents +} + +// filterParentRefs returns the subset of parent references on a route that point to the given gateway. +func filterParentRefs(gateway types.NamespacedName, namespace string, refs []gwv1beta1.ParentReference) []gwv1beta1.ParentReference { + references := []gwv1beta1.ParentReference{} + for _, ref := range refs { + if nilOrEqual(ref.Group, betaGroup) && + nilOrEqual(ref.Kind, kindGateway) && + gateway.Namespace == valueOr(ref.Namespace, namespace) && + gateway.Name == string(ref.Name) { + references = append(references, ref) + } + } + + return references +} + +// listenersFor returns the listeners corresponding the given section name. If the section +// name is actually specified, the returned set should just have one listener, if it is +// unspecified, the all gatweway listeners should be returned. +func listenersFor(gateway *gwv1beta1.Gateway, name *gwv1beta1.SectionName) []gwv1beta1.Listener { + listeners := []gwv1beta1.Listener{} + for _, listener := range gateway.Spec.Listeners { + if name == nil { + listeners = append(listeners, listener) + continue + } + if listener.Name == *name { + listeners = append(listeners, listener) + } + } + return listeners +} diff --git a/control-plane/api-gateway/binding/setter.go b/control-plane/api-gateway/binding/setter.go new file mode 100644 index 0000000000..2d1f058501 --- /dev/null +++ b/control-plane/api-gateway/binding/setter.go @@ -0,0 +1,177 @@ +package binding + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// setter wraps the status setting logic for routes. +type setter struct { + controllerName string +} + +// newSetter constructs a status setter with the given controller name. +func newSetter(controllerName string) *setter { + return &setter{controllerName: controllerName} +} + +// setHTTPRouteCondition sets an HTTPRoute condition on its status with the given parent. +func (s *setter) setHTTPRouteCondition(route *gwv1beta1.HTTPRoute, parent *gwv1beta1.ParentReference, condition metav1.Condition) bool { + condition.LastTransitionTime = metav1.Now() + condition.ObservedGeneration = route.Generation + + status := s.getParentStatus(route.Status.Parents, parent) + conditions, modified := setCondition(status.Conditions, condition) + if modified { + status.Conditions = conditions + route.Status.Parents = s.setParentStatus(route.Status.Parents, status) + } + return modified +} + +// removeHTTPRouteReferences removes the given parent reference sections from an HTTPRoute's status. +func (s *setter) removeHTTPRouteReferences(route *gwv1beta1.HTTPRoute, refs []gwv1beta1.ParentReference) bool { + modified := false + for _, parent := range refs { + parents, removed := s.removeParentStatus(route.Status.Parents, parent) + route.Status.Parents = parents + if removed { + modified = true + } + } + return modified +} + +// setTCPRouteCondition sets a TCPRoute condition on its status with the given parent. +func (s *setter) setTCPRouteCondition(route *gwv1alpha2.TCPRoute, parent *gwv1beta1.ParentReference, condition metav1.Condition) bool { + condition.LastTransitionTime = metav1.Now() + condition.ObservedGeneration = route.Generation + + status := s.getParentStatus(route.Status.Parents, parent) + conditions, modified := setCondition(status.Conditions, condition) + if modified { + status.Conditions = conditions + route.Status.Parents = s.setParentStatus(route.Status.Parents, status) + } + return modified +} + +// removeTCPRouteReferences removes the given parent reference sections from a TCPRoute's status. +func (s *setter) removeTCPRouteReferences(route *gwv1alpha2.TCPRoute, refs []gwv1beta1.ParentReference) bool { + modified := false + for _, parent := range refs { + parents, removed := s.removeParentStatus(route.Status.Parents, parent) + route.Status.Parents = parents + if removed { + modified = true + } + } + return modified +} + +// removeHTTPStatuses removes all statuses set by the given controller from an HTTPRoute's status. +func (s *setter) removeHTTPStatuses(route *gwv1beta1.HTTPRoute) bool { + modified := false + filtered := []gwv1beta1.RouteParentStatus{} + for _, status := range route.Status.Parents { + if string(status.ControllerName) == s.controllerName { + modified = true + continue + } + filtered = append(filtered, status) + } + + if modified { + route.Status.Parents = filtered + } + return modified +} + +// removeTCPStatuses removes all statuses set by the given controller from a TCPRoute's status. +func (s *setter) removeTCPStatuses(route *gwv1alpha2.TCPRoute) bool { + modified := false + filtered := []gwv1beta1.RouteParentStatus{} + for _, status := range route.Status.Parents { + if string(status.ControllerName) == s.controllerName { + modified = true + continue + } + filtered = append(filtered, status) + } + + if modified { + route.Status.Parents = filtered + } + return modified +} + +// getParentStatus returns the section of a status referenced by the given parent reference. +func (s *setter) getParentStatus(statuses []gwv1beta1.RouteParentStatus, parent *gwv1beta1.ParentReference) gwv1beta1.RouteParentStatus { + var parentRef gwv1beta1.ParentReference + if parent != nil { + parentRef = *parent + } + + for _, status := range statuses { + if parentsEqual(status.ParentRef, parentRef) && string(status.ControllerName) == s.controllerName { + return status + } + } + return gwv1beta1.RouteParentStatus{ + ParentRef: parentRef, + ControllerName: gwv1beta1.GatewayController(s.controllerName), + } +} + +// removeParentStatus removes the section of a status referenced by the given parent reference. +func (s *setter) removeParentStatus(statuses []gwv1beta1.RouteParentStatus, parent gwv1beta1.ParentReference) ([]gwv1beta1.RouteParentStatus, bool) { + found := false + filtered := []gwv1beta1.RouteParentStatus{} + for _, status := range statuses { + if parentsEqual(status.ParentRef, parent) && string(status.ControllerName) == s.controllerName { + found = true + continue + } + filtered = append(filtered, status) + } + return filtered, found +} + +// setCondition overrides or appends a condition to the list of conditions, returning if a modification +// to the condition set was made or not. Modifications only occur if a field other than the observation +// timestamp is modified. +func setCondition(conditions []metav1.Condition, condition metav1.Condition) ([]metav1.Condition, bool) { + for i, existing := range conditions { + if existing.Type == condition.Type { + // no-op if we have the exact same thing + if condition.Reason == existing.Reason && condition.Message == existing.Message && condition.ObservedGeneration == existing.ObservedGeneration { + return conditions, false + } + + conditions[i] = condition + return conditions, true + } + } + return append(conditions, condition), true +} + +// setParentStatus updates or inserts the set of parent statuses with the newly modified parent. +func (s *setter) setParentStatus(statuses []gwv1beta1.RouteParentStatus, parent gwv1beta1.RouteParentStatus) []gwv1beta1.RouteParentStatus { + for i, status := range statuses { + if parentsEqual(status.ParentRef, parent.ParentRef) && status.ControllerName == parent.ControllerName { + statuses[i] = parent + return statuses + } + } + return append(statuses, parent) +} + +// parentsEqual checks for equality between two parent references. +func parentsEqual(one, two gwv1beta1.ParentReference) bool { + return bothNilOrEqual(one.Group, two.Group) && + bothNilOrEqual(one.Kind, two.Kind) && + bothNilOrEqual(one.SectionName, two.SectionName) && + bothNilOrEqual(one.Port, two.Port) && + one.Name == two.Name +} diff --git a/control-plane/api-gateway/binding/setter_test.go b/control-plane/api-gateway/binding/setter_test.go new file mode 100644 index 0000000000..f1bb0fc833 --- /dev/null +++ b/control-plane/api-gateway/binding/setter_test.go @@ -0,0 +1,39 @@ +package binding + +import ( + "testing" + + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +func TestSetter(t *testing.T) { + setter := newSetter("test") + parentRef := gwv1beta1.ParentReference{ + Name: "test", + } + parentRefDup := gwv1beta1.ParentReference{ + Name: "test", + } + condition := metav1.Condition{ + Type: "Accepted", + Status: metav1.ConditionTrue, + Reason: "Accepted", + Message: "route accepted", + } + route := &gwv1beta1.HTTPRoute{ + Spec: gwv1beta1.HTTPRouteSpec{ + CommonRouteSpec: gwv1beta1.CommonRouteSpec{ + ParentRefs: []gwv1beta1.ParentReference{parentRef}, + }, + }, + } + require.True(t, setter.setHTTPRouteCondition(route, &parentRef, condition)) + require.False(t, setter.setHTTPRouteCondition(route, &parentRefDup, condition)) + require.False(t, setter.setHTTPRouteCondition(route, &parentRefDup, condition)) + require.False(t, setter.setHTTPRouteCondition(route, &parentRefDup, condition)) + + require.Len(t, route.Status.Parents, 1) + require.Len(t, route.Status.Parents[0].Conditions, 1) +} diff --git a/control-plane/api-gateway/binding/snapshot.go b/control-plane/api-gateway/binding/snapshot.go new file mode 100644 index 0000000000..ff43cd2d13 --- /dev/null +++ b/control-plane/api-gateway/binding/snapshot.go @@ -0,0 +1,43 @@ +package binding + +import ( + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul/api" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// KubernetesSnapshot contains all the operations +// required in Kubernetes to complete reconciliation. +type KubernetesSnapshot struct { + // Updates is the list of objects that need to have + // aspects of their metadata or spec updated in Kubernetes + // (i.e. for finalizers or annotations) + Updates []client.Object + // StatusUpdates is the list of objects that need + // to have their statuses updated in Kubernetes + StatusUpdates []client.Object +} + +// ConsulSnapshot contains all the operations required +// in Consul to complete reconciliation. +type ConsulSnapshot struct { + // Updates is the list of ConfigEntry objects that should + // either be updated or created in Consul + Updates []api.ConfigEntry + // Deletions is a list of references that ought to be + // deleted in Consul + Deletions []api.ResourceReference +} + +// Snapshot contains all Kubernetes and Consul operations +// needed to complete reconciliation. +type Snapshot struct { + // Kubernetes holds the snapshot of required Kubernetes operations + Kubernetes KubernetesSnapshot + // Consul holds the snapshot of required Consul operations + Consul ConsulSnapshot + // GatewayClassConfig is the configuration to use for determining + // a Gateway deployment, if it is not set, a deployment should be + // deleted instead of updated + GatewayClassConfig *v1alpha1.GatewayClassConfig +} diff --git a/control-plane/api-gateway/binding/utils.go b/control-plane/api-gateway/binding/utils.go new file mode 100644 index 0000000000..d112d894c9 --- /dev/null +++ b/control-plane/api-gateway/binding/utils.go @@ -0,0 +1,102 @@ +package binding + +import ( + "reflect" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// pointerTo is a convenience method for taking a pointer +// of an object without having to declare an intermediate variable. +// It's also useful for making sure we don't accidentally take +// the pointer of a range variable directly. +func pointerTo[T any](v T) *T { + return &v +} + +// isNil checks if the argument is nil. It's mainly used to +// check if a generic conforming to a nullable interface is +// actually nil. +func isNil(arg interface{}) bool { + return arg == nil || reflect.ValueOf(arg).IsNil() +} + +// bothNilOrEqual is used to determine if two pointers to comparable +// object are either nil or both point to the same value. +func bothNilOrEqual[T comparable](one, two *T) bool { + if one == nil && two == nil { + return true + } + if one == nil { + return false + } + if two == nil { + return false + } + return *one == *two +} + +// valueOr checks if a string-like pointer is nil, and if it is, +// returns the given value instead. +func valueOr[T ~string](v *T, fallback string) string { + if v == nil { + return fallback + } + return string(*v) +} + +// nilOrEqual checks if a string-like pointer is nil or if it is +// equal to the value provided. +func nilOrEqual[T ~string](v *T, check string) bool { + return v == nil || string(*v) == check +} + +// objectToMeta returns the NamespacedName for the given object. +func objectToMeta[T metav1.Object](object T) types.NamespacedName { + return types.NamespacedName{ + Namespace: object.GetNamespace(), + Name: object.GetName(), + } +} + +// isDeleted checks if the deletion timestamp is set for an object. +func isDeleted(object client.Object) bool { + return !object.GetDeletionTimestamp().IsZero() +} + +// ensureFinalizer ensures that our finalizer is set on an object +// returning whether or not it modified the object. +func ensureFinalizer(object client.Object) bool { + if !object.GetDeletionTimestamp().IsZero() { + return false + } + + finalizers := object.GetFinalizers() + for _, f := range finalizers { + if f == gatewayFinalizer { + return false + } + } + + object.SetFinalizers(append(finalizers, gatewayFinalizer)) + return true +} + +// removeFinalizer ensures that our finalizer is absent from an object +// returning whether or not it modified the object. +func removeFinalizer(object client.Object) bool { + found := false + filtered := []string{} + for _, f := range object.GetFinalizers() { + if f == gatewayFinalizer { + found = true + continue + } + filtered = append(filtered, f) + } + + object.SetFinalizers(filtered) + return found +} diff --git a/control-plane/api-gateway/binding/utils_test.go b/control-plane/api-gateway/binding/utils_test.go new file mode 100644 index 0000000000..a8da3f9a76 --- /dev/null +++ b/control-plane/api-gateway/binding/utils_test.go @@ -0,0 +1,236 @@ +package binding + +import ( + "testing" + + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +func TestIsNil(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + value interface{} + expected bool + }{ + "nil pointer": { + value: (*string)(nil), + expected: true, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, isNil(tt.value)) + }) + } +} + +func TestBothNilOrEqual(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + first *string + second *string + expected bool + }{ + "both nil": { + first: nil, + second: nil, + expected: true, + }, + "second nil": { + first: pointerTo(""), + second: nil, + expected: false, + }, + "first nil": { + first: nil, + second: pointerTo(""), + expected: false, + }, + "both equal": { + first: pointerTo(""), + second: pointerTo(""), + expected: true, + }, + "both not equal": { + first: pointerTo("1"), + second: pointerTo("2"), + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, bothNilOrEqual(tt.first, tt.second)) + }) + } +} + +func TestValueOr(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + value *string + or string + expected string + }{ + "nil value": { + value: nil, + or: "test", + expected: "test", + }, + "set value": { + value: pointerTo("value"), + or: "test", + expected: "value", + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, valueOr(tt.value, tt.or)) + }) + } +} + +func TestNilOrEqual(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + value *string + check string + expected bool + }{ + "nil value": { + value: nil, + check: "test", + expected: true, + }, + "equal values": { + value: pointerTo("test"), + check: "test", + expected: true, + }, + "unequal values": { + value: pointerTo("value"), + check: "test", + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, nilOrEqual(tt.value, tt.check)) + }) + } +} + +func TestObjectToMeta(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + object metav1.Object + expected types.NamespacedName + }{ + "gateway": { + object: &gwv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{Namespace: "test", Name: "test"}}, + expected: types.NamespacedName{Namespace: "test", Name: "test"}, + }, + "secret": { + object: &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Namespace: "secret", Name: "secret"}}, + expected: types.NamespacedName{Namespace: "secret", Name: "secret"}, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, objectToMeta(tt.object)) + }) + } +} + +func TestIsDeleted(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + object client.Object + expected bool + }{ + "deleted gateway": { + object: &gwv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{DeletionTimestamp: pointerTo(metav1.Now())}}, + expected: true, + }, + "non-deleted http route": { + object: &gwv1beta1.HTTPRoute{}, + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, isDeleted(tt.object)) + }) + } +} + +func TestEnsureFinalizer(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + object client.Object + expected bool + finalizers []string + }{ + "gateway no finalizer": { + object: &gwv1beta1.Gateway{}, + expected: true, + finalizers: []string{gatewayFinalizer}, + }, + "gateway other finalizer": { + object: &gwv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{"other"}}}, + expected: true, + finalizers: []string{"other", gatewayFinalizer}, + }, + "gateway already has finalizer": { + object: &gwv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{gatewayFinalizer}}}, + expected: false, + finalizers: []string{gatewayFinalizer}, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, ensureFinalizer(tt.object)) + require.Equal(t, tt.finalizers, tt.object.GetFinalizers()) + }) + } +} + +func TestRemoveFinalizer(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + object client.Object + expected bool + finalizers []string + }{ + "gateway no finalizer": { + object: &gwv1beta1.Gateway{}, + expected: false, + finalizers: []string{}, + }, + "gateway other finalizer": { + object: &gwv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{"other"}}}, + expected: false, + finalizers: []string{"other"}, + }, + "gateway multiple finalizers": { + object: &gwv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{gatewayFinalizer, gatewayFinalizer}}}, + expected: true, + finalizers: []string{}, + }, + "gateway mixed finalizers": { + object: &gwv1beta1.Gateway{ObjectMeta: metav1.ObjectMeta{Finalizers: []string{"other", gatewayFinalizer}}}, + expected: true, + finalizers: []string{"other"}, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, removeFinalizer(tt.object)) + require.Equal(t, tt.finalizers, tt.object.GetFinalizers()) + }) + } +} diff --git a/control-plane/api-gateway/binding/validation.go b/control-plane/api-gateway/binding/validation.go new file mode 100644 index 0000000000..f18c980cba --- /dev/null +++ b/control-plane/api-gateway/binding/validation.go @@ -0,0 +1,298 @@ +package binding + +import ( + "strings" + + "github.com/hashicorp/consul/api" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + klabels "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +// validateRefs validates backend references for a route, determining whether or +// not they were found in the list of known connect-injected services. +func validateRefs(namespace string, refs []gwv1beta1.BackendRef, services map[types.NamespacedName]api.CatalogService) routeValidationResults { + var result routeValidationResults + for _, ref := range refs { + nsn := types.NamespacedName{ + Name: string(ref.BackendObjectReference.Name), + Namespace: valueOr(ref.BackendObjectReference.Namespace, namespace), + } + + // TODO: check reference grants + + if !nilOrEqual(ref.BackendObjectReference.Group, "") || + !nilOrEqual(ref.BackendObjectReference.Kind, "Service") { + result = append(result, routeValidationResult{ + namespace: nsn.Namespace, + backend: ref, + err: errRouteInvalidKind, + }) + continue + } + + if _, found := services[nsn]; !found { + result = append(result, routeValidationResult{ + namespace: nsn.Namespace, + backend: ref, + err: errRouteBackendNotFound, + }) + continue + } + + result = append(result, routeValidationResult{ + namespace: nsn.Namespace, + backend: ref, + }) + } + return result +} + +// validateGateway validates that a gateway is semantically valid given +// the set of features that we support. +func validateGateway(gateway gwv1beta1.Gateway) gatewayValidationResult { + var result gatewayValidationResult + + if len(gateway.Spec.Addresses) > 0 { + result.acceptedErr = errGatewayUnsupportedAddress + } + + return result +} + +// mergedListener associates a listener with its indexed position +// in the gateway spec, it's used to re-associate a status with +// a listener after we merge compatible listeners together and then +// validate their conflicts. +type mergedListener struct { + index int + listener gwv1beta1.Listener +} + +// mergedListeners is a set of a listeners that are considered "merged" +// due to referencing the same listener port. +type mergedListeners []mergedListener + +// validateProtocol validates that the protocols used across all merged +// listeners are compatible. +func (m mergedListeners) validateProtocol() error { + var protocol *gwv1beta1.ProtocolType + for _, l := range m { + if protocol == nil { + protocol = pointerTo(l.listener.Protocol) + } + if *protocol != l.listener.Protocol { + return errListenerProtocolConflict + } + } + return nil +} + +// validateHostname validates that the merged listeners don't use the same +// hostnames as per the spec. +func (m mergedListeners) validateHostname(index int, listener gwv1beta1.Listener) error { + for _, l := range m { + if l.index == index { + continue + } + if bothNilOrEqual(listener.Hostname, l.listener.Hostname) { + return errListenerHostnameConflict + } + } + return nil +} + +// validateTLS validates that the TLS configuration for a given listener is valid and that +// the certificates that it references exist. +func validateTLS(namespace string, tls *gwv1beta1.GatewayTLSConfig, certificates []corev1.Secret) (error, error) { + if tls == nil { + return nil, nil + } + + // TODO: Resource Grants + + var err error +MAIN_LOOP: + for _, ref := range tls.CertificateRefs { + // break on the first error + if !nilOrEqual(ref.Group, "") || !nilOrEqual(ref.Kind, "Secret") { + err = errListenerInvalidCertificateRef_NotSupported + break MAIN_LOOP + } + ns := valueOr(ref.Namespace, namespace) + + for _, secret := range certificates { + if secret.Namespace == ns && secret.Name == string(ref.Name) { + continue MAIN_LOOP + } + } + + // not found, set error + err = errListenerInvalidCertificateRef_NotFound + break MAIN_LOOP + } + + if tls.Mode != nil && *tls.Mode == gwv1beta1.TLSModePassthrough { + return errListenerNoTLSPassthrough, err + } + + // TODO: validate tls options + return nil, err +} + +// validateListeners validates the given listeners both internally and with respect to each +// other for purposes of setting "Conflicted" status conditions. +func validateListeners(namespace string, listeners []gwv1beta1.Listener, secrets []corev1.Secret) listenerValidationResults { + var results listenerValidationResults + merged := make(map[gwv1beta1.PortNumber]mergedListeners) + for i, listener := range listeners { + merged[listener.Port] = append(merged[listener.Port], mergedListener{ + index: i, + listener: listener, + }) + } + + for i, listener := range listeners { + var result listenerValidationResult + + err, refErr := validateTLS(namespace, listener.TLS, secrets) + result.refErr = refErr + if err != nil { + result.acceptedErr = err + } else { + _, supported := supportedKindsForProtocol[listener.Protocol] + if !supported { + result.acceptedErr = errListenerUnsupportedProtocol + } else if listener.Port == 20000 { //admin port + result.acceptedErr = errListenerPortUnavailable + } + } + + if err := merged[listener.Port].validateProtocol(); err != nil { + result.conflictedErr = err + } else { + result.conflictedErr = merged[listener.Port].validateHostname(i, listener) + } + + results = append(results, result) + } + return results +} + +// routeAllowedForListenerNamespaces determines whether the route is allowed +// to bind to the Gateway based on the AllowedRoutes namespace selectors. +func routeAllowedForListenerNamespaces(gatewayNamespace string, allowedRoutes *gwv1beta1.AllowedRoutes, namespace corev1.Namespace) bool { + var namespaceSelector *gwv1beta1.RouteNamespaces + if allowedRoutes != nil { + // check gateway namespace + namespaceSelector = allowedRoutes.Namespaces + } + + // set default if namespace selector is nil + from := gwv1beta1.NamespacesFromSame + if namespaceSelector != nil && namespaceSelector.From != nil && *namespaceSelector.From != "" { + from = *namespaceSelector.From + } + + switch from { + case gwv1beta1.NamespacesFromAll: + return true + case gwv1beta1.NamespacesFromSame: + return gatewayNamespace == namespace.Name + case gwv1beta1.NamespacesFromSelector: + namespaceSelector, err := metav1.LabelSelectorAsSelector(namespaceSelector.Selector) + if err != nil { + // log the error here, the label selector is invalid + return false + } + + return namespaceSelector.Matches(toNamespaceSet(namespace.GetName(), namespace.GetLabels())) + default: + return false + } +} + +// routeAllowedForListenerHostname checks that a hostname specified on a route and the hostname specified +// on the gateway listener are compatible. +func routeAllowedForListenerHostname(hostname *gwv1beta1.Hostname, hostnames []gwv1beta1.Hostname) bool { + if hostname == nil || len(hostnames) == 0 { + return true + } + + for _, name := range hostnames { + if hostnamesMatch(name, *hostname) { + return true + } + } + return false +} + +// hostnameMatch checks that an individual hostname matches another hostname for +// compatibility. +func hostnamesMatch(a gwv1alpha2.Hostname, b gwv1beta1.Hostname) bool { + if a == "" || a == "*" || b == "" || b == "*" { + // any wildcard always matches + return true + } + + if strings.HasPrefix(string(a), "*.") || strings.HasPrefix(string(b), "*.") { + aLabels, bLabels := strings.Split(string(a), "."), strings.Split(string(b), ".") + if len(aLabels) != len(bLabels) { + return false + } + + for i := 1; i < len(aLabels); i++ { + if !strings.EqualFold(aLabels[i], bLabels[i]) { + return false + } + } + return true + } + + return string(a) == string(b) +} + +// routeKindIsAllowedForListener checks that the given route kind is present in the allowed set. +func routeKindIsAllowedForListener(kinds []gwv1beta1.RouteGroupKind, gk schema.GroupKind) bool { + if kinds == nil { + return true + } + + for _, kind := range kinds { + if string(kind.Kind) == gk.Kind && nilOrEqual(kind.Group, gk.Group) { + return true + } + } + + return false +} + +// routeKindIsAllowedForListenerExplicit checks that a route is allowed by the kinds specified explicitly +// on the listener. +func routeKindIsAllowedForListenerExplicit(allowedRoutes *gwv1alpha2.AllowedRoutes, gk schema.GroupKind) bool { + if allowedRoutes == nil { + return true + } + + return routeKindIsAllowedForListener(allowedRoutes.Kinds, gk) +} + +// toNamespaceSet constructs a list of labels used to match a Namespace. +func toNamespaceSet(name string, labels map[string]string) klabels.Labels { + // If namespace label is not set, implicitly insert it to support older Kubernetes versions + if labels[namespaceNameLabel] == name { + // Already set, avoid copies + return klabels.Set(labels) + } + // First we need a copy to not modify the underlying object + ret := make(map[string]string, len(labels)+1) + for k, v := range labels { + ret[k] = v + } + ret[namespaceNameLabel] = name + return klabels.Set(ret) +} diff --git a/control-plane/api-gateway/binding/validation_test.go b/control-plane/api-gateway/binding/validation_test.go new file mode 100644 index 0000000000..ee0ff8ba1b --- /dev/null +++ b/control-plane/api-gateway/binding/validation_test.go @@ -0,0 +1,644 @@ +package binding + +import ( + "testing" + + "github.com/hashicorp/consul/api" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" +) + +func TestValidateRefs(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + namespace string + refs []gwv1beta1.BackendObjectReference + services map[types.NamespacedName]api.CatalogService + expectedErrors []error + }{ + "all pass no namespaces": { + namespace: "test", + refs: []gwv1beta1.BackendObjectReference{{Name: "1"}, {Name: "2"}}, + services: map[types.NamespacedName]api.CatalogService{ + {Name: "1", Namespace: "test"}: {}, + {Name: "2", Namespace: "test"}: {}, + {Name: "3", Namespace: "test"}: {}, + }, + expectedErrors: []error{nil, nil}, + }, + "all pass namespaces": { + namespace: "test", + refs: []gwv1beta1.BackendObjectReference{ + {Name: "1", Namespace: pointerTo[gwv1beta1.Namespace]("other")}, + {Name: "2", Namespace: pointerTo[gwv1beta1.Namespace]("other")}, + }, + services: map[types.NamespacedName]api.CatalogService{ + {Name: "1", Namespace: "other"}: {}, + {Name: "2", Namespace: "other"}: {}, + {Name: "3", Namespace: "other"}: {}, + }, + expectedErrors: []error{nil, nil}, + }, + "all pass mixed": { + namespace: "test", + refs: []gwv1beta1.BackendObjectReference{ + {Name: "1", Namespace: pointerTo[gwv1beta1.Namespace]("other")}, + {Name: "2"}, + }, + services: map[types.NamespacedName]api.CatalogService{ + {Name: "1", Namespace: "other"}: {}, + {Name: "2", Namespace: "test"}: {}, + {Name: "3", Namespace: "other"}: {}, + }, + expectedErrors: []error{nil, nil}, + }, + "all fail mixed": { + namespace: "test", + refs: []gwv1beta1.BackendObjectReference{ + {Name: "1"}, + {Name: "2", Namespace: pointerTo[gwv1beta1.Namespace]("other")}, + }, + services: map[types.NamespacedName]api.CatalogService{ + {Name: "1", Namespace: "other"}: {}, + {Name: "2", Namespace: "test"}: {}, + {Name: "3", Namespace: "other"}: {}, + }, + expectedErrors: []error{errRouteBackendNotFound, errRouteBackendNotFound}, + }, + "all fail no namespaces": { + namespace: "test", + refs: []gwv1beta1.BackendObjectReference{ + {Name: "1"}, + {Name: "2"}, + }, + services: map[types.NamespacedName]api.CatalogService{ + {Name: "1", Namespace: "other"}: {}, + {Name: "2", Namespace: "other"}: {}, + {Name: "3", Namespace: "other"}: {}, + }, + expectedErrors: []error{errRouteBackendNotFound, errRouteBackendNotFound}, + }, + "all fail namespaces": { + namespace: "test", + refs: []gwv1beta1.BackendObjectReference{ + {Name: "1", Namespace: pointerTo[gwv1beta1.Namespace]("other")}, + {Name: "2", Namespace: pointerTo[gwv1beta1.Namespace]("other")}, + }, + services: map[types.NamespacedName]api.CatalogService{ + {Name: "1", Namespace: "test"}: {}, + {Name: "2", Namespace: "test"}: {}, + {Name: "3", Namespace: "test"}: {}, + }, + expectedErrors: []error{errRouteBackendNotFound, errRouteBackendNotFound}, + }, + "type failures": { + namespace: "test", + refs: []gwv1beta1.BackendObjectReference{ + {Name: "1", Group: pointerTo[gwv1beta1.Group]("test")}, + {Name: "2"}, + }, + services: map[types.NamespacedName]api.CatalogService{ + {Name: "1", Namespace: "test"}: {}, + {Name: "2", Namespace: "test"}: {}, + {Name: "3", Namespace: "test"}: {}, + }, + expectedErrors: []error{errRouteInvalidKind, nil}, + }, + } { + t.Run(name, func(t *testing.T) { + refs := make([]gwv1beta1.BackendRef, len(tt.refs)) + for i, ref := range tt.refs { + refs[i] = gwv1beta1.BackendRef{BackendObjectReference: ref} + } + + actual := validateRefs(tt.namespace, refs, tt.services) + require.Equal(t, len(actual), len(tt.refs)) + require.Equal(t, len(actual), len(tt.expectedErrors)) + for i, err := range tt.expectedErrors { + require.Equal(t, err, actual[i].err) + } + }) + } +} + +func TestValidateGateway(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + object gwv1beta1.Gateway + expected error + }{ + "valid": { + object: gwv1beta1.Gateway{}, + expected: nil, + }, + "invalid": { + object: gwv1beta1.Gateway{Spec: gwv1beta1.GatewaySpec{Addresses: []gwv1beta1.GatewayAddress{ + {Value: "1"}, + }}}, + expected: errGatewayUnsupportedAddress, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, validateGateway(tt.object).acceptedErr) + }) + } +} + +func TestMergedListeners_ValidateProtocol(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + mergedListeners mergedListeners + expected error + }{ + "valid": { + mergedListeners: []mergedListener{ + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + }, + expected: nil, + }, + "invalid": { + mergedListeners: []mergedListener{ + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.TCPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + }, + expected: errListenerProtocolConflict, + }, + "big list": { + mergedListeners: []mergedListener{ + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPSProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + {listener: gwv1beta1.Listener{Protocol: gwv1beta1.HTTPProtocolType}}, + }, + expected: errListenerProtocolConflict, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, tt.mergedListeners.validateProtocol()) + }) + } +} + +func TestMergedListeners_ValidateHostname(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + mergedListeners mergedListeners + expected error + }{ + "valid": { + mergedListeners: []mergedListener{ + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("1")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("2")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("3")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("4")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("5")}}, + {}, + }, + expected: nil, + }, + "invalid nil": { + mergedListeners: []mergedListener{ + {}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("1")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("2")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("3")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("4")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("5")}}, + {}, + }, + expected: errListenerHostnameConflict, + }, + "invalid set": { + mergedListeners: []mergedListener{ + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("1")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("2")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("3")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("4")}}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("5")}}, + {}, + {listener: gwv1beta1.Listener{Hostname: pointerTo[gwv1beta1.Hostname]("1")}}, + }, + expected: errListenerHostnameConflict, + }, + } { + t.Run(name, func(t *testing.T) { + for i, l := range tt.mergedListeners { + l.index = i + tt.mergedListeners[i] = l + } + + require.Equal(t, tt.expected, tt.mergedListeners.validateHostname(0, tt.mergedListeners[0].listener)) + }) + } +} + +func TestValidateTLS(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + namespace string + tls *gwv1beta1.GatewayTLSConfig + certificates []corev1.Secret + expectedResolvedRefsErr error + expectedAcceptedErr error + }{ + "no tls": { + namespace: "test", + tls: nil, + certificates: nil, + expectedResolvedRefsErr: nil, + expectedAcceptedErr: nil, + }, + "not supported certificate": { + namespace: "test", + tls: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + {Name: "foo", Namespace: pointerTo[gwv1beta1.Namespace]("other"), Group: pointerTo[gwv1beta1.Group]("test")}, + }, + }, + certificates: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "other"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "bar", Namespace: "other"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "baz", Namespace: "other"}}, + }, + expectedResolvedRefsErr: errListenerInvalidCertificateRef_NotSupported, + expectedAcceptedErr: nil, + }, + "not found certificate": { + namespace: "test", + tls: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + {Name: "zoiks", Namespace: pointerTo[gwv1beta1.Namespace]("other")}, + }, + }, + certificates: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "other"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "bar", Namespace: "other"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "baz", Namespace: "other"}}, + }, + expectedResolvedRefsErr: errListenerInvalidCertificateRef_NotFound, + expectedAcceptedErr: nil, + }, + "not found certificate mismatched namespace": { + namespace: "test", + tls: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + {Name: "foo", Namespace: pointerTo[gwv1beta1.Namespace]("1")}, + }, + }, + certificates: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "other"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "bar", Namespace: "other"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "baz", Namespace: "other"}}, + }, + expectedResolvedRefsErr: errListenerInvalidCertificateRef_NotFound, + expectedAcceptedErr: nil, + }, + "passthrough mode": { + namespace: "test", + tls: &gwv1beta1.GatewayTLSConfig{ + Mode: pointerTo(gwv1beta1.TLSModePassthrough), + }, + certificates: nil, + expectedResolvedRefsErr: nil, + expectedAcceptedErr: errListenerNoTLSPassthrough, + }, + "valid targeted namespace": { + namespace: "test", + tls: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + {Name: "foo", Namespace: pointerTo[gwv1beta1.Namespace]("1")}, + {Name: "bar", Namespace: pointerTo[gwv1beta1.Namespace]("2")}, + {Name: "baz", Namespace: pointerTo[gwv1beta1.Namespace]("3")}, + }, + }, + certificates: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "1"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "bar", Namespace: "2"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "baz", Namespace: "3"}}, + }, + expectedResolvedRefsErr: nil, + expectedAcceptedErr: nil, + }, + "valid same namespace": { + namespace: "test", + tls: &gwv1beta1.GatewayTLSConfig{ + CertificateRefs: []gwv1beta1.SecretObjectReference{ + {Name: "foo"}, + {Name: "bar"}, + {Name: "baz"}, + }, + }, + certificates: []corev1.Secret{ + {ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "test"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "bar", Namespace: "test"}}, + {ObjectMeta: metav1.ObjectMeta{Name: "baz", Namespace: "test"}}, + }, + expectedResolvedRefsErr: nil, + expectedAcceptedErr: nil, + }, + "valid empty certs": { + namespace: "test", + tls: &gwv1beta1.GatewayTLSConfig{}, + certificates: nil, + expectedResolvedRefsErr: nil, + expectedAcceptedErr: nil, + }, + } { + t.Run(name, func(t *testing.T) { + actualAcceptedError, actualResolvedRefsError := validateTLS(tt.namespace, tt.tls, tt.certificates) + require.Equal(t, tt.expectedResolvedRefsErr, actualResolvedRefsError) + require.Equal(t, tt.expectedAcceptedErr, actualAcceptedError) + }) + } +} + +func TestValidateListeners(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + listeners []gwv1beta1.Listener + expectedAcceptedErr error + }{ + "valid protocol HTTP": { + listeners: []gwv1beta1.Listener{ + {Protocol: gwv1beta1.HTTPProtocolType}, + }, + expectedAcceptedErr: nil, + }, + "valid protocol HTTPS": { + listeners: []gwv1beta1.Listener{ + {Protocol: gwv1beta1.HTTPSProtocolType}, + }, + expectedAcceptedErr: nil, + }, + "valid protocol TCP": { + listeners: []gwv1beta1.Listener{ + {Protocol: gwv1beta1.TCPProtocolType}, + }, + expectedAcceptedErr: nil, + }, + "invalid protocol UDP": { + listeners: []gwv1beta1.Listener{ + {Protocol: gwv1beta1.UDPProtocolType}, + }, + expectedAcceptedErr: errListenerUnsupportedProtocol, + }, + "invalid port": { + listeners: []gwv1beta1.Listener{ + {Protocol: gwv1beta1.TCPProtocolType, Port: 20000}, + }, + expectedAcceptedErr: errListenerPortUnavailable, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expectedAcceptedErr, validateListeners("", tt.listeners, nil)[0].acceptedErr) + }) + } +} + +func TestRouteAllowedForListenerNamespaces(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + allowedRoutes *gwv1beta1.AllowedRoutes + gatewayNamespace string + routeNamespace corev1.Namespace + expected bool + }{ + "default same namespace allowed": { + allowedRoutes: nil, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, + expected: true, + }, + "default same namespace not allowed": { + allowedRoutes: nil, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other"}}, + expected: false, + }, + "explicit same namespace allowed": { + allowedRoutes: &gwv1beta1.AllowedRoutes{Namespaces: &gwv1beta1.RouteNamespaces{From: pointerTo(gwv1beta1.NamespacesFromSame)}}, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, + expected: true, + }, + "explicit same namespace not allowed": { + allowedRoutes: &gwv1beta1.AllowedRoutes{Namespaces: &gwv1beta1.RouteNamespaces{From: pointerTo(gwv1beta1.NamespacesFromSame)}}, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other"}}, + expected: false, + }, + "all namespace allowed": { + allowedRoutes: &gwv1beta1.AllowedRoutes{Namespaces: &gwv1beta1.RouteNamespaces{From: pointerTo(gwv1beta1.NamespacesFromAll)}}, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other"}}, + expected: true, + }, + "invalid namespace from not allowed": { + allowedRoutes: &gwv1beta1.AllowedRoutes{Namespaces: &gwv1beta1.RouteNamespaces{From: pointerTo[gwv1beta1.FromNamespaces]("other")}}, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "test"}}, + expected: false, + }, + "labeled namespace allowed": { + allowedRoutes: &gwv1beta1.AllowedRoutes{Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromSelector), + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}, + }}, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other", Labels: map[string]string{ + "foo": "bar", + }}}, + expected: true, + }, + "labeled namespace not allowed": { + allowedRoutes: &gwv1beta1.AllowedRoutes{Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromSelector), + Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}}, + }}, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other", Labels: map[string]string{ + "foo": "baz", + }}}, + expected: false, + }, + "invalid labeled namespace": { + allowedRoutes: &gwv1beta1.AllowedRoutes{Namespaces: &gwv1beta1.RouteNamespaces{ + From: pointerTo(gwv1beta1.NamespacesFromSelector), + Selector: &metav1.LabelSelector{MatchExpressions: []metav1.LabelSelectorRequirement{ + {Key: "foo", Operator: "junk", Values: []string{"1"}}, + }}, + }}, + gatewayNamespace: "test", + routeNamespace: corev1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: "other", Labels: map[string]string{ + "foo": "bar", + }}}, + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, routeAllowedForListenerNamespaces(tt.gatewayNamespace, tt.allowedRoutes, tt.routeNamespace)) + }) + } +} + +func TestRouteAllowedForListenerHostname(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + hostname *gwv1beta1.Hostname + hostnames []gwv1beta1.Hostname + expected bool + }{ + "empty hostnames": { + hostname: nil, + hostnames: []gwv1beta1.Hostname{"foo", "bar"}, + expected: true, + }, + "empty hostname": { + hostname: pointerTo[gwv1beta1.Hostname]("foo"), + hostnames: nil, + expected: true, + }, + "any hostname match": { + hostname: pointerTo[gwv1beta1.Hostname]("foo"), + hostnames: []gwv1beta1.Hostname{"foo", "bar"}, + expected: true, + }, + "no match": { + hostname: pointerTo[gwv1beta1.Hostname]("foo"), + hostnames: []gwv1beta1.Hostname{"bar"}, + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, routeAllowedForListenerHostname(tt.hostname, tt.hostnames)) + }) + } +} + +func TestHostnamesMatch(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + one gwv1beta1.Hostname + two gwv1beta1.Hostname + expected bool + }{ + "wildcard one": { + one: "*", + two: "foo", + expected: true, + }, + "wildcard two": { + one: "foo", + two: "*", + expected: true, + }, + "empty one": { + one: "", + two: "foo", + expected: true, + }, + "empty two": { + one: "foo", + two: "", + expected: true, + }, + "subdomain one": { + one: "*.foo", + two: "sub.foo", + expected: true, + }, + "subdomain two": { + one: "sub.foo", + two: "*.foo", + expected: true, + }, + "exact match": { + one: "foo", + two: "foo", + expected: true, + }, + "no match": { + one: "foo", + two: "bar", + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, hostnamesMatch(tt.one, tt.two)) + }) + } +} + +func TestRouteKindIsAllowedForListener(t *testing.T) { + t.Parallel() + + for name, tt := range map[string]struct { + kinds []gwv1beta1.RouteGroupKind + gk schema.GroupKind + expected bool + }{ + "empty kinds": { + kinds: nil, + gk: schema.GroupKind{Group: "a", Kind: "b"}, + expected: true, + }, + "group specified": { + kinds: []gwv1beta1.RouteGroupKind{ + {Group: pointerTo[gwv1beta1.Group]("a"), Kind: "b"}, + }, + gk: schema.GroupKind{Group: "a", Kind: "b"}, + expected: true, + }, + "group unspecified": { + kinds: []gwv1beta1.RouteGroupKind{ + {Kind: "b"}, + }, + gk: schema.GroupKind{Group: "a", Kind: "b"}, + expected: true, + }, + "kind mismatch": { + kinds: []gwv1beta1.RouteGroupKind{ + {Kind: "b"}, + }, + gk: schema.GroupKind{Group: "a", Kind: "c"}, + expected: false, + }, + "group mismatch": { + kinds: []gwv1beta1.RouteGroupKind{ + {Group: pointerTo[gwv1beta1.Group]("a"), Kind: "b"}, + }, + gk: schema.GroupKind{Group: "d", Kind: "b"}, + expected: false, + }, + } { + t.Run(name, func(t *testing.T) { + require.Equal(t, tt.expected, routeKindIsAllowedForListener(tt.kinds, tt.gk)) + }) + } +} diff --git a/control-plane/api-gateway/controllers/gateway_class_config_controller_test.go b/control-plane/api-gateway/controllers/gateway_class_config_controller_test.go index b0f8d27636..e3aad69019 100644 --- a/control-plane/api-gateway/controllers/gateway_class_config_controller_test.go +++ b/control-plane/api-gateway/controllers/gateway_class_config_controller_test.go @@ -15,8 +15,10 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" ) @@ -95,8 +97,11 @@ func TestGatewayClassConfigReconcile(t *testing.T) { for _, tt := range cases { t.Run(tt.name, func(t *testing.T) { s := runtime.NewScheme() - k8sSchemaObjects := append(tt.k8sObjects(), &gwv1beta1.GatewayClass{}, &gwv1beta1.GatewayClassList{}, &v1alpha1.GatewayClassConfig{}) - s.AddKnownTypes(v1alpha1.GroupVersion, k8sSchemaObjects...) + require.NoError(t, clientgoscheme.AddToScheme(s)) + require.NoError(t, gwv1alpha2.Install(s)) + require.NoError(t, gwv1beta1.Install(s)) + require.NoError(t, v1alpha1.AddToScheme(s)) + fakeClient := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(tt.k8sObjects()...).Build() // Create the gateway class config controller. diff --git a/control-plane/api-gateway/controllers/gateway_class_serializer.go b/control-plane/api-gateway/controllers/gateway_class_serializer.go deleted file mode 100644 index 3d65dac54f..0000000000 --- a/control-plane/api-gateway/controllers/gateway_class_serializer.go +++ /dev/null @@ -1,91 +0,0 @@ -package controllers - -import ( - "context" - "encoding/json" - "errors" - - k8serrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" - "sigs.k8s.io/controller-runtime/pkg/client" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" -) - -const ( - Group = "api-gateway.consul.hashicorp.com" - annotationConfigKey = "api-gateway.consul.hashicorp.com/config" -) - -var ErrUnmarshallingGatewayClassConfig = errors.New("error unmarshalling GatewayClassConfig annotation, skipping") - -func SerializeGatewayClassConfig(ctx context.Context, client client.Client, gw *gwv1beta1.Gateway, gwc *gwv1beta1.GatewayClass) (bool, error) { - var ( - config v1alpha1.GatewayClassConfig - err error - annotated bool - managed bool - ) - - if gw.Annotations == nil { - gw.Annotations = make(map[string]string) - } - - if annotatedConfig, ok := gw.Annotations[annotationConfigKey]; ok { - if err := json.Unmarshal([]byte(annotatedConfig), &config.Spec); err != nil { - return false, ErrUnmarshallingGatewayClassConfig - } - annotated = true - managed = true - } - - if !managed { - // check if we own the gateway - config, managed, err = getConfigForGatewayClass(ctx, client, gwc) - if err != nil { - gw.Annotations[annotationConfigKey] = "" - if k8serrors.IsNotFound(err) { - // invalid config which means an invalid gatewayclass - // so pretend we don't exist - return false, nil - } - return false, err - } - if !managed { - gw.Annotations[annotationConfigKey] = "" - // we don't own this gateway so we pretend it doesn't exist - return false, nil - } - } - - marshaled, err := json.Marshal(config.Spec) - if err != nil { - return false, err - } - - gw.Annotations[annotationConfigKey] = string(marshaled) - - if !annotated { - // we annotated for the first time - return true, client.Update(ctx, gw) - } - - return false, nil -} - -func getConfigForGatewayClass(ctx context.Context, client client.Client, gwc *gwv1beta1.GatewayClass) (config v1alpha1.GatewayClassConfig, managed bool, err error) { - if ref := gwc.Spec.ParametersRef; ref != nil { - if string(ref.Group) != Group || ref.Kind != v1alpha1.GatewayClassConfigKind { - // pretend we have nothing because we don't support an untyped configuration - return config, false, nil - } - - err := client.Get(ctx, types.NamespacedName{Name: ref.Name}, &config) - if err != nil { - return config, false, err - } - - } - return config, true, nil -} diff --git a/control-plane/api-gateway/controllers/gateway_class_serializer_test.go b/control-plane/api-gateway/controllers/gateway_class_serializer_test.go deleted file mode 100644 index c9d93cd882..0000000000 --- a/control-plane/api-gateway/controllers/gateway_class_serializer_test.go +++ /dev/null @@ -1,486 +0,0 @@ -package controllers - -import ( - "context" - "encoding/json" - "testing" - - "github.com/google/go-cmp/cmp" - "github.com/stretchr/testify/require" - corev1 "k8s.io/api/core/v1" - v1 "k8s.io/api/core/v1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client/fake" - gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" - - "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" -) - -func TestSerializeGatewayClassConfig_HappyPath(t *testing.T) { - t.Parallel() - type args struct { - ctx context.Context - gw *gwv1beta1.Gateway - gwc *gwv1beta1.GatewayClass - } - tests := []struct { - name string - args args - gwcConfig *v1alpha1.GatewayClassConfig - expectedDidUpdate bool - wantErr bool - }{ - { - name: "when gateway has not been annotated yet and annotations are nil", - args: args{ - ctx: context.Background(), - gw: &gwv1beta1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw", - }, - Spec: gwv1beta1.GatewaySpec{}, - Status: gwv1beta1.GatewayStatus{}, - }, - gwc: &gwv1beta1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw-class", - }, - Spec: gwv1beta1.GatewayClassSpec{ - ControllerName: "", - ParametersRef: &gwv1beta1.ParametersReference{ - Group: Group, - Kind: v1alpha1.GatewayClassConfigKind, - Name: "the config", - }, - Description: new(string), - }, - }, - }, - gwcConfig: &v1alpha1.GatewayClassConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: "the config", - }, - Spec: v1alpha1.GatewayClassConfigSpec{ - ServiceType: pointerTo(corev1.ServiceType("serviceType")), - NodeSelector: map[string]string{ - "selector": "of node", - }, - Tolerations: []v1.Toleration{ - { - Key: "key", - Operator: "op", - Value: "120", - Effect: "to the moon", - TolerationSeconds: new(int64), - }, - }, - CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ - Service: []string{"service"}, - }, - }, - }, - expectedDidUpdate: true, - wantErr: false, - }, - { - name: "when gateway has not been annotated yet but annotations are empty", - args: args{ - ctx: context.Background(), - gw: &gwv1beta1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw", - Annotations: make(map[string]string), - }, - Spec: gwv1beta1.GatewaySpec{}, - Status: gwv1beta1.GatewayStatus{}, - }, - gwc: &gwv1beta1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw-class", - }, - Spec: gwv1beta1.GatewayClassSpec{ - ControllerName: "", - ParametersRef: &gwv1beta1.ParametersReference{ - Group: Group, - Kind: v1alpha1.GatewayClassConfigKind, - Name: "the config", - }, - Description: new(string), - }, - }, - }, - gwcConfig: &v1alpha1.GatewayClassConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: "the config", - }, - Spec: v1alpha1.GatewayClassConfigSpec{ - ServiceType: pointerTo(corev1.ServiceType("serviceType")), - NodeSelector: map[string]string{ - "selector": "of node", - }, - Tolerations: []v1.Toleration{ - { - Key: "key", - Operator: "op", - Value: "120", - Effect: "to the moon", - TolerationSeconds: new(int64), - }, - }, - CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ - Service: []string{"service"}, - }, - }, - }, - expectedDidUpdate: true, - wantErr: false, - }, - { - name: "when gateway has been annotated", - args: args{ - ctx: context.Background(), - gw: &gwv1beta1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw", - Annotations: map[string]string{ - annotationConfigKey: `{"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"copyAnnotations":{"service":["service"]}}`, - }, - }, - Spec: gwv1beta1.GatewaySpec{}, - Status: gwv1beta1.GatewayStatus{}, - }, - gwc: &gwv1beta1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw-class", - }, - Spec: gwv1beta1.GatewayClassSpec{ - ControllerName: "", - ParametersRef: &gwv1beta1.ParametersReference{ - Group: Group, - Kind: v1alpha1.GatewayClassConfigKind, - Name: "the config", - }, - Description: new(string), - }, - }, - }, - gwcConfig: &v1alpha1.GatewayClassConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: "the config", - }, - Spec: v1alpha1.GatewayClassConfigSpec{ - ServiceType: pointerTo(corev1.ServiceType("serviceType")), - NodeSelector: map[string]string{ - "selector": "of node", - }, - Tolerations: []v1.Toleration{ - { - Key: "key", - Operator: "op", - Value: "120", - Effect: "to the moon", - TolerationSeconds: new(int64), - }, - }, - CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ - Service: []string{"service"}, - }, - }, - }, - expectedDidUpdate: false, - wantErr: false, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := runtime.NewScheme() - require.NoError(t, gwv1beta1.Install(s)) - require.NoError(t, v1alpha1.AddToScheme(s)) - objs := []runtime.Object{tt.args.gw, tt.args.gwc, tt.gwcConfig} - fakeClient := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build() - - actualDidUpdate, err := SerializeGatewayClassConfig(tt.args.ctx, fakeClient, tt.args.gw, tt.args.gwc) - if (err != nil) != tt.wantErr { - t.Errorf("SerializeGatewayClassConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - if actualDidUpdate != tt.expectedDidUpdate { - t.Errorf("SerializeGatewayClassConfig() = %v, want %v", actualDidUpdate, tt.expectedDidUpdate) - } - - var config v1alpha1.GatewayClassConfig - err = json.Unmarshal([]byte(tt.args.gw.Annotations[annotationConfigKey]), &config.Spec) - require.NoError(t, err) - - if diff := cmp.Diff(config.Spec, tt.gwcConfig.Spec); diff != "" { - t.Errorf("Expected gwconfig spec to match serialized version (-want,+got):\n%s", diff) - } - }) - } -} - -func TestSerializeGatewayClassConfig_SadPath(t *testing.T) { - t.Parallel() - type args struct { - ctx context.Context - gw *gwv1beta1.Gateway - gwc *gwv1beta1.GatewayClass - } - tests := []struct { - name string - args args - gwcConfig *v1alpha1.GatewayClassConfig - expectedDidUpdate bool - wantErr bool - }{ - { - name: "when gateway has been annotated but the serialization was invalid", - args: args{ - ctx: context.Background(), - gw: &gwv1beta1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw", - Annotations: map[string]string{ - // we remove the opening brace to make unmarshalling fail - annotationConfigKey: `"serviceType":"serviceType","nodeSelector":{"selector":"of node"},"tolerations":[{"key":"key","operator":"op","value":"120","effect":"to the moon","tolerationSeconds":0}],"copyAnnotations":{"service":["service"]}}`, - }, - }, - Spec: gwv1beta1.GatewaySpec{}, - Status: gwv1beta1.GatewayStatus{}, - }, - gwc: &gwv1beta1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw-class", - }, - Spec: gwv1beta1.GatewayClassSpec{ - ControllerName: "", - ParametersRef: &gwv1beta1.ParametersReference{ - Group: Group, - Kind: v1alpha1.GatewayClassConfigKind, - Name: "the config", - }, - Description: new(string), - }, - }, - }, - gwcConfig: &v1alpha1.GatewayClassConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: "the config", - }, - Spec: v1alpha1.GatewayClassConfigSpec{ - ServiceType: pointerTo(corev1.ServiceType("serviceType")), - NodeSelector: map[string]string{ - "selector": "of node", - }, - Tolerations: []v1.Toleration{ - { - Key: "key", - Operator: "op", - Value: "120", - Effect: "to the moon", - TolerationSeconds: new(int64), - }, - }, - CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ - Service: []string{"service"}, - }, - }, - }, - expectedDidUpdate: false, - wantErr: true, - }, - { - name: "No Annotation and GatewayConfig is missing the Group field", - args: args{ - ctx: context.Background(), - gw: &gwv1beta1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw", - Annotations: map[string]string{}, - }, - Spec: gwv1beta1.GatewaySpec{}, - Status: gwv1beta1.GatewayStatus{}, - }, - gwc: &gwv1beta1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw-class", - }, - Spec: gwv1beta1.GatewayClassSpec{ - ControllerName: "", - ParametersRef: &gwv1beta1.ParametersReference{ - Group: "", - Kind: v1alpha1.GatewayClassConfigKind, - Name: "the config", - }, - Description: new(string), - }, - }, - }, - gwcConfig: &v1alpha1.GatewayClassConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: "the config", - }, - Spec: v1alpha1.GatewayClassConfigSpec{ - ServiceType: pointerTo(corev1.ServiceType("serviceType")), - NodeSelector: map[string]string{ - "selector": "of node", - }, - Tolerations: []v1.Toleration{ - { - Key: "key", - Operator: "op", - Value: "120", - Effect: "to the moon", - TolerationSeconds: new(int64), - }, - }, - CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ - Service: []string{"service"}, - }, - }, - }, - expectedDidUpdate: false, - wantErr: false, - }, - { - name: "No Annotation and GatewayConfig is missing the Kind field we get an error", - args: args{ - ctx: context.Background(), - gw: &gwv1beta1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw", - Annotations: map[string]string{}, - }, - Spec: gwv1beta1.GatewaySpec{}, - Status: gwv1beta1.GatewayStatus{}, - }, - gwc: &gwv1beta1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw-class", - }, - Spec: gwv1beta1.GatewayClassSpec{ - ControllerName: "", - ParametersRef: &gwv1beta1.ParametersReference{ - Group: Group, - Kind: "", - Name: "the config", - }, - Description: new(string), - }, - }, - }, - gwcConfig: &v1alpha1.GatewayClassConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: "the config", - }, - Spec: v1alpha1.GatewayClassConfigSpec{ - ServiceType: pointerTo(corev1.ServiceType("serviceType")), - NodeSelector: map[string]string{ - "selector": "of node", - }, - Tolerations: []v1.Toleration{ - { - Key: "key", - Operator: "op", - Value: "120", - Effect: "to the moon", - TolerationSeconds: new(int64), - }, - }, - CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ - Service: []string{"service"}, - }, - }, - }, - expectedDidUpdate: false, - wantErr: false, - }, - { - name: "No Annotation and GatewayConfig is not able to be found", - args: args{ - ctx: context.Background(), - gw: &gwv1beta1.Gateway{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw", - Annotations: map[string]string{}, - }, - Spec: gwv1beta1.GatewaySpec{}, - Status: gwv1beta1.GatewayStatus{}, - }, - gwc: &gwv1beta1.GatewayClass{ - ObjectMeta: metav1.ObjectMeta{ - Name: "my-gw-class", - }, - Spec: gwv1beta1.GatewayClassSpec{ - ControllerName: "", - ParametersRef: &gwv1beta1.ParametersReference{ - Group: Group, - Kind: v1alpha1.GatewayClassConfigKind, - Name: "NOT GONNA FIND ME", - }, - Description: new(string), - }, - }, - }, - gwcConfig: &v1alpha1.GatewayClassConfig{ - TypeMeta: metav1.TypeMeta{}, - ObjectMeta: metav1.ObjectMeta{ - Name: "the config", - }, - Spec: v1alpha1.GatewayClassConfigSpec{ - ServiceType: pointerTo(corev1.ServiceType("serviceType")), - NodeSelector: map[string]string{ - "selector": "of node", - }, - Tolerations: []v1.Toleration{ - { - Key: "key", - Operator: "op", - Value: "120", - Effect: "to the moon", - TolerationSeconds: new(int64), - }, - }, - CopyAnnotations: v1alpha1.CopyAnnotationsSpec{ - Service: []string{"service"}, - }, - }, - }, - expectedDidUpdate: false, - wantErr: false, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - s := runtime.NewScheme() - require.NoError(t, gwv1beta1.Install(s)) - require.NoError(t, v1alpha1.AddToScheme(s)) - objs := []runtime.Object{tt.args.gw, tt.args.gwc, tt.gwcConfig} - fakeClient := fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...).Build() - - actualDidUpdate, err := SerializeGatewayClassConfig(tt.args.ctx, fakeClient, tt.args.gw, tt.args.gwc) - if (err != nil) != tt.wantErr { - t.Errorf("SerializeGatewayClassConfig() error = %v, wantErr %v", err, tt.wantErr) - return - } - if actualDidUpdate != tt.expectedDidUpdate { - t.Errorf("SerializeGatewayClassConfig() = %v, want %v", actualDidUpdate, tt.expectedDidUpdate) - } - - // don't bother checking the annotation if there was an error - if err != nil { - return - } - - require.Empty(t, tt.args.gw.Annotations[annotationConfigKey]) - }) - } -} diff --git a/control-plane/api-gateway/controllers/gateway_controller.go b/control-plane/api-gateway/controllers/gateway_controller.go index 0d1c2cc4f9..ce9818fcf6 100644 --- a/control-plane/api-gateway/controllers/gateway_controller.go +++ b/control-plane/api-gateway/controllers/gateway_controller.go @@ -2,6 +2,7 @@ package controllers import ( "context" + "reflect" "github.com/go-logr/logr" appsv1 "k8s.io/api/apps/v1" @@ -20,8 +21,9 @@ import ( gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" apigateway "github.com/hashicorp/consul-k8s/control-plane/api-gateway" - + "github.com/hashicorp/consul-k8s/control-plane/api-gateway/binding" "github.com/hashicorp/consul-k8s/control-plane/api-gateway/translation" + "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" "github.com/hashicorp/consul-k8s/control-plane/cache" "github.com/hashicorp/consul-k8s/control-plane/consul" "github.com/hashicorp/consul/api" @@ -47,6 +49,7 @@ type GatewayControllerConfig struct { type GatewayController struct { HelmConfig apigateway.HelmConfig Log logr.Logger + Translator translation.K8sToConsulTranslator cache *cache.Cache client.Client } @@ -57,8 +60,8 @@ func (r *GatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ct log.Info("Reconciling the Gateway: ", req.Name) // If gateway does not exist, log an error. - gw := &gwv1beta1.Gateway{} - err := r.Client.Get(ctx, req.NamespacedName, gw) + var gw gwv1beta1.Gateway + err := r.Client.Get(ctx, req.NamespacedName, &gw) if err != nil { if k8serrors.IsNotFound(err) { return ctrl.Result{}, nil @@ -71,64 +74,198 @@ func (r *GatewayController) Reconcile(ctx context.Context, req ctrl.Request) (ct gwc := &gwv1beta1.GatewayClass{} err = r.Client.Get(ctx, types.NamespacedName{Name: string(gw.Spec.GatewayClassName)}, gwc) if err != nil { - if k8serrors.IsNotFound(err) { - return r.cleanupGatewayResources(ctx, log, gw) + if !k8serrors.IsNotFound(err) { + log.Error(err, "unable to get GatewayClass") + return ctrl.Result{}, err } - log.Error(err, "unable to get GatewayClass") + gwc = nil + } + + gwcc, err := getConfigForGatewayClass(ctx, r.Client, gwc) + if err != nil { + log.Error(err, "error fetching the gateway class config") return ctrl.Result{}, err } - if string(gwc.Spec.ControllerName) != GatewayClassControllerName || !gw.ObjectMeta.DeletionTimestamp.IsZero() { - // This Gateway is not for this controller or the gateway is being deleted. - return r.cleanupGatewayResources(ctx, log, gw) + // fetch all namespaces + namespaceList := &corev1.NamespaceList{} + if err := r.Client.List(ctx, namespaceList); err != nil { + log.Error(err, "unable to list Namespaces") + return ctrl.Result{}, err + } + namespaces := map[string]corev1.Namespace{} + for _, namespace := range namespaceList.Items { + namespaces[namespace.Name] = namespace } - didUpdateForSerialize, err := SerializeGatewayClassConfig(ctx, r.Client, gw, gwc) - if err != nil { - log.Error(err, "unable to add serialize gateway class config") - // we probably should just continue here right and not exit early? - // return ctrl.Result{}, err + // fetch all gateways we control for reference counting + gwcList := &gwv1beta1.GatewayClassList{} + if err := r.Client.List(ctx, gwcList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(GatewayClass_ControllerNameIndex, GatewayClassControllerName), + }); err != nil { + log.Error(err, "unable to list GatewayClasses") + return ctrl.Result{}, err } - didUpdateForFinalizer, err := EnsureFinalizer(ctx, r.Client, gw, gatewayFinalizer) - if err != nil { - log.Error(err, "unable to add finalizer") + gwList := &gwv1beta1.GatewayList{} + if err := r.Client.List(ctx, gwList); err != nil { + log.Error(err, "unable to list Gateways") return ctrl.Result{}, err } - if didUpdateForSerialize || didUpdateForFinalizer { - // We updated the Gateway, requeue to avoid another update. - return ctrl.Result{}, nil + + controlled := map[types.NamespacedName]gwv1beta1.Gateway{} + for _, gwc := range gwcList.Items { + for _, gw := range gwList.Items { + if string(gw.Spec.GatewayClassName) == gwc.Name { + controlled[types.NamespacedName{Namespace: gw.Namespace, Name: gw.Name}] = gw + } + } } - /* TODO: + // fetch all secrets referenced by this gateway + secretList := &corev1.SecretList{} + if err := r.Client.List(ctx, secretList); err != nil { + log.Error(err, "unable to list Secrets") + return ctrl.Result{}, err + } + + listenerCerts := make(map[types.NamespacedName]struct{}) + for _, listener := range gw.Spec.Listeners { + if listener.TLS != nil { + for _, ref := range listener.TLS.CertificateRefs { + if nilOrEqual(ref.Group, "") && nilOrEqual(ref.Kind, "Secret") { + listenerCerts[indexedNamespacedNameWithDefault(ref.Name, ref.Namespace, gw.Namespace)] = struct{}{} + } + } + } + } + + filteredSecrets := []corev1.Secret{} + for _, secret := range secretList.Items { + namespacedName := types.NamespacedName{Namespace: secret.Namespace, Name: secret.Name} + if _, ok := listenerCerts[namespacedName]; ok { + filteredSecrets = append(filteredSecrets, secret) + } + } + + // fetch all http routes referencing this gateway + httpRouteList := &gwv1beta1.HTTPRouteList{} + if err := r.Client.List(ctx, httpRouteList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(HTTPRoute_GatewayIndex, req.String()), + }); err != nil { + log.Error(err, "unable to list HTTPRoutes") + return ctrl.Result{}, err + } + + // fetch all tcp routes referencing this gateway + tcpRouteList := &gwv1alpha2.TCPRouteList{} + if err := r.Client.List(ctx, tcpRouteList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(TCPRoute_GatewayIndex, req.String()), + }); err != nil { + log.Error(err, "unable to list TCPRoutes") + return ctrl.Result{}, err + } + + httpRoutes := r.cache.List(api.HTTPRoute) + tcpRoutes := r.cache.List(api.TCPRoute) + inlineCertificates := r.cache.List(api.InlineCertificate) + services := r.cache.ListServices() + + binder := binding.NewBinder(binding.BinderConfig{ + Translator: r.Translator, + ControllerName: GatewayClassControllerName, + GatewayClassConfig: gwcc, + GatewayClass: gwc, + Gateway: gw, + HTTPRoutes: httpRouteList.Items, + TCPRoutes: tcpRouteList.Items, + Secrets: filteredSecrets, + ConsulHTTPRoutes: derefAll(configEntriesTo[*api.HTTPRouteConfigEntry](httpRoutes)), + ConsulTCPRoutes: derefAll(configEntriesTo[*api.TCPRouteConfigEntry](tcpRoutes)), + ConsulInlineCertificates: derefAll(configEntriesTo[*api.InlineCertificateConfigEntry](inlineCertificates)), + ConnectInjectedServices: services, + Namespaces: namespaces, + ControlledGateways: controlled, + }) + + updates := binder.Snapshot() + + // do something with deployments, gwcc and such here, if the following exists on + // the snapshot it means we should attempt to enforce the deployment, if it's nil + // then we should delete the deployment + _ = updates.GatewayClassConfig + + for _, deletion := range updates.Consul.Deletions { + log.Info("deleting from Consul", "kind", deletion.Kind, "namespace", deletion.Namespace, "name", deletion.Name) + // TODO: the actual delete + } + + for _, update := range updates.Consul.Updates { + log.Info("updating in Consul", "kind", update.GetKind(), "namespace", update.GetNamespace(), "name", update.GetName()) + // TODO: the actual update + } - 1. Get all resources from Kubernetes which refer to this Gateway: - - HTTPRoutes - - TCPRoutes - - Secrets - - Services which refer to routes. - Pull in the deployments that have been created through Gatekeeper previously to check their statuses. + for _, update := range updates.Kubernetes.Updates { + log.Info("update in Kubernetes", "kind", update.GetObjectKind().GroupVersionKind().Kind, "namespace", update.GetNamespace(), "name", update.GetName()) + if err := r.updateAndResetStatus(ctx, update); err != nil { + log.Error(err, "error updating object") + return ctrl.Result{}, err + } + } + + for _, update := range updates.Kubernetes.StatusUpdates { + log.Info("update status in Kubernetes", "kind", update.GetObjectKind().GroupVersionKind().Kind, "namespace", update.GetNamespace(), "name", update.GetName()) + if err := r.Client.Status().Update(ctx, update); err != nil { + log.Error(err, "error updating status") + return ctrl.Result{}, err + } + } + + /* TODO: + 1.Pull in the deployments that have been created through Gatekeeper previously to check their statuses. Leverage health-checking. OG impl: Any state change for the deployment/pods we subscribe to and set statuses on the gateway. Do a health check on the service. - 2. Compile the resources into Consul config entries, while respecting the requirement for ReferenceGrants when - moving across namespace. - We need to check if binding can occur outside of ReferenceGrants. - See "AllowedRoutes" on Listeners. - https://gateway-api.sigs.k8s.io/references/spec/#gateway.networking.k8s.io/v1beta1.AllowedRoutes - Error out if someone uses an unsupported feature: + 2.ReferenceGrants + Error out if someone uses an unsupported feature: - TLS mode type pass through - 3. Sync the config entries into Consul. - Sync the health status of the deployment to Consul at the same time. - 4. Run Gatekeeper Upsert with the GW, GWCC, HelmConfig. - + 3. Run Gatekeeper Upsert with the GW, GWCC, HelmConfig. */ return ctrl.Result{}, nil } +func (r *GatewayController) updateAndResetStatus(ctx context.Context, o client.Object) error { + // we create a copy so that we can re-update its status if need be + status := reflect.ValueOf(o.DeepCopyObject()).Elem().FieldByName("Status") + if err := r.Client.Update(ctx, o); err != nil { + return err + } + // reset the status in case it needs to be updated below + reflect.ValueOf(o).Elem().FieldByName("Status").Set(status) + return nil +} + +func derefAll[T any](vs []*T) []T { + e := make([]T, len(vs)) + for _, v := range vs { + e = append(e, *v) + } + return e +} + +func configEntriesTo[T api.ConfigEntry](entries []api.ConfigEntry) []T { + es := []T{} + for _, e := range entries { + es = append(es, e.(T)) + } + return es +} + // SetupWithGatewayControllerManager registers the controller with the given manager. func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, config GatewayControllerConfig) (*cache.Cache, error) { + logger := mgr.GetLogger() + c := cache.New(cache.Config{ ConsulClientConfig: config.ConsulClientConfig, ConsulServerConnMgr: config.ConsulServerConnMgr, @@ -137,14 +274,14 @@ func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, co Logger: mgr.GetLogger(), }) + translator := translation.NewConsulToNamespaceNameTranslator(c) + r := &GatewayController{ Client: mgr.GetClient(), cache: c, Log: mgr.GetLogger(), } - translator := translation.NewConsulToNamespaceNameTranslator(c) - return c, ctrl.NewControllerManagedBy(mgr). For(&gwv1beta1.Gateway{}). Owns(&appsv1.Deployment{}). @@ -170,6 +307,50 @@ func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, co source.NewKindWithCache(&gwv1beta1.ReferenceGrant{}, mgr.GetCache()), handler.EnqueueRequestsFromMapFunc(r.transformReferenceGrant(ctx)), ). + Watches( + // Subscribe to changes from Consul Connect Services + &source.Channel{Source: c.SubscribeServices(ctx, func(service *api.CatalogService) []types.NamespacedName { + nsn := serviceToNamespacedName(service) + + if nsn.Namespace != "" && nsn.Name != "" { + key := nsn.String() + + requestSet := make(map[types.NamespacedName]struct{}) + tcpRouteList := &gwv1alpha2.TCPRouteList{} + if err := r.Client.List(ctx, tcpRouteList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(TCPRoute_ServiceIndex, key), + }); err != nil { + logger.Error(err, "unable to list TCPRoutes") + } + for _, route := range tcpRouteList.Items { + for _, ref := range parentRefs(gwv1beta1.GroupVersion.Group, kindGateway, route.Namespace, route.Spec.ParentRefs) { + requestSet[ref] = struct{}{} + } + } + + httpRouteList := &gwv1alpha2.HTTPRouteList{} + if err := r.Client.List(ctx, httpRouteList, &client.ListOptions{ + FieldSelector: fields.OneTermEqualSelector(HTTPRoute_ServiceIndex, key), + }); err != nil { + logger.Error(err, "unable to list HTTPRoutes") + } + for _, route := range httpRouteList.Items { + for _, ref := range parentRefs(gwv1beta1.GroupVersion.Group, kindGateway, route.Namespace, route.Spec.ParentRefs) { + requestSet[ref] = struct{}{} + } + } + + requests := []types.NamespacedName{} + for request := range requestSet { + requests = append(requests, request) + } + return requests + } + + return nil + }).Events()}, + &handler.EnqueueRequestForObject{}, + ). Watches( // Subscribe to changes from Consul for APIGateways &source.Channel{Source: c.Subscribe(ctx, api.APIGateway, translator.BuildConsulGatewayTranslator(ctx)).Events()}, @@ -192,16 +373,15 @@ func SetupGatewayControllerWithManager(ctx context.Context, mgr ctrl.Manager, co ).Complete(r) } -func (r *GatewayController) cleanupGatewayResources(ctx context.Context, log logr.Logger, gw *gwv1beta1.Gateway) (ctrl.Result, error) { - // TODO: Delete configuration in Consul servers. - // TODO: Call gatekeeper delete. - - _, err := RemoveFinalizer(ctx, r.Client, gw, gatewayFinalizer) - if err != nil { - log.Error(err, "unable to remove finalizer") +func serviceToNamespacedName(s *api.CatalogService) types.NamespacedName { + var ( + metaKeyKubeNS = "k8s-namespace" + metaKeyKubeServiceName = "k8s-service-name" + ) + return types.NamespacedName{ + Namespace: s.ServiceMeta[metaKeyKubeNS], + Name: s.ServiceMeta[metaKeyKubeServiceName], } - - return ctrl.Result{}, err } // transformGatewayClass will check the list of GatewayClass objects for a matching @@ -380,3 +560,30 @@ func (r *GatewayController) getAllRefsForGateway(ctx context.Context, gw *gwv1be return objs, nil } + +// getConfigForGatewayClass returns the relevant GatewayClassConfig for the GatewayClass. +func getConfigForGatewayClass(ctx context.Context, client client.Client, gwc *gwv1beta1.GatewayClass) (*v1alpha1.GatewayClassConfig, error) { + if gwc == nil { + // if we don't have a gateway class we can't fetch the corresponding config + return nil, nil + } + + config := &v1alpha1.GatewayClassConfig{} + if ref := gwc.Spec.ParametersRef; ref != nil { + if string(ref.Group) != v1alpha1.GroupVersion.Group || + ref.Kind != v1alpha1.GatewayClassConfigKind || + gwc.Spec.ControllerName != GatewayClassControllerName { + // we don't have supported params, so return nil + return nil, nil + } + + err := client.Get(ctx, types.NamespacedName{Name: ref.Name}, config) + if err != nil { + if k8serrors.IsNotFound(err) { + return nil, nil + } + return nil, err + } + } + return config, nil +} diff --git a/control-plane/api-gateway/controllers/gateway_controller_test.go b/control-plane/api-gateway/controllers/gateway_controller_test.go index f07e01cfc5..d4ff452395 100644 --- a/control-plane/api-gateway/controllers/gateway_controller_test.go +++ b/control-plane/api-gateway/controllers/gateway_controller_test.go @@ -11,13 +11,16 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/reconcile" - "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" + "github.com/hashicorp/consul-k8s/control-plane/cache" + "github.com/hashicorp/consul-k8s/control-plane/consul" ) func TestGatewayReconciler(t *testing.T) { @@ -78,8 +81,9 @@ func TestGatewayReconciler(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { s := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(s)) + require.NoError(t, gwv1alpha2.Install(s)) require.NoError(t, gwv1beta1.Install(s)) - require.NoError(t, v1alpha2.Install(s)) require.NoError(t, v1alpha1.AddToScheme(s)) objs := tc.k8sObjects @@ -90,6 +94,7 @@ func TestGatewayReconciler(t *testing.T) { fakeClient := registerFieldIndexersForTest(fake.NewClientBuilder().WithScheme(s).WithRuntimeObjects(objs...)).Build() r := &GatewayController{ + cache: cache.New(cache.Config{Logger: logrtest.New(t), ConsulClientConfig: &consul.Config{}}), Client: fakeClient, Log: logrtest.New(t), } @@ -161,7 +166,7 @@ func TestGatewayController_getAllRefsForGateway(t *testing.T) { t.Parallel() s := runtime.NewScheme() require.NoError(t, gwv1beta1.Install(s)) - require.NoError(t, v1alpha2.Install(s)) + require.NoError(t, gwv1alpha2.Install(s)) require.NoError(t, corev1.AddToScheme(s)) require.NoError(t, v1alpha1.AddToScheme(s)) @@ -272,12 +277,12 @@ func TestGatewayController_getAllRefsForGateway(t *testing.T) { Status: gwv1beta1.HTTPRouteStatus{}, } - tcpRoute := &v1alpha2.TCPRoute{ + tcpRoute := &gwv1alpha2.TCPRoute{ TypeMeta: metav1.TypeMeta{}, ObjectMeta: metav1.ObjectMeta{ Name: "tcp route", }, - Spec: v1alpha2.TCPRouteSpec{ + Spec: gwv1alpha2.TCPRouteSpec{ CommonRouteSpec: gwv1beta1.CommonRouteSpec{ ParentRefs: []gwv1beta1.ParentReference{ { diff --git a/control-plane/api-gateway/controllers/gatewayclass_controller_test.go b/control-plane/api-gateway/controllers/gatewayclass_controller_test.go index 76c18546b5..6adec89389 100644 --- a/control-plane/api-gateway/controllers/gatewayclass_controller_test.go +++ b/control-plane/api-gateway/controllers/gatewayclass_controller_test.go @@ -11,10 +11,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" + clientgoscheme "k8s.io/client-go/kubernetes/scheme" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" - "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" "sigs.k8s.io/gateway-api/apis/v1beta1" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" @@ -228,8 +229,9 @@ func TestGatewayClassReconciler(t *testing.T) { for name, tc := range cases { t.Run(name, func(t *testing.T) { s := runtime.NewScheme() + require.NoError(t, clientgoscheme.AddToScheme(s)) + require.NoError(t, gwv1alpha2.Install(s)) require.NoError(t, gwv1beta1.Install(s)) - require.NoError(t, v1alpha2.Install(s)) require.NoError(t, v1alpha1.AddToScheme(s)) objs := tc.k8sObjects diff --git a/control-plane/api-gateway/controllers/index.go b/control-plane/api-gateway/controllers/index.go index fa90ccffa0..17aeb38506 100644 --- a/control-plane/api-gateway/controllers/index.go +++ b/control-plane/api-gateway/controllers/index.go @@ -3,10 +3,9 @@ package controllers import ( "context" - "k8s.io/apimachinery/pkg/types" ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" - "sigs.k8s.io/gateway-api/apis/v1alpha2" + gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/hashicorp/consul-k8s/control-plane/api/v1alpha1" @@ -15,6 +14,7 @@ import ( const ( // Naming convention: TARGET_REFERENCE. GatewayClass_GatewayClassConfigIndex = "__gatewayclass_referencing_gatewayclassconfig" + GatewayClass_ControllerNameIndex = "__gatewayclass_controller_name" Gateway_GatewayClassIndex = "__gateway_referencing_gatewayclass" HTTPRoute_GatewayIndex = "__httproute_referencing_gateway" HTTPRoute_ServiceIndex = "__httproute_referencing_service" @@ -47,6 +47,11 @@ var indexes = []index{ target: &gwv1beta1.GatewayClass{}, indexerFunc: gatewayClassConfigForGatewayClass, }, + { + name: GatewayClass_ControllerNameIndex, + target: &gwv1beta1.GatewayClass{}, + indexerFunc: gatewayClassControllerName, + }, { name: Gateway_GatewayClassIndex, target: &gwv1beta1.Gateway{}, @@ -60,12 +65,22 @@ var indexes = []index{ { name: HTTPRoute_GatewayIndex, target: &gwv1beta1.HTTPRoute{}, - indexerFunc: gatewayForHTTPRoute, + indexerFunc: gatewaysForHTTPRoute, + }, + { + name: HTTPRoute_ServiceIndex, + target: &gwv1beta1.HTTPRoute{}, + indexerFunc: servicesForHTTPRoute, }, { name: TCPRoute_GatewayIndex, - target: &v1alpha2.TCPRoute{}, - indexerFunc: gatewayForTCPRoute, + target: &gwv1alpha2.TCPRoute{}, + indexerFunc: gatewaysForTCPRoute, + }, + { + name: TCPRoute_ServiceIndex, + target: &gwv1alpha2.TCPRoute{}, + indexerFunc: servicesForTCPRoute, }, } @@ -81,6 +96,16 @@ func gatewayClassConfigForGatewayClass(o client.Object) []string { return []string{} } +func gatewayClassControllerName(o client.Object) []string { + gc := o.(*gwv1beta1.GatewayClass) + + if gc.Spec.ControllerName != "" { + return []string{string(gc.Spec.ControllerName)} + } + + return []string{} +} + // gatewayClassForGateway creates an index of every GatewayClass referenced by a Gateway. func gatewayClassForGateway(o client.Object) []string { g := o.(*gwv1beta1.Gateway) @@ -104,38 +129,63 @@ func gatewayForSecret(o client.Object) []string { return secretReferences } -func gatewayForHTTPRoute(o client.Object) []string { - httpRoute := o.(*gwv1beta1.HTTPRoute) - refSet := make(map[types.NamespacedName]struct{}, len(httpRoute.Spec.ParentRefs)) - for _, parent := range httpRoute.Spec.ParentRefs { - namespace := "" - if parent.Namespace != nil { - namespace = string(*parent.Namespace) - } - refSet[types.NamespacedName{Name: string(parent.Name), Namespace: namespace}] = struct{}{} - } +func gatewaysForHTTPRoute(o client.Object) []string { + route := o.(*gwv1beta1.HTTPRoute) + return gatewaysForRoute(route.Namespace, route.Spec.ParentRefs) +} - refs := make([]string, 0, len(refSet)) - for namespaceName := range refSet { - refs = append(refs, namespaceName.String()) +func gatewaysForTCPRoute(o client.Object) []string { + route := o.(*gwv1alpha2.TCPRoute) + return gatewaysForRoute(route.Namespace, route.Spec.ParentRefs) +} + +func servicesForHTTPRoute(o client.Object) []string { + route := o.(*gwv1beta1.HTTPRoute) + refs := []string{} + for _, rule := range route.Spec.Rules { + BACKEND_LOOP: + for _, ref := range rule.BackendRefs { + if nilOrEqual(ref.Group, "") && nilOrEqual(ref.Kind, "Service") { + backendRef := indexedNamespacedNameWithDefault(ref.Name, ref.Namespace, route.Namespace).String() + for _, member := range refs { + if member == backendRef { + continue BACKEND_LOOP + } + } + refs = append(refs, backendRef) + } + } } return refs } -func gatewayForTCPRoute(o client.Object) []string { - httpRoute := o.(*v1alpha2.TCPRoute) - refSet := make(map[types.NamespacedName]struct{}, len(httpRoute.Spec.ParentRefs)) - for _, parent := range httpRoute.Spec.ParentRefs { - namespace := "" - if parent.Namespace != nil { - namespace = string(*parent.Namespace) +func servicesForTCPRoute(o client.Object) []string { + route := o.(*gwv1alpha2.TCPRoute) + refs := []string{} + for _, rule := range route.Spec.Rules { + BACKEND_LOOP: + for _, ref := range rule.BackendRefs { + if nilOrEqual(ref.Group, "") && nilOrEqual(ref.Kind, "Service") { + backendRef := indexedNamespacedNameWithDefault(ref.Name, ref.Namespace, route.Namespace).String() + for _, member := range refs { + if member == backendRef { + continue BACKEND_LOOP + } + } + refs = append(refs, backendRef) + } } - refSet[types.NamespacedName{Name: string(parent.Name), Namespace: namespace}] = struct{}{} } + return refs +} - refs := make([]string, 0, len(refSet)) - for namespaceName := range refSet { - refs = append(refs, namespaceName.String()) +func gatewaysForRoute(namespace string, refs []gwv1beta1.ParentReference) []string { + var references []string + for _, parent := range refs { + if nilOrEqual(parent.Group, gwv1beta1.GroupVersion.Group) && nilOrEqual(parent.Kind, "Gateway") { + // If an explicit Gateway namespace is not provided, use the Route namespace to lookup the provided Gateway Namespace. + references = append(references, indexedNamespacedNameWithDefault(parent.Name, parent.Namespace, namespace).String()) + } } - return refs + return references } diff --git a/control-plane/api-gateway/gatekeeper/gatekeeper_test.go b/control-plane/api-gateway/gatekeeper/gatekeeper_test.go index bcd78003ca..44196b77aa 100644 --- a/control-plane/api-gateway/gatekeeper/gatekeeper_test.go +++ b/control-plane/api-gateway/gatekeeper/gatekeeper_test.go @@ -800,6 +800,11 @@ func configureRole(name, namespace string, labels map[string]string, resourceVer APIGroups: []string{"policy"}, Resources: []string{"podsecuritypolicies"}, Verbs: []string{"use"}, + }, { + APIGroups: []string{"security.openshift.io"}, + Resources: []string{"securitycontextconstraints"}, + ResourceNames: []string{"name-of-the-security-context-constraints"}, + Verbs: []string{"use"}, }}, } } diff --git a/control-plane/api-gateway/translation/config_entry_translation.go b/control-plane/api-gateway/translation/config_entry_translation.go index 415da6b854..58844cc879 100644 --- a/control-plane/api-gateway/translation/config_entry_translation.go +++ b/control-plane/api-gateway/translation/config_entry_translation.go @@ -7,11 +7,13 @@ package translation import ( "strings" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" "github.com/hashicorp/consul-k8s/control-plane/namespaces" + "github.com/hashicorp/consul/api" capi "github.com/hashicorp/consul/api" ) @@ -32,12 +34,6 @@ const ( AnnotationInlineCertificate = "consul.hashicorp.com/inline-certificate" ) -type consulIdentifier struct { - name string - namespace string - partition string -} - // K8sToConsulTranslator handles translating K8s resources into Consul config entries. type K8sToConsulTranslator struct { EnableConsulNamespaces bool @@ -48,33 +44,40 @@ type K8sToConsulTranslator struct { } // GatewayToAPIGateway translates a kuberenetes API gateway into a Consul APIGateway Config Entry. -func (t K8sToConsulTranslator) GatewayToAPIGateway(k8sGW gwv1beta1.Gateway, certs map[types.NamespacedName]consulIdentifier) capi.APIGatewayConfigEntry { +func (t K8sToConsulTranslator) GatewayToAPIGateway(k8sGW gwv1beta1.Gateway, certs map[types.NamespacedName]api.ResourceReference) capi.APIGatewayConfigEntry { listeners := make([]capi.APIGatewayListener, 0, len(k8sGW.Spec.Listeners)) for _, listener := range k8sGW.Spec.Listeners { - certificates := make([]capi.ResourceReference, 0, len(listener.TLS.CertificateRefs)) - for _, certificate := range listener.TLS.CertificateRefs { - k8sNS := "" - if certificate.Namespace != nil { - k8sNS = string(*certificate.Namespace) - } - nsn := types.NamespacedName{Name: string(certificate.Name), Namespace: k8sNS} - certRef, ok := certs[nsn] - if !ok { - // we don't have a ref for this certificate in consul - // drop the ref from the created gateway - continue + var certificates []capi.ResourceReference + if listener.TLS != nil { + certificates = make([]capi.ResourceReference, 0, len(listener.TLS.CertificateRefs)) + for _, certificate := range listener.TLS.CertificateRefs { + k8sNS := "" + if certificate.Namespace != nil { + k8sNS = string(*certificate.Namespace) + } + nsn := types.NamespacedName{Name: string(certificate.Name), Namespace: k8sNS} + certRef, ok := certs[nsn] + if !ok { + // we don't have a ref for this certificate in consul + // drop the ref from the created gateway + continue + } + c := capi.ResourceReference{ + Kind: capi.InlineCertificate, + Name: certRef.Name, + Partition: certRef.Partition, + Namespace: certRef.Namespace, + } + certificates = append(certificates, c) } - c := capi.ResourceReference{ - Kind: capi.InlineCertificate, - Name: certRef.name, - Partition: certRef.partition, - Namespace: certRef.namespace, - } - certificates = append(certificates, c) + } + hostname := "" + if listener.Hostname != nil { + hostname = string(*listener.Hostname) } l := capi.APIGatewayListener{ Name: string(listener.Name), - Hostname: string(*listener.Hostname), + Hostname: hostname, Port: int(listener.Port), Protocol: string(listener.Protocol), TLS: capi.APIGatewayTLSConfiguration{ @@ -105,14 +108,26 @@ func (t K8sToConsulTranslator) GatewayToAPIGateway(k8sGW gwv1beta1.Gateway, cert } } +func (t K8sToConsulTranslator) ReferenceForGateway(k8sGW *gwv1beta1.Gateway) api.ResourceReference { + gwName := k8sGW.Name + if gwNameFromAnnotation, ok := k8sGW.Annotations[AnnotationGateway]; ok && gwNameFromAnnotation != "" && !strings.Contains(gwNameFromAnnotation, ",") { + gwName = gwNameFromAnnotation + } + return api.ResourceReference{ + Kind: api.APIGateway, + Name: gwName, + Namespace: t.getConsulNamespace(k8sGW.GetObjectMeta().GetNamespace()), + } +} + // HTTPRouteToHTTPRoute translates a k8s HTTPRoute into a Consul HTTPRoute Config Entry. -func (t K8sToConsulTranslator) HTTPRouteToHTTPRoute(k8sHTTPRoute gwv1beta1.HTTPRoute, parentRefs map[types.NamespacedName]consulIdentifier) capi.HTTPRouteConfigEntry { +func (t K8sToConsulTranslator) HTTPRouteToHTTPRoute(k8sHTTPRoute *gwv1beta1.HTTPRoute, parentRefs map[types.NamespacedName]api.ResourceReference) *capi.HTTPRouteConfigEntry { routeName := k8sHTTPRoute.Name if routeNameFromAnnotation, ok := k8sHTTPRoute.Annotations[AnnotationHTTPRoute]; ok && routeNameFromAnnotation != "" && !strings.Contains(routeNameFromAnnotation, ",") { routeName = routeNameFromAnnotation } - consulHTTPRoute := capi.HTTPRouteConfigEntry{ + consulHTTPRoute := &capi.HTTPRouteConfigEntry{ Kind: capi.HTTPRoute, Name: routeName, Meta: map[string]string{ @@ -142,11 +157,27 @@ func (t K8sToConsulTranslator) HTTPRouteToHTTPRoute(k8sHTTPRoute gwv1beta1.HTTPR return consulHTTPRoute } +func (t K8sToConsulTranslator) ReferenceForHTTPRoute(k8sHTTPRoute *gwv1beta1.HTTPRoute) api.ResourceReference { + routeName := k8sHTTPRoute.Name + if routeNameFromAnnotation, ok := k8sHTTPRoute.Annotations[AnnotationHTTPRoute]; ok && routeNameFromAnnotation != "" && !strings.Contains(routeNameFromAnnotation, ",") { + routeName = routeNameFromAnnotation + } + return api.ResourceReference{ + Kind: api.HTTPRoute, + Name: routeName, + Namespace: t.getConsulNamespace(k8sHTTPRoute.GetObjectMeta().GetNamespace()), + } +} + // translates parent refs for Routes into Consul Resource References. -func translateRouteParentRefs(k8sParentRefs []gwv1beta1.ParentReference, parentRefs map[types.NamespacedName]consulIdentifier) []capi.ResourceReference { +func translateRouteParentRefs(k8sParentRefs []gwv1beta1.ParentReference, parentRefs map[types.NamespacedName]api.ResourceReference) []capi.ResourceReference { parents := make([]capi.ResourceReference, 0, len(k8sParentRefs)) for _, k8sParentRef := range k8sParentRefs { - parentRef, ok := parentRefs[types.NamespacedName{Name: string(k8sParentRef.Name), Namespace: string(*k8sParentRef.Namespace)}] + namespace := "" + if k8sParentRef.Namespace != nil { + namespace = string(*k8sParentRef.Namespace) + } + parentRef, ok := parentRefs[types.NamespacedName{Name: string(k8sParentRef.Name), Namespace: namespace}] if !(ok && isRefAPIGateway(k8sParentRef)) { // we drop any parent refs that consul does not know about continue @@ -157,10 +188,10 @@ func translateRouteParentRefs(k8sParentRefs []gwv1beta1.ParentReference, parentR } ref := capi.ResourceReference{ Kind: capi.APIGateway, // Will this ever not be a gateway? is that something we need to handle? - Name: parentRef.name, + Name: parentRef.Name, SectionName: sectionName, - Partition: parentRef.partition, - Namespace: parentRef.namespace, + Partition: parentRef.Partition, + Namespace: parentRef.Namespace, } parents = append(parents, ref) } @@ -215,10 +246,12 @@ func translateHTTPMatches(k8sMatches []gwv1beta1.HTTPRouteMatch) []capi.HTTPMatc headers := make([]capi.HTTPHeaderMatch, 0, len(k8sMatch.Headers)) for _, k8sHeader := range k8sMatch.Headers { header := capi.HTTPHeaderMatch{ - Match: headerMatchTypeTranslation[*k8sHeader.Type], Name: string(k8sHeader.Name), Value: k8sHeader.Value, } + if k8sHeader.Type != nil { + header.Match = headerMatchTypeTranslation[*k8sHeader.Type] + } headers = append(headers, header) } @@ -226,21 +259,29 @@ func translateHTTPMatches(k8sMatches []gwv1beta1.HTTPRouteMatch) []capi.HTTPMatc queries := make([]capi.HTTPQueryMatch, 0, len(k8sMatch.QueryParams)) for _, k8sQuery := range k8sMatch.QueryParams { query := capi.HTTPQueryMatch{ - Match: queryMatchTypeTranslation[*k8sQuery.Type], Name: k8sQuery.Name, Value: k8sQuery.Value, } + if k8sQuery.Type != nil { + query.Match = queryMatchTypeTranslation[*k8sQuery.Type] + } queries = append(queries, query) } match := capi.HTTPMatch{ Headers: headers, - Method: capi.HTTPMatchMethod(*k8sMatch.Method), - Path: capi.HTTPPathMatch{ - Match: headerPathMatchTypeTranslation[*k8sMatch.Path.Type], - Value: string(*k8sMatch.Path.Value), - }, - Query: queries, + Query: queries, + } + if k8sMatch.Method != nil { + match.Method = capi.HTTPMatchMethod(*k8sMatch.Method) + } + if k8sMatch.Path != nil { + if k8sMatch.Path.Type != nil { + match.Path.Match = headerPathMatchTypeTranslation[*k8sMatch.Path.Type] + } + if k8sMatch.Path.Value != nil { + match.Path.Value = string(*k8sMatch.Path.Value) + } } matches = append(matches, match) } @@ -290,10 +331,14 @@ func (t K8sToConsulTranslator) translateHTTPServices(k8sBackendRefs []gwv1beta1. for _, k8sRef := range k8sBackendRefs { service := capi.HTTPService{ - Name: string(k8sRef.Name), - Weight: int(*k8sRef.Weight), - Filters: translateHTTPFilters(k8sRef.Filters), - Namespace: t.getConsulNamespace(string(*k8sRef.Namespace)), + Name: string(k8sRef.Name), + Filters: translateHTTPFilters(k8sRef.Filters), + } + if k8sRef.Weight != nil { + service.Weight = int(*k8sRef.Weight) + } + if k8sRef.Namespace != nil { + service.Namespace = t.getConsulNamespace(string(*k8sRef.Namespace)) } services = append(services, service) } @@ -302,13 +347,13 @@ func (t K8sToConsulTranslator) translateHTTPServices(k8sBackendRefs []gwv1beta1. } // TCPRouteToTCPRoute translates a Kuberenetes TCPRoute into a Consul TCPRoute Config Entry. -func (t K8sToConsulTranslator) TCPRouteToTCPRoute(k8sRoute gwv1alpha2.TCPRoute, parentRefs map[types.NamespacedName]consulIdentifier) capi.TCPRouteConfigEntry { +func (t K8sToConsulTranslator) TCPRouteToTCPRoute(k8sRoute *gwv1alpha2.TCPRoute, parentRefs map[types.NamespacedName]api.ResourceReference) *capi.TCPRouteConfigEntry { routeName := k8sRoute.Name if routeNameFromAnnotation, ok := k8sRoute.Annotations[AnnotationTCPRoute]; ok && routeNameFromAnnotation != "" && !strings.Contains(routeNameFromAnnotation, ",") { routeName = routeNameFromAnnotation } - consulRoute := capi.TCPRouteConfigEntry{ + consulRoute := &capi.TCPRouteConfigEntry{ Kind: capi.TCPRoute, Name: routeName, Meta: map[string]string{ @@ -345,30 +390,50 @@ func (t K8sToConsulTranslator) TCPRouteToTCPRoute(k8sRoute gwv1alpha2.TCPRoute, return consulRoute } +func (t K8sToConsulTranslator) ReferenceForTCPRoute(k8sTCPRoute *gwv1alpha2.TCPRoute) api.ResourceReference { + routeName := k8sTCPRoute.Name + if routeNameFromAnnotation, ok := k8sTCPRoute.Annotations[AnnotationTCPRoute]; ok && routeNameFromAnnotation != "" && !strings.Contains(routeNameFromAnnotation, ",") { + routeName = routeNameFromAnnotation + } + return api.ResourceReference{ + Kind: api.TCPRoute, + Name: routeName, + Namespace: t.getConsulNamespace(k8sTCPRoute.GetObjectMeta().GetNamespace()), + } +} + // SecretToInlineCertificate translates a Kuberenetes Secret into a Consul Inline Certificate Config Entry. -func (t K8sToConsulTranslator) SecretToInlineCertificate(k8sSecret gwv1beta1.SecretObjectReference, certs map[types.NamespacedName]consulIdentifier) capi.InlineCertificateConfigEntry { - inlineCert := capi.InlineCertificateConfigEntry{Kind: capi.InlineCertificate} +func (t K8sToConsulTranslator) SecretToInlineCertificate(k8sSecret corev1.Secret) capi.InlineCertificateConfigEntry { + namespace := t.getConsulNamespace(k8sSecret.GetObjectMeta().GetNamespace()) + return capi.InlineCertificateConfigEntry{ + Kind: capi.InlineCertificate, + Namespace: namespace, + Name: k8sSecret.Name, + Certificate: k8sSecret.StringData[corev1.TLSCertKey], + PrivateKey: k8sSecret.StringData[corev1.TLSPrivateKeyKey], + Meta: map[string]string{ + metaKeyManagedBy: metaValueManagedBy, + metaKeyKubeNS: namespace, + metaKeyKubeServiceName: string(k8sSecret.Name), + metaKeyKubeName: string(k8sSecret.Name), + }, + } +} - for namespaceName, consulIdentifier := range certs { - k8sSecretNS := "" - if k8sSecret.Namespace != nil { - k8sSecretNS = string(*k8sSecret.Namespace) - } - nsn := types.NamespacedName{Name: string(k8sSecret.Name), Namespace: k8sSecretNS} - if namespaceName == nsn { - inlineCert.Name = consulIdentifier.name - inlineCert.Namespace = consulIdentifier.namespace - inlineCert.Partition = consulIdentifier.partition - inlineCert.Meta = map[string]string{ - metaKeyManagedBy: metaValueManagedBy, - metaKeyKubeNS: k8sSecretNS, - metaKeyKubeServiceName: string(k8sSecret.Name), - metaKeyKubeName: string(k8sSecret.Name), - } - return inlineCert - } +func (t K8sToConsulTranslator) ReferenceForSecret(k8sSecret corev1.Secret) api.ResourceReference { + return api.ResourceReference{ + Kind: api.InlineCertificate, + Name: k8sSecret.Name, + Namespace: t.getConsulNamespace(k8sSecret.GetObjectMeta().GetNamespace()), + } +} + +func EntryToNamespacedName(entry capi.ConfigEntry) types.NamespacedName { + meta := entry.GetMeta() + return types.NamespacedName{ + Name: meta[metaKeyKubeName], + Namespace: meta[metaKeyKubeNS], } - return inlineCert } func (t K8sToConsulTranslator) getConsulNamespace(k8sNS string) string { diff --git a/control-plane/api-gateway/translation/config_entry_translation_test.go b/control-plane/api-gateway/translation/config_entry_translation_test.go index 0df03a3f75..dd1ccf011d 100644 --- a/control-plane/api-gateway/translation/config_entry_translation_test.go +++ b/control-plane/api-gateway/translation/config_entry_translation_test.go @@ -13,6 +13,7 @@ import ( gwv1alpha2 "sigs.k8s.io/gateway-api/apis/v1alpha2" gwv1beta1 "sigs.k8s.io/gateway-api/apis/v1beta1" + "github.com/hashicorp/consul/api" capi "github.com/hashicorp/consul/api" ) @@ -227,14 +228,14 @@ func TestTranslator_GatewayToAPIGateway(t *testing.T) { MirroringPrefix: "", } - certs := map[types.NamespacedName]consulIdentifier{ + certs := map[types.NamespacedName]api.ResourceReference{ {Name: listenerOneCertName, Namespace: listenerOneCertK8sNamespace}: { - name: listenerOneCertName, - namespace: listenerOneCertConsulNamespace, + Name: listenerOneCertName, + Namespace: listenerOneCertConsulNamespace, }, {Name: listenerTwoCertName, Namespace: listenerTwoCertK8sNamespace}: { - name: listenerTwoCertName, - namespace: listenerTwoCertConsulNamespace, + Name: listenerTwoCertName, + Namespace: listenerTwoCertConsulNamespace, }, } @@ -251,7 +252,7 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) { t.Parallel() type args struct { k8sHTTPRoute gwv1beta1.HTTPRoute - parentRefs map[types.NamespacedName]consulIdentifier + parentRefs map[types.NamespacedName]api.ResourceReference } tests := map[string]struct { args args @@ -378,8 +379,8 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) { }, }, }, - parentRefs: map[types.NamespacedName]consulIdentifier{ - {Name: "api-gw", Namespace: "k8s-gw-ns"}: {name: "api-gw", partition: "part-1", namespace: "ns"}, + parentRefs: map[types.NamespacedName]api.ResourceReference{ + {Name: "api-gw", Namespace: "k8s-gw-ns"}: {Name: "api-gw", Partition: "part-1", Namespace: "ns"}, }, }, want: capi.HTTPRouteConfigEntry{ @@ -596,8 +597,8 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) { }, }, }, - parentRefs: map[types.NamespacedName]consulIdentifier{ - {Name: "api-gw", Namespace: "k8s-gw-ns"}: {name: "api-gw", partition: "part-1", namespace: "ns"}, + parentRefs: map[types.NamespacedName]api.ResourceReference{ + {Name: "api-gw", Namespace: "k8s-gw-ns"}: {Name: "api-gw", Partition: "part-1", Namespace: "ns"}, }, }, want: capi.HTTPRouteConfigEntry{ @@ -815,8 +816,8 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) { }, }, }, - parentRefs: map[types.NamespacedName]consulIdentifier{ - {Name: "api-gw", Namespace: "k8s-gw-ns"}: {name: "api-gw", partition: "part-1", namespace: "ns"}, + parentRefs: map[types.NamespacedName]api.ResourceReference{ + {Name: "api-gw", Namespace: "k8s-gw-ns"}: {Name: "api-gw", Partition: "part-1", Namespace: "ns"}, }, }, want: capi.HTTPRouteConfigEntry{ @@ -1038,8 +1039,8 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) { }, }, }, - parentRefs: map[types.NamespacedName]consulIdentifier{ - {Name: "api-gw", Namespace: "k8s-gw-ns"}: {name: "api-gw", partition: "part-1", namespace: "ns"}, + parentRefs: map[types.NamespacedName]api.ResourceReference{ + {Name: "api-gw", Namespace: "k8s-gw-ns"}: {Name: "api-gw", Partition: "part-1", Namespace: "ns"}, }, }, want: capi.HTTPRouteConfigEntry{ @@ -1253,8 +1254,8 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) { }, }, }, - parentRefs: map[types.NamespacedName]consulIdentifier{ - {Name: "api-gw", Namespace: "k8s-gw-ns"}: {name: "api-gw", partition: "part-1", namespace: "ns"}, + parentRefs: map[types.NamespacedName]api.ResourceReference{ + {Name: "api-gw", Namespace: "k8s-gw-ns"}: {Name: "api-gw", Partition: "part-1", Namespace: "ns"}, }, }, want: capi.HTTPRouteConfigEntry{ @@ -1355,8 +1356,8 @@ func TestTranslator_HTTPRouteToHTTPRoute(t *testing.T) { EnableConsulNamespaces: true, EnableK8sMirroring: true, } - got := tr.HTTPRouteToHTTPRoute(tc.args.k8sHTTPRoute, tc.args.parentRefs) - if diff := cmp.Diff(tc.want, got); diff != "" { + got := tr.HTTPRouteToHTTPRoute(&tc.args.k8sHTTPRoute, tc.args.parentRefs) + if diff := cmp.Diff(&tc.want, got); diff != "" { t.Errorf("Translator.HTTPRouteToHTTPRoute() mismatch (-want +got):\n%s", diff) } }) @@ -1367,7 +1368,7 @@ func TestTranslator_TCPRouteToTCPRoute(t *testing.T) { t.Parallel() type args struct { k8sRoute gwv1alpha2.TCPRoute - parentRefs map[types.NamespacedName]consulIdentifier + parentRefs map[types.NamespacedName]api.ResourceReference } tests := map[string]struct { args args @@ -1417,14 +1418,14 @@ func TestTranslator_TCPRouteToTCPRoute(t *testing.T) { }, }, }, - parentRefs: map[types.NamespacedName]consulIdentifier{ + parentRefs: map[types.NamespacedName]api.ResourceReference{ { Namespace: "another-ns", Name: "mygw", }: { - name: "mygw", - namespace: "another-ns", - partition: "", + Name: "mygw", + Namespace: "another-ns", + Partition: "", }, }, }, @@ -1509,14 +1510,14 @@ func TestTranslator_TCPRouteToTCPRoute(t *testing.T) { }, }, }, - parentRefs: map[types.NamespacedName]consulIdentifier{ + parentRefs: map[types.NamespacedName]api.ResourceReference{ { Namespace: "another-ns", Name: "mygw", }: { - name: "mygw", - namespace: "another-ns", - partition: "", + Name: "mygw", + Namespace: "another-ns", + Partition: "", }, }, }, @@ -1561,64 +1562,10 @@ func TestTranslator_TCPRouteToTCPRoute(t *testing.T) { EnableK8sMirroring: true, } - got := tr.TCPRouteToTCPRoute(tt.args.k8sRoute, tt.args.parentRefs) - if diff := cmp.Diff(got, tt.want); diff != "" { + got := tr.TCPRouteToTCPRoute(&tt.args.k8sRoute, tt.args.parentRefs) + if diff := cmp.Diff(got, &tt.want); diff != "" { t.Errorf("Translator.TCPRouteToTCPRoute() mismatch (-want +got):\n%s", diff) } }) } } - -func TestTranslator_SecretToInlineCertificate(t *testing.T) { - t.Parallel() - type args struct { - k8sSecret gwv1beta1.SecretObjectReference - certs map[types.NamespacedName]consulIdentifier - } - tests := map[string]struct { - args args - want capi.InlineCertificateConfigEntry - }{ - "base test": { - args: args{ - k8sSecret: gwv1beta1.SecretObjectReference{ - Name: "my-secret", - Namespace: ptrTo(gwv1beta1.Namespace("k8s-ns")), - }, - certs: map[types.NamespacedName]consulIdentifier{ - { - Namespace: "k8s-ns", - Name: "my-secret", - }: { - name: "inline certs", - namespace: "my-ns", - partition: "", - }, - }, - }, - want: capi.InlineCertificateConfigEntry{ - Kind: capi.InlineCertificate, - Name: "inline certs", - Meta: map[string]string{ - metaKeyManagedBy: metaValueManagedBy, - metaKeyKubeNS: "k8s-ns", - metaKeyKubeServiceName: "my-secret", - metaKeyKubeName: "my-secret", - }, - Namespace: "my-ns", - }, - }, - } - for name, tt := range tests { - t.Run(name, func(t *testing.T) { - tr := K8sToConsulTranslator{ - EnableConsulNamespaces: true, - EnableK8sMirroring: true, - } - got := tr.SecretToInlineCertificate(tt.args.k8sSecret, tt.args.certs) - if diff := cmp.Diff(got, tt.want); diff != "" { - t.Errorf("Translator.SecretToInlineCertificate() mismatch (-want +got):\n%s", diff) - } - }) - } -} diff --git a/control-plane/cache/consul.go b/control-plane/cache/consul.go index b92c909475..cd5721887d 100644 --- a/control-plane/cache/consul.go +++ b/control-plane/cache/consul.go @@ -56,6 +56,38 @@ func (oldCache resourceCache) diff(newCache resourceCache) []api.ConfigEntry { } } + // get all deleted entries, these are entries present in the old cache + // that are not present in the new + for ref, entry := range oldCache { + if _, ok := newCache[ref]; !ok { + diffs = append(diffs, entry) + } + } + + return diffs +} + +type serviceCache map[api.ResourceReference]*api.CatalogService + +func (oldCache serviceCache) diff(newCache serviceCache) []*api.CatalogService { + diffs := make([]*api.CatalogService, 0) + + for ref, entry := range newCache { + oldRef, ok := oldCache[ref] + // ref from the new cache doesn't exist in the old one + // this means a resource was added + if !ok { + diffs = append(diffs, entry) + continue + } + + // the entry in the old cache has an older modify index than the ref + // from the new cache + if oldRef.ModifyIndex < entry.ModifyIndex { + diffs = append(diffs, entry) + } + } + // get all deleted entries, these are entries present in the old cache // that are not present in the new for ref, entry := range oldCache { @@ -105,6 +137,24 @@ func (s *Subscription) Events() chan event.GenericEvent { return s.events } +type ServiceTranslatorFn func(*api.CatalogService) []types.NamespacedName + +// ServiceSubscription represents a watcher for events on a specific kind. +type ServiceSubscription struct { + translator ServiceTranslatorFn + ctx context.Context + cancelCtx context.CancelFunc + events chan event.GenericEvent +} + +func (s *ServiceSubscription) Cancel() { + s.cancelCtx() +} + +func (s *ServiceSubscription) Events() chan event.GenericEvent { + return s.events +} + // Cache subscribes to and caches Consul objects, it also responsible for mainting subscriptions to // resources that it caches. type Cache struct { @@ -112,11 +162,13 @@ type Cache struct { serverMgr consul.ServerConnectionManager logger logr.Logger - cache map[string]resourceCache - cacheMutex *sync.Mutex + cache map[string]resourceCache + serviceCache serviceCache + cacheMutex *sync.Mutex - subscribers map[string][]*Subscription - subscriberMutex *sync.Mutex + subscribers map[string][]*Subscription + serviceSubscribers []*ServiceSubscription + subscriberMutex *sync.Mutex partition string namespacesEnabled bool @@ -134,17 +186,21 @@ func New(config Config) *Cache { config.ConsulClientConfig.APITimeout = apiTimeout return &Cache{ - config: config.ConsulClientConfig, - serverMgr: config.ConsulServerConnMgr, - namespacesEnabled: config.NamespacesEnabled, - partition: config.Partition, - cache: cache, - cacheMutex: &sync.Mutex{}, - subscribers: make(map[string][]*Subscription), - subscriberMutex: &sync.Mutex{}, - kinds: Kinds, - synced: make(chan struct{}, len(Kinds)), - logger: config.Logger, + config: config.ConsulClientConfig, + serverMgr: config.ConsulServerConnMgr, + namespacesEnabled: config.NamespacesEnabled, + partition: config.Partition, + cache: cache, + serviceCache: make(serviceCache), + cacheMutex: &sync.Mutex{}, + subscribers: make(map[string][]*Subscription), + serviceSubscribers: make([]*ServiceSubscription, 0), + subscriberMutex: &sync.Mutex{}, + kinds: Kinds, + // we make a buffered channel that is the length of the kinds which + // are subscribed to + 1 for services + synced: make(chan struct{}, len(Kinds)+1), + logger: config.Logger, } } @@ -157,6 +213,12 @@ func (c *Cache) WaitSynced(ctx context.Context) { return } } + // one more for service subscribers + select { + case <-c.synced: + case <-ctx.Done(): + return + } } // Subscribe handles adding a new subscription for resources of a given kind. @@ -186,6 +248,25 @@ func (c *Cache) Subscribe(ctx context.Context, kind string, translator translati subscribers = append(subscribers, sub) c.subscribers[kind] = subscribers + + return sub +} + +// SubscribeServices handles adding a new subscription for resources of a given kind. +func (c *Cache) SubscribeServices(ctx context.Context, translator ServiceTranslatorFn) *ServiceSubscription { + c.subscriberMutex.Lock() + defer c.subscriberMutex.Unlock() + + ctx, cancel := context.WithCancel(ctx) + events := make(chan event.GenericEvent) + sub := &ServiceSubscription{ + translator: translator, + ctx: ctx, + cancelCtx: cancel, + events: events, + } + + c.serviceSubscribers = append(c.serviceSubscribers, sub) return sub } @@ -203,6 +284,12 @@ func (c *Cache) Run(ctx context.Context) { }() } + wg.Add(1) + go func() { + defer wg.Done() + c.subscribeToConsulServices(ctx) + }() + wg.Wait() } @@ -219,13 +306,19 @@ func (c *Cache) subscribeToConsul(ctx context.Context, kind string) { } for { + select { + case <-ctx.Done(): + return + default: + } + client, err := consul.NewClientFromConnMgr(c.config, c.serverMgr) if err != nil { c.logger.Error(err, "error initializing consul client") continue } - entries, meta, err := client.ConfigEntries().List(kind, opts) + entries, meta, err := client.ConfigEntries().List(kind, opts.WithContext(ctx)) if err != nil { c.logger.Error(err, fmt.Sprintf("error fetching config entries for kind: %s", kind)) continue @@ -244,6 +337,60 @@ func (c *Cache) subscribeToConsul(ctx context.Context, kind string) { } } +func (c *Cache) subscribeToConsulServices(ctx context.Context) { + once := &sync.Once{} + + opts := &api.QueryOptions{Connect: true} + if c.namespacesEnabled { + opts.Namespace = namespaceWildcard + } + + if c.partition != "" { + opts.Partition = c.partition + } + +MAIN_LOOP: + for { + select { + case <-ctx.Done(): + return + default: + } + + client, err := consul.NewClientFromConnMgr(c.config, c.serverMgr) + if err != nil { + c.logger.Error(err, "error initializing consul client") + continue + } + + services, meta, err := client.Catalog().Services(opts.WithContext(ctx)) + if err != nil { + c.logger.Error(err, "error fetching services") + continue + } + + flattened := []*api.CatalogService{} + for service := range services { + serviceList, _, err := client.Catalog().Service(service, "", opts.WithContext(ctx)) + if err != nil { + c.logger.Error(err, fmt.Sprintf("error fetching service: %s", service)) + continue MAIN_LOOP + } + flattened = append(flattened, serviceList...) + } + + opts.WaitIndex = meta.LastIndex + c.updateAndNotifyServices(ctx, once, flattened) + + select { + case <-ctx.Done(): + return + default: + continue + } + } +} + func (c *Cache) updateAndNotify(ctx context.Context, once *sync.Once, kind string, entries []api.ConfigEntry) { c.cacheMutex.Lock() defer c.cacheMutex.Unlock() @@ -267,6 +414,66 @@ func (c *Cache) updateAndNotify(ctx context.Context, once *sync.Once, kind strin c.notifySubscribers(ctx, kind, diffs) } +func (c *Cache) updateAndNotifyServices(ctx context.Context, once *sync.Once, services []*api.CatalogService) { + c.cacheMutex.Lock() + defer c.cacheMutex.Unlock() + + cache := make(serviceCache) + + for _, service := range services { + cache[api.ResourceReference{Name: service.ServiceName, Namespace: service.Namespace, Partition: service.Partition}] = service + } + + diffs := c.serviceCache.diff(cache) + + c.serviceCache = cache + + // we run this the first time the cache is filled to notify the waiter + once.Do(func() { + c.synced <- struct{}{} + }) + + // now notify all subscribers + c.notifyServiceSubscribers(ctx, diffs) +} + +// notifyServiceSubscribers notifies each subscriber for a given kind on changes to a config entry of that kind. It also +// handles removing any subscribers that have marked themselves as done. +func (c *Cache) notifyServiceSubscribers(ctx context.Context, services []*api.CatalogService) { + c.subscriberMutex.Lock() + defer c.subscriberMutex.Unlock() + + for _, service := range services { + // this will hold the new list of current subscribers after we finish notifying + subscribers := make([]*ServiceSubscription, 0, len(c.serviceSubscribers)) + for _, subscriber := range c.serviceSubscribers { + addSubscriber := false + + for _, namespaceName := range subscriber.translator(service) { + event := event.GenericEvent{ + Object: newConfigEntryObject(namespaceName), + } + + select { + case <-ctx.Done(): + return + case <-subscriber.ctx.Done(): + // don't add this subscriber to current list because it is done + addSubscriber = false + case subscriber.events <- event: + // keep this one since we can send events to it + addSubscriber = true + } + } + + if addSubscriber { + subscribers = append(subscribers, subscriber) + } + } + c.serviceSubscribers = subscribers + } +} + // notifySubscribers notifies each subscriber for a given kind on changes to a config entry of that kind. It also // handles removing any subscribers that have marked themselves as done. func (c *Cache) notifySubscribers(ctx context.Context, kind string, entries []api.ConfigEntry) { @@ -354,3 +561,33 @@ func (c *Cache) Get(ref api.ResourceReference) api.ConfigEntry { return entry } + +// List returns a list of config entries from the cache that corresponds to the given kind. +func (c *Cache) List(kind string) []api.ConfigEntry { + c.cacheMutex.Lock() + defer c.cacheMutex.Unlock() + + entryMap, ok := c.cache[kind] + if !ok { + return nil + } + entries := make([]api.ConfigEntry, len(entryMap)) + for _, entry := range entryMap { + entries = append(entries, entry) + } + + return entries +} + +// ListServices returns a list of services from the cache that corresponds to the given kind. +func (c *Cache) ListServices() []api.CatalogService { + c.cacheMutex.Lock() + defer c.cacheMutex.Unlock() + + entries := make([]api.CatalogService, len(c.serviceCache)) + for _, service := range c.serviceCache { + entries = append(entries, *service) + } + + return entries +} diff --git a/control-plane/cache/consul_test.go b/control-plane/cache/consul_test.go index 78d1c22a22..4a4b0968df 100644 --- a/control-plane/cache/consul_test.go +++ b/control-plane/cache/consul_test.go @@ -1285,7 +1285,6 @@ func TestCache_Subscribe(t *testing.T) { if expectedSubscriberCount != actualSubscriberCount { t.Errorf("Expected there to be %d subscribers, there were %d", expectedSubscriberCount, actualSubscriberCount) } - } }) } @@ -1322,6 +1321,8 @@ func TestCache_Write(t *testing.T) { switch r.URL.Path { case "/v1/config": tt.responseFn(w) + case "/v1/catalog/services": + fmt.Fprintln(w, `{}`) default: w.WriteHeader(500) fmt.Fprintln(w, "Mock Server not configured for this route: "+r.URL.Path) @@ -1605,6 +1606,8 @@ func Test_Run(t *testing.T) { return } fmt.Fprintln(w, string(val)) + case "/v1/catalog/services": + fmt.Fprintln(w, `{}`) default: w.WriteHeader(500) fmt.Fprintln(w, "Mock Server not configured for this route: "+r.URL.Path) @@ -1712,6 +1715,8 @@ func Test_Run(t *testing.T) { } }) + c.SubscribeServices(ctx, func(cs *api.CatalogService) []types.NamespacedName { return nil }).Cancel() + // mark this subscription as ended canceledSub.Cancel() diff --git a/control-plane/subcommand/inject-connect/command.go b/control-plane/subcommand/inject-connect/command.go index 0247227195..4a6901d788 100644 --- a/control-plane/subcommand/inject-connect/command.go +++ b/control-plane/subcommand/inject-connect/command.go @@ -498,7 +498,9 @@ func (c *Command) Run(args []string) int { go cache.Run(ctx) // wait for the cache to fill + setupLog.Info("waiting for Consul cache sync") cache.WaitSynced(ctx) + setupLog.Info("Consul cache synced") configEntryReconciler := &controller.ConfigEntryController{ ConsulClientConfig: c.consul.ConsulClientConfig(),