From 9545cf023cd129ae9548a7fcfbf3d683cbb5720a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Grzegorz=20Burzy=C5=84ski?= Date: Thu, 26 Sep 2024 14:34:34 +0200 Subject: [PATCH] feat(konnect): add KongKey - KongKeySet binding --- config/samples/konnect_kongkeyset.yaml | 61 +++++ controller/konnect/conditions/conditions.go | 14 + controller/konnect/ops/ops_kongkey.go | 8 + controller/konnect/ops/ops_kongkey_test.go | 166 ++++++++--- controller/konnect/reconciler_generic.go | 288 ++++++++++++++++++++ test/envtest/deploy_resources.go | 9 +- test/envtest/konnect_entities_key_test.go | 174 +++++++++--- test/envtest/update_status.go | 18 ++ 8 files changed, 651 insertions(+), 87 deletions(-) diff --git a/config/samples/konnect_kongkeyset.yaml b/config/samples/konnect_kongkeyset.yaml index 8ca4bc0b6..89c50e3c3 100644 --- a/config/samples/konnect_kongkeyset.yaml +++ b/config/samples/konnect_kongkeyset.yaml @@ -37,3 +37,64 @@ spec: name: key-set-1 tags: - production +--- +kind: KongKey +apiVersion: configuration.konghq.com/v1alpha1 +metadata: + name: key-1 + namespace: default + annotations: + konghq.com/tags: "infra" +spec: + controlPlaneRef: + type: konnectNamespacedRef + konnectNamespacedRef: + name: test1 + keySetRef: + type: namespacedRef + namespacedRef: + name: key-set-1 + tags: + - production + kid: kid + name: key-1 + pem: + private_key: | + -----BEGIN PRIVATE KEY----- + MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQCdLY9b3Pb/Fsxo + cIJAKxZQKEtqTU8k72BOvMBtzcIN9ekGFrUYB8GQgIJsPIUw4+c4XK99YNh0tVBE + /9W8OdyXwopzqNn+nRfrhXpxDu+BVvjQ/AENAHKqg8pJKhNTd4W6dAzxelLO/t7y + rlXbjGX/Ry/3ednPq6PpDcxvqgc+v7Rcmh+5dEKdIoIrppjUc2X38+LXcy9xOuML + FtxNtx+NB+5bNq31eooT9OKk3r7mA0gX4Su2DnIL+SLsdTIb0dnCBIydUpbLdYfd + dew1UGy2XtlWsxux3zoXjGe+RBtndUzPBvyb/k6g2QFAaIEwndPbwQ4fi9y4FrB7 + hqjQa+OLAgMBAAECggEAMkWruCydHarLl04BAwgk+19f+7Cdc0lTCuwJOjWY70Er + loR1yKlWamMIFBlpWmFSh67xfE8Y/H8vnNodITZ6jVmuUd78VpklWPHY30dxKHPK + YoFvzppJkqtTbIJWKxir/551s1i2GrnfUkybbnzh9Lvuph9loKwb4YNF06NU7OcA + tgCk78oA/JpVa01PCJYmVy8zI4UERt/2mBzuummk8kJhPl+A7K9gVkNz6KSeQDGM + QUZ6gtiYtyg7nT+kI1H6LfwokxCljQ+MBuB62eehUsie7EmpgmJqbzesqnWfdbFp + IjCDn174R45o0FUD1QpcbQWxa39cdo4f6oP4My8szQKBgQDJT8Z7yfYAXeAyVeRD + tTrOWhXqKzj3DOO65n+Evwen9O4NlWKtbC6LeaogcrlJSuHhYlAShdgrBy6DLWi8 + DEwozbK5YvpKbQ8u03rJYnfM6nN57gvm49SgsaoUPO4FlZMt1V3VC6kG2K4YbP8Y + OWy5FCdYPRlOtPp4CsFQ4xzbjQKBgQDH4IIMBT667V+7fWC/YyvUqJoIimuZcVzP + zmxICWVP9u4VKCHw46sbqukCw56bMYD8X7zu16Sbkkc3YzeOP6n4NGcLUzIFkweq + nzKdxZ6wj00x+mHT0/i/B8IZDYSkRFHF7ISV3Z8B9FuJXfsk5xGHVc47jVOTyKPb + XuLzcAlpdwKBgAsij37/X80LZEBEgfjAyHzrfLTUKTV5EAuhfkIwctL2eEhmD+w5 + xKVQWHms/tSwAKh/0KAFqTxQDGGTHGzyXTAQmKcqc1+0gpd7eRo0iR3bhgGjiiL+ + TR+KVDcEW8IRUO/DEoqbN4E6cP7G4KFNY9ck5zw5PPIejpAfQCwiM9FtAoGAW8Kn + EWurA9gMFiAWNWcK7UNGC9u4UCZqDIDg1yVxHIfpf08AXf23RSludbVm8CqG49Xz + /9aCHGXIShZDoAt8NZWhJOLZ2RNJ9rvFWgcqtjXjo6kmFkB/NvwR0LyTA3LV876E + k+S9pgEPsP2zWZq3QmFTH6XfE76N8x0ZpdbuizsCgYBBDNh8AfKbaEdo90bQi8No + sNqbHFAc12H6qxqnRl/pDvoY34wBVeZP3QEfb/XeOO2BVrcx6tGvIosy2lkOJtrh + ckY/QO1OLvcDtDgMA6qOr1rAROP/aWhuhJg1Aw50vCuy3z96CfUVSJBG+r0v7HvO + ZNgrh9kB0qmomKcjwwJlKQ== + -----END PRIVATE KEY----- + public_key: | + -----BEGIN PUBLIC KEY----- + MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAnS2PW9z2/xbMaHCCQCsW + UChLak1PJO9gTrzAbc3CDfXpBha1GAfBkICCbDyFMOPnOFyvfWDYdLVQRP/VvDnc + l8KKc6jZ/p0X64V6cQ7vgVb40PwBDQByqoPKSSoTU3eFunQM8XpSzv7e8q5V24xl + /0cv93nZz6uj6Q3Mb6oHPr+0XJofuXRCnSKCK6aY1HNl9/Pi13MvcTrjCxbcTbcf + jQfuWzat9XqKE/TipN6+5gNIF+Ertg5yC/ki7HUyG9HZwgSMnVKWy3WH3XXsNVBs + tl7ZVrMbsd86F4xnvkQbZ3VMzwb8m/5OoNkBQGiBMJ3T28EOH4vcuBawe4ao0Gvj + iwIDAQAB + -----END PUBLIC KEY----- diff --git a/controller/konnect/conditions/conditions.go b/controller/konnect/conditions/conditions.go index 7452bcdd7..532809f09 100644 --- a/controller/konnect/conditions/conditions.go +++ b/controller/konnect/conditions/conditions.go @@ -128,6 +128,20 @@ const ( KongUpstreamRefReasonInvalid = "Invalid" ) +const ( + // KeySetRefValidConditionType is the type of the condition that indicates + // whether the KeySet reference is valid and points to an existing + // KeySet. + KeySetRefValidConditionType = "KeySetRefValid" + + // KeySetRefReasonValid is the reason used with the KeySetRefValid + // condition type indicating that the KeySet reference is valid. + KeySetRefReasonValid = "Valid" + // KeySetRefReasonInvalid is the reason used with the KeySetRefValid + // condition type indicating that the KeySet reference is invalid. + KeySetRefReasonInvalid = "Invalid" +) + const ( // KongCertificateRefValidConditionType is the type of the condition that indicates // whether the KongCertificate reference is valid and points to an existing KongCertificate diff --git a/controller/konnect/ops/ops_kongkey.go b/controller/konnect/ops/ops_kongkey.go index d3fa8eace..f40ab1572 100644 --- a/controller/konnect/ops/ops_kongkey.go +++ b/controller/konnect/ops/ops_kongkey.go @@ -138,6 +138,7 @@ func kongKeyToKeyInput(key *configurationv1alpha1.KongKey) sdkkonnectcomp.KeyInp specTags = key.Spec.Tags k8sMetaTags = GenerateKubernetesMetadataTags(key) ) + k := sdkkonnectcomp.KeyInput{ Jwk: key.Spec.JWK, Kid: key.Spec.KID, @@ -151,5 +152,12 @@ func kongKeyToKeyInput(key *configurationv1alpha1.KongKey) sdkkonnectcomp.KeyInp PublicKey: lo.ToPtr(key.Spec.PEM.PublicKey), } } + if konnectStatus := key.Status.Konnect; konnectStatus != nil { + if keySetID := konnectStatus.GetKeySetID(); keySetID != "" { + k.Set = &sdkkonnectcomp.Set{ + ID: lo.ToPtr(konnectStatus.GetKeySetID()), + } + } + } return k } diff --git a/controller/konnect/ops/ops_kongkey_test.go b/controller/konnect/ops/ops_kongkey_test.go index 5852c6657..08a19f4cb 100644 --- a/controller/konnect/ops/ops_kongkey_test.go +++ b/controller/konnect/ops/ops_kongkey_test.go @@ -1,9 +1,10 @@ package ops import ( + "sort" "testing" - "github.com/google/uuid" + sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components" "github.com/samber/lo" "github.com/stretchr/testify/require" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -12,55 +13,138 @@ import ( konnectconsts "github.com/kong/gateway-operator/controller/konnect/consts" configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" ) func TestKongKeyToKeyInput(t *testing.T) { - key := &configurationv1alpha1.KongKey{ - TypeMeta: metav1.TypeMeta{ - Kind: "KongKey", - APIVersion: "configuration.konghq.com/v1alpha1", - }, - ObjectMeta: metav1.ObjectMeta{ - Name: "key-1", - Namespace: "default", - Generation: 2, - UID: k8stypes.UID(uuid.NewString()), - Annotations: map[string]string{ - konnectconsts.AnnotationTags: "tag1,tag2,duplicate", + testCases := []struct { + name string + key *configurationv1alpha1.KongKey + expectedOutput sdkkonnectcomp.KeyInput + }{ + { + name: "kong key with all fields set without key set", + key: &configurationv1alpha1.KongKey{ + TypeMeta: metav1.TypeMeta{ + Kind: "KongKey", + APIVersion: "configuration.konghq.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "key-1", + Namespace: "default", + Generation: 2, + UID: k8stypes.UID("key-uid"), + Annotations: map[string]string{ + konnectconsts.AnnotationTags: "tag1,tag2,duplicate", + }, + }, + Spec: configurationv1alpha1.KongKeySpec{ + KongKeyAPISpec: configurationv1alpha1.KongKeyAPISpec{ + KID: "kid", + Name: lo.ToPtr("name"), + JWK: lo.ToPtr("jwk"), + PEM: &configurationv1alpha1.PEMKeyPair{ + PublicKey: "public", + PrivateKey: "private", + }, + Tags: []string{"tag3", "tag4", "duplicate"}, + }, + }, + }, + expectedOutput: sdkkonnectcomp.KeyInput{ + Kid: "kid", + Name: lo.ToPtr("name"), + Jwk: lo.ToPtr("jwk"), + Pem: &sdkkonnectcomp.Pem{ + PublicKey: lo.ToPtr("public"), + PrivateKey: lo.ToPtr("private"), + }, + Tags: []string{ + "duplicate", + "k8s-generation:2", + "k8s-group:configuration.konghq.com", + "k8s-kind:KongKey", + "k8s-name:key-1", + "k8s-namespace:default", + "k8s-uid:key-uid", + "k8s-version:v1alpha1", + "tag1", + "tag2", + "tag3", + "tag4", + }, }, }, - Spec: configurationv1alpha1.KongKeySpec{ - KongKeyAPISpec: configurationv1alpha1.KongKeyAPISpec{ - KID: "kid", + { + name: "kong key with all fields set with key set", + key: &configurationv1alpha1.KongKey{ + TypeMeta: metav1.TypeMeta{ + Kind: "KongKey", + APIVersion: "configuration.konghq.com/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "key-1", + Namespace: "default", + Generation: 2, + UID: k8stypes.UID("key-uid"), + Annotations: map[string]string{ + konnectconsts.AnnotationTags: "tag1,tag2,duplicate", + }, + }, + Spec: configurationv1alpha1.KongKeySpec{ + KongKeyAPISpec: configurationv1alpha1.KongKeyAPISpec{ + KID: "kid", + Name: lo.ToPtr("name"), + JWK: lo.ToPtr("jwk"), + PEM: &configurationv1alpha1.PEMKeyPair{ + PublicKey: "public", + PrivateKey: "private", + }, + Tags: []string{"tag3", "tag4", "duplicate"}, + }, + }, + Status: configurationv1alpha1.KongKeyStatus{ + Konnect: &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndKeySetRef{ + KeySetID: "key-set-id", + }, + }, + }, + expectedOutput: sdkkonnectcomp.KeyInput{ + Kid: "kid", Name: lo.ToPtr("name"), - JWK: lo.ToPtr("jwk"), - PEM: &configurationv1alpha1.PEMKeyPair{ - PublicKey: "public", - PrivateKey: "private", + Jwk: lo.ToPtr("jwk"), + Pem: &sdkkonnectcomp.Pem{ + PublicKey: lo.ToPtr("public"), + PrivateKey: lo.ToPtr("private"), + }, + Set: &sdkkonnectcomp.Set{ + ID: lo.ToPtr("key-set-id"), + }, + Tags: []string{ + "duplicate", + "k8s-generation:2", + "k8s-group:configuration.konghq.com", + "k8s-kind:KongKey", + "k8s-name:key-1", + "k8s-namespace:default", + "k8s-uid:key-uid", + "k8s-version:v1alpha1", + "tag1", + "tag2", + "tag3", + "tag4", }, - Tags: []string{"tag3", "tag4", "duplicate"}, }, }, } - output := kongKeyToKeyInput(key) - expectedTags := []string{ - "k8s-generation:2", - "k8s-kind:KongKey", - "k8s-name:key-1", - "k8s-uid:" + string(key.GetUID()), - "k8s-version:v1alpha1", - "k8s-group:configuration.konghq.com", - "k8s-namespace:default", - "tag1", - "tag2", - "tag3", - "tag4", - "duplicate", + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + output := kongKeyToKeyInput(tc.key) + + // Tags order is not guaranteed, so we need to sort them before comparing. + sort.Strings(output.Tags) + require.Equal(t, tc.expectedOutput, output) + }) } - require.ElementsMatch(t, expectedTags, output.Tags) - require.Equal(t, "kid", output.Kid) - require.Equal(t, "name", *output.Name) - require.Equal(t, "jwk", *output.Jwk) - require.Equal(t, "public", *output.Pem.PublicKey) - require.Equal(t, "private", *output.Pem.PrivateKey) } diff --git a/controller/konnect/reconciler_generic.go b/controller/konnect/reconciler_generic.go index 18f5a50e1..26987693c 100644 --- a/controller/konnect/reconciler_generic.go +++ b/controller/konnect/reconciler_generic.go @@ -288,6 +288,12 @@ func (r *KonnectEntityReconciler[T, TEnt]) Reconcile( return res, nil } + // If a type has a KongKeySet ref, handle it. + res, err = handleKongKeySetRef(ctx, r.Client, ent) + if err != nil || !res.IsZero() { + return res, err + } + apiAuthRef, err := getAPIAuthRefNN(ctx, r.Client, ent) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to get APIAuth ref for %s: %w", client.ObjectKeyFromObject(ent), err) @@ -723,6 +729,189 @@ func getConsumerRef[T constraints.SupportedKonnectEntityType, TEnt constraints.E } } +func getServiceRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + e TEnt, +) mo.Option[configurationv1alpha1.ServiceRef] { + switch e := any(e).(type) { + case *configurationv1alpha1.KongRoute: + if e.Spec.ServiceRef == nil { + return mo.None[configurationv1alpha1.ServiceRef]() + } + return mo.Some(*e.Spec.ServiceRef) + default: + return mo.None[configurationv1alpha1.ServiceRef]() + } +} + +func getKeySetRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + e TEnt, +) mo.Option[configurationv1alpha1.KeySetRef] { + switch e := any(e).(type) { + case *configurationv1alpha1.KongKey: + if e.Spec.KeySetRef == nil { + return mo.None[configurationv1alpha1.KeySetRef]() + } + return mo.Some(*e.Spec.KeySetRef) + default: + return mo.None[configurationv1alpha1.KeySetRef]() + } +} + +// handleKongServiceRef handles the ServiceRef for the given entity. +// It sets the owner reference to the referenced KongService and updates the +// status of the entity based on the referenced KongService status. +func handleKongServiceRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + ctx context.Context, + cl client.Client, + ent TEnt, +) (ctrl.Result, error) { + kongServiceRef, ok := getServiceRef(ent).Get() + if !ok { + return ctrl.Result{}, nil + } + switch kongServiceRef.Type { + case configurationv1alpha1.ServiceRefNamespacedRef: + svc := configurationv1alpha1.KongService{} + nn := types.NamespacedName{ + Name: kongServiceRef.NamespacedRef.Name, + // TODO: handle cross namespace refs + Namespace: ent.GetNamespace(), + } + + if err := cl.Get(ctx, nn, &svc); err != nil { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongServiceRefValidConditionType, + metav1.ConditionFalse, + conditions.KongServiceRefReasonInvalid, + err.Error(), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{}, fmt.Errorf("can't get the referenced KongService %s: %w", nn, err) + } + + // If referenced KongService is being deleted, return an error so that we + // can remove the entity from Konnect first. + if delTimestamp := svc.GetDeletionTimestamp(); !delTimestamp.IsZero() { + return ctrl.Result{}, ReferencedKongServiceIsBeingDeleted{ + Reference: nn, + } + } + + cond, ok := k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, &svc) + if !ok || cond.Status != metav1.ConditionTrue { + ent.SetKonnectID("") + if res, err := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongServiceRefValidConditionType, + metav1.ConditionFalse, + conditions.KongServiceRefReasonInvalid, + fmt.Sprintf("Referenced KongService %s is not programmed yet", nn), + ); err != nil || res.Requeue { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + old := ent.DeepCopyObject().(TEnt) + if err := controllerutil.SetOwnerReference(&svc, ent, cl.Scheme(), controllerutil.WithBlockOwnerDeletion(true)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to set owner reference: %w", err) + } + if err := cl.Patch(ctx, ent, client.MergeFrom(old)); err != nil { + if k8serrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err) + } + + // TODO(pmalek): make this generic. + // Service ID is not stored in KonnectEntityStatus because not all entities + // have a ServiceRef, hence the type constraints in the reconciler can't be used. + if route, ok := any(ent).(*configurationv1alpha1.KongRoute); ok { + if route.Status.Konnect == nil { + route.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndServiceRefs{} + } + route.Status.Konnect.ServiceID = svc.Status.Konnect.GetKonnectID() + } + + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongServiceRefValidConditionType, + metav1.ConditionTrue, + conditions.KongServiceRefReasonValid, + fmt.Sprintf("Referenced KongService %s programmed", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + cpRef, ok := getControlPlaneRef(&svc).Get() + if !ok { + return ctrl.Result{}, fmt.Errorf( + "KongRoute references a KongService %s which does not have a ControlPlane ref", + client.ObjectKeyFromObject(&svc), + ) + } + cp, err := getCPForRef(ctx, cl, cpRef, ent.GetNamespace()) + if err != nil { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionFalse, + conditions.ControlPlaneRefReasonInvalid, + err.Error(), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + if k8serrors.IsNotFound(err) { + return ctrl.Result{}, ReferencedControlPlaneDoesNotExistError{ + Reference: nn, + Err: err, + } + } + return ctrl.Result{}, err + } + + cond, ok = k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, cp) + if !ok || cond.Status != metav1.ConditionTrue || cond.ObservedGeneration != cp.GetGeneration() { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionFalse, + conditions.ControlPlaneRefReasonInvalid, + fmt.Sprintf("Referenced ControlPlane %s is not programmed yet", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{Requeue: true}, nil + } + + // TODO(pmalek): make this generic. + // CP ID is not stored in KonnectEntityStatus because not all entities + // have a ControlPlaneRef, hence the type constraints in the reconciler can't be used. + if resource, ok := any(ent).(EntityWithControlPlaneRef); ok { + resource.SetControlPlaneID(cp.Status.ID) + } + + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionTrue, + conditions.ControlPlaneRefReasonValid, + fmt.Sprintf("Referenced ControlPlane %s is programmed", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + default: + return ctrl.Result{}, fmt.Errorf("unimplemented KongService ref type %q", kongServiceRef.Type) + } + + return ctrl.Result{}, nil +} + // handleKongConsumerRef handles the ConsumerRef for the given entity. // It sets the owner reference to the referenced KongConsumer and updates the // status of the entity based on the referenced KongConsumer status. @@ -1048,6 +1237,105 @@ func handleControlPlaneRef[T constraints.SupportedKonnectEntityType, TEnt constr } } +// handleKongKeySetRef handles the KeySetRef for the given entity. +func handleKongKeySetRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + ctx context.Context, + cl client.Client, + ent TEnt, +) (ctrl.Result, error) { + keySetRef, ok := getKeySetRef(ent).Get() + if !ok { + if key, ok := any(ent).(*configurationv1alpha1.KongKey); ok { + // If the entity has a resolved reference, but the spec has changed, we need to adjust the status. + if key.Status.Konnect != nil && key.Status.Konnect.GetKeySetID() != "" { + key.Status.Konnect.KeySetID = "" + if res, err := updateStatusWithCondition(ctx, cl, key, + conditions.KeySetRefValidConditionType, + metav1.ConditionTrue, + conditions.KeySetRefReasonValid, + "KeySetRef is nil", + ); err != nil || !res.IsZero() { + return res, err + } + } + } + return ctrl.Result{}, nil + } + + if keySetRef.Type != configurationv1alpha1.KeySetRefNamespacedRef { + return ctrl.Result{}, fmt.Errorf("unsupported KeySet ref type %q", keySetRef.Type) + } + + keySet := configurationv1alpha1.KongKeySet{} + nn := types.NamespacedName{ + Name: keySetRef.NamespacedRef.Name, + Namespace: ent.GetNamespace(), + } + + if err := cl.Get(ctx, nn, &keySet); err != nil { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KeySetRefValidConditionType, + metav1.ConditionFalse, + conditions.KeySetRefReasonInvalid, + err.Error(), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + // If it was not found, let's requeue. + if k8serrors.IsNotFound(err) { + return ctrl.Result{Requeue: true}, nil + } + + return ctrl.Result{}, fmt.Errorf("failed getting KongKeySet %s: %w", nn, err) + } + + // If referenced KongKeySet is being deleted, requeue. + if delTimestamp := keySet.GetDeletionTimestamp(); !delTimestamp.IsZero() { + return ctrl.Result{ + RequeueAfter: time.Until(delTimestamp.Time), + }, nil + } + + // Verify that the KongKeySet is programmed. + cond, ok := k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, &keySet) + if !ok || cond.Status != metav1.ConditionTrue { + if res, err := updateStatusWithCondition( + ctx, cl, ent, + conditions.KeySetRefValidConditionType, + metav1.ConditionFalse, + conditions.KeySetRefReasonInvalid, + fmt.Sprintf("Referenced KongKeySet %s is not programmed yet", nn), + ); err != nil || res.Requeue { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + // TODO: make this generic. + // KongKeySet ID is not stored in KonnectEntityStatus because not all entities + // have a KeySetRef, hence the type constraints in the reconciler can't be used. + if key, ok := any(ent).(*configurationv1alpha1.KongKey); ok { + if key.Status.Konnect == nil { + key.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndKeySetRef{} + } + key.Status.Konnect.KeySetID = keySet.Status.Konnect.GetKonnectID() + } + + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KeySetRefValidConditionType, + metav1.ConditionTrue, + conditions.KeySetRefReasonValid, + fmt.Sprintf("Referenced KongKeySet %s programmed", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{}, nil +} + func conditionMessageReferenceKonnectAPIAuthConfigurationInvalid(apiAuthRef types.NamespacedName) string { return fmt.Sprintf("referenced KonnectAPIAuthConfiguration %s is invalid", apiAuthRef) } diff --git a/test/envtest/deploy_resources.go b/test/envtest/deploy_resources.go index 92ea69b6f..111365ba3 100644 --- a/test/envtest/deploy_resources.go +++ b/test/envtest/deploy_resources.go @@ -479,6 +479,9 @@ func deployKongVaultAttachedToCP( return vault } +// kongKeyOptFn is a function that modifies a KongKey resource. +type kongKeyOptFn func(*configurationv1alpha1.KongKey) + // deployKongKeyAttachedToCP deploys a KongKey resource attached to a CP and returns the resource. func deployKongKeyAttachedToCP( t *testing.T, @@ -486,6 +489,7 @@ func deployKongKeyAttachedToCP( cl client.Client, kid, name string, cp *konnectv1alpha1.KonnectGatewayControlPlane, + opts ...kongKeyOptFn, ) *configurationv1alpha1.KongKey { t.Helper() @@ -507,6 +511,9 @@ func deployKongKeyAttachedToCP( }, }, } + for _, opt := range opts { + opt(key) + } require.NoError(t, cl.Create(ctx, key)) t.Logf("deployed new KongKey %s", client.ObjectKeyFromObject(key)) return key @@ -547,7 +554,7 @@ func deployKongKeySetAttachedToCP( keySet := &configurationv1alpha1.KongKeySet{ ObjectMeta: metav1.ObjectMeta{ - GenerateName: "key-set-", + Name: name, }, Spec: configurationv1alpha1.KongKeySetSpec{ ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ diff --git a/test/envtest/konnect_entities_key_test.go b/test/envtest/konnect_entities_key_test.go index 87996deca..9c8ae6e2c 100644 --- a/test/envtest/konnect_entities_key_test.go +++ b/test/envtest/konnect_entities_key_test.go @@ -27,8 +27,10 @@ func TestKongKey(t *testing.T) { keyKid = "key-kid" keyName = "key-name" keyID = "key-id" - ) + keySetName = "key-set-name" + keySetID = "key-set-id" + ) t.Parallel() ctx, cancel := Context(t, context.Background()) defer cancel() @@ -70,50 +72,132 @@ func TestKongKey(t *testing.T) { t.Log("Setting up a watch for KongKey events") w := setupWatch[configurationv1alpha1.KongKeyList](t, ctx, cl, client.InNamespace(ns.Name)) - t.Log("Creating KongKey") - createdKey := deployKongKeyAttachedToCP(t, ctx, clientNamespaced, keyKid, keyName, cp) + t.Run("without KongKeySet", func(t *testing.T) { + t.Log("Creating KongKey") + createdKey := deployKongKeyAttachedToCP(t, ctx, clientNamespaced, keyKid, keyName, cp) + + t.Log("Waiting for KongKey to be programmed") + watchFor(t, ctx, w, watch.Modified, func(c *configurationv1alpha1.KongKey) bool { + if c.GetName() != createdKey.GetName() { + return false + } + return lo.ContainsBy(c.Status.Conditions, func(condition metav1.Condition) bool { + return condition.Type == conditions.KonnectEntityProgrammedConditionType && + condition.Status == metav1.ConditionTrue + }) + }, "KongKey's Programmed condition should be true eventually") + + t.Log("Waiting for KongKey to be created in the SDK") + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + t.Log("Setting up SDK expectations on KongKey update") + sdk.KeysSDK.EXPECT().UpsertKey(mock.Anything, mock.MatchedBy(func(r sdkkonnectops.UpsertKeyRequest) bool { + return r.KeyID == keyID && + lo.Contains(r.Key.Tags, "addedTag") + })).Return(&sdkkonnectops.UpsertKeyResponse{}, nil) + + t.Log("Patching KongKey") + certToPatch := createdKey.DeepCopy() + certToPatch.Spec.Tags = append(certToPatch.Spec.Tags, "addedTag") + require.NoError(t, clientNamespaced.Patch(ctx, certToPatch, client.MergeFrom(createdKey))) + + t.Log("Waiting for KongKey to be updated in the SDK") + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + t.Log("Setting up SDK expectations on KongKey deletion") + sdk.KeysSDK.EXPECT().DeleteKey(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), keyID). + Return(&sdkkonnectops.DeleteKeyResponse{}, nil) + + t.Log("Deleting KongKey") + require.NoError(t, cl.Delete(ctx, createdKey)) + + t.Log("Waiting for KongKey to be deleted in the SDK") + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) - t.Log("Waiting for KongKey to be programmed") - watchFor(t, ctx, w, watch.Modified, func(c *configurationv1alpha1.KongKey) bool { - if c.GetName() != createdKey.GetName() { - return false + t.Run("with KongKeySet", func(t *testing.T) { + t.Log("Creating KongKey") + withKeySetRef := func(key *configurationv1alpha1.KongKey) { + key.Spec.KeySetRef = &configurationv1alpha1.KeySetRef{ + Type: configurationv1alpha1.KeySetRefNamespacedRef, + NamespacedRef: lo.ToPtr(configurationv1alpha1.KeySetNamespacedRef{ + Name: keySetName, + }), + } } - return lo.ContainsBy(c.Status.Conditions, func(condition metav1.Condition) bool { - return condition.Type == conditions.KonnectEntityProgrammedConditionType && - condition.Status == metav1.ConditionTrue - }) - }, "KongKey's Programmed condition should be true eventually") - - t.Log("Waiting for KongKey to be created in the SDK") - require.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) - }, waitTime, tickTime) - - t.Log("Setting up SDK expectations on KongKey update") - sdk.KeysSDK.EXPECT().UpsertKey(mock.Anything, mock.MatchedBy(func(r sdkkonnectops.UpsertKeyRequest) bool { - return r.KeyID == keyID && - lo.Contains(r.Key.Tags, "addedTag") - })).Return(&sdkkonnectops.UpsertKeyResponse{}, nil) - - t.Log("Patching KongKey") - certToPatch := createdKey.DeepCopy() - certToPatch.Spec.Tags = append(certToPatch.Spec.Tags, "addedTag") - require.NoError(t, clientNamespaced.Patch(ctx, certToPatch, client.MergeFrom(createdKey))) - - t.Log("Waiting for KongKey to be updated in the SDK") - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) - }, waitTime, tickTime) - - t.Log("Setting up SDK expectations on KongKey deletion") - sdk.KeysSDK.EXPECT().DeleteKey(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), keyID). - Return(&sdkkonnectops.DeleteKeyResponse{}, nil) - - t.Log("Deleting KongKey") - require.NoError(t, cl.Delete(ctx, createdKey)) - - t.Log("Waiting for KongKey to be deleted in the SDK") - assert.EventuallyWithT(t, func(c *assert.CollectT) { - assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) - }, waitTime, tickTime) + createdKey := deployKongKeyAttachedToCP(t, ctx, clientNamespaced, keyKid, keyName, cp, withKeySetRef) + + t.Log("Waiting for KeySetRefValid condition to be false") + watchFor(t, ctx, w, watch.Modified, func(c *configurationv1alpha1.KongKey) bool { + if c.GetName() != createdKey.GetName() { + return false + } + return lo.ContainsBy(c.Status.Conditions, func(condition metav1.Condition) bool { + return condition.Type == conditions.KeySetRefValidConditionType && + condition.Status == metav1.ConditionFalse + }) + }, "KongKey's KeySetRefValid condition should be false eventually as the KongKeySet is not created yet") + + t.Log("Setting up SDK expectations on KongKey creation with KeySetRef") + sdk.KeysSDK.EXPECT().CreateKey(mock.Anything, cp.GetKonnectStatus().GetKonnectID(), + mock.MatchedBy(func(input sdkkonnectcomp.KeyInput) bool { + return input.Kid == keyKid && + input.Name != nil && *input.Name == keyName && + input.Set != nil && input.Set.GetID() != nil && *input.Set.GetID() == keySetID + }), + ).Return(&sdkkonnectops.CreateKeyResponse{ + Key: &sdkkonnectcomp.Key{ + ID: lo.ToPtr(keyID), + }, + }, nil) + + t.Log("Creating KongKeySet") + keySet := deployKongKeySetAttachedToCP(t, ctx, clientNamespaced, keySetName, cp) + updateKongKeySetStatusWithProgrammed(t, ctx, clientNamespaced, keySet, keySetID, cp.GetKonnectStatus().GetKonnectID()) + + t.Log("Waiting for KongKey to be programmed and associated with KongKeySet") + watchFor(t, ctx, w, watch.Modified, func(c *configurationv1alpha1.KongKey) bool { + if c.GetName() != createdKey.GetName() { + return false + } + programmed := lo.ContainsBy(c.Status.Conditions, func(condition metav1.Condition) bool { + return condition.Type == conditions.KonnectEntityProgrammedConditionType && + condition.Status == metav1.ConditionTrue + }) + associated := lo.ContainsBy(c.Status.Conditions, func(condition metav1.Condition) bool { + return condition.Type == conditions.KeySetRefValidConditionType && + condition.Status == metav1.ConditionTrue + }) + keySetIDPopulated := c.Status.Konnect != nil && c.Status.Konnect.KeySetID != "" + + return programmed && associated && keySetIDPopulated + }, "KongKey's Programmed and KeySetRefValid conditions should be true eventually") + + t.Log("Waiting for KongKey to be created in the SDK") + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + t.Log("Setting up SDK expectations on KongKeySet deattachment") + sdk.KeysSDK.EXPECT().UpsertKey(mock.Anything, mock.MatchedBy(func(r sdkkonnectops.UpsertKeyRequest) bool { + return r.KeyID == keyID && + r.Key.Set == nil + })).Return(&sdkkonnectops.UpsertKeyResponse{}, nil) + + t.Log("Patching KongKey to deattach from KongKeySet") + keyToPatch := createdKey.DeepCopy() + keyToPatch.Spec.KeySetRef = nil + require.NoError(t, clientNamespaced.Patch(ctx, keyToPatch, client.MergeFrom(createdKey))) + + t.Log("Waiting for KongKey to be deattached from KongKeySet in the SDK") + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, factory.SDK.KeysSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) } diff --git a/test/envtest/update_status.go b/test/envtest/update_status.go index 3034b14c6..2dbde5af7 100644 --- a/test/envtest/update_status.go +++ b/test/envtest/update_status.go @@ -89,6 +89,24 @@ func updateKongRouteStatusWithProgrammed( require.NoError(t, cl.Status().Update(ctx, obj)) } +func updateKongKeySetStatusWithProgrammed( + t *testing.T, + ctx context.Context, + cl client.Client, + obj *configurationv1alpha1.KongKeySet, + id, cpID string, +) { + obj.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + ControlPlaneID: cpID, + KonnectEntityStatus: konnectEntityStatus(id), + } + obj.Status.Conditions = []metav1.Condition{ + programmedCondition(obj.GetGeneration()), + } + + require.NoError(t, cl.Status().Update(ctx, obj)) +} + func konnectEntityStatus(id string) konnectv1alpha1.KonnectEntityStatus { return konnectv1alpha1.KonnectEntityStatus{ ID: id,