From c84710bb41bdd52d4a0ed822aa1eadaed26d1568 Mon Sep 17 00:00:00 2001 From: Tao Yi Date: Mon, 30 Sep 2024 23:11:36 +0800 Subject: [PATCH] feat(konnect): add KongSNI reconciler (#653) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * KongSNI reconciler * add tests and sample * Update modules/manager/controller_setup.go * Update test/envtest/konnect_entities_sni_test.go * feat(konnect): add index for KongSNI on referenced certificate name * feat(konnect): fallback to create on not found error when upserting the SNI --------- Co-authored-by: Patryk Małek --- .mockery.yaml | 1 + config/rbac/role/role.yaml | 2 + config/samples/konnect_sni.yaml | 98 +++++ controller/konnect/conditions/conditions.go | 16 +- controller/konnect/constraints/constraints.go | 3 +- controller/konnect/errors.go | 25 ++ controller/konnect/index.go | 2 + controller/konnect/index_sni.go | 31 ++ controller/konnect/ops/kongsni.go | 14 + controller/konnect/ops/kongsni_mock.go | 259 +++++++++++ controller/konnect/ops/ops.go | 8 + controller/konnect/ops/ops_kongsni.go | 164 +++++++ controller/konnect/ops/ops_kongtarget.go | 4 +- controller/konnect/ops/sdkfactory.go | 6 + controller/konnect/ops/sdkfactory_mock.go | 6 + .../konnect/reconciler_certificateref.go | 188 ++++++++ .../konnect/reconciler_certificateref_test.go | 401 ++++++++++++++++++ controller/konnect/reconciler_generic.go | 68 ++- controller/konnect/reconciler_generic_rbac.go | 3 + .../konnect/reconciler_upstreamref_test.go | 11 +- controller/konnect/watch.go | 2 + controller/konnect/watch_kongsni.go | 96 +++++ modules/manager/controller_setup.go | 14 + test/envtest/deploy_resources.go | 28 ++ test/envtest/konnect_entities_sni_test.go | 143 +++++++ test/integration/test_konnect_entities.go | 130 ++++++ 26 files changed, 1706 insertions(+), 17 deletions(-) create mode 100644 config/samples/konnect_sni.yaml create mode 100644 controller/konnect/index_sni.go create mode 100644 controller/konnect/ops/kongsni.go create mode 100644 controller/konnect/ops/kongsni_mock.go create mode 100644 controller/konnect/ops/ops_kongsni.go create mode 100644 controller/konnect/reconciler_certificateref.go create mode 100644 controller/konnect/reconciler_certificateref_test.go create mode 100644 controller/konnect/watch_kongsni.go create mode 100644 test/envtest/konnect_entities_sni_test.go diff --git a/.mockery.yaml b/.mockery.yaml index 7ab0cc5e0..43cab5626 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -28,3 +28,4 @@ packages: CertificatesSDK: KeysSDK: KeySetsSDK: + SNIsSDK: diff --git a/config/rbac/role/role.yaml b/config/rbac/role/role.yaml index 0b108c417..5163dc8b5 100644 --- a/config/rbac/role/role.yaml +++ b/config/rbac/role/role.yaml @@ -154,6 +154,7 @@ rules: - kongplugins/status - kongroutes/status - kongservices/status + - kongsnis/status - kongtargets/status - kongupstreampolicies/status - kongupstreams/status @@ -182,6 +183,7 @@ rules: resources: - kongroutes - kongservices + - kongsnis - kongtargets - kongupstreams - kongvaults diff --git a/config/samples/konnect_sni.yaml b/config/samples/konnect_sni.yaml new file mode 100644 index 000000000..8d364f96e --- /dev/null +++ b/config/samples/konnect_sni.yaml @@ -0,0 +1,98 @@ +kind: KonnectAPIAuthConfiguration +apiVersion: konnect.konghq.com/v1alpha1 +metadata: + name: konnect-api-auth-dev-1 + namespace: default +spec: + type: token + token: kpat_XXXXXXXXXXXXXXXXXXX + serverURL: us.api.konghq.tech +--- +kind: KonnectGatewayControlPlane +apiVersion: konnect.konghq.com/v1alpha1 +metadata: + name: test1 + namespace: default +spec: + name: test1 + labels: + app: test1 + key1: test1 + konnect: + authRef: + name: konnect-api-auth-dev-1 +--- +kind: KongCertificate +apiVersion: configuration.konghq.com/v1alpha1 +metadata: + name: cert-1 + namespace: default + annotations: + konghq.com/tags: "infra" +spec: + controlPlaneRef: + type: konnectNamespacedRef + konnectNamespacedRef: + name: test1 + tags: + - production + cert: | + -----BEGIN CERTIFICATE----- + MIIDPTCCAiUCFG5IolqRiKPMfzTI8peXlaF6cZODMA0GCSqGSIb3DQEBCwUAMFsx + CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5 + MRIwEAYDVQQKDAlLb25nIEluYy4xFDASBgNVBAMMC2tvbmdocS50ZWNoMB4XDTI0 + MDkyNTA3MjIzOFoXDTM0MDkyMzA3MjIzOFowWzELMAkGA1UEBhMCVVMxCzAJBgNV + BAgMAkNBMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxEjAQBgNVBAoMCUtvbmcgSW5j + LjEUMBIGA1UEAwwLa29uZ2hxLnRlY2gwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw + ggEKAoIBAQDXmNBzpWyJ0YUdfCamZpJiwRQMn5vVY8iKQrd3dD03DWyPHu/fXlrL + +QPTRip5d1SrxjzQ4S3fgme442BTlElF9d1w1rhg+DIg6NsW1jd+3IZaICnq7BZH + rJGlW+IWJSKHmNQ39nfVQwgL/QdylrYpbB7uwdEDMa78GfXteiXTcuNobCr7VWVz + rY6rQXo/dImWE1PtMp/EZEMsEbgbQpK5+fUnKTmFncVlDAZ2Q3s2MPikV5UhMVyQ + dKQydU0Ev0LRtpsjW8pQdshMG1ilMq6Yg6YU95gakLVjRXMoDlIJOu08mdped+2Y + VIUSXhRyRt1hbkFP0fXG0THfZ3DjH7jRAgMBAAEwDQYJKoZIhvcNAQELBQADggEB + ANEXlNaQKLrB+jsnNjybsTRkvRRmwjnXaQV0jHzjseGsTJoKY5ABBsSRDiHtqB+9 + LPTpHhLYJWsHSLwawIJ3aWDDpF4MNTRsvO12v7wM8Q42OSgkP23O6a5ESkyHRBAb + dLVEp+0Z3kjYwPIglIK37PcgDci6Zim73GOfapDEASNbnCu8js2g/ucYPPXkGMxl + PSUER7MTNf9wRbXrroCE+tZw4kUyUh+6taNlU4ialAJLO1x6UGVRHvPgEx0fAAxA + seBH+A9QMvVl2cKcvrOgZ0CWY01aFRO9ROQ7PrYXqRFvOZu8K3QzLw7xYoK1DTp+ + kkO/oPy+WIbqEvj7QrhUXpo= + -----END CERTIFICATE----- + key: | + -----BEGIN PRIVATE KEY----- + MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDXmNBzpWyJ0YUd + fCamZpJiwRQMn5vVY8iKQrd3dD03DWyPHu/fXlrL+QPTRip5d1SrxjzQ4S3fgme4 + 42BTlElF9d1w1rhg+DIg6NsW1jd+3IZaICnq7BZHrJGlW+IWJSKHmNQ39nfVQwgL + /QdylrYpbB7uwdEDMa78GfXteiXTcuNobCr7VWVzrY6rQXo/dImWE1PtMp/EZEMs + EbgbQpK5+fUnKTmFncVlDAZ2Q3s2MPikV5UhMVyQdKQydU0Ev0LRtpsjW8pQdshM + G1ilMq6Yg6YU95gakLVjRXMoDlIJOu08mdped+2YVIUSXhRyRt1hbkFP0fXG0THf + Z3DjH7jRAgMBAAECggEAOSZ4h1dgDK5+H2FEK5MAFe6BnpEGsYu4YrIpySAGhBvq + XYwBYRA1eGFjmrM8WiOATeKIR4SRcPC0BwY7CBzESafRkfJRQN86BpBDV2vknRve + /3AMPIplo41CtHdFWMJyQ0iHZOhQPrd8oBTsTvtVgWh4UKkO+05FyO0mzFM3SLPs + pqRwMZjLlKVZhbI1l0Ur787tzWpMQQHmd8csAvlak+GIciQWELbVK+5kr/FDpJbq + joIeHX7DCmIqrD/Okwa8SfJu1sutmRX+nrxkDi7trPYcpqriDoWs2jMcnS2GHq9M + lsy2XHn+qLjCpl3/XU61xenWs+Rmmj6J/oIs1zYXCwKBgQDywRS/MNgwMst703Wh + ERJO0rNSR0XVzzoKOqc/ghPkeA8mVNwfNkbgWks+83tuAb6IunMIeyQJ3g/jRhjz + KImsqJmO+DoZCioeaR3PeIWibi9I9Irg6dtoNMwxSmmOtCKD0rccxM1V9OnYkn5a + 0Fb+irQSgJYiHrF2SLAT0NoWEwKBgQDjXGLHCu/WEy49vROdkTia133Wc7m71/D5 + RDUqvSmAEHJyhTlzCbTO+JcNhC6cx3s102GNcVYHlAq3WoE5EV1YykUNJwUg4XPn + AggNkYOiXs6tf+uePmT8MddixFFgFxZ2bIqFhvnY+WqypHuxtwIepqKJjq5xZTiB + +lfp7SziCwKBgAivofdpXwLyfldy7I2T18zcOzBhfn01CgWdrahXFjEhnqEnfizb + u1OBx5l8CtmX1GJ+EWmnRlXYDUd7lZ71v19fNQdpmGKW+4TVDA0Fafqy6Jw6q9F6 + bLBg20GUQQyrI2UGICk2XYaK2ec27rB/Le2zttfGpBiaco0h8rLy0SrjAoGBAM4/ + UY/UOQsOrUTuT2wBf8LfNtUid9uSIZRNrplNrebxhJCkkB/uLyoN0iE9xncMcpW6 + YmVH6c3IGwyHOnBFc1OHcapjukBApL5rVljQpwPVU1GKmHgdi8hHgmajRlqPtx3I + isRkVCPi5kqV8WueY3rgmNOGLnLJasBmE/gt4ihPAoGAG3v93R5tAeSrn7DMHaZt + p+udsNw9mDPYHAzlYtnw1OE/I0ceR5UyCFSzCd00Q8ZYBLf9jRvsO/GUA4F51isE + 8/7xyqSxJqDwzv9N8EGkqf/SfMKA3kK3Sc8u+ovhzJu8OxcY+qrpo4+vYWYeW42n + 5XBwvWV2ovRMx7Ntw7FUc24= + -----END PRIVATE KEY----- +--- +apiVersion: configuration.konghq.com/v1alpha1 +kind: KongSNI +metadata: + name: sni-1 + namespace: default +spec: + certificateRef: + name: cert-1 + name: "test-sni.konghq.com" diff --git a/controller/konnect/conditions/conditions.go b/controller/konnect/conditions/conditions.go index 555328207..7452bcdd7 100644 --- a/controller/konnect/conditions/conditions.go +++ b/controller/konnect/conditions/conditions.go @@ -117,8 +117,7 @@ const ( const ( // KongUpstreamRefValidConditionType is the type of the condition that indicates - // whether the KongUpstream reference is valid and points to an existing - // KongUpstreamRefValid. + // whether the KongUpstream reference is valid and points to an existing KongUpstream. KongUpstreamRefValidConditionType = "KongUpstreamRefValid" // KongUpstreamRefReasonValid is the reason used with the KongUpstreamRefValid @@ -128,3 +127,16 @@ const ( // condition type indicating that the KongUpstream reference is invalid. KongUpstreamRefReasonInvalid = "Invalid" ) + +const ( + // KongCertificateRefValidConditionType is the type of the condition that indicates + // whether the KongCertificate reference is valid and points to an existing KongCertificate + KongCertificateRefValidConditionType = "KongCertificateRefValid" + + // KongCertificateRefReasonValid is the reason used with the KongCertificateRefValid + // condition type indicating that the KongCertificate reference is valid. + KongCertificateRefReasonValid = "Valid" + // KongCertificateRefReasonInvalid is the reason used with the KongCertificateRefValid + // condition type indicating that the KongCertificate reference is invalid. + KongCertificateRefReasonInvalid = "Invalid" +) diff --git a/controller/konnect/constraints/constraints.go b/controller/konnect/constraints/constraints.go index e455aaa06..d3b74abab 100644 --- a/controller/konnect/constraints/constraints.go +++ b/controller/konnect/constraints/constraints.go @@ -28,7 +28,8 @@ type SupportedKonnectEntityType interface { configurationv1alpha1.KongTarget | configurationv1alpha1.KongVault | configurationv1alpha1.KongKey | - configurationv1alpha1.KongKeySet + configurationv1alpha1.KongKeySet | + configurationv1alpha1.KongSNI // TODO: add other types GetTypeName() string diff --git a/controller/konnect/errors.go b/controller/konnect/errors.go index 44d5d875e..133cf8ee7 100644 --- a/controller/konnect/errors.go +++ b/controller/konnect/errors.go @@ -86,3 +86,28 @@ type ReferencedKongUpstreamDoesNotExist struct { func (e ReferencedKongUpstreamDoesNotExist) Error() string { return fmt.Sprintf("referenced Kong Upstream %s does not exist: %v", e.Reference, e.Err) } + +// ReferencedKongCertificateIsBeingDeleted is an error type that is returned when +// a Konnect entity references a Kong Certificate which is being deleted. +type ReferencedKongCertificateIsBeingDeleted struct { + Reference types.NamespacedName + DeletionTimestamp time.Time +} + +// Error implements the error interface. +func (e ReferencedKongCertificateIsBeingDeleted) Error() string { + return fmt.Sprintf("referenced Kong Certificate %s is being deleted (deletion timestamp: %s)", + e.Reference, e.DeletionTimestamp) +} + +// ReferencedKongCertificateDoesNotExist is an error type that is returned when +// a Konnect entity references a Kong Certificate which does not exist. +type ReferencedKongCertificateDoesNotExist struct { + Reference types.NamespacedName + Err error +} + +// Error implements the error interface. +func (e ReferencedKongCertificateDoesNotExist) Error() string { + return fmt.Sprintf("referenced Kong Certificate %s does not exist: %v", e.Reference, e.Err) +} diff --git a/controller/konnect/index.go b/controller/konnect/index.go index f7289753c..1a39c14eb 100644 --- a/controller/konnect/index.go +++ b/controller/konnect/index.go @@ -33,6 +33,8 @@ func ReconciliationIndexOptionsForEntity[ return IndexOptionsForCredentialsBasicAuth() case *configurationv1.KongConsumer: return IndexOptionsForKongConsumer() + case *configurationv1alpha1.KongSNI: + return IndexOptionsForKongSNI() } return nil } diff --git a/controller/konnect/index_sni.go b/controller/konnect/index_sni.go new file mode 100644 index 000000000..a73f2b250 --- /dev/null +++ b/controller/konnect/index_sni.go @@ -0,0 +1,31 @@ +package konnect + +import ( + "sigs.k8s.io/controller-runtime/pkg/client" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" +) + +const ( + // IndexFieldKongSNIOnCertificateRefNmae is the index field for KongSNI -> Certificate. + IndexFieldKongSNIOnCertificateRefNmae = "kongSNICertificateRefName" +) + +// IndexOptionsForKongSNI returns required Index options for KongSNI reconciler. +func IndexOptionsForKongSNI() []ReconciliationIndexOption { + return []ReconciliationIndexOption{ + { + IndexObject: &configurationv1alpha1.KongSNI{}, + IndexField: IndexFieldKongSNIOnCertificateRefNmae, + ExtractValue: kongSNIReferencesCertificate, + }, + } +} + +func kongSNIReferencesCertificate(object client.Object) []string { + sni, ok := object.(*configurationv1alpha1.KongSNI) + if !ok { + return nil + } + return []string{sni.Spec.CertificateRef.Name} +} diff --git a/controller/konnect/ops/kongsni.go b/controller/konnect/ops/kongsni.go new file mode 100644 index 000000000..395506592 --- /dev/null +++ b/controller/konnect/ops/kongsni.go @@ -0,0 +1,14 @@ +package ops + +import ( + "context" + + sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations" +) + +// SNIsSDK is the interface to operate Kong SNIs. +type SNIsSDK interface { + CreateSniWithCertificate(context.Context, sdkkonnectops.CreateSniWithCertificateRequest, ...sdkkonnectops.Option) (*sdkkonnectops.CreateSniWithCertificateResponse, error) + UpsertSniWithCertificate(ctx context.Context, request sdkkonnectops.UpsertSniWithCertificateRequest, opts ...sdkkonnectops.Option) (*sdkkonnectops.UpsertSniWithCertificateResponse, error) + DeleteSniWithCertificate(ctx context.Context, request sdkkonnectops.DeleteSniWithCertificateRequest, opts ...sdkkonnectops.Option) (*sdkkonnectops.DeleteSniWithCertificateResponse, error) +} diff --git a/controller/konnect/ops/kongsni_mock.go b/controller/konnect/ops/kongsni_mock.go new file mode 100644 index 000000000..275f0c790 --- /dev/null +++ b/controller/konnect/ops/kongsni_mock.go @@ -0,0 +1,259 @@ +// Code generated by mockery. DO NOT EDIT. + +package ops + +import ( + context "context" + + operations "github.com/Kong/sdk-konnect-go/models/operations" + mock "github.com/stretchr/testify/mock" +) + +// MockSNIsSDK is an autogenerated mock type for the SNIsSDK type +type MockSNIsSDK struct { + mock.Mock +} + +type MockSNIsSDK_Expecter struct { + mock *mock.Mock +} + +func (_m *MockSNIsSDK) EXPECT() *MockSNIsSDK_Expecter { + return &MockSNIsSDK_Expecter{mock: &_m.Mock} +} + +// CreateSniWithCertificate provides a mock function with given fields: _a0, _a1, _a2 +func (_m *MockSNIsSDK) CreateSniWithCertificate(_a0 context.Context, _a1 operations.CreateSniWithCertificateRequest, _a2 ...operations.Option) (*operations.CreateSniWithCertificateResponse, error) { + _va := make([]interface{}, len(_a2)) + for _i := range _a2 { + _va[_i] = _a2[_i] + } + var _ca []interface{} + _ca = append(_ca, _a0, _a1) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for CreateSniWithCertificate") + } + + var r0 *operations.CreateSniWithCertificateResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, operations.CreateSniWithCertificateRequest, ...operations.Option) (*operations.CreateSniWithCertificateResponse, error)); ok { + return rf(_a0, _a1, _a2...) + } + if rf, ok := ret.Get(0).(func(context.Context, operations.CreateSniWithCertificateRequest, ...operations.Option) *operations.CreateSniWithCertificateResponse); ok { + r0 = rf(_a0, _a1, _a2...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*operations.CreateSniWithCertificateResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, operations.CreateSniWithCertificateRequest, ...operations.Option) error); ok { + r1 = rf(_a0, _a1, _a2...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockSNIsSDK_CreateSniWithCertificate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'CreateSniWithCertificate' +type MockSNIsSDK_CreateSniWithCertificate_Call struct { + *mock.Call +} + +// CreateSniWithCertificate is a helper method to define mock.On call +// - _a0 context.Context +// - _a1 operations.CreateSniWithCertificateRequest +// - _a2 ...operations.Option +func (_e *MockSNIsSDK_Expecter) CreateSniWithCertificate(_a0 interface{}, _a1 interface{}, _a2 ...interface{}) *MockSNIsSDK_CreateSniWithCertificate_Call { + return &MockSNIsSDK_CreateSniWithCertificate_Call{Call: _e.mock.On("CreateSniWithCertificate", + append([]interface{}{_a0, _a1}, _a2...)...)} +} + +func (_c *MockSNIsSDK_CreateSniWithCertificate_Call) Run(run func(_a0 context.Context, _a1 operations.CreateSniWithCertificateRequest, _a2 ...operations.Option)) *MockSNIsSDK_CreateSniWithCertificate_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]operations.Option, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(operations.Option) + } + } + run(args[0].(context.Context), args[1].(operations.CreateSniWithCertificateRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockSNIsSDK_CreateSniWithCertificate_Call) Return(_a0 *operations.CreateSniWithCertificateResponse, _a1 error) *MockSNIsSDK_CreateSniWithCertificate_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockSNIsSDK_CreateSniWithCertificate_Call) RunAndReturn(run func(context.Context, operations.CreateSniWithCertificateRequest, ...operations.Option) (*operations.CreateSniWithCertificateResponse, error)) *MockSNIsSDK_CreateSniWithCertificate_Call { + _c.Call.Return(run) + return _c +} + +// DeleteSniWithCertificate provides a mock function with given fields: ctx, request, opts +func (_m *MockSNIsSDK) DeleteSniWithCertificate(ctx context.Context, request operations.DeleteSniWithCertificateRequest, opts ...operations.Option) (*operations.DeleteSniWithCertificateResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, request) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for DeleteSniWithCertificate") + } + + var r0 *operations.DeleteSniWithCertificateResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, operations.DeleteSniWithCertificateRequest, ...operations.Option) (*operations.DeleteSniWithCertificateResponse, error)); ok { + return rf(ctx, request, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, operations.DeleteSniWithCertificateRequest, ...operations.Option) *operations.DeleteSniWithCertificateResponse); ok { + r0 = rf(ctx, request, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*operations.DeleteSniWithCertificateResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, operations.DeleteSniWithCertificateRequest, ...operations.Option) error); ok { + r1 = rf(ctx, request, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockSNIsSDK_DeleteSniWithCertificate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'DeleteSniWithCertificate' +type MockSNIsSDK_DeleteSniWithCertificate_Call struct { + *mock.Call +} + +// DeleteSniWithCertificate is a helper method to define mock.On call +// - ctx context.Context +// - request operations.DeleteSniWithCertificateRequest +// - opts ...operations.Option +func (_e *MockSNIsSDK_Expecter) DeleteSniWithCertificate(ctx interface{}, request interface{}, opts ...interface{}) *MockSNIsSDK_DeleteSniWithCertificate_Call { + return &MockSNIsSDK_DeleteSniWithCertificate_Call{Call: _e.mock.On("DeleteSniWithCertificate", + append([]interface{}{ctx, request}, opts...)...)} +} + +func (_c *MockSNIsSDK_DeleteSniWithCertificate_Call) Run(run func(ctx context.Context, request operations.DeleteSniWithCertificateRequest, opts ...operations.Option)) *MockSNIsSDK_DeleteSniWithCertificate_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]operations.Option, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(operations.Option) + } + } + run(args[0].(context.Context), args[1].(operations.DeleteSniWithCertificateRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockSNIsSDK_DeleteSniWithCertificate_Call) Return(_a0 *operations.DeleteSniWithCertificateResponse, _a1 error) *MockSNIsSDK_DeleteSniWithCertificate_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockSNIsSDK_DeleteSniWithCertificate_Call) RunAndReturn(run func(context.Context, operations.DeleteSniWithCertificateRequest, ...operations.Option) (*operations.DeleteSniWithCertificateResponse, error)) *MockSNIsSDK_DeleteSniWithCertificate_Call { + _c.Call.Return(run) + return _c +} + +// UpsertSniWithCertificate provides a mock function with given fields: ctx, request, opts +func (_m *MockSNIsSDK) UpsertSniWithCertificate(ctx context.Context, request operations.UpsertSniWithCertificateRequest, opts ...operations.Option) (*operations.UpsertSniWithCertificateResponse, error) { + _va := make([]interface{}, len(opts)) + for _i := range opts { + _va[_i] = opts[_i] + } + var _ca []interface{} + _ca = append(_ca, ctx, request) + _ca = append(_ca, _va...) + ret := _m.Called(_ca...) + + if len(ret) == 0 { + panic("no return value specified for UpsertSniWithCertificate") + } + + var r0 *operations.UpsertSniWithCertificateResponse + var r1 error + if rf, ok := ret.Get(0).(func(context.Context, operations.UpsertSniWithCertificateRequest, ...operations.Option) (*operations.UpsertSniWithCertificateResponse, error)); ok { + return rf(ctx, request, opts...) + } + if rf, ok := ret.Get(0).(func(context.Context, operations.UpsertSniWithCertificateRequest, ...operations.Option) *operations.UpsertSniWithCertificateResponse); ok { + r0 = rf(ctx, request, opts...) + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(*operations.UpsertSniWithCertificateResponse) + } + } + + if rf, ok := ret.Get(1).(func(context.Context, operations.UpsertSniWithCertificateRequest, ...operations.Option) error); ok { + r1 = rf(ctx, request, opts...) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + +// MockSNIsSDK_UpsertSniWithCertificate_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'UpsertSniWithCertificate' +type MockSNIsSDK_UpsertSniWithCertificate_Call struct { + *mock.Call +} + +// UpsertSniWithCertificate is a helper method to define mock.On call +// - ctx context.Context +// - request operations.UpsertSniWithCertificateRequest +// - opts ...operations.Option +func (_e *MockSNIsSDK_Expecter) UpsertSniWithCertificate(ctx interface{}, request interface{}, opts ...interface{}) *MockSNIsSDK_UpsertSniWithCertificate_Call { + return &MockSNIsSDK_UpsertSniWithCertificate_Call{Call: _e.mock.On("UpsertSniWithCertificate", + append([]interface{}{ctx, request}, opts...)...)} +} + +func (_c *MockSNIsSDK_UpsertSniWithCertificate_Call) Run(run func(ctx context.Context, request operations.UpsertSniWithCertificateRequest, opts ...operations.Option)) *MockSNIsSDK_UpsertSniWithCertificate_Call { + _c.Call.Run(func(args mock.Arguments) { + variadicArgs := make([]operations.Option, len(args)-2) + for i, a := range args[2:] { + if a != nil { + variadicArgs[i] = a.(operations.Option) + } + } + run(args[0].(context.Context), args[1].(operations.UpsertSniWithCertificateRequest), variadicArgs...) + }) + return _c +} + +func (_c *MockSNIsSDK_UpsertSniWithCertificate_Call) Return(_a0 *operations.UpsertSniWithCertificateResponse, _a1 error) *MockSNIsSDK_UpsertSniWithCertificate_Call { + _c.Call.Return(_a0, _a1) + return _c +} + +func (_c *MockSNIsSDK_UpsertSniWithCertificate_Call) RunAndReturn(run func(context.Context, operations.UpsertSniWithCertificateRequest, ...operations.Option) (*operations.UpsertSniWithCertificateResponse, error)) *MockSNIsSDK_UpsertSniWithCertificate_Call { + _c.Call.Return(run) + return _c +} + +// NewMockSNIsSDK creates a new instance of MockSNIsSDK. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewMockSNIsSDK(t interface { + mock.TestingT + Cleanup(func()) +}) *MockSNIsSDK { + mock := &MockSNIsSDK{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/controller/konnect/ops/ops.go b/controller/konnect/ops/ops.go index 3468ce5dc..a543dcf39 100644 --- a/controller/konnect/ops/ops.go +++ b/controller/konnect/ops/ops.go @@ -84,6 +84,9 @@ func Create[ return e, createKey(ctx, sdk.GetKeysSDK(), ent) case *configurationv1alpha1.KongKeySet: return e, createKeySet(ctx, sdk.GetKeySetsSDK(), ent) + case *configurationv1alpha1.KongSNI: + return e, createSNI(ctx, sdk.GetSNIsSDK(), ent) + // --------------------------------------------------------------------- // TODO: add other Konnect types @@ -141,6 +144,9 @@ func Delete[ return deleteKey(ctx, sdk.GetKeysSDK(), ent) case *configurationv1alpha1.KongKeySet: return deleteKeySet(ctx, sdk.GetKeySetsSDK(), ent) + case *configurationv1alpha1.KongSNI: + return deleteSNI(ctx, sdk.GetSNIsSDK(), ent) + // --------------------------------------------------------------------- // TODO: add other Konnect types @@ -243,6 +249,8 @@ func Update[ return ctrl.Result{}, updateKey(ctx, sdk.GetKeysSDK(), ent) case *configurationv1alpha1.KongKeySet: return ctrl.Result{}, updateKeySet(ctx, sdk.GetKeySetsSDK(), ent) + case *configurationv1alpha1.KongSNI: + return ctrl.Result{}, updateSNI(ctx, sdk.GetSNIsSDK(), ent) // --------------------------------------------------------------------- // TODO: add other Konnect types diff --git a/controller/konnect/ops/ops_kongsni.go b/controller/konnect/ops/ops_kongsni.go new file mode 100644 index 000000000..2bedd1c5d --- /dev/null +++ b/controller/konnect/ops/ops_kongsni.go @@ -0,0 +1,164 @@ +package ops + +import ( + "context" + "errors" + "fmt" + "net/http" + "slices" + + sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components" + sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations" + sdkkonnecterrs "github.com/Kong/sdk-konnect-go/models/sdkerrors" + "github.com/samber/lo" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrllog "sigs.k8s.io/controller-runtime/pkg/log" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + "github.com/kong/kubernetes-configuration/pkg/metadata" +) + +func createSNI( + ctx context.Context, + sdk SNIsSDK, + sni *configurationv1alpha1.KongSNI, +) error { + cpID := sni.GetControlPlaneID() + if cpID == "" { + return fmt.Errorf("can't create %T %s without a Konnect ControlPlane ID", sni, client.ObjectKeyFromObject(sni)) + } + if sni.Status.Konnect == nil || sni.Status.Konnect.CertificateID == "" { + return fmt.Errorf("can't create %T %s without a Konnect Certificate ID", sni, client.ObjectKeyFromObject(sni)) + } + + resp, err := sdk.CreateSniWithCertificate(ctx, sdkkonnectops.CreateSniWithCertificateRequest{ + ControlPlaneID: cpID, + CertificateID: sni.Status.Konnect.CertificateID, + SNIWithoutParents: kongSNIToSNIWithoutParents(sni), + }) + + if errWrapped := wrapErrIfKonnectOpFailed(err, CreateOp, sni); errWrapped != nil { + SetKonnectEntityProgrammedConditionFalse(sni, "FailedToCreate", errWrapped.Error()) + return errWrapped + } + + sni.Status.Konnect.SetKonnectID(*resp.Sni.ID) + SetKonnectEntityProgrammedCondition(sni) + + return nil +} + +func updateSNI( + ctx context.Context, + sdk SNIsSDK, + sni *configurationv1alpha1.KongSNI, +) error { + cpID := sni.GetControlPlaneID() + if cpID == "" { + return fmt.Errorf("can't update %T %s without a Konnect ControlPlane ID", sni, client.ObjectKeyFromObject(sni)) + } + if sni.Status.Konnect == nil || sni.Status.Konnect.CertificateID == "" { + return fmt.Errorf("can't update %T %s without a Konnect Certificate ID", sni, client.ObjectKeyFromObject(sni)) + } + id := sni.GetKonnectID() + + _, err := sdk.UpsertSniWithCertificate(ctx, sdkkonnectops.UpsertSniWithCertificateRequest{ + ControlPlaneID: cpID, + CertificateID: sni.Status.Konnect.CertificateID, + SNIID: id, + SNIWithoutParents: kongSNIToSNIWithoutParents(sni), + }) + + // TODO: handle already exists + // Can't adopt it as it will cause conflicts between the controller + // that created that entity and already manages it, hm + if errWrap := wrapErrIfKonnectOpFailed(err, UpdateOp, sni); errWrap != nil { + // SNI update operation returns an SDKError instead of a NotFoundError. + var sdkError *sdkkonnecterrs.SDKError + if errors.As(errWrap, &sdkError) { + switch sdkError.StatusCode { + case 404: + if err := createSNI(ctx, sdk, sni); err != nil { + return FailedKonnectOpError[configurationv1alpha1.KongSNI]{ + Op: UpdateOp, + Err: err, + } + } + // Create succeeded, createSNI sets the status so no need to do this here. + + return nil + default: + return FailedKonnectOpError[configurationv1alpha1.KongSNI]{ + Op: UpdateOp, + Err: sdkError, + } + } + } + + SetKonnectEntityProgrammedConditionFalse(sni, "FailedToUpdate", errWrap.Error()) + return errWrap + } + + SetKonnectEntityProgrammedCondition(sni) + return nil +} + +func deleteSNI( + ctx context.Context, + sdk SNIsSDK, + sni *configurationv1alpha1.KongSNI, +) error { + cpID := sni.GetControlPlaneID() + if cpID == "" { + return fmt.Errorf("can't delete %T %s without a Konnect ControlPlane ID", sni, client.ObjectKeyFromObject(sni)) + } + if sni.Status.Konnect == nil || sni.Status.Konnect.CertificateID == "" { + return fmt.Errorf("can't delete %T %s without a Konnect Certificate ID", sni, client.ObjectKeyFromObject(sni)) + } + id := sni.GetKonnectID() + + _, err := sdk.DeleteSniWithCertificate(ctx, sdkkonnectops.DeleteSniWithCertificateRequest{ + ControlPlaneID: cpID, + CertificateID: sni.Status.Konnect.CertificateID, + SNIID: id, + }) + + if errWrapped := wrapErrIfKonnectOpFailed(err, DeleteOp, sni); errWrapped != nil { + // Service delete operation returns an SDKError instead of a NotFoundError. + var sdkError *sdkkonnecterrs.SDKError + if errors.As(errWrapped, &sdkError) { + if sdkError.StatusCode == http.StatusNotFound { + ctrllog.FromContext(ctx). + Info("entity not found in Konnect, skipping delete", + "op", DeleteOp, "type", sni.GetTypeName(), "id", id, + ) + return nil + } + return FailedKonnectOpError[configurationv1alpha1.KongSNI]{ + Op: DeleteOp, + Err: sdkError, + } + } + return FailedKonnectOpError[configurationv1alpha1.KongSNI]{ + Op: DeleteOp, + Err: errWrapped, + } + } + + return nil +} + +func kongSNIToSNIWithoutParents(sni *configurationv1alpha1.KongSNI) sdkkonnectcomp.SNIWithoutParents { + var ( + specTags = sni.Spec.Tags + annotationTags = metadata.ExtractTags(sni) + k8sTags = GenerateKubernetesMetadataTags(sni) + ) + // Deduplicate tags to avoid rejection by Konnect. + tags := lo.Uniq(slices.Concat(specTags, annotationTags, k8sTags)) + + return sdkkonnectcomp.SNIWithoutParents{ + Name: sni.Spec.Name, + Tags: tags, + } +} diff --git a/controller/konnect/ops/ops_kongtarget.go b/controller/konnect/ops/ops_kongtarget.go index bb01ea60b..53bd97812 100644 --- a/controller/konnect/ops/ops_kongtarget.go +++ b/controller/konnect/ops/ops_kongtarget.go @@ -84,10 +84,10 @@ func deleteTarget( ) error { cpID := target.GetControlPlaneID() if cpID == "" { - return fmt.Errorf("can't update %T %s without a Konnect ControlPlane ID", target, client.ObjectKeyFromObject(target)) + return fmt.Errorf("can't delete %T %s without a Konnect ControlPlane ID", target, client.ObjectKeyFromObject(target)) } if target.Status.Konnect == nil || target.Status.Konnect.UpstreamID == "" { - return fmt.Errorf("can't update %T %s without a Konnect Upstream ID", target, client.ObjectKeyFromObject(target)) + return fmt.Errorf("can't delete %T %s without a Konnect Upstream ID", target, client.ObjectKeyFromObject(target)) } id := target.GetKonnectID() diff --git a/controller/konnect/ops/sdkfactory.go b/controller/konnect/ops/sdkfactory.go index 09928e32d..5a559228a 100644 --- a/controller/konnect/ops/sdkfactory.go +++ b/controller/konnect/ops/sdkfactory.go @@ -24,6 +24,7 @@ type SDKWrapper interface { GetCertificatesSDK() CertificatesSDK GetKeysSDK() KeysSDK GetKeySetsSDK() KeySetsSDK + GetSNIsSDK() SNIsSDK } type sdkWrapper struct { @@ -92,6 +93,11 @@ func (w sdkWrapper) GetCertificatesSDK() CertificatesSDK { return w.sdk.Certificates } +// GetSNIsSDK returns the SDK to operate SNIs. +func (w sdkWrapper) GetSNIsSDK() SNIsSDK { + return w.sdk.SNIs +} + // GetBasicAuthCredentialsSDK returns the BasicAuthCredentials SDK to get current organization. func (w sdkWrapper) GetBasicAuthCredentialsSDK() KongCredentialBasicAuthSDK { return w.sdk.BasicAuthCredentials diff --git a/controller/konnect/ops/sdkfactory_mock.go b/controller/konnect/ops/sdkfactory_mock.go index e15d9cef0..e618b3b40 100644 --- a/controller/konnect/ops/sdkfactory_mock.go +++ b/controller/konnect/ops/sdkfactory_mock.go @@ -24,6 +24,7 @@ type MockSDKWrapper struct { VaultSDK *MockVaultSDK KeysSDK *MockKeysSDK KeySetsSDK *MockKeySetsSDK + SNIsSDK *MockSNIsSDK } var _ SDKWrapper = MockSDKWrapper{} @@ -47,6 +48,7 @@ func NewMockSDKWrapperWithT(t *testing.T) *MockSDKWrapper { VaultSDK: NewMockVaultSDK(t), KeysSDK: NewMockKeysSDK(t), KeySetsSDK: NewMockKeySetsSDK(t), + SNIsSDK: NewMockSNIsSDK(t), } } @@ -118,6 +120,10 @@ func (m MockSDKWrapper) GetKeySetsSDK() KeySetsSDK { return m.KeySetsSDK } +func (m MockSDKWrapper) GetSNIsSDK() SNIsSDK { + return m.SNIsSDK +} + type MockSDKFactory struct { t *testing.T SDK *MockSDKWrapper diff --git a/controller/konnect/reconciler_certificateref.go b/controller/konnect/reconciler_certificateref.go new file mode 100644 index 000000000..9f3916c82 --- /dev/null +++ b/controller/konnect/reconciler_certificateref.go @@ -0,0 +1,188 @@ +package konnect + +import ( + "context" + "fmt" + + "github.com/samber/mo" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + "github.com/kong/gateway-operator/controller/konnect/conditions" + "github.com/kong/gateway-operator/controller/konnect/constraints" + k8sutils "github.com/kong/gateway-operator/pkg/utils/kubernetes" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" +) + +// getKongCertificateRef gets the reference of KongCertificate. +func getKongCertificateRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + e TEnt, +) mo.Option[configurationv1alpha1.KongObjectRef] { + switch e := any(e).(type) { + case *configurationv1alpha1.KongSNI: + // Since certificateRef is required for KongSNI, we directly return spec.CertificateRef here. + return mo.Some(e.Spec.CertificateRef) + default: + return mo.None[configurationv1alpha1.KongObjectRef]() + } +} + +func handleKongCertificateRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + ctx context.Context, + cl client.Client, + ent TEnt, +) (ctrl.Result, error) { + certRef, ok := getKongCertificateRef(ent).Get() + if !ok { + return ctrl.Result{}, nil + } + + cert := &configurationv1alpha1.KongCertificate{} + nn := types.NamespacedName{ + Name: certRef.Name, + // TODO: handle cross namespace refs + Namespace: ent.GetNamespace(), + } + err := cl.Get(ctx, nn, cert) + if err != nil { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongCertificateRefValidConditionType, + metav1.ConditionFalse, + conditions.KongCertificateRefReasonInvalid, + err.Error(), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{}, ReferencedKongCertificateDoesNotExist{ + Reference: nn, + Err: err, + } + } + + // If referenced KongCertificate is being deleted, return an error so that we + // can remove the entity from Konnect first. + if delTimestamp := cert.GetDeletionTimestamp(); !delTimestamp.IsZero() { + return ctrl.Result{}, ReferencedKongCertificateIsBeingDeleted{ + Reference: nn, + DeletionTimestamp: delTimestamp.Time, + } + } + + // requeue it if referenced KongCertificate is not programmed yet so we cannot do the following work. + cond, ok := k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, cert) + if !ok || cond.Status != metav1.ConditionTrue { + ent.SetKonnectID("") + if res, err := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongCertificateRefValidConditionType, + metav1.ConditionFalse, + conditions.KongCertificateRefReasonInvalid, + fmt.Sprintf("Referenced KongCertificate %s is not programmed yet", nn), + ); err != nil || res.Requeue { + return ctrl.Result{}, err + } + return ctrl.Result{Requeue: true}, nil + } + + // Set owner reference of referenced KongCertificate and the reconciled entity. + old := ent.DeepCopyObject().(TEnt) + if err := controllerutil.SetOwnerReference(cert, ent, cl.Scheme(), controllerutil.WithBlockOwnerDeletion(true)); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to set owner reference: %w", err) + } + if err := cl.Patch(ctx, ent, client.MergeFrom(old)); err != nil { + if k8serrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to update status: %w", err) + } + + // TODO: make this more generic. + if sni, ok := any(ent).(*configurationv1alpha1.KongSNI); ok { + if sni.Status.Konnect == nil { + sni.Status.Konnect = &konnectv1alpha1.KonnectEntityStatusWithControlPlaneAndCertificateRefs{} + } + sni.Status.Konnect.CertificateID = cert.GetKonnectID() + } + + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.KongCertificateRefValidConditionType, + metav1.ConditionTrue, + conditions.KongCertificateRefReasonValid, + fmt.Sprintf("Referenced KongCertificate %s programmed", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + cpRef, ok := getControlPlaneRef(cert).Get() + // TODO: ignore the entity if referenced KongCertificate does not have a Konnect control plane reference + // because this situation is likely to mean that they are not controlled by us: + // https://github.com/Kong/gateway-operator/issues/629 + if !ok { + return ctrl.Result{}, fmt.Errorf( + "%T references a KongCertificate %s which does not have a ControlPlane ref", + ent, client.ObjectKeyFromObject(cert), + ) + } + cp, err := getCPForRef(ctx, cl, cpRef, ent.GetNamespace()) + if err != nil { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionFalse, + conditions.ControlPlaneRefReasonInvalid, + err.Error(), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + if k8serrors.IsNotFound(err) { + return ctrl.Result{}, ReferencedControlPlaneDoesNotExistError{ + Reference: types.NamespacedName{ + Namespace: ent.GetNamespace(), + Name: cpRef.KonnectNamespacedRef.Name, + }, + Err: err, + } + } + return ctrl.Result{}, err + } + + cond, ok = k8sutils.GetCondition(conditions.KonnectEntityProgrammedConditionType, cp) + if !ok || cond.Status != metav1.ConditionTrue || cond.ObservedGeneration != cp.GetGeneration() { + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionFalse, + conditions.ControlPlaneRefReasonInvalid, + fmt.Sprintf("Referenced ControlPlane %s is not programmed yet", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{Requeue: true}, nil + } + + if resource, ok := any(ent).(EntityWithControlPlaneRef); ok { + resource.SetControlPlaneID(cp.Status.ID) + } + + if res, errStatus := updateStatusWithCondition( + ctx, cl, ent, + conditions.ControlPlaneRefValidConditionType, + metav1.ConditionTrue, + conditions.ControlPlaneRefReasonValid, + fmt.Sprintf("Referenced ControlPlane %s is programmed", nn), + ); errStatus != nil || res.Requeue { + return res, errStatus + } + + return ctrl.Result{}, nil +} diff --git a/controller/konnect/reconciler_certificateref_test.go b/controller/konnect/reconciler_certificateref_test.go new file mode 100644 index 000000000..f5ef373dd --- /dev/null +++ b/controller/konnect/reconciler_certificateref_test.go @@ -0,0 +1,401 @@ +package konnect + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/samber/lo" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/kong/gateway-operator/controller/konnect/conditions" + "github.com/kong/gateway-operator/controller/konnect/constraints" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" +) + +type handleCertRefTestCase[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]] struct { + name string + ent TEnt + objects []client.Object + expectResult ctrl.Result + expectError bool + expectErrorContains string + // Returns true if the updated entity satisfy the assertion. + // Returns false and error message if entity fails to satisfy it. + updatedEntAssertions []func(TEnt) (ok bool, message string) +} + +var testKongCertOK = &configurationv1alpha1.KongCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-ok", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongCertificateSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: "cp-ok", + }, + }, + KongCertificateAPISpec: configurationv1alpha1.KongCertificateAPISpec{ + Cert: "===== BEGIN CERTIFICATE", + Key: "===== BEGIN PRIVATE KEY", + }, + }, + Status: configurationv1alpha1.KongCertificateStatus{ + Konnect: &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + KonnectEntityStatus: konnectv1alpha1.KonnectEntityStatus{ + ID: "12345", + }, + ControlPlaneID: "123456789", + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testKongCertNotProgrammed = &configurationv1alpha1.KongCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-not-programmed", + Namespace: "default", + }, + Status: configurationv1alpha1.KongCertificateStatus{ + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionFalse, + }, + }, + }, +} + +var testKongCertNoControlPlaneRef = &configurationv1alpha1.KongCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-no-cp-ref", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongCertificateSpec{ + KongCertificateAPISpec: configurationv1alpha1.KongCertificateAPISpec{ + Cert: "===== BEGIN CERTIFICATE", + Key: "===== BEGIN PRIVATE KEY", + }, + }, + Status: configurationv1alpha1.KongCertificateStatus{ + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testKongCertBeingDeleted = &configurationv1alpha1.KongCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-being-deleted", + Namespace: "default", + DeletionTimestamp: &metav1.Time{Time: time.Now()}, + Finalizers: []string{"sni-0"}, + }, +} + +var testKongCertificateControlPlaneRefNotFound = &configurationv1alpha1.KongCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-cpref-not-found", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongCertificateSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: "cp-not-found", + }, + }, + }, + Status: configurationv1alpha1.KongCertificateStatus{ + Konnect: &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + KonnectEntityStatus: konnectv1alpha1.KonnectEntityStatus{ + ID: "12345", + }, + ControlPlaneID: "123456789", + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +var testKongCertControlPlaneRefNotProgrammed = &configurationv1alpha1.KongCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-cpref-not-programmed", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongCertificateSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: "cp-not-programmed", + }, + }, + }, + Status: configurationv1alpha1.KongCertificateStatus{ + Konnect: &konnectv1alpha1.KonnectEntityStatusWithControlPlaneRef{ + KonnectEntityStatus: konnectv1alpha1.KonnectEntityStatus{ + ID: "12345", + }, + ControlPlaneID: "123456789", + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + }, + }, + }, +} + +func TestHandleCertificateRef(t *testing.T) { + testCases := []handleCertRefTestCase[configurationv1alpha1.KongSNI, *configurationv1alpha1.KongSNI]{ + { + name: "has certificate ref and control plane ref", + ent: &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sni-ok", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: "cert-ok", + }, + }, + }, + objects: []client.Object{ + testKongCertOK, + testControlPlaneOK, + }, + expectResult: ctrl.Result{}, + expectError: false, + updatedEntAssertions: []func(*configurationv1alpha1.KongSNI) (bool, string){ + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongCertificateRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongSNI does not have KongCertificateRefValid condition set to True" + }, + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.ControlPlaneRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongSNI does not have ControlPlaneRefValid condition set to True" + }, + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.OwnerReferences, func(o metav1.OwnerReference) bool { + return o.Kind == "KongCertificate" && o.Name == "cert-ok" + }), "OwnerReference of KongSNI is not set" + }, + }, + }, + { + name: "certificate ref not found", + ent: &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cert-ref-not-found", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: "cert-nonexist", + }, + }, + }, + expectError: true, + expectErrorContains: "referenced Kong Certificate default/cert-nonexist does not exist", + updatedEntAssertions: []func(*configurationv1alpha1.KongSNI) (bool, string){ + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongCertificateRefValidConditionType && c.Status == metav1.ConditionFalse + }), "KongSNI does not have KongCertificateRefValid condition set to False" + }, + }, + }, + { + name: "referenced KongCertificate not programmed", + ent: &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sni-cert-ref-not-programmed", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: "cert-not-programmed", + }, + }, + }, + objects: []client.Object{ + testKongCertNotProgrammed, + }, + expectError: false, + expectResult: ctrl.Result{Requeue: true}, + updatedEntAssertions: []func(*configurationv1alpha1.KongSNI) (bool, string){ + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.GetConditions(), func(c metav1.Condition) bool { + return c.Type == conditions.KongCertificateRefValidConditionType && c.Status == metav1.ConditionFalse && + c.Message == fmt.Sprintf("Referenced KongCertificate %s/%s is not programmed yet", + testKongCertNotProgrammed.Namespace, testKongCertNotProgrammed.Name) + }), "KongSNI does not have KongCertificateRefValid condition set to False" + }, + }, + }, + { + name: "referenced KongCertificate has no ControlPlaneRef", + ent: &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sni-cert-no-cpref", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: "cert-no-cp-ref", + }, + }, + }, + objects: []client.Object{ + testKongCertNoControlPlaneRef, + }, + expectError: true, + expectErrorContains: fmt.Sprintf("references a KongCertificate %s/%s which does not have a ControlPlane ref", + testKongCertNoControlPlaneRef.Namespace, testKongCertNoControlPlaneRef.Name), + updatedEntAssertions: []func(*configurationv1alpha1.KongSNI) (bool, string){ + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongCertificateRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongSNI does not have KongCertificateRefValid condition set to True" + }, + }, + }, + { + name: "referenced KongCertificate is being deleted", + ent: &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Name: "sni-cert-being-deleted", + Namespace: "default", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: "cert-being-deleted", + }, + }, + }, + objects: []client.Object{ + testKongCertBeingDeleted, + }, + expectError: true, + expectErrorContains: fmt.Sprintf("referenced Kong Certificate %s/%s is being deleted", testKongCertBeingDeleted.Namespace, testKongCertBeingDeleted.Name), + }, + { + name: "ControlPlaneRef not found", + ent: &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "sni-cp-ref-not-found", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: "cert-cpref-not-found", + }, + }, + }, + objects: []client.Object{ + testKongCertificateControlPlaneRefNotFound, + }, + expectError: true, + expectErrorContains: fmt.Sprintf("referenced Control Plane %s/%s does not exist", + testKongCertificateControlPlaneRefNotFound.Namespace, + testKongCertificateControlPlaneRefNotFound.Spec.ControlPlaneRef.KonnectNamespacedRef.Name, + ), + }, + { + name: "ControlPlaneRef not programmed", + ent: &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "sni-cp-ref-not-programmed", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: "cert-cpref-not-programmed", + }, + }, + }, + objects: []client.Object{ + testKongCertControlPlaneRefNotProgrammed, + testControlPlaneNotProgrammed, + }, + expectError: false, + expectResult: ctrl.Result{Requeue: true}, + updatedEntAssertions: []func(*configurationv1alpha1.KongSNI) (bool, string){ + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.KongCertificateRefValidConditionType && c.Status == metav1.ConditionTrue + }), "KongSNI does not have KongCertificateRefValid condition set to True" + }, + func(ks *configurationv1alpha1.KongSNI) (bool, string) { + return lo.ContainsBy(ks.Status.Conditions, func(c metav1.Condition) bool { + return c.Type == conditions.ControlPlaneRefValidConditionType && c.Status == metav1.ConditionFalse + }), "KongSNI does not have ControlPlaneRefValid condition set to False" + }, + }, + }, + } + + testHandleCertificateRef(t, testCases) + +} + +func testHandleCertificateRef[T constraints.SupportedKonnectEntityType, TEnt constraints.EntityType[T]]( + t *testing.T, testCases []handleCertRefTestCase[T, TEnt]) { + t.Helper() + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + scheme := runtime.NewScheme() + require.NoError(t, configurationv1alpha1.AddToScheme(scheme)) + require.NoError(t, konnectv1alpha1.AddToScheme(scheme)) + fakeClient := fake.NewClientBuilder().WithScheme(scheme). + WithObjects(tc.ent).WithObjects(tc.objects...). + // WithStatusSubresource is required for updating status of handled entity. + WithStatusSubresource(tc.ent).Build() + require.NoError(t, fakeClient.SubResource("status").Update(context.Background(), tc.ent)) + + res, err := handleKongCertificateRef(context.Background(), fakeClient, tc.ent) + + var updatedEnt TEnt = tc.ent.DeepCopyObject().(TEnt) + require.NoError(t, fakeClient.Get(context.Background(), client.ObjectKeyFromObject(tc.ent), updatedEnt)) + for _, assertion := range tc.updatedEntAssertions { + ok, msg := assertion(updatedEnt) + require.True(t, ok, msg) + } + + if tc.expectError { + require.Error(t, err) + require.Contains(t, err.Error(), tc.expectErrorContains) + return + } + + require.NoError(t, err) + require.Equal(t, tc.expectResult, res) + }) + } +} diff --git a/controller/konnect/reconciler_generic.go b/controller/konnect/reconciler_generic.go index bee185b8c..18f5a50e1 100644 --- a/controller/konnect/reconciler_generic.go +++ b/controller/konnect/reconciler_generic.go @@ -250,6 +250,44 @@ func (r *KonnectEntityReconciler[T, TEnt]) Reconcile( return res, nil } + // If a type has a KongCertificateRef (KongCertificate), handle it. + res, err = handleKongCertificateRef(ctx, r.Client, ent) + if err != nil { + // If the referenced KongCertificate is being deleted and the object + // is not being deleted yet then requeue until it will + // get the deletion timestamp set due to having the owner set to KongCertificate. + if errDel := (&ReferencedKongCertificateIsBeingDeleted{}); errors.As(err, errDel) && + ent.GetDeletionTimestamp().IsZero() { + return ctrl.Result{ + RequeueAfter: time.Until(errDel.DeletionTimestamp), + }, nil + } + + // If the referenced KongCertificate is not found or is being deleted + // and the object is being deleted, remove the finalizer and let the + // deletion proceed without trying to delete the entity from Konnect + // as the KongCertificate deletion will take care of it on the Konnect side. + if errors.As(err, &ReferencedKongCertificateIsBeingDeleted{}) || + errors.As(err, &ReferencedKongCertificateDoesNotExist{}) { + if !ent.GetDeletionTimestamp().IsZero() { + if controllerutil.RemoveFinalizer(ent, KonnectCleanupFinalizer) { + if err := r.Client.Update(ctx, ent); err != nil { + if k8serrors.IsConflict(err) { + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, fmt.Errorf("failed to remove finalizer %s: %w", KonnectCleanupFinalizer, err) + } + log.Debug(logger, "finalizer removed as the owning KongCertificate is being deleted or is already gone", ent, + "finalizer", KonnectCleanupFinalizer, + ) + } + } + } + return ctrl.Result{}, nil + } else if res.Requeue { + return res, nil + } + apiAuthRef, err := getAPIAuthRefNN(ctx, r.Client, ent) if err != nil { return ctrl.Result{}, fmt.Errorf("failed to get APIAuth ref for %s: %w", client.ObjectKeyFromObject(ent), err) @@ -505,10 +543,7 @@ func updateStatusWithCondition[T interface { if k8serrors.IsConflict(err) { return ctrl.Result{Requeue: true}, nil } - return ctrl.Result{}, fmt.Errorf( - "failed to update status with %s condition: %w", - conditions.KonnectEntityAPIAuthConfigurationResolvedRefConditionType, err, - ) + return ctrl.Result{}, fmt.Errorf("failed to update status with %s condition: %w", conditionType, err) } return ctrl.Result{}, nil @@ -619,10 +654,10 @@ func getAPIAuthRefNN[T constraints.SupportedKonnectEntityType, TEnt constraints. // If the entity has a KongUpstreamRef, get the KonnectAPIAuthConfiguration // ref from the referenced KongUpstream. - upsteramRef, ok := getKongUpstreamRef(ent).Get() + upstreamRef, ok := getKongUpstreamRef(ent).Get() if ok { nn := types.NamespacedName{ - Name: upsteramRef.Name, + Name: upstreamRef.Name, Namespace: ent.GetNamespace(), } @@ -646,6 +681,27 @@ func getAPIAuthRefNN[T constraints.SupportedKonnectEntityType, TEnt constraints. }, nil } + // If the entity has a KongCertificateRef, get the KonnectAPIAuthConfiguration + // ref from the referenced KongUpstream. + certificateRef, ok := getKongCertificateRef(ent).Get() + if ok { + nn := types.NamespacedName{ + Name: certificateRef.Name, + Namespace: ent.GetNamespace(), + } + + var cert configurationv1alpha1.KongCertificate + if err := cl.Get(ctx, nn, &cert); err != nil { + return types.NamespacedName{}, fmt.Errorf("failed to get KongCertificate %s", nn) + } + + cpRef, ok := getControlPlaneRef(&cert).Get() + if !ok { + return types.NamespacedName{}, fmt.Errorf("KongCertificate %s does not have a ControlPlaneRef", nn) + } + return getCPAuthRefForRef(ctx, cl, cpRef, ent.GetNamespace()) + } + return types.NamespacedName{}, fmt.Errorf( "cannot get KonnectAPIAuthConfiguration for entity type %T %s", client.ObjectKeyFromObject(ent), ent, diff --git a/controller/konnect/reconciler_generic_rbac.go b/controller/konnect/reconciler_generic_rbac.go index 43f42f454..45396bba1 100644 --- a/controller/konnect/reconciler_generic_rbac.go +++ b/controller/konnect/reconciler_generic_rbac.go @@ -33,4 +33,7 @@ package konnect //+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongvaults,verbs=get;list;watch;update;patch //+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongvaults/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongsnis,verbs=get;list;watch;update;patch +//+kubebuilder:rbac:groups=configuration.konghq.com,resources=kongsnis/status,verbs=get;update;patch + //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch diff --git a/controller/konnect/reconciler_upstreamref_test.go b/controller/konnect/reconciler_upstreamref_test.go index 20629ba2d..9389361c2 100644 --- a/controller/konnect/reconciler_upstreamref_test.go +++ b/controller/konnect/reconciler_upstreamref_test.go @@ -230,7 +230,7 @@ func TestHandleUpstreamRef(t *testing.T) { func(kt *configurationv1alpha1.KongTarget) (bool, string) { return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionTrue - }), "KongTarget does not have KongUpsteamRefValid condition set to True" + }), "KongTarget does not have KongUpstreamRefValid condition set to True" }, func(kt *configurationv1alpha1.KongTarget) (bool, string) { return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { @@ -263,7 +263,7 @@ func TestHandleUpstreamRef(t *testing.T) { func(kt *configurationv1alpha1.KongTarget) (bool, string) { return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionFalse - }), "KongTarget does not have KongUpsteamRefValid condition set to False" + }), "KongTarget does not have KongUpstreamRefValid condition set to False" }, }, }, @@ -289,7 +289,7 @@ func TestHandleUpstreamRef(t *testing.T) { return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionFalse && c.Message == fmt.Sprintf("Referenced KongUpstream %s/%s is not programmed yet", testKongUpstreamNotProgrammed.Namespace, testKongUpstreamNotProgrammed.Name) - }), "KongTarget does not have KongUpsteamRefValid condition set to False" + }), "KongTarget does not have KongUpstreamRefValid condition set to False" }, }, }, @@ -314,7 +314,7 @@ func TestHandleUpstreamRef(t *testing.T) { func(kt *configurationv1alpha1.KongTarget) (bool, string) { return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionTrue - }), "KongTarget does not have KongUpsteamRefValid condition set to True" + }), "KongTarget does not have KongUpstreamRefValid condition set to True" }, }, }, @@ -378,7 +378,7 @@ func TestHandleUpstreamRef(t *testing.T) { func(kt *configurationv1alpha1.KongTarget) (bool, string) { return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { return c.Type == conditions.KongUpstreamRefValidConditionType && c.Status == metav1.ConditionTrue - }), "KongTarget does not have KongUpsteamRefValid condition set to True" + }), "KongTarget does not have KongUpstreamRefValid condition set to True" }, func(kt *configurationv1alpha1.KongTarget) (bool, string) { return lo.ContainsBy(kt.Status.Conditions, func(c metav1.Condition) bool { @@ -418,7 +418,6 @@ func testHandleUpstreamRef[T constraints.SupportedKonnectEntityType, TEnt constr if tc.expectError { require.Error(t, err) require.Contains(t, err.Error(), tc.expectErrorContains) - t.Logf("%#v", err) return } diff --git a/controller/konnect/watch.go b/controller/konnect/watch.go index e8bfcfd56..3e9b9ed61 100644 --- a/controller/konnect/watch.go +++ b/controller/konnect/watch.go @@ -60,6 +60,8 @@ func ReconciliationWatchOptionsForEntity[ return KongKeyReconciliationWatchOptions(cl) case *configurationv1alpha1.KongKeySet: return KongKeySetReconciliationWatchOptions(cl) + case *configurationv1alpha1.KongSNI: + return KongSNIReconciliationWatchOptions(cl) default: panic(fmt.Sprintf("unsupported entity type %T", ent)) } diff --git a/controller/konnect/watch_kongsni.go b/controller/konnect/watch_kongsni.go new file mode 100644 index 000000000..029dd4e44 --- /dev/null +++ b/controller/konnect/watch_kongsni.go @@ -0,0 +1,96 @@ +package konnect + +import ( + "context" + + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" +) + +// KongSNIReconciliationWatchOptions returns the watch options for +// the KongSNI. +func KongSNIReconciliationWatchOptions(cl client.Client, +) []func(*ctrl.Builder) *ctrl.Builder { + return []func(*ctrl.Builder) *ctrl.Builder{ + func(b *ctrl.Builder) *ctrl.Builder { + return b.For( + &configurationv1alpha1.KongSNI{}, + builder.WithPredicates( + predicate.NewPredicateFuncs(kongSNIRefersToKonnectGatewayControlPlane(cl)), + ), + ) + }, + + func(b *ctrl.Builder) *ctrl.Builder { + return b.Watches( + &configurationv1alpha1.KongCertificate{}, + handler.EnqueueRequestsFromMapFunc(enqueueKongSNIForKongCertificate(cl)), + ) + }, + } +} + +func kongSNIRefersToKonnectGatewayControlPlane( + cl client.Client, +) func(client.Object) bool { + return func(obj client.Object) bool { + sni, ok := obj.(*configurationv1alpha1.KongSNI) + if !ok { + return false + } + + certNN := types.NamespacedName{ + Namespace: sni.Namespace, + Name: sni.Spec.CertificateRef.Name, + } + cert := configurationv1alpha1.KongCertificate{} + if err := cl.Get(context.Background(), certNN, &cert); err != nil { + return true + } + + return cert.Spec.ControlPlaneRef != nil && cert.Spec.ControlPlaneRef.Type == configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef + } +} + +func enqueueKongSNIForKongCertificate( + cl client.Client, +) func(context.Context, client.Object) []reconcile.Request { + return func(ctx context.Context, obj client.Object) []reconcile.Request { + cert, ok := obj.(*configurationv1alpha1.KongCertificate) + if !ok { + return nil + } + + cpRef := cert.Spec.ControlPlaneRef + if cpRef == nil || cpRef.Type != configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef { + return nil + } + + sniList := configurationv1alpha1.KongSNIList{} + if err := cl.List(ctx, &sniList, client.InNamespace(cert.Namespace), + client.MatchingFields{ + IndexFieldKongPluginBindingKongServiceReference: cert.Name, + }, + ); err != nil { + return nil + } + + ret := make([]reconcile.Request, 0, len(sniList.Items)) + for _, sni := range sniList.Items { + ret = append(ret, reconcile.Request{ + NamespacedName: types.NamespacedName{ + Namespace: sni.Namespace, + Name: sni.Name, + }, + }) + } + return ret + } +} diff --git a/modules/manager/controller_setup.go b/modules/manager/controller_setup.go index 8790fb1c4..f14ba58dc 100644 --- a/modules/manager/controller_setup.go +++ b/modules/manager/controller_setup.go @@ -100,6 +100,8 @@ const ( KongKeyControllerName = "KongKey" // KongKeySetControllerName is the name of KongKeySet controller. KongKeySetControllerName = "KongKeySet" + // KongSNIControllerName is the name of KongSNI controller. + KongSNIControllerName = "KongSNI" ) // SetupControllersShim runs SetupControllers and returns its result as a slice of the map values. @@ -524,6 +526,15 @@ func SetupControllers(mgr manager.Manager, c *Config) (map[string]ControllerDef, konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongVault](c.KonnectSyncPeriod), ), }, + KongSNIControllerName: { + Enabled: c.KonnectControllersEnabled, + Controller: konnect.NewKonnectEntityReconciler( + sdkFactory, + c.DevelopmentMode, + mgr.GetClient(), + konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongSNI](c.KonnectSyncPeriod), + ), + }, } // Merge Konnect controllers into the controllers map. This is done this way instead of directly assigning @@ -558,6 +569,9 @@ func SetupCacheIndicesForKonnectTypes(ctx context.Context, mgr manager.Manager, if err := setupCacheIndicesForKonnectType[configurationv1alpha1.KongRoute](ctx, mgr, developmentMode); err != nil { return err } + if err := setupCacheIndicesForKonnectType[configurationv1alpha1.KongSNI](ctx, mgr, developmentMode); err != nil { + return err + } return nil } diff --git a/test/envtest/deploy_resources.go b/test/envtest/deploy_resources.go index 5ca4afbe9..92ea69b6f 100644 --- a/test/envtest/deploy_resources.go +++ b/test/envtest/deploy_resources.go @@ -566,3 +566,31 @@ func deployKongKeySetAttachedToCP( return keySet } + +func deploySNIAttachedToCertificate( + t *testing.T, + ctx context.Context, + cl client.Client, + name string, tags []string, + cert *configurationv1alpha1.KongCertificate, +) *configurationv1alpha1.KongSNI { + t.Helper() + + sni := &configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + GenerateName: "sni-", + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: cert.Name, + }, + KongSNIAPISpec: configurationv1alpha1.KongSNIAPISpec{ + Name: name, + Tags: tags, + }, + }, + } + require.NoError(t, cl.Create(ctx, sni)) + t.Logf("deployed KongSNI %s/%s", sni.Namespace, sni.Name) + return sni +} diff --git a/test/envtest/konnect_entities_sni_test.go b/test/envtest/konnect_entities_sni_test.go new file mode 100644 index 000000000..526098946 --- /dev/null +++ b/test/envtest/konnect_entities_sni_test.go @@ -0,0 +1,143 @@ +package envtest + +import ( + "context" + "testing" + + sdkkonnectcomp "github.com/Kong/sdk-konnect-go/models/components" + sdkkonnectops "github.com/Kong/sdk-konnect-go/models/operations" + "github.com/samber/lo" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/watch" + "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kong/gateway-operator/controller/konnect" + "github.com/kong/gateway-operator/controller/konnect/conditions" + "github.com/kong/gateway-operator/controller/konnect/ops" + "github.com/kong/gateway-operator/modules/manager/scheme" + + configurationv1alpha1 "github.com/kong/kubernetes-configuration/api/configuration/v1alpha1" + konnectalpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" +) + +func TestKongSNI(t *testing.T) { + t.Parallel() + ctx, cancel := Context(t, context.Background()) + defer cancel() + cfg, ns := Setup(t, ctx, scheme.Get()) + + t.Log("Setting up the manager with reconcilers") + mgr, logs := NewManager(t, ctx, cfg, scheme.Get()) + factory := ops.NewMockSDKFactory(t) + sdk := factory.SDK + StartReconcilers(ctx, t, mgr, logs, + konnect.NewKonnectEntityReconciler(factory, false, mgr.GetClient(), + konnect.WithKonnectEntitySyncPeriod[configurationv1alpha1.KongSNI](konnectInfiniteSyncTime), + ), + ) + + t.Log("Setting up clients") + cl, err := client.NewWithWatch(mgr.GetConfig(), client.Options{ + Scheme: scheme.Get(), + }) + require.NoError(t, err) + clientNamespaced := client.NewNamespacedClient(mgr.GetClient(), ns.Name) + + t.Log("Creating KonnectAPIAuthConfiguration and KonnectGatewayControlPlane") + apiAuth := deployKonnectAPIAuthConfigurationWithProgrammed(t, ctx, clientNamespaced) + cp := deployKonnectGatewayControlPlaneWithID(t, ctx, clientNamespaced, apiAuth) + + t.Run("adding, patching and deleting KongSNI", func(t *testing.T) { + t.Log("Creating KongCertificate and setting it to Programmed") + createdCert := deployKongCertificateAttachedToCP(t, ctx, clientNamespaced, cp) + createdCert.Status = configurationv1alpha1.KongCertificateStatus{ + Konnect: &konnectalpha1.KonnectEntityStatusWithControlPlaneRef{ + KonnectEntityStatus: konnectEntityStatus("cert-12345"), + ControlPlaneID: cp.Status.GetKonnectID(), + }, + Conditions: []metav1.Condition{ + { + Type: conditions.KonnectEntityProgrammedConditionType, + Status: metav1.ConditionTrue, + Reason: conditions.KonnectEntityProgrammedReasonProgrammed, + ObservedGeneration: createdCert.GetGeneration(), + LastTransitionTime: metav1.Now(), + }, + }, + } + require.NoError(t, clientNamespaced.Status().Update(ctx, createdCert)) + + t.Log("Setting up a watch for KongSNI events") + w := setupWatch[configurationv1alpha1.KongSNIList](t, ctx, cl, client.InNamespace(ns.Name)) + + t.Log("Setting up SDK for creating SNI") + sdk.SNIsSDK.EXPECT().CreateSniWithCertificate( + mock.Anything, + mock.MatchedBy(func(req sdkkonnectops.CreateSniWithCertificateRequest) bool { + return req.ControlPlaneID == cp.Status.ID && + req.CertificateID == createdCert.GetKonnectID() && + req.SNIWithoutParents.Name == "test.kong-sni.example.com" + }), + ).Return(&sdkkonnectops.CreateSniWithCertificateResponse{ + Sni: &sdkkonnectcomp.Sni{ + ID: lo.ToPtr("sni-12345"), + }, + }, nil) + + t.Log("Creating KongSNI") + createdSNI := deploySNIAttachedToCertificate(t, ctx, + clientNamespaced, + "test.kong-sni.example.com", nil, + createdCert, + ) + + t.Log("Waiting for SNI to be programmed and get Konnect ID") + watchFor(t, ctx, w, watch.Modified, func(s *configurationv1alpha1.KongSNI) bool { + return s.GetKonnectID() == "sni-12345" && lo.ContainsBy(s.Status.Conditions, + func(c metav1.Condition) bool { + return c.Type == "Programmed" && c.Status == metav1.ConditionTrue + }) + }, "SNI didn't get Programmed status condition or didn't get the correct (sni-12345) Konnect ID assigned") + + t.Log("Set up SDK for SNI update") + sdk.SNIsSDK.EXPECT().UpsertSniWithCertificate( + mock.Anything, + mock.MatchedBy(func(req sdkkonnectops.UpsertSniWithCertificateRequest) bool { + return req.CertificateID == createdCert.GetKonnectID() && + req.ControlPlaneID == cp.Status.ID && + req.SNIWithoutParents.Name == "test2.kong-sni.example.com" + }), + ).Return(&sdkkonnectops.UpsertSniWithCertificateResponse{}, nil) + + t.Log("Patching KongSNI") + sniToPatch := createdSNI.DeepCopy() + sniToPatch.Spec.KongSNIAPISpec.Name = "test2.kong-sni.example.com" + require.NoError(t, clientNamespaced.Patch(ctx, sniToPatch, client.MergeFrom(createdSNI))) + + t.Log("Waiting for KongSNI to be updated in the SDK") + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, factory.SDK.SNIsSDK.AssertExpectations(t)) + }, waitTime, tickTime) + + t.Log("Setting up SDK for deleting SNI") + sdk.SNIsSDK.EXPECT().DeleteSniWithCertificate( + mock.Anything, + sdkkonnectops.DeleteSniWithCertificateRequest{ + ControlPlaneID: cp.Status.ID, + CertificateID: createdCert.GetKonnectID(), + SNIID: "sni-12345", + }, + ).Return(&sdkkonnectops.DeleteSniWithCertificateResponse{}, nil) + + t.Log("Deleting KongSNI") + require.NoError(t, clientNamespaced.Delete(ctx, createdSNI)) + + t.Log("Waiting for SNI to be deleted in SDK") + assert.EventuallyWithT(t, func(c *assert.CollectT) { + assert.True(c, factory.SDK.SNIsSDK.AssertExpectations(t)) + }, waitTime, tickTime) + }) +} diff --git a/test/integration/test_konnect_entities.go b/test/integration/test_konnect_entities.go index 57c898fdc..4efb4884d 100644 --- a/test/integration/test_konnect_entities.go +++ b/test/integration/test_konnect_entities.go @@ -26,6 +26,61 @@ import ( konnectv1alpha1 "github.com/kong/kubernetes-configuration/api/konnect/v1alpha1" ) +const ( + // dummyValidCertPEM is a dummy valid certificate PEM to be used in tests. + dummyValidCertPEM = `-----BEGIN CERTIFICATE----- +MIIDPTCCAiUCFG5IolqRiKPMfzTI8peXlaF6cZODMA0GCSqGSIb3DQEBCwUAMFsx +CzAJBgNVBAYTAlVTMQswCQYDVQQIDAJDQTEVMBMGA1UEBwwMRGVmYXVsdCBDaXR5 +MRIwEAYDVQQKDAlLb25nIEluYy4xFDASBgNVBAMMC2tvbmdocS50ZWNoMB4XDTI0 +MDkyNTA3MjIzOFoXDTM0MDkyMzA3MjIzOFowWzELMAkGA1UEBhMCVVMxCzAJBgNV +BAgMAkNBMRUwEwYDVQQHDAxEZWZhdWx0IENpdHkxEjAQBgNVBAoMCUtvbmcgSW5j +LjEUMBIGA1UEAwwLa29uZ2hxLnRlY2gwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAw +ggEKAoIBAQDXmNBzpWyJ0YUdfCamZpJiwRQMn5vVY8iKQrd3dD03DWyPHu/fXlrL ++QPTRip5d1SrxjzQ4S3fgme442BTlElF9d1w1rhg+DIg6NsW1jd+3IZaICnq7BZH +rJGlW+IWJSKHmNQ39nfVQwgL/QdylrYpbB7uwdEDMa78GfXteiXTcuNobCr7VWVz +rY6rQXo/dImWE1PtMp/EZEMsEbgbQpK5+fUnKTmFncVlDAZ2Q3s2MPikV5UhMVyQ +dKQydU0Ev0LRtpsjW8pQdshMG1ilMq6Yg6YU95gakLVjRXMoDlIJOu08mdped+2Y +VIUSXhRyRt1hbkFP0fXG0THfZ3DjH7jRAgMBAAEwDQYJKoZIhvcNAQELBQADggEB +ANEXlNaQKLrB+jsnNjybsTRkvRRmwjnXaQV0jHzjseGsTJoKY5ABBsSRDiHtqB+9 +LPTpHhLYJWsHSLwawIJ3aWDDpF4MNTRsvO12v7wM8Q42OSgkP23O6a5ESkyHRBAb +dLVEp+0Z3kjYwPIglIK37PcgDci6Zim73GOfapDEASNbnCu8js2g/ucYPPXkGMxl +PSUER7MTNf9wRbXrroCE+tZw4kUyUh+6taNlU4ialAJLO1x6UGVRHvPgEx0fAAxA +seBH+A9QMvVl2cKcvrOgZ0CWY01aFRO9ROQ7PrYXqRFvOZu8K3QzLw7xYoK1DTp+ +kkO/oPy+WIbqEvj7QrhUXpo= +-----END CERTIFICATE----- +` + // dummyValidCertKeyPEM is a dummy valid certificate key PEM to be used in tests. + dummyValidCertKeyPEM = `-----BEGIN PRIVATE KEY----- +MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDXmNBzpWyJ0YUd +fCamZpJiwRQMn5vVY8iKQrd3dD03DWyPHu/fXlrL+QPTRip5d1SrxjzQ4S3fgme4 +42BTlElF9d1w1rhg+DIg6NsW1jd+3IZaICnq7BZHrJGlW+IWJSKHmNQ39nfVQwgL +/QdylrYpbB7uwdEDMa78GfXteiXTcuNobCr7VWVzrY6rQXo/dImWE1PtMp/EZEMs +EbgbQpK5+fUnKTmFncVlDAZ2Q3s2MPikV5UhMVyQdKQydU0Ev0LRtpsjW8pQdshM +G1ilMq6Yg6YU95gakLVjRXMoDlIJOu08mdped+2YVIUSXhRyRt1hbkFP0fXG0THf +Z3DjH7jRAgMBAAECggEAOSZ4h1dgDK5+H2FEK5MAFe6BnpEGsYu4YrIpySAGhBvq +XYwBYRA1eGFjmrM8WiOATeKIR4SRcPC0BwY7CBzESafRkfJRQN86BpBDV2vknRve +/3AMPIplo41CtHdFWMJyQ0iHZOhQPrd8oBTsTvtVgWh4UKkO+05FyO0mzFM3SLPs +pqRwMZjLlKVZhbI1l0Ur787tzWpMQQHmd8csAvlak+GIciQWELbVK+5kr/FDpJbq +joIeHX7DCmIqrD/Okwa8SfJu1sutmRX+nrxkDi7trPYcpqriDoWs2jMcnS2GHq9M +lsy2XHn+qLjCpl3/XU61xenWs+Rmmj6J/oIs1zYXCwKBgQDywRS/MNgwMst703Wh +ERJO0rNSR0XVzzoKOqc/ghPkeA8mVNwfNkbgWks+83tuAb6IunMIeyQJ3g/jRhjz +KImsqJmO+DoZCioeaR3PeIWibi9I9Irg6dtoNMwxSmmOtCKD0rccxM1V9OnYkn5a +0Fb+irQSgJYiHrF2SLAT0NoWEwKBgQDjXGLHCu/WEy49vROdkTia133Wc7m71/D5 +RDUqvSmAEHJyhTlzCbTO+JcNhC6cx3s102GNcVYHlAq3WoE5EV1YykUNJwUg4XPn +AggNkYOiXs6tf+uePmT8MddixFFgFxZ2bIqFhvnY+WqypHuxtwIepqKJjq5xZTiB ++lfp7SziCwKBgAivofdpXwLyfldy7I2T18zcOzBhfn01CgWdrahXFjEhnqEnfizb +u1OBx5l8CtmX1GJ+EWmnRlXYDUd7lZ71v19fNQdpmGKW+4TVDA0Fafqy6Jw6q9F6 +bLBg20GUQQyrI2UGICk2XYaK2ec27rB/Le2zttfGpBiaco0h8rLy0SrjAoGBAM4/ +UY/UOQsOrUTuT2wBf8LfNtUid9uSIZRNrplNrebxhJCkkB/uLyoN0iE9xncMcpW6 +YmVH6c3IGwyHOnBFc1OHcapjukBApL5rVljQpwPVU1GKmHgdi8hHgmajRlqPtx3I +isRkVCPi5kqV8WueY3rgmNOGLnLJasBmE/gt4ihPAoGAG3v93R5tAeSrn7DMHaZt +p+udsNw9mDPYHAzlYtnw1OE/I0ceR5UyCFSzCd00Q8ZYBLf9jRvsO/GUA4F51isE +8/7xyqSxJqDwzv9N8EGkqf/SfMKA3kK3Sc8u+ovhzJu8OxcY+qrpo4+vYWYeW42n +5XBwvWV2ovRMx7Ntw7FUc24= +-----END PRIVATE KEY----- +` +) + func TestKonnectEntities(t *testing.T) { // A cleaner is created underneath anyway, and a whole namespace is deleted eventually. // We can't use a cleaner to delete objects because it handles deletes in FIFO order and that won't work in this @@ -358,6 +413,81 @@ func TestKonnectEntities(t *testing.T) { assert.NotEmpty(t, kv.Status.Konnect.KonnectEntityStatus.GetOrgID()) assert.NotEmpty(t, kv.Status.Konnect.KonnectEntityStatus.GetServerURL()) }, testutils.ObjectUpdateTimeout, time.Second) + + t.Logf("Creating KongCertificate") + kcertName := "kcert-" + testID + kcert := configurationv1alpha1.KongCertificate{ + ObjectMeta: metav1.ObjectMeta{ + Name: kcertName, + Namespace: ns.Name, + }, + Spec: configurationv1alpha1.KongCertificateSpec{ + ControlPlaneRef: &configurationv1alpha1.ControlPlaneRef{ + Type: configurationv1alpha1.ControlPlaneRefKonnectNamespacedRef, + KonnectNamespacedRef: &configurationv1alpha1.KonnectNamespacedRef{ + Name: cp.Name, + Namespace: ns.Name, + }, + }, + KongCertificateAPISpec: configurationv1alpha1.KongCertificateAPISpec{ + Cert: dummyValidCertPEM, + Key: dummyValidCertKeyPEM, + }, + }, + } + require.NoError(t, GetClients().MgrClient.Create(GetCtx(), &kcert)) + + t.Logf("Waiting for KongCertificate to get Konnect ID") + require.EventuallyWithT(t, func(t *assert.CollectT) { + err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{ + Name: kcertName, + Namespace: ns.Name, + }, &kcert) + require.NoError(t, err) + + if !assert.NotNil(t, kcert.Status.Konnect) { + return + } + assert.NotEmpty(t, kcert.Status.Konnect.KonnectEntityStatus.GetKonnectID()) + assert.NotEmpty(t, kcert.Status.Konnect.KonnectEntityStatus.GetOrgID()) + assert.NotEmpty(t, kcert.Status.Konnect.KonnectEntityStatus.GetServerURL()) + }, testutils.ObjectUpdateTimeout, time.Second) + + t.Log("Creating a KongSNI attached to KongCertificate") + ksniName := "ksni-" + testID + ksni := configurationv1alpha1.KongSNI{ + ObjectMeta: metav1.ObjectMeta{ + Name: ksniName, + Namespace: ns.Name, + }, + Spec: configurationv1alpha1.KongSNISpec{ + CertificateRef: configurationv1alpha1.KongObjectRef{ + Name: kcertName, + }, + KongSNIAPISpec: configurationv1alpha1.KongSNIAPISpec{ + Name: "test.kong-sni.example.com", + }, + }, + } + require.NoError(t, GetClients().MgrClient.Create(GetCtx(), &ksni)) + + t.Logf("Waiting for KongSNI to get Konnect ID") + require.EventuallyWithT(t, func(t *assert.CollectT) { + err := GetClients().MgrClient.Get(GetCtx(), types.NamespacedName{ + Name: ksniName, + Namespace: ns.Name, + }, &ksni) + require.NoError(t, err) + + if !assert.NotNil(t, ksni.Status.Konnect) { + return + } + assert.NotEmpty(t, ksni.Status.Konnect.KonnectEntityStatus.GetKonnectID()) + assert.NotEmpty(t, ksni.Status.Konnect.KonnectEntityStatus.GetOrgID()) + assert.NotEmpty(t, ksni.Status.Konnect.KonnectEntityStatus.GetServerURL()) + assert.Equal(t, kcert.GetKonnectID(), ksni.Status.Konnect.CertificateID) + }, testutils.ObjectUpdateTimeout, time.Second) + } // deleteObjectAndWaitForDeletionFn returns a function that deletes the given object and waits for it to be gone.