diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index 60953311991..7d693d0d49c 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -36,7 +36,8 @@ jobs: "purge-controller", "purge-metrics", "module-upgrade", - "certificate-rotation" + "ca-certificate-rotation" + "self-signed-certificate-rotation" ] name: "E2E" needs: [wait-for-img] @@ -133,8 +134,23 @@ jobs: cat purge_finalizer.yaml kustomize edit add patch --path purge_finalizer.yaml --kind Deployment popd + - name: Patch self signed certificate lifetime + if: ${{matrix.e2e-test == 'self-signed-certificate-rotation'}} + working-directory: lifecycle-manager + run: | + pushd config/watcher_local_test + echo \ + "- op: add + path: /spec/template/spec/containers/0/args/- + value: --self-signed-cert-duration=1h + - op: add + path: /spec/template/spec/containers/0/args/- + value: --self-signed-cert-renew-before=59m" >> self-signed-cert.yaml + cat self-signed-cert.yaml + kustomize edit add patch --path self-signed-cert.yaml --kind Deployment + popd - name: Patch CA certificate renewBefore - if: ${{matrix.e2e-test == 'certificate-rotation'}} + if: ${{matrix.e2e-test == 'ca-certificate-rotation'}} working-directory: lifecycle-manager run: | pushd config/watcher_local_test @@ -235,7 +251,10 @@ jobs: kubectl apply -f template.yaml - name: Expose Metrics Endpoint working-directory: lifecycle-manager - if: ${{ matrix.e2e-test == 'kyma-metrics' || matrix.e2e-test == 'purge-metrics' }} + if: ${{ matrix.e2e-test == 'kyma-metrics' || + matrix.e2e-test == 'purge-metrics' || + matrix.e2e-test == 'self-signed-certificate-rotation' + }} run: | kubectl patch svc klm-metrics-service -p '{"spec": {"type": "LoadBalancer"}}' -n kcp-system - name: Run ${{ matrix.e2e-test }} diff --git a/cmd/flags.go b/cmd/flags.go index de71a59f9a0..4e77516b287 100644 --- a/cmd/flags.go +++ b/cmd/flags.go @@ -32,12 +32,12 @@ const ( defaultIstioNamespace = "istio-system" defaultCaCertName = "klm-watcher-serving-cert" defaultCaCertCacheTTL time.Duration = 1 * time.Hour - defaultCertificateDuration time.Duration = 90 * 24 * time.Hour - defaultCertificateRenewBefore time.Duration = 60 * 24 * time.Hour + defaultSelfSignedCertDuration time.Duration = 90 * 24 * time.Hour + defaultSelfSignedCertRenewBefore time.Duration = 60 * 24 * time.Hour ) //nolint:funlen -func defineFlagVar() *FlagVar { +func DefineFlagVar() *FlagVar { flagVar := new(FlagVar) flag.StringVar(&flagVar.metricsAddr, "metrics-bind-address", ":8080", "The address the metric endpoint binds to.") @@ -139,11 +139,12 @@ func defineFlagVar() *FlagVar { "Name of the CA Certificate in Istio Namespace which is used to sign SKR Certificates") flag.DurationVar(&flagVar.caCertCacheTTL, "ca-cert-cache-ttl", defaultCaCertCacheTTL, "The ttl for the CA Certificate Cache") - flag.DurationVar(&flagVar.certificateDuration, "cert-duration", defaultCertificateDuration, - "The lifetime duration of certificate") - flag.DurationVar(&flagVar.certificateRenewBefore, "cert-renew-before", defaultCertificateRenewBefore, - "The duration time to renew certificate") - flag.BoolVar(&flagVar.isKymaManaged, "is-kyma-managed", false, "indicates whether Kyma is managed") + flag.DurationVar(&flagVar.SelfSignedCertDuration, "self-signed-cert-duration", defaultSelfSignedCertDuration, + "The lifetime duration of self-signed certificate") + flag.DurationVar(&flagVar.SelfSignedCertRenewBefore, "self-signed-cert-renew-before", + defaultSelfSignedCertRenewBefore, + "The duration time to renew self-signed certificate") + flag.BoolVar(&flagVar.IsKymaManaged, "is-kyma-managed", false, "indicates whether Kyma is managed") return flagVar } @@ -193,7 +194,7 @@ type FlagVar struct { caCertName string caCertCacheTTL time.Duration enableVerification bool - isKymaManaged bool - certificateDuration time.Duration - certificateRenewBefore time.Duration + IsKymaManaged bool + SelfSignedCertDuration time.Duration + SelfSignedCertRenewBefore time.Duration } diff --git a/cmd/main.go b/cmd/main.go index 88f2a8bfb8f..6df9cf12107 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -86,7 +86,7 @@ func init() { } func main() { - flagVar := defineFlagVar() + flagVar := DefineFlagVar() flag.Parse() ctrl.SetLogger(log.ConfigLogger(int8(flagVar.logLevel), zapcore.Lock(os.Stdout))) if flagVar.pprof { @@ -249,7 +249,7 @@ func setupKymaReconciler(mgr ctrl.Manager, remoteClientCache *remote.ClientCache }, InKCPMode: flagVar.inKCPMode, RemoteSyncNamespace: flagVar.remoteSyncNamespace, - IsManagedKyma: flagVar.isKymaManaged, + IsManagedKyma: flagVar.IsKymaManaged, KymaMetrics: metrics.NewKymaMetrics(), }).SetupWithManager( mgr, options, controller.SetupUpSetting{ @@ -264,7 +264,7 @@ func setupKymaReconciler(mgr ctrl.Manager, remoteClientCache *remote.ClientCache } func createSkrWebhookManager(mgr ctrl.Manager, flagVar *FlagVar) (*watcher.SKRWebhookManifestManager, error) { - caCertificateCache := watcher.NewCertificateCache(flagVar.caCertCacheTTL) + caCertificateCache := watcher.NewCACertificateCache(flagVar.caCertCacheTTL) return watcher.NewSKRWebhookManifestManager(mgr.GetConfig(), mgr.GetScheme(), caCertificateCache, watcher.SkrWebhookManagerConfig{ SKRWatcherPath: flagVar.skrWatcherPath, @@ -277,8 +277,8 @@ func createSkrWebhookManager(mgr ctrl.Manager, flagVar *FlagVar) (*watcher.SKRWe RemoteSyncNamespace: flagVar.remoteSyncNamespace, CACertificateName: flagVar.caCertName, AdditionalDNSNames: strings.Split(flagVar.additionalDNSNames, ","), - Duration: apimetav1.Duration{Duration: flagVar.certificateDuration}, - RenewBefore: apimetav1.Duration{Duration: flagVar.certificateRenewBefore}, + Duration: apimetav1.Duration{Duration: flagVar.SelfSignedCertDuration}, + RenewBefore: apimetav1.Duration{Duration: flagVar.SelfSignedCertRenewBefore}, }, watcher.GatewayConfig{ IstioGatewayName: flagVar.istioGatewayName, IstioGatewayNamespace: flagVar.istioGatewayNamespace, @@ -302,7 +302,7 @@ func setupPurgeReconciler(mgr ctrl.Manager, ResolveRemoteClient: resolveRemoteClientFunc, PurgeFinalizerTimeout: flagVar.purgeFinalizerTimeout, SkipCRDs: matcher.CreateCRDMatcherFrom(flagVar.skipPurgingFor), - IsManagedKyma: flagVar.isKymaManaged, + IsManagedKyma: flagVar.IsKymaManaged, Metrics: metrics.NewPurgeMetrics(), }).SetupWithManager( mgr, options, diff --git a/internal/controller/watcher_controller.go b/internal/controller/watcher_controller.go index 259e4269d7d..828714fc457 100644 --- a/internal/controller/watcher_controller.go +++ b/internal/controller/watcher_controller.go @@ -147,7 +147,7 @@ func (r *WatcherReconciler) handleDeletingState(ctx context.Context, watcherCR * func (r *WatcherReconciler) handleProcessingState(ctx context.Context, watcherCR *v1beta2.Watcher, ) (ctrl.Result, error) { - // Create virtualService in Memory + // CreateSelfSignedCert virtualService in Memory virtualSvc, err := r.IstioClient.NewVirtualService(ctx, watcherCR) if err != nil { return r.updateWatcherState(ctx, watcherCR, shared.StateError, err) diff --git a/internal/declarative/v2/factory.go b/internal/declarative/v2/factory.go index 679aff68f67..21cae91d27a 100644 --- a/internal/declarative/v2/factory.go +++ b/internal/declarative/v2/factory.go @@ -83,7 +83,7 @@ func NewSingletonClients(info *ClusterInfo) (*SingletonClients, error) { discoveryRESTMapper := restmapper.NewDeferredDiscoveryRESTMapper(cachedDiscoveryClient) discoveryShortcutExpander := restmapper.NewShortcutExpander(discoveryRESTMapper, cachedDiscoveryClient) - // Create target cluster client only if not passed. + // CreateSelfSignedCert target cluster client only if not passed. // Clients should be passed only in two cases: // 1. Single cluster mode is enabled. // Since such clients are similar to the root client instance. diff --git a/internal/pkg/metrics/common.go b/internal/pkg/metrics/common.go index a5776d7bc09..b01ad8a6ba7 100644 --- a/internal/pkg/metrics/common.go +++ b/internal/pkg/metrics/common.go @@ -11,7 +11,7 @@ import ( const ( shootIDLabel = "shoot" instanceIDLabel = "instance_id" - kymaNameLabel = "kyma_name" + KymaNameLabel = "kyma_name" ) var ( diff --git a/internal/pkg/metrics/kyma.go b/internal/pkg/metrics/kyma.go index 37cbf6a5b62..693d6d7167f 100644 --- a/internal/pkg/metrics/kyma.go +++ b/internal/pkg/metrics/kyma.go @@ -11,8 +11,8 @@ import ( ) const ( - metricKymaState = "lifecycle_mgr_kyma_state" - metricModuleState = "lifecycle_mgr_module_state" + MetricKymaState = "lifecycle_mgr_kyma_state" + MetricModuleState = "lifecycle_mgr_module_state" stateLabel = "state" moduleNameLabel = "module_name" ) @@ -25,14 +25,14 @@ type KymaMetrics struct { func NewKymaMetrics() *KymaMetrics { kymaMetrics := &KymaMetrics{ kymaStateGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: metricKymaState, + Name: MetricKymaState, Help: "Indicates the Status.state for a given Kyma object", - }, []string{kymaNameLabel, stateLabel, shootIDLabel, instanceIDLabel}), + }, []string{KymaNameLabel, stateLabel, shootIDLabel, instanceIDLabel}), moduleStateGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: metricModuleState, + Name: MetricModuleState, Help: "Indicates the Status.state for modules of Kyma", - }, []string{moduleNameLabel, kymaNameLabel, stateLabel, shootIDLabel, instanceIDLabel}), + }, []string{moduleNameLabel, KymaNameLabel, stateLabel, shootIDLabel, instanceIDLabel}), } ctrlmetrics.Registry.MustRegister(kymaMetrics.kymaStateGauge) ctrlmetrics.Registry.MustRegister(kymaMetrics.moduleStateGauge) @@ -61,10 +61,10 @@ func (k *KymaMetrics) UpdateAll(kyma *v1beta2.Kyma) error { // 'lifecycle_mgr_module_state' metrics for the matching Kyma. func (k *KymaMetrics) CleanupMetrics(kymaName string) { k.kymaStateGauge.DeletePartialMatch(prometheus.Labels{ - kymaNameLabel: kymaName, + KymaNameLabel: kymaName, }) k.moduleStateGauge.DeletePartialMatch(prometheus.Labels{ - kymaNameLabel: kymaName, + KymaNameLabel: kymaName, }) } @@ -72,7 +72,7 @@ func (k *KymaMetrics) CleanupMetrics(kymaName string) { func (k *KymaMetrics) RemoveModuleStateMetrics(kyma *v1beta2.Kyma, moduleName string) { k.moduleStateGauge.DeletePartialMatch(prometheus.Labels{ moduleNameLabel: moduleName, - kymaNameLabel: kyma.Name, + KymaNameLabel: kyma.Name, }) } @@ -81,7 +81,7 @@ func (k *KymaMetrics) setKymaStateGauge(newState shared.State, kymaName, shootID for _, state := range states { newValue := calcStateValue(state, newState) k.kymaStateGauge.With(prometheus.Labels{ - kymaNameLabel: kymaName, + KymaNameLabel: kymaName, shootIDLabel: shootID, instanceIDLabel: instanceID, stateLabel: string(state), @@ -95,7 +95,7 @@ func (k *KymaMetrics) setModuleStateGauge(newState shared.State, moduleName, kym newValue := calcStateValue(state, newState) k.moduleStateGauge.With(prometheus.Labels{ moduleNameLabel: moduleName, - kymaNameLabel: kymaName, + KymaNameLabel: kymaName, shootIDLabel: shootID, instanceIDLabel: instanceID, stateLabel: string(state), diff --git a/internal/pkg/metrics/purge.go b/internal/pkg/metrics/purge.go index 9fff2631356..31d7a0ee1b8 100644 --- a/internal/pkg/metrics/purge.go +++ b/internal/pkg/metrics/purge.go @@ -40,7 +40,7 @@ func NewPurgeMetrics() *PurgeMetrics { purgeErrorGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ Name: metricPurgeError, Help: "Indicates purge errors", - }, []string{kymaNameLabel, shootIDLabel, instanceIDLabel, errorReasonLabel}), + }, []string{KymaNameLabel, shootIDLabel, instanceIDLabel, errorReasonLabel}), } ctrlmetrics.Registry.MustRegister(purgeMetrics.purgeTimeGauge) ctrlmetrics.Registry.MustRegister(purgeMetrics.purgeRequestsCounter) @@ -66,7 +66,7 @@ func (p *PurgeMetrics) UpdatePurgeError(kyma *v1beta2.Kyma, purgeError PurgeErro return fmt.Errorf("%w: %w", errMetric, err) } metric, err := p.purgeErrorGauge.GetMetricWith(prometheus.Labels{ - kymaNameLabel: kyma.Name, + KymaNameLabel: kyma.Name, shootIDLabel: shootID, instanceIDLabel: instanceID, errorReasonLabel: string(purgeError), diff --git a/internal/pkg/metrics/watcher.go b/internal/pkg/metrics/watcher.go index e4a7f7496b8..01be2db872f 100644 --- a/internal/pkg/metrics/watcher.go +++ b/internal/pkg/metrics/watcher.go @@ -7,7 +7,7 @@ import ( ) const ( - CertNotRenewMetrics = "lifecycle_mgr_cert_not_renew" + SelfSignedCertNotRenewMetrics = "lifecycle_mgr_self_signed_cert_not_renew" ) type WatcherMetrics struct { @@ -17,9 +17,9 @@ type WatcherMetrics struct { func NewWatcherMetrics() *WatcherMetrics { watcherMetrics := &WatcherMetrics{ certNotRenewGauge: prometheus.NewGaugeVec(prometheus.GaugeOpts{ - Name: CertNotRenewMetrics, - Help: "Indicates the Certificate CR of related Kyma is not renewed yet", - }, []string{kymaNameLabel}), + Name: SelfSignedCertNotRenewMetrics, + Help: "Indicates the self-signed Certificate of related Kyma is not renewed yet", + }, []string{KymaNameLabel}), } ctrlmetrics.Registry.MustRegister(watcherMetrics.certNotRenewGauge) watchermetrics.Init(ctrlmetrics.Registry) @@ -28,12 +28,12 @@ func NewWatcherMetrics() *WatcherMetrics { func (w *WatcherMetrics) CleanupMetrics(kymaName string) { w.certNotRenewGauge.DeletePartialMatch(prometheus.Labels{ - kymaNameLabel: kymaName, + KymaNameLabel: kymaName, }) } func (w *WatcherMetrics) SetCertNotRenew(kymaName string) { w.certNotRenewGauge.With(prometheus.Labels{ - kymaNameLabel: kymaName, + KymaNameLabel: kymaName, }).Set(1) } diff --git a/pkg/testutils/deployment.go b/pkg/testutils/deployment.go index 5af544d6ca6..2ae3a190450 100644 --- a/pkg/testutils/deployment.go +++ b/pkg/testutils/deployment.go @@ -7,9 +7,14 @@ import ( apiappsv1 "k8s.io/api/apps/v1" "sigs.k8s.io/controller-runtime/pkg/client" + + "github.com/kyma-project/lifecycle-manager/api/v1beta2" ) -var ErrDeploymentNotReady = errors.New("deployment is not ready") +var ( + ErrDeploymentNotReady = errors.New("deployment is not ready") + ErrDeploymentNotStopped = errors.New("deployment is not stopped") +) func DeploymentIsReady(ctx context.Context, name, namespace string, clnt client.Client) error { deploy := &apiappsv1.Deployment{} @@ -23,3 +28,25 @@ func DeploymentIsReady(ctx context.Context, name, namespace string, clnt client. } return ErrDeploymentNotReady } + +func StopDeployment(ctx context.Context, clnt client.Client, + name, namespace string, +) error { + deploy := &apiappsv1.Deployment{} + if err := clnt.Get(ctx, client.ObjectKey{Name: name, Namespace: namespace}, deploy); err != nil { + return fmt.Errorf("could not get deployment: %w", err) + } + if deploy.Status.AvailableReplicas == 0 { + return nil + } + deploy.Spec.Replicas = int32Ptr(0) + err := clnt.Patch(ctx, deploy, client.Apply, + client.ForceOwnership, + client.FieldOwner(v1beta2.OperatorName)) + if err != nil { + return err + } + return ErrDeploymentNotStopped +} + +func int32Ptr(i int32) *int32 { return &i } diff --git a/pkg/testutils/metrics.go b/pkg/testutils/metrics.go new file mode 100644 index 00000000000..8b502a6744c --- /dev/null +++ b/pkg/testutils/metrics.go @@ -0,0 +1,116 @@ +package testutils + +import ( + "context" + "fmt" + "io" + "net/http" + "regexp" + "strconv" + + "github.com/kyma-project/lifecycle-manager/internal/pkg/metrics" +) + +func GetKymaStateMetricCount(ctx context.Context, kymaName, state string) (int, error) { + bodyString, err := getMetricsBody(ctx) + if err != nil { + return 0, err + } + + re := regexp.MustCompile( + metrics.MetricKymaState + `{instance_id="[^"]+",kyma_name="` + kymaName + + `",shoot="[^"]+",state="` + state + + `"} (\d+)`) + return getCount(re, bodyString) +} + +func GetModuleStateMetricCount(ctx context.Context, kymaName, moduleName, state string) (int, error) { + bodyString, err := getMetricsBody(ctx) + if err != nil { + return 0, err + } + re := regexp.MustCompile( + metrics.MetricModuleState + `{instance_id="[^"]+",kyma_name="` + kymaName + + `",module_name="` + moduleName + + `",shoot="[^"]+",state="` + state + + `"} (\d+)`) + return getCount(re, bodyString) +} + +func PurgeMetricsAreAsExpected(ctx context.Context, + timeShouldBeMoreThan float64, + expectedRequests int, +) bool { + correctCount := false + correctTime := false + bodyString, err := getMetricsBody(ctx) + if err != nil { + return false + } + reg := regexp.MustCompile(`lifecycle_mgr_purgectrl_time ([0-9]*\.?[0-9]+)`) + match := reg.FindStringSubmatch(bodyString) + + if len(match) > 1 { + duration, err := strconv.ParseFloat(match[1], 64) + if err == nil && duration > timeShouldBeMoreThan { + correctTime = true + } + } + + reg = regexp.MustCompile(`lifecycle_mgr_purgectrl_requests_total (\d+)`) + match = reg.FindStringSubmatch(bodyString) + + if len(match) > 1 { + count, err := strconv.Atoi(match[1]) + if err == nil && count == expectedRequests { + correctCount = true + } + } + + return correctTime && correctCount +} + +func GetSelfSignedCertNotRenewMetricsGauge(ctx context.Context, kymaName string) (int, error) { + bodyString, err := getMetricsBody(ctx) + if err != nil { + return 0, err + } + + re := regexp.MustCompile(fmt.Sprintf(`%s{%s="%s"} (\d+)`, metrics.SelfSignedCertNotRenewMetrics, + metrics.KymaNameLabel, + kymaName)) + return getCount(re, bodyString) +} + +func getMetricsBody(ctx context.Context) (string, error) { + clnt := &http.Client{} + request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:9081/metrics", nil) + if err != nil { + return "", err + } + response, err := clnt.Do(request) + if err != nil { + return "", err + } + defer response.Body.Close() + bodyBytes, err := io.ReadAll(response.Body) + if err != nil { + return "", err + } + bodyString := string(bodyBytes) + + return bodyString, nil +} + +func getCount(re *regexp.Regexp, bodyString string) (int, error) { + match := re.FindStringSubmatch(bodyString) + if len(match) > 1 { + count, err := strconv.Atoi(match[1]) + if err != nil { + return 0, err + } + return count, nil + } + + return 0, nil +} diff --git a/pkg/testutils/watcher.go b/pkg/testutils/watcher.go index 167989cfa1d..fa2a2be1d6e 100644 --- a/pkg/testutils/watcher.go +++ b/pkg/testutils/watcher.go @@ -97,7 +97,7 @@ func DeleteCertificateSecret(ctx context.Context, namespacedSecretName types.Nam return nil } -func GetCACertificate(ctx context.Context, namespacedCertName types.NamespacedName, k8sClient client.Client, +func GetCertificate(ctx context.Context, namespacedCertName types.NamespacedName, k8sClient client.Client, ) (*certmanagerv1.Certificate, error) { caCert := &certmanagerv1.Certificate{} if err := k8sClient.Get(ctx, namespacedCertName, caCert); err != nil { diff --git a/pkg/watcher/certificate_cache.go b/pkg/watcher/ca_certificate_cache.go similarity index 58% rename from pkg/watcher/certificate_cache.go rename to pkg/watcher/ca_certificate_cache.go index d5f73c7a55b..caed5ca2c73 100644 --- a/pkg/watcher/certificate_cache.go +++ b/pkg/watcher/ca_certificate_cache.go @@ -7,18 +7,18 @@ import ( "github.com/jellydator/ttlcache/v3" ) -type CertificateCache struct { +type CACertificateCache struct { TTL time.Duration *ttlcache.Cache[string, *certmanagerv1.Certificate] } -func NewCertificateCache(ttl time.Duration) *CertificateCache { +func NewCACertificateCache(ttl time.Duration) *CACertificateCache { cache := ttlcache.New[string, *certmanagerv1.Certificate]() go cache.Start() - return &CertificateCache{Cache: cache, TTL: ttl} + return &CACertificateCache{Cache: cache, TTL: ttl} } -func (c *CertificateCache) GetCACertFromCache(caCertName string) *certmanagerv1.Certificate { +func (c *CACertificateCache) GetCACertFromCache(caCertName string) *certmanagerv1.Certificate { value := c.Cache.Get(caCertName) if value != nil { cert := value.Value() @@ -28,6 +28,6 @@ func (c *CertificateCache) GetCACertFromCache(caCertName string) *certmanagerv1. return nil } -func (c *CertificateCache) SetCACertToCache(cert *certmanagerv1.Certificate) { +func (c *CACertificateCache) SetCACertToCache(cert *certmanagerv1.Certificate) { c.Cache.Set(cert.Name, cert, c.TTL) } diff --git a/pkg/watcher/certificate.go b/pkg/watcher/certificate_manager.go similarity index 96% rename from pkg/watcher/certificate.go rename to pkg/watcher/certificate_manager.go index 9883c2242db..afe92073a64 100644 --- a/pkg/watcher/certificate.go +++ b/pkg/watcher/certificate_manager.go @@ -44,7 +44,7 @@ type SubjectAltName struct { type CertificateManager struct { kcpClient client.Client - caCertCache *CertificateCache + caCertCache *CACertificateCache certificateName string secretName string config CertificateConfig @@ -74,7 +74,7 @@ type CertificateSecret struct { // NewCertificateManager returns a new CertificateManager, which can be used for creating a cert-manager Certificates. func NewCertificateManager(kcpClient client.Client, kymaName string, config CertificateConfig, - caCertCache *CertificateCache, + caCertCache *CACertificateCache, ) *CertificateManager { return &CertificateManager{ kcpClient: kcpClient, @@ -85,8 +85,10 @@ func NewCertificateManager(kcpClient client.Client, kymaName string, } } -// Create creates a cert-manager Certificate with a sufficient set of Subject-Alternative-Names. -func (c *CertificateManager) Create(ctx context.Context, kyma *v1beta2.Kyma) (*certmanagerv1.Certificate, error) { +// CreateSelfSignedCert creates a cert-manager Certificate with a sufficient set of Subject-Alternative-Names. +func (c *CertificateManager) CreateSelfSignedCert(ctx context.Context, kyma *v1beta2.Kyma) (*certmanagerv1.Certificate, + error, +) { subjectAltNames, err := c.getSubjectAltNames(kyma) if err != nil { return nil, fmt.Errorf("error get Subject Alternative Name from KymaCR: %w", err) @@ -141,8 +143,6 @@ func (c *CertificateManager) GetSecret(ctx context.Context) (*CertificateSecret, func (c *CertificateManager) patchCertificate(ctx context.Context, subjectAltName *SubjectAltName, ) (*certmanagerv1.Certificate, error) { - // Default Duration 90 days - // Default RenewBefore default 2/3 of Duration issuer, err := c.getIssuer(ctx) if err != nil { return nil, fmt.Errorf("error getting issuer: %w", err) diff --git a/pkg/watcher/skr_webhook_manifest_manager.go b/pkg/watcher/skr_webhook_manifest_manager.go index faa5b3b210f..53f2662a0fd 100644 --- a/pkg/watcher/skr_webhook_manifest_manager.go +++ b/pkg/watcher/skr_webhook_manifest_manager.go @@ -29,7 +29,7 @@ type SKRWebhookManifestManager struct { config SkrWebhookManagerConfig kcpAddr string baseResources []*unstructured.Unstructured - caCertificateCache *CertificateCache + caCertificateCache *CACertificateCache WatcherMetrics *metrics.WatcherMetrics certificateConfig CertificateConfig } @@ -49,7 +49,7 @@ const rawManifestFilePathTpl = "%s/resources.yaml" func NewSKRWebhookManifestManager(kcpConfig *rest.Config, schema *machineryruntime.Scheme, - caCertificateCache *CertificateCache, + caCertificateCache *CACertificateCache, managerConfig SkrWebhookManagerConfig, certificateConfig CertificateConfig, gatewayConfig GatewayConfig, @@ -92,11 +92,11 @@ func (m *SKRWebhookManifestManager) Install(ctx context.Context, kyma *v1beta2.K return fmt.Errorf("failed to get syncContext: %w", err) } - // Create CertificateCR which will be used for mTLS connection from SKR to KCP + // CreateSelfSignedCert CertificateCR which will be used for mTLS connection from SKR to KCP certificateMgr := NewCertificateManager(syncContext.ControlPlaneClient, kyma.Name, m.certificateConfig, m.caCertificateCache) - certificate, err := certificateMgr.Create(ctx, kyma) + certificate, err := certificateMgr.CreateSelfSignedCert(ctx, kyma) if err != nil { return fmt.Errorf("error while patching certificate: %w", err) } diff --git a/tests/e2e_test/Makefile b/tests/e2e_test/Makefile index 9704bc8eaf9..00d323490f0 100644 --- a/tests/e2e_test/Makefile +++ b/tests/e2e_test/Makefile @@ -96,5 +96,8 @@ purge-metrics: module-upgrade: go test -timeout 20m -ginkgo.v -ginkgo.focus "Module Upgrade" -certificate-rotation: - go test -timeout 20m -ginkgo.v -ginkgo.focus "Certificate Rotation" +ca-certificate-rotation: + go test -timeout 20m -ginkgo.v -ginkgo.focus "CA Certificate Rotation" + +self-signed-certificate-rotation: + go test -timeout 20m -ginkgo.v -ginkgo.focus "Self Signed Certificate Rotation" diff --git a/tests/e2e_test/certificate_rotation_test.go b/tests/e2e_test/ca_certificate_rotation_test.go similarity index 93% rename from tests/e2e_test/certificate_rotation_test.go rename to tests/e2e_test/ca_certificate_rotation_test.go index 4d325a0a8dc..22779cd2a5a 100644 --- a/tests/e2e_test/certificate_rotation_test.go +++ b/tests/e2e_test/ca_certificate_rotation_test.go @@ -17,7 +17,7 @@ import ( . "github.com/kyma-project/lifecycle-manager/pkg/testutils" ) -var _ = Describe("Certificate Rotation", Ordered, func() { +var _ = Describe("CA Certificate Rotation", Ordered, func() { kyma := NewKymaWithSyncLabel("kyma-sample", "kcp-system", v1beta2.DefaultChannel, v1beta2.SyncStrategyLocalSecret) InitEmptyKymaBeforeAll(kyma) @@ -53,7 +53,7 @@ var _ = Describe("Certificate Rotation", Ordered, func() { Name: caCertName, Namespace: "istio-system", } - caCertificate, err = GetCACertificate(ctx, namespacedCertName, controlPlaneClient) + caCertificate, err = GetCertificate(ctx, namespacedCertName, controlPlaneClient) Expect(err).NotTo(HaveOccurred()) Eventually(CertificateSecretIsCreatedAfter). WithContext(ctx). diff --git a/tests/e2e_test/self_signed_certificate_rotation_test.go b/tests/e2e_test/self_signed_certificate_rotation_test.go new file mode 100644 index 00000000000..d96a08c2d46 --- /dev/null +++ b/tests/e2e_test/self_signed_certificate_rotation_test.go @@ -0,0 +1,49 @@ +package e2e_test + +import ( + "fmt" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/types" + + "github.com/kyma-project/lifecycle-manager/api/shared" + "github.com/kyma-project/lifecycle-manager/api/v1beta2" + "github.com/kyma-project/lifecycle-manager/internal/pkg/metrics" + . "github.com/kyma-project/lifecycle-manager/pkg/testutils" + "github.com/kyma-project/lifecycle-manager/pkg/watcher" +) + +var _ = Describe("Self Signed Certificate Rotation", Ordered, func() { + kyma := NewKymaWithSyncLabel("kyma-sample", "kcp-system", v1beta2.DefaultChannel, + v1beta2.SyncStrategyLocalSecret) + InitEmptyKymaBeforeAll(kyma) + CleanupKymaAfterAll(kyma) + + Context("Given Kyma deployed in KCP", func() { + It("When self signed certificate exists", func() { + namespacedCertName := types.NamespacedName{ + Name: watcher.ResolveTLSCertName(kyma.Name), + Namespace: "istio-system", + } + Eventually(GetCertificate). + WithContext(ctx). + WithArguments(namespacedCertName, controlPlaneClient). + Should(Succeed()) + }) + It("Then disable cert manager operator to prevent certificate auto renewed", func() { + Eventually(StopDeployment). + WithContext(ctx). + WithArguments("cert-manager", "cert-manager"). + Should(Succeed()) + }) + It(fmt.Sprintf("Then %s metric increased to 1", metrics.SelfSignedCertNotRenewMetrics), func() { + Eventually(GetSelfSignedCertNotRenewMetricsGauge). + WithContext(ctx). + WithTimeout(5*time.Minute). + WithArguments(kyma.GetName(), string(shared.StateReady)). + Should(Equal(1)) + }) + }) +}) diff --git a/tests/e2e_test/utils_test.go b/tests/e2e_test/utils_test.go index 052d0e5d943..274f7cfb8ef 100644 --- a/tests/e2e_test/utils_test.go +++ b/tests/e2e_test/utils_test.go @@ -4,10 +4,6 @@ import ( "context" "errors" "fmt" - "io" - "net/http" - "regexp" - "strconv" "strings" "time" @@ -219,50 +215,6 @@ func SetFinalizer(name, namespace, group, version, kind string, finalizers []str return clnt.Update(ctx, resourceCR) } -func GetKymaStateMetricCount(ctx context.Context, kymaName, state string) (int, error) { - bodyString, err := getMetricsBody(ctx) - if err != nil { - return 0, err - } - - re := regexp.MustCompile( - `lifecycle_mgr_kyma_state{instance_id="[^"]+",kyma_name="` + kymaName + `",shoot="[^"]+",state="` + state + - `"} (\d+)`) - match := re.FindStringSubmatch(bodyString) - if len(match) > 1 { - count, err := strconv.Atoi(match[1]) - if err != nil { - return 0, err - } - return count, nil - } - - return 0, nil -} - -func GetModuleStateMetricCount(ctx context.Context, kymaName, moduleName, state string) (int, error) { - bodyString, err := getMetricsBody(ctx) - if err != nil { - return 0, err - } - - re := regexp.MustCompile( - `lifecycle_mgr_module_state{instance_id="[^"]+",kyma_name="` + kymaName + - `",module_name="` + moduleName + - `",shoot="[^"]+",state="` + state + - `"} (\d+)`) - match := re.FindStringSubmatch(bodyString) - if len(match) > 1 { - count, err := strconv.Atoi(match[1]) - if err != nil { - return 0, err - } - return count, nil - } - - return 0, nil -} - func CheckSampleCRIsInState(ctx context.Context, name, namespace string, clnt client.Client, expectedState string, ) error { @@ -273,56 +225,3 @@ func CheckSampleCRIsInState(ctx context.Context, name, namespace string, clnt cl clnt, expectedState) } - -func getMetricsBody(ctx context.Context) (string, error) { - clnt := &http.Client{} - request, err := http.NewRequestWithContext(ctx, http.MethodGet, "http://localhost:9081/metrics", nil) - if err != nil { - return "", err - } - response, err := clnt.Do(request) - if err != nil { - return "", err - } - defer response.Body.Close() - bodyBytes, err := io.ReadAll(response.Body) - if err != nil { - return "", err - } - bodyString := string(bodyBytes) - - return bodyString, nil -} - -func PurgeMetricsAreAsExpected(ctx context.Context, - timeShouldBeMoreThan float64, - expectedRequests int, -) bool { - correctCount := false - correctTime := false - bodyString, err := getMetricsBody(ctx) - if err != nil { - return false - } - reg := regexp.MustCompile(`lifecycle_mgr_purgectrl_time ([0-9]*\.?[0-9]+)`) - match := reg.FindStringSubmatch(bodyString) - - if len(match) > 1 { - duration, err := strconv.ParseFloat(match[1], 64) - if err == nil && duration > timeShouldBeMoreThan { - correctTime = true - } - } - - reg = regexp.MustCompile(`lifecycle_mgr_purgectrl_requests_total (\d+)`) - match = reg.FindStringSubmatch(bodyString) - - if len(match) > 1 { - count, err := strconv.Atoi(match[1]) - if err == nil && count == expectedRequests { - correctCount = true - } - } - - return correctTime && correctCount -} diff --git a/tests/integration/controller/controlplane/kyma_remote_sync_test.go b/tests/integration/controller/controlplane/kyma_remote_sync_test.go index c3fa94f1ebd..2487a1a7cef 100644 --- a/tests/integration/controller/controlplane/kyma_remote_sync_test.go +++ b/tests/integration/controller/controlplane/kyma_remote_sync_test.go @@ -175,7 +175,8 @@ var _ = Describe("Kyma sync into Remote Cluster", Ordered, func() { By("Remote Kyma contains correct conditions for Modules") Eventually(kymaHasCondition, Timeout, Interval). - WithArguments(runtimeClient, v1beta2.ConditionTypeModules, string(v1beta2.ConditionReason), apimetav1.ConditionTrue, + WithArguments(runtimeClient, v1beta2.ConditionTypeModules, string(v1beta2.ConditionReason), + apimetav1.ConditionTrue, remoteKyma.GetName(), remoteKyma.GetNamespace()). Should(Succeed()) }) @@ -208,7 +209,7 @@ var _ = Describe("Kyma sync into Remote Cluster", Ordered, func() { }) It("Enable Custom ModuleTemplate in SKR", func() { - By("Create SKRCustomTemplate in SKR") + By("CreateSelfSignedCert SKRCustomTemplate in SKR") SKRCustomTemplate.Namespace = kyma.Namespace Eventually(runtimeClient.Create, Timeout, Interval). WithContext(ctx). diff --git a/tests/integration/controller/purge/purge_controller_test.go b/tests/integration/controller/purge/purge_controller_test.go index f1d9e0f872f..53fee19be22 100644 --- a/tests/integration/controller/purge/purge_controller_test.go +++ b/tests/integration/controller/purge/purge_controller_test.go @@ -28,7 +28,7 @@ var _ = Describe("When kyma is not deleted within configured timeout", Ordered, var issuer1 *unstructured.Unstructured var issuer2 *unstructured.Unstructured - By("Create the Kyma object", func() { + By("CreateSelfSignedCert the Kyma object", func() { Expect(controlPlaneClient.Create(ctx, kyma)).Should(Succeed()) if updateRequired := kyma.EnsureLabelsAndFinalizers(); updateRequired { var err error @@ -44,7 +44,7 @@ var _ = Describe("When kyma is not deleted within configured timeout", Ordered, } }) - By("Create some CR with finalizer(s)", func() { + By("CreateSelfSignedCert some CR with finalizer(s)", func() { issuer1 = createIssuerFor(kyma, "1") Expect(issuer1).NotTo(BeNil()) Expect(controlPlaneClient.Create(ctx, issuer1)).Should(Succeed()) @@ -107,7 +107,7 @@ var _ = Describe("When kyma is deleted before configured timeout", Ordered, func } }) - By("Create some CR with finalizer(s)", func() { + By("CreateSelfSignedCert some CR with finalizer(s)", func() { issuer1 = createIssuerFor(kyma, "1") Expect(issuer1).NotTo(BeNil()) Expect(controlPlaneClient.Create(ctx, issuer1)).Should(Succeed()) @@ -167,7 +167,7 @@ var _ = Describe("When some important CRDs should be skipped", Ordered, func() { } }) - By("Create some CR with finalizer(s)", func() { + By("CreateSelfSignedCert some CR with finalizer(s)", func() { issuer1 = createIssuerFor(kyma, "1") Expect(issuer1).NotTo(BeNil()) Expect(controlPlaneClient.Create(ctx, issuer1)).Should(Succeed()) diff --git a/tests/integration/controller/withwatcher/suite_with_watcher_test.go b/tests/integration/controller/withwatcher/suite_with_watcher_test.go index 78f3924a4ec..c8c23c26b8d 100644 --- a/tests/integration/controller/withwatcher/suite_with_watcher_test.go +++ b/tests/integration/controller/withwatcher/suite_with_watcher_test.go @@ -199,7 +199,7 @@ var _ = BeforeSuite(func() { LocalGatewayPortOverwrite: "", } - caCertCache := watcher.NewCertificateCache(5 * time.Minute) + caCertCache := watcher.NewCACertificateCache(5 * time.Minute) skrWebhookChartManager, err := watcher.NewSKRWebhookManifestManager( restCfg, k8sclientscheme.Scheme, diff --git a/tests/integration/declarative/declarative_test.go b/tests/integration/declarative/declarative_test.go index 411bcb0e424..d47a4ea8146 100644 --- a/tests/integration/declarative/declarative_test.go +++ b/tests/integration/declarative/declarative_test.go @@ -124,20 +124,22 @@ var _ = Describe( fmt.Sprintf("Starting Controller and Testing Declarative Reconciler (Run %s)", runID), tableTest, Entry( - "Create simple raw manifest with a different Control Plane and Runtime Client", + "CreateSelfSignedCert simple raw manifest with a different Control Plane and Runtime Client", declarativetest.TestAPISpec{ManifestName: "custom-client"}, DefaultSpec(filepath.Join(testSamplesDir, "raw-manifest.yaml"), ocirefSynced, RenderModeRaw), - []Option{WithRemoteTargetCluster( - func(context.Context, Object) (*ClusterInfo, error) { - return &ClusterInfo{ - Config: cfg, - }, nil - }, - )}, + []Option{ + WithRemoteTargetCluster( + func(context.Context, Object) (*ClusterInfo, error) { + return &ClusterInfo{ + Config: cfg, + }, nil + }, + ), + }, nil, ), Entry( - "Create simple Raw manifest", + "CreateSelfSignedCert simple Raw manifest", declarativetest.TestAPISpec{ManifestName: "simple-raw"}, DefaultSpec(filepath.Join(testSamplesDir, "raw-manifest.yaml"), ocirefSynced, RenderModeRaw), []Option{}, @@ -164,13 +166,15 @@ var _ = Describe("Test Manifest Reconciliation for module deletion", Ordered, fu key := client.ObjectKeyFromObject(obj) - opts := []Option{WithRemoteTargetCluster( - func(context.Context, Object) (*ClusterInfo, error) { - return &ClusterInfo{ - Config: cfg, - }, nil - }, - )} + opts := []Option{ + WithRemoteTargetCluster( + func(context.Context, Object) (*ClusterInfo, error) { + return &ClusterInfo{ + Config: cfg, + }, nil + }, + ), + } source := WithSpecResolver(DefaultSpec(filepath.Join(testSamplesDir, "raw-manifest.yaml"), ocirefSynced, RenderModeRaw)) oldDeployedResources, err := internal.ParseManifestToObjects(path.Join(testSamplesDir, "raw-manifest.yaml")) @@ -289,9 +293,11 @@ func StartDeclarativeReconcilerForRun( Expect( ctrl.NewControllerManagedBy(mgr).WithEventFilter(testWatchPredicate). WithOptions( - ctrlruntime.Options{RateLimiter: workqueue.NewMaxOfRateLimiter( - &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(30), 200)}, - )}, + ctrlruntime.Options{ + RateLimiter: workqueue.NewMaxOfRateLimiter( + &workqueue.BucketRateLimiter{Limiter: rate.NewLimiter(rate.Limit(30), 200)}, + ), + }, ). For(&declarativetest.TestAPI{}).Complete(reconciler), ).To(Succeed()) diff --git a/tests/integration/security/san_pinning_test.go b/tests/integration/security/san_pinning_test.go index 117629305ec..21f37f0a5fc 100644 --- a/tests/integration/security/san_pinning_test.go +++ b/tests/integration/security/san_pinning_test.go @@ -48,9 +48,11 @@ func TestRequestVerifier_verifySAN(t *testing.T) { args: args{ certificate: &x509.Certificate{ IPAddresses: nil, - URIs: []*url.URL{{ - Path: "example.domain.com", - }}, + URIs: []*url.URL{ + { + Path: "example.domain.com", + }, + }, DNSNames: nil, }, kymaDomain: "example.domain.com", @@ -74,9 +76,11 @@ func TestRequestVerifier_verifySAN(t *testing.T) { args: args{ certificate: &x509.Certificate{ IPAddresses: []net.IP{{192, 168, 127, 1}, {192, 168, 127, 2}}, - URIs: []*url.URL{{ - Path: "wrong.domain.com", - }}, + URIs: []*url.URL{ + { + Path: "wrong.domain.com", + }, + }, DNSNames: []string{"wrong.domain.com"}, }, kymaDomain: "example.domain.com", @@ -167,13 +171,13 @@ var _ = Describe("Verify Request using SAN", Ordered, func() { for _, tt := range tests { test := tt It(test.name, func() { - // Create Request Verifier + // CreateSelfSignedCert Request Verifier verifier := &security.RequestVerifier{ Client: k8sClient, Log: zapr.NewLogger(zapLog), } - // Create Kyma CR + // CreateSelfSignedCert Kyma CR if test.kyma != nil { Expect(k8sClient.Create(context.TODO(), test.kyma)).Should(Succeed()) } diff --git a/tests/integration/watcher/certificate_test.go b/tests/integration/watcher/certificate_test.go index 2a341e1643b..b61bfeac241 100644 --- a/tests/integration/watcher/certificate_test.go +++ b/tests/integration/watcher/certificate_test.go @@ -15,7 +15,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("Create Watcher Certificates", Ordered, func() { +var _ = Describe("CreateSelfSignedCert Watcher Certificates", Ordered, func() { const caCertName = "klm-watcher-serving-cert" tests := []struct { @@ -84,9 +84,9 @@ var _ = Describe("Create Watcher Certificates", Ordered, func() { RenewBefore: apimetav1.Duration{Duration: 5 * time.Minute}, } cert := watcher.NewCertificateManager(controlPlaneClient, - test.kyma.Name, config, watcher.NewCertificateCache(1*time.Minute)) + test.kyma.Name, config, watcher.NewCACertificateCache(1*time.Minute)) - _, err := cert.Create(ctx, test.kyma) + _, err := cert.CreateSelfSignedCert(ctx, test.kyma) if test.wantCreateErr { Expect(err).Should(HaveOccurred()) return