From fe579712c75996c554a44f76c6085732e0d51f9d Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Wed, 24 Jul 2024 15:50:12 -0700 Subject: [PATCH 1/2] Use higher QPS for secrets reencryption Signed-off-by: Derek Nola --- pkg/daemons/control/server.go | 3 +-- pkg/secretsencrypt/controller.go | 15 +++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/pkg/daemons/control/server.go b/pkg/daemons/control/server.go index b99fc7cf1eb3..7dd5ae64fa87 100644 --- a/pkg/daemons/control/server.go +++ b/pkg/daemons/control/server.go @@ -68,8 +68,7 @@ func Server(ctx context.Context, cfg *config.Control) error { if err := secretsencrypt.Register(ctx, controllerName, cfg, - cfg.Runtime.Core.Core().V1().Node(), - cfg.Runtime.Core.Core().V1().Secret()); err != nil { + cfg.Runtime.Core.Core().V1().Node()); err != nil { logrus.Errorf("Failed to register %s controller: %v", controllerName, err) } } diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go index 03976d7f9e02..824081d5c85c 100644 --- a/pkg/secretsencrypt/controller.go +++ b/pkg/secretsencrypt/controller.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + typev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/pager" "k8s.io/client-go/tools/record" @@ -38,7 +39,7 @@ type handler struct { ctx context.Context controlConfig *config.Control nodes coreclient.NodeController - secrets coreclient.SecretController + cclient typev1.CoreV1Interface recorder record.EventRecorder } @@ -47,12 +48,14 @@ func Register( controllerName string, controlConfig *config.Control, nodes coreclient.NodeController, - secrets coreclient.SecretController, ) error { restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor) if err != nil { return err } + // For secrets we need a much higher QPS than what wrangler provides, so we create a new clientset + restConfig.QPS = 200 + restConfig.Burst = 200 k8s, err := kubernetes.NewForConfig(restConfig) if err != nil { return err @@ -62,7 +65,7 @@ func Register( ctx: ctx, controlConfig: controlConfig, nodes: nodes, - secrets: secrets, + cclient: k8s.CoreV1(), recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName, metav1.NamespaceDefault), } @@ -217,7 +220,7 @@ func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) ( func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error { secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { - return h.secrets.List(metav1.NamespaceAll, opts) + return h.cclient.Secrets(metav1.NamespaceAll).List(h.ctx, metav1.ListOptions{}) })) secretPager.PageSize = secretListPageSize @@ -227,10 +230,10 @@ func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error { if !ok { return errors.New("failed to convert object to Secret") } - if _, err := h.secrets.Update(secret); err != nil && !apierrors.IsConflict(err) { + if _, err := h.cclient.Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { return fmt.Errorf("failed to update secret: %v", err) } - if i != 0 && i%10 == 0 { + if i != 0 && i%50 == 0 { h.recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i) } i++ From 9615189ad70fdcd8439a402d08750cb8b5e22f59 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 25 Jul 2024 09:33:12 -0700 Subject: [PATCH 2/2] Address feedback Signed-off-by: Derek Nola --- pkg/secretsencrypt/controller.go | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go index 824081d5c85c..ac820fdb798f 100644 --- a/pkg/secretsencrypt/controller.go +++ b/pkg/secretsencrypt/controller.go @@ -18,7 +18,6 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" - typev1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/pager" "k8s.io/client-go/tools/record" @@ -39,7 +38,7 @@ type handler struct { ctx context.Context controlConfig *config.Control nodes coreclient.NodeController - cclient typev1.CoreV1Interface + k8s *kubernetes.Clientset recorder record.EventRecorder } @@ -65,7 +64,7 @@ func Register( ctx: ctx, controlConfig: controlConfig, nodes: nodes, - cclient: k8s.CoreV1(), + k8s: k8s, recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName, metav1.NamespaceDefault), } @@ -220,7 +219,7 @@ func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) ( func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error { secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { - return h.cclient.Secrets(metav1.NamespaceAll).List(h.ctx, metav1.ListOptions{}) + return h.k8s.CoreV1().Secrets(metav1.NamespaceAll).List(h.ctx, opts) })) secretPager.PageSize = secretListPageSize @@ -230,7 +229,7 @@ func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error { if !ok { return errors.New("failed to convert object to Secret") } - if _, err := h.cclient.Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { + if _, err := h.k8s.CoreV1().Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { return fmt.Errorf("failed to update secret: %v", err) } if i != 0 && i%50 == 0 {