Use higher QPS for secrets reencryption (k3s-io#10571)

* Use higher QPS for secrets reencryption Signed-off-by: Derek Nola <derek.nola@suse.com>
dereknola · Aug 5, 2024 · e5789d1 · e5789d1
1 parent e5854fe
commit e5789d1
Show file tree

Hide file tree

Showing 2 changed files with 20 additions and 10 deletions.
diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go
@@ -18,6 +18,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/pager"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/retry"
@@ -37,22 +38,33 @@ type handler struct {
 	ctx           context.Context
 	controlConfig *config.Control
 	nodes         coreclient.NodeController
-	secrets       coreclient.SecretController
+	k8s           *kubernetes.Clientset
 	recorder      record.EventRecorder
 }
 
 func Register(
 	ctx context.Context,
-	k8s kubernetes.Interface,
 	controlConfig *config.Control,
 	nodes coreclient.NodeController,
-	secrets coreclient.SecretController,
 ) error {
+
+	restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor)
+	if err != nil {
+		return err
+	}
+	// For secrets we need a much higher QPS than what wrangler provides, so we create a new clientset
+	restConfig.QPS = 200
+	restConfig.Burst = 200
+	k8s, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
 	h := &handler{
 		ctx:           ctx,
 		controlConfig: controlConfig,
 		nodes:         nodes,
-		secrets:       secrets,
+		k8s:           k8s,
 		recorder:      util.BuildControllerEventRecorder(k8s, controllerAgentName, metav1.NamespaceDefault),
 	}
 
@@ -210,7 +222,7 @@ func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (
 
 func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error {
 	secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
-		return h.secrets.List(metav1.NamespaceAll, opts)
+		return h.k8s.CoreV1().Secrets(metav1.NamespaceAll).List(h.ctx, opts)
 	}))
 	secretPager.PageSize = secretListPageSize
 
@@ -220,10 +232,10 @@ func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error {
 		if !ok {
 			return errors.New("failed to convert object to Secret")
 		}
-		if _, err := h.secrets.Update(secret); err != nil && !apierrors.IsConflict(err) {
+		if _, err := h.k8s.CoreV1().Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) {
 			return fmt.Errorf("failed to update secret: %v", err)
 		}
-		if i != 0 && i%10 == 0 {
+		if i != 0 && i%50 == 0 {
 			h.recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i)
 		}
 		i++

diff --git a/pkg/server/server.go b/pkg/server/server.go
@@ -247,10 +247,8 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 
 	if config.ControlConfig.EncryptSecrets {
 		if err := secretsencrypt.Register(ctx,
-			sc.K8s,
 			&config.ControlConfig,
-			sc.Core.Core().V1().Node(),
-			sc.Core.Core().V1().Secret()); err != nil {
+			sc.Core.Core().V1().Node()); err != nil {
 			return err
 		}
 	}