[Release-1.28] Backports for August 2024

pkg/agent/run.go

@@ -284,6 +284,18 @@ func RunStandalone(ctx context.Context, cfg cmds.Agent) error {
 		return err
 	}
 
+	if nodeConfig.SupervisorMetrics {
+		if err := metrics.DefaultMetrics.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to serve metrics")
+		}
+	}
+
+	if nodeConfig.EnablePProf {
+		if err := profile.DefaultProfiler.Start(ctx, nodeConfig); err != nil {
+			return errors.Wrap(err, "failed to serve pprof")
+		}
+	}
+
 	<-ctx.Done()
 	return ctx.Err()
 }

pkg/secretsencrypt/controller.go

@@ -18,6 +18,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/tools/clientcmd"
 	"k8s.io/client-go/tools/pager"
 	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/retry"
@@ -37,22 +38,33 @@ type handler struct {
 	ctx           context.Context
 	controlConfig *config.Control
 	nodes         coreclient.NodeController
-	secrets       coreclient.SecretController
+	k8s           *kubernetes.Clientset
 	recorder      record.EventRecorder
 }
 
 func Register(
 	ctx context.Context,
-	k8s kubernetes.Interface,
 	controlConfig *config.Control,
 	nodes coreclient.NodeController,
-	secrets coreclient.SecretController,
 ) error {
+
+	restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor)
+	if err != nil {
+		return err
+	}
+	// For secrets we need a much higher QPS than what wrangler provides, so we create a new clientset
+	restConfig.QPS = 200
+	restConfig.Burst = 200
+	k8s, err := kubernetes.NewForConfig(restConfig)
+	if err != nil {
+		return err
+	}
+
 	h := &handler{
 		ctx:           ctx,
 		controlConfig: controlConfig,
 		nodes:         nodes,
-		secrets:       secrets,
+		k8s:           k8s,
 		recorder:      util.BuildControllerEventRecorder(k8s, controllerAgentName, metav1.NamespaceDefault),
 	}
 
@@ -210,7 +222,7 @@ func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) (
 
 func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error {
 	secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) {
-		return h.secrets.List(metav1.NamespaceAll, opts)
+		return h.k8s.CoreV1().Secrets(metav1.NamespaceAll).List(h.ctx, opts)
 	}))
 	secretPager.PageSize = secretListPageSize
 
@@ -220,10 +232,10 @@ func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error {
 		if !ok {
 			return errors.New("failed to convert object to Secret")
 		}
-		if _, err := h.secrets.Update(secret); err != nil && !apierrors.IsConflict(err) {
+		if _, err := h.k8s.CoreV1().Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) {
 			return fmt.Errorf("failed to update secret: %v", err)
 		}
-		if i != 0 && i%10 == 0 {
+		if i != 0 && i%50 == 0 {
 			h.recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i)
 		}
 		i++

pkg/server/server.go

@@ -247,10 +247,8 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error {
 
 	if config.ControlConfig.EncryptSecrets {
 		if err := secretsencrypt.Register(ctx,
-			sc.K8s,
 			&config.ControlConfig,
-			sc.Core.Core().V1().Node(),
-			sc.Core.Core().V1().Secret()); err != nil {
+			sc.Core.Core().V1().Node()); err != nil {
 			return err
 		}
 	}

tests/docker/test-run-hardened

@@ -23,8 +23,6 @@ export SERVER_ARGS="--selinux=true \
 --kube-apiserver-arg=audit-log-maxage=30 \
 --kube-apiserver-arg=audit-log-maxbackup=10 \
 --kube-apiserver-arg=audit-log-maxsize=100 \
---kube-apiserver-arg=request-timeout=300s \
---kube-apiserver-arg=service-account-lookup=true \
 --kube-apiserver-arg=enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount \
 --kube-apiserver-arg=admission-control-config-file=/opt/rancher/k3s/cluster-level-pss.yaml \
 --kube-controller-manager-arg=terminated-pod-gc-threshold=10 \

tests/e2e/vagrantdefaults.rb

@@ -53,29 +53,36 @@ def getHardenedArg(vm, hardened, scripts_location)
     secrets-encryption: true
     kube-controller-manager-arg:
       - 'terminated-pod-gc-threshold=10'
-      - 'use-service-account-credentials=true'
     kubelet-arg:
       - 'streaming-connection-idle-timeout=5m'
       - 'make-iptables-util-chains=true'
       - 'event-qps=0'
+      - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305"
     kube-apiserver-arg:
       - 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log'
       - 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml'
       - 'audit-log-maxage=30'
       - 'audit-log-maxbackup=10'
       - 'audit-log-maxsize=100'
-      - 'service-account-lookup=true'
   HARD
-  if hardened == "psp"
-    vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
-    hardened_arg += "  - 'enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount,PodSecurityPolicy'"
-  elsif hardened == "psa"
+
+  if hardened == "psa" || hardened == "true"
     vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh", args: [ "psa" ]
     hardened_arg += "  - 'admission-control-config-file=/var/lib/rancher/k3s/server/psa.yaml'"
+  elsif hardened == "psp"
+      vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh"
+      hardened_arg += "  - 'enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount,PodSecurityPolicy'"
   else 
     puts "Invalid E2E_HARDENED option"
     exit 1
   end
+  if vm.box.to_s.include?("generic/ubuntu")
+    vm.provision "Install kube-bench", type: "shell", inline: <<-SHELL
+    export KBV=0.8.0
+    curl -L "https://github.com/aquasecurity/kube-bench/releases/download/v${KBV}/kube-bench_${KBV}_linux_amd64.deb" -o "kube-bench_${KBV}_linux_amd64.deb"
+    dpkg -i "./kube-bench_${KBV}_linux_amd64.deb"
+    SHELL
+  end
   return hardened_arg
 end