From e5854fe499a06130d174a72282422e985ece3560 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Thu, 25 Jul 2024 09:37:03 -0700 Subject: [PATCH 1/3] Enhance E2E Hardened option (#10558) * Remove unnecessary hardened arguments Signed-off-by: Derek Nola * slim down hardened arguments Signed-off-by: Derek Nola --------- Signed-off-by: Derek Nola --- tests/docker/test-run-hardened | 2 -- tests/e2e/vagrantdefaults.rb | 19 +++++++++++++------ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/tests/docker/test-run-hardened b/tests/docker/test-run-hardened index 53aac273fd44..d8e101ec323a 100755 --- a/tests/docker/test-run-hardened +++ b/tests/docker/test-run-hardened @@ -23,8 +23,6 @@ export SERVER_ARGS="--selinux=true \ --kube-apiserver-arg=audit-log-maxage=30 \ --kube-apiserver-arg=audit-log-maxbackup=10 \ --kube-apiserver-arg=audit-log-maxsize=100 \ ---kube-apiserver-arg=request-timeout=300s \ ---kube-apiserver-arg=service-account-lookup=true \ --kube-apiserver-arg=enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount \ --kube-apiserver-arg=admission-control-config-file=/opt/rancher/k3s/cluster-level-pss.yaml \ --kube-controller-manager-arg=terminated-pod-gc-threshold=10 \ diff --git a/tests/e2e/vagrantdefaults.rb b/tests/e2e/vagrantdefaults.rb index 4238725cf638..17d3e370bb4f 100644 --- a/tests/e2e/vagrantdefaults.rb +++ b/tests/e2e/vagrantdefaults.rb @@ -53,29 +53,36 @@ def getHardenedArg(vm, hardened, scripts_location) secrets-encryption: true kube-controller-manager-arg: - 'terminated-pod-gc-threshold=10' - - 'use-service-account-credentials=true' kubelet-arg: - 'streaming-connection-idle-timeout=5m' - 'make-iptables-util-chains=true' - 'event-qps=0' + - "tls-cipher-suites=TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305,TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305" kube-apiserver-arg: - 'audit-log-path=/var/lib/rancher/k3s/server/logs/audit.log' - 'audit-policy-file=/var/lib/rancher/k3s/server/audit.yaml' - 'audit-log-maxage=30' - 'audit-log-maxbackup=10' - 'audit-log-maxsize=100' - - 'service-account-lookup=true' HARD - if hardened == "psp" - vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh" - hardened_arg += " - 'enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount,PodSecurityPolicy'" - elsif hardened == "psa" + + if hardened == "psa" || hardened == "true" vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh", args: [ "psa" ] hardened_arg += " - 'admission-control-config-file=/var/lib/rancher/k3s/server/psa.yaml'" + elsif hardened == "psp" + vm.provision "Set kernel parameters", type: "shell", path: scripts_location + "/harden.sh" + hardened_arg += " - 'enable-admission-plugins=NodeRestriction,NamespaceLifecycle,ServiceAccount,PodSecurityPolicy'" else puts "Invalid E2E_HARDENED option" exit 1 end + if vm.box.to_s.include?("generic/ubuntu") + vm.provision "Install kube-bench", type: "shell", inline: <<-SHELL + export KBV=0.8.0 + curl -L "https://github.com/aquasecurity/kube-bench/releases/download/v${KBV}/kube-bench_${KBV}_linux_amd64.deb" -o "kube-bench_${KBV}_linux_amd64.deb" + dpkg -i "./kube-bench_${KBV}_linux_amd64.deb" + SHELL + end return hardened_arg end From e5789d19d026d4efcb036c157e1c41fc1d2d7544 Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 26 Jul 2024 12:07:26 -0700 Subject: [PATCH 2/3] Use higher QPS for secrets reencryption (#10571) * Use higher QPS for secrets reencryption Signed-off-by: Derek Nola --- pkg/secretsencrypt/controller.go | 26 +++++++++++++++++++------- pkg/server/server.go | 4 +--- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/pkg/secretsencrypt/controller.go b/pkg/secretsencrypt/controller.go index 3a9d7018ec03..3f7ff0087849 100644 --- a/pkg/secretsencrypt/controller.go +++ b/pkg/secretsencrypt/controller.go @@ -18,6 +18,7 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/pager" "k8s.io/client-go/tools/record" "k8s.io/client-go/util/retry" @@ -37,22 +38,33 @@ type handler struct { ctx context.Context controlConfig *config.Control nodes coreclient.NodeController - secrets coreclient.SecretController + k8s *kubernetes.Clientset recorder record.EventRecorder } func Register( ctx context.Context, - k8s kubernetes.Interface, controlConfig *config.Control, nodes coreclient.NodeController, - secrets coreclient.SecretController, ) error { + + restConfig, err := clientcmd.BuildConfigFromFlags("", controlConfig.Runtime.KubeConfigSupervisor) + if err != nil { + return err + } + // For secrets we need a much higher QPS than what wrangler provides, so we create a new clientset + restConfig.QPS = 200 + restConfig.Burst = 200 + k8s, err := kubernetes.NewForConfig(restConfig) + if err != nil { + return err + } + h := &handler{ ctx: ctx, controlConfig: controlConfig, nodes: nodes, - secrets: secrets, + k8s: k8s, recorder: util.BuildControllerEventRecorder(k8s, controllerAgentName, metav1.NamespaceDefault), } @@ -210,7 +222,7 @@ func (h *handler) validateReencryptStage(node *corev1.Node, annotation string) ( func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error { secretPager := pager.New(pager.SimplePageFunc(func(opts metav1.ListOptions) (runtime.Object, error) { - return h.secrets.List(metav1.NamespaceAll, opts) + return h.k8s.CoreV1().Secrets(metav1.NamespaceAll).List(h.ctx, opts) })) secretPager.PageSize = secretListPageSize @@ -220,10 +232,10 @@ func (h *handler) updateSecrets(nodeRef *corev1.ObjectReference) error { if !ok { return errors.New("failed to convert object to Secret") } - if _, err := h.secrets.Update(secret); err != nil && !apierrors.IsConflict(err) { + if _, err := h.k8s.CoreV1().Secrets(secret.Namespace).Update(h.ctx, secret, metav1.UpdateOptions{}); err != nil && !apierrors.IsConflict(err) { return fmt.Errorf("failed to update secret: %v", err) } - if i != 0 && i%10 == 0 { + if i != 0 && i%50 == 0 { h.recorder.Eventf(nodeRef, corev1.EventTypeNormal, secretsProgressEvent, "reencrypted %d secrets", i) } i++ diff --git a/pkg/server/server.go b/pkg/server/server.go index 29dc25fd285d..d12b3009c749 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -247,10 +247,8 @@ func coreControllers(ctx context.Context, sc *Context, config *Config) error { if config.ControlConfig.EncryptSecrets { if err := secretsencrypt.Register(ctx, - sc.K8s, &config.ControlConfig, - sc.Core.Core().V1().Node(), - sc.Core.Core().V1().Secret()); err != nil { + sc.Core.Core().V1().Node()); err != nil { return err } } From 6ad921767f736a08d4a4dfd7256311799f190d4c Mon Sep 17 00:00:00 2001 From: Derek Nola Date: Fri, 26 Jul 2024 11:23:57 -0700 Subject: [PATCH 3/3] Allow Pprof and Superisor metrics in standalone mode (#10576) * Allow pprof to run on server with `--disable-agent` * Allow supervisor metrics to run on server with `--disable-agent` Signed-off-by: Derek Nola --- pkg/agent/run.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pkg/agent/run.go b/pkg/agent/run.go index 93b4e27b6230..b0f9398086d6 100644 --- a/pkg/agent/run.go +++ b/pkg/agent/run.go @@ -284,6 +284,18 @@ func RunStandalone(ctx context.Context, cfg cmds.Agent) error { return err } + if nodeConfig.SupervisorMetrics { + if err := metrics.DefaultMetrics.Start(ctx, nodeConfig); err != nil { + return errors.Wrap(err, "failed to serve metrics") + } + } + + if nodeConfig.EnablePProf { + if err := profile.DefaultProfiler.Start(ctx, nodeConfig); err != nil { + return errors.Wrap(err, "failed to serve pprof") + } + } + <-ctx.Done() return ctx.Err() }