Stabilization Improvement for the Helm Charts (for Resource-Limited E…

…nvironments) (#933) * attempt 2 Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * add priorityclassname Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * add health check to spire agent Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * increase wait timeout Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * add probes to spire-server Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * add controller manager health checks Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * attempt 2 Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * attemp 3 Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * BundleEndpoint update Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * add notifier Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * whitespace Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * updated agent and server versions Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * 1.9.4 Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * version change Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * update spire controller manager Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> * spiffe csi driver update Signed-off-by: Volkan Özçelik <ovolkan@vmware.com> --------- Signed-off-by: Volkan Özçelik <ovolkan@vmware.com>
vmware-tanzu · Apr 24, 2024 · d7e1387 · d7e1387
1 parent 9ae41ac
commit d7e1387
Show file tree

Hide file tree

Showing 32 changed files with 879 additions and 436 deletions.
diff --git a/hack/install-vsecm-to-eks.sh b/hack/install-vsecm-to-eks.sh
@@ -20,8 +20,8 @@
 helm install vsecm vsecm/vsecm
 
 echo "verifying vsecm installation"
-kubectl wait --for=condition=Available deployment -n vsecm-system vsecm-sentinel
+kubectl wait --timeout=60s --for=condition=Available deployment -n vsecm-system vsecm-sentinel
 echo "vsecm-sentinel: deployment available"
-kubectl wait --for=condition=Available deployment -n vsecm-system vsecm-safe
+kubectl wait --timeout=60s --for=condition=Available deployment -n vsecm-system vsecm-safe
 echo "vsecm-safe: deployment available"
 echo "vsecm installation successful"
diff --git a/helm-charts/0.24.5/charts/keystone/templates/Deployment.yaml b/helm-charts/0.24.5/charts/keystone/templates/Deployment.yaml
@@ -38,6 +38,9 @@ spec:
  serviceAccountName: {{ include "keystone.serviceAccountName" . }}
  securityContext:
  {{- toYaml .Values.podSecurityContext | nindent 8 }}
+
+ priorityClassName: system-cluster-critical
+
  initContainers:
  - name: init-container
  image: "{{ .Values.global.registry }}/{{ .Values.global.images.initContainer.repository }}:{{ .Values.global.images.initContainer.tag }}"

diff --git a/helm-charts/0.24.5/charts/safe/templates/Deployment.yaml b/helm-charts/0.24.5/charts/safe/templates/Deployment.yaml
@@ -38,6 +38,9 @@ spec:
  serviceAccountName: {{ include "safe.serviceAccountName" . }}
  securityContext:
  {{- toYaml .Values.podSecurityContext | nindent 8 }}
+
+ priorityClassName: system-cluster-critical
+
  containers:
  - name: main
  image: "{{ .Values.global.registry }}/{{- include "safe.repository" .}}:{{ .Values.global.images.safe.tag }}"

diff --git a/helm-charts/0.24.5/charts/sentinel/templates/Deployment.yaml b/helm-charts/0.24.5/charts/sentinel/templates/Deployment.yaml
@@ -38,6 +38,9 @@ spec:
  serviceAccountName: {{ include "sentinel.serviceAccountName" . }}
  securityContext:
  {{- toYaml .Values.podSecurityContext | nindent 8 }}
+
+ priorityClassName: system-cluster-critical
+
  containers:
  - name: main
  image: "{{ .Values.global.registry }}/{{- include "sentinel.repository" .}}:{{ .Values.global.images.sentinel.tag }}"

diff --git a/helm-charts/0.24.5/charts/spire/templates/spire-agent-cluster-role-binding.yaml b/helm-charts/0.24.5/charts/spire/templates/spire-agent-cluster-role-binding.yaml
@@ -0,0 +1,23 @@
+# /*
+# | Protect your secrets, protect your sensitive data.
+# : Explore VMware Secrets Manager docs at https://vsecm.com/
+# </
+# <>/ keep your secrets... secret
+# >/
+# <>/' Copyright 2023-present VMware Secrets Manager contributors.
+# >/' SPDX-License-Identifier: BSD-2-Clause
+# */
+
+# Binds above cluster role to spire-agent service account
+kind: ClusterRoleBinding
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: spire-agent-cluster-role-binding
+subjects:
+ - kind: ServiceAccount
+ name: spire-agent
+ namespace: {{ .Values.global.spire.namespace }}
+roleRef:
+ kind: ClusterRole
+ name: spire-agent-cluster-role
+ apiGroup: rbac.authorization.k8s.io
diff --git a/helm-charts/0.24.5/charts/spire/templates/spire-agent-cluster-role.yaml b/helm-charts/0.24.5/charts/spire/templates/spire-agent-cluster-role.yaml
@@ -0,0 +1,19 @@
+# /*
+# | Protect your secrets, protect your sensitive data.
+# : Explore VMware Secrets Manager docs at https://vsecm.com/
+# </
+# <>/ keep your secrets... secret
+# >/
+# <>/' Copyright 2023-present VMware Secrets Manager contributors.
+# >/' SPDX-License-Identifier: BSD-2-Clause
+# */
+
+# Required cluster role to allow spire-agent to query k8s API server
+kind: ClusterRole
+apiVersion: rbac.authorization.k8s.io/v1
+metadata:
+ name: spire-agent-cluster-role
+rules:
+ - apiGroups: [""]
+ resources: ["pods","nodes","nodes/proxy"]
+ verbs: ["get"]
diff --git a/helm-charts/0.24.5/charts/spire/templates/spire-agent-config-map.yaml b/helm-charts/0.24.5/charts/spire/templates/spire-agent-config-map.yaml
@@ -0,0 +1,56 @@
+# /*
+# | Protect your secrets, protect your sensitive data.
+# : Explore VMware Secrets Manager docs at https://vsecm.com/
+# </
+# <>/ keep your secrets... secret
+# >/
+# <>/' Copyright 2023-present VMware Secrets Manager contributors.
+# >/' SPDX-License-Identifier: BSD-2-Clause
+# */
+
+# ConfigMap for the SPIRE agent featuring:
+# 1) PSAT node attestation
+# 2) K8S Workload Attestation over the secure kubelet port
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: spire-agent
+ namespace: {{ .Values.global.spire.namespace }}
+data:
+ agent.conf: |
+ agent {
+ data_dir = "/run/spire"
+ log_level = {{ .Values.global.spire.logLevel | quote }}
+ server_address = "spire-server"
+ server_port = {{ .Values.global.spire.serverPort | quote }}
+ socket_path = "/run/spire/sockets/agent.sock"
+ trust_bundle_path = "/run/spire/bundle/bundle.crt"
+ trust_domain = {{ .Values.global.spire.trustDomain | quote }}
+ }
+
+ health_checks {
+ bind_address = "0.0.0.0"
+ bind_port = "9982"
+ listener_enabled = true
+ live_path = "/live"
+ ready_path = "/ready"
+ }
+
+ plugins {
+ NodeAttestor "k8s_psat" {
+ plugin_data {
+ cluster = "vsecm-cluster"
+ }
+ }
+
+ KeyManager "memory" {
+ plugin_data {
+ }
+ }
+
+ WorkloadAttestor "k8s" {
+ plugin_data {
+ skip_kubelet_verification = true
+ }
+ }
+ }
diff --git a/...5/charts/spire/templates/spire-agent.yaml → ...pire/templates/spire-agent-daemonset.yaml b/...5/charts/spire/templates/spire-agent.yaml → ...pire/templates/spire-agent-daemonset.yaml
@@ -8,85 +8,6 @@
 # >/' SPDX-License-Identifier: BSD-2-Clause
 # */
 
-# ServiceAccount for the SPIRE agent
-apiVersion: v1
-kind: ServiceAccount
-metadata:
- name: spire-agent
- namespace: {{ .Values.global.spire.namespace }}
-
----
-
-# Required cluster role to allow spire-agent to query k8s API server
-kind: ClusterRole
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- name: spire-agent-cluster-role
-rules:
- - apiGroups: [""]
- resources: ["pods","nodes","nodes/proxy"]
- verbs: ["get"]
-
----
-
-# Binds above cluster role to spire-agent service account
-kind: ClusterRoleBinding
-apiVersion: rbac.authorization.k8s.io/v1
-metadata:
- name: spire-agent-cluster-role-binding
-subjects:
- - kind: ServiceAccount
- name: spire-agent
- namespace: {{ .Values.global.spire.namespace }}
-roleRef:
- kind: ClusterRole
- name: spire-agent-cluster-role
- apiGroup: rbac.authorization.k8s.io
-
-
----
-
-# ConfigMap for the SPIRE agent featuring:
-# 1) PSAT node attestation
-# 2) K8S Workload Attestation over the secure kubelet port
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: spire-agent
- namespace: {{ .Values.global.spire.namespace }}
-data:
- agent.conf: |
- agent {
- data_dir = "/run/spire"
- log_level = {{ .Values.global.spire.logLevel | quote }}
- server_address = "spire-server"
- server_port = {{ .Values.global.spire.serverPort | quote }}
- socket_path = "/run/spire/sockets/agent.sock"
- trust_bundle_path = "/run/spire/bundle/bundle.crt"
- trust_domain = {{ .Values.global.spire.trustDomain | quote }}
- }
-
- plugins {
- NodeAttestor "k8s_psat" {
- plugin_data {
- cluster = "vsecm-cluster"
- }
- }
-
- KeyManager "memory" {
- plugin_data {
- }
- }
-
- WorkloadAttestor "k8s" {
- plugin_data {
- skip_kubelet_verification = true
- }
- }
- }
-
----
-
 apiVersion: apps/v1
 kind: DaemonSet
 metadata:
@@ -114,6 +35,7 @@ spec:
  hostNetwork: true
  dnsPolicy: ClusterFirstWithHostNet
  serviceAccountName: spire-agent
+ priorityClassName: system-node-critical
  containers:
  - name: spire-agent
  image: {{ .Values.global.images.spireAgent.repository }}:{{ .Values.global.images.spireAgent.tag }}
@@ -123,6 +45,23 @@ spec:
  requests:
  memory: {{ .Values.resources.agent.requests.memory }}
  cpu: {{ .Values.resources.agent.requests.cpu }}
+
+ ports:
+ - containerPort: 9982
+ name: healthz
+ livenessProbe:
+ httpGet:
+ path: /live
+ port: healthz
+ initialDelaySeconds: 15
+ periodSeconds: 60
+ readinessProbe:
+ httpGet:
+ path: /ready
+ port: healthz
+ initialDelaySeconds: 10
+ periodSeconds: 30
+
  volumeMounts:
  - name: spire-config
  mountPath: /run/spire/config

diff --git a/helm-charts/0.24.5/charts/spire/templates/spire-agent-service-account.yaml b/helm-charts/0.24.5/charts/spire/templates/spire-agent-service-account.yaml
@@ -0,0 +1,16 @@
+# /*
+# | Protect your secrets, protect your sensitive data.
+# : Explore VMware Secrets Manager docs at https://vsecm.com/
+# </
+# <>/ keep your secrets... secret
+# >/
+# <>/' Copyright 2023-present VMware Secrets Manager contributors.
+# >/' SPDX-License-Identifier: BSD-2-Clause
+# */
+
+# ServiceAccount for the SPIRE agent
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: spire-agent
+ namespace: {{ .Values.global.spire.namespace }}
diff --git a/helm-charts/0.24.5/charts/spire/templates/spire-controller-manager-config.yaml b/helm-charts/0.24.5/charts/spire/templates/spire-controller-manager-config.yaml
@@ -19,8 +19,8 @@ data:
  kind: ControllerManagerConfig
  metrics:
  bindAddress: 127.0.0.1:8082
- healthProbe:
- bindAddress: 127.0.0.1:8083
+ health:
+ healthProbeBindAddress: 0.0.0.0:8083
  leaderElection:
  leaderElect: true
  resourceName: 98c9c988.spiffe.io