fix controller can't restart in helm for dependent secret not found

Signed-off-by: chaosi-zju <chaosi@zju.edu.cn>
karmada-io · Aug 8, 2024 · 1661e5a · 1661e5a
1 parent 04a4d84
commit 1661e5a
Show file tree

Hide file tree

Showing 13 changed files with 27 additions and 53 deletions.
diff --git a/charts/karmada/templates/_helpers.tpl b/charts/karmada/templates/_helpers.tpl
@@ -586,40 +586,25 @@ Return the proper Docker Image Registry Secret Names
 {{- end }}
 {{- end -}}
 
-{{- define "karmada.init-sa-secret.volume" -}}
-{{- $name := include "karmada.name" . -}}
-- name: init-sa-secret
-  secret:
-    secretName: {{ $name }}-hook-job
-{{- end -}}
-
-{{- define "karmada.init-sa-secret.volumeMount" -}}
-- name: init-sa-secret
-  mountPath: /opt/mount
-{{- end -}}
-
-{{- define "karmada.initContainer.build-kubeconfig" -}}
-TOKEN=$(cat /opt/mount/token)
-kubectl config set-cluster karmada-host --server=https://${KUBERNETES_SERVICE_HOST}:${KUBERNETES_SERVICE_PORT} --certificate-authority=/opt/mount/ca.crt
-kubectl config set-credentials default --token=$TOKEN
-kubectl config set-context karmada-host-context --cluster=karmada-host --user=default --namespace=default
-kubectl config use-context karmada-host-context
-{{- end -}}
-
 {{- define "karmada.initContainer.waitEtcd" -}}
 - name: wait
-  image: {{ include "karmada.kubectl.image" . }}
+  image: {{ include "karmada.cfssl.image" . }}
   imagePullPolicy: {{ .Values.kubectl.image.pullPolicy }}
   command:
     - /bin/sh
     - -c
     - |
       bash <<'EOF'
-      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
-      kubectl rollout status statefulset etcd -n {{ include "karmada.namespace" . }}
+      set -ex
+      while true; do
+        if curl --connect-timeout 2 ${ETCD_CLIENT_SERVICE_HOST}":"${ETCD_CLIENT_SERVICE_PORT} || [ $? -eq 52 ]; then
+          break
+        fi
+        echo "failed to connect to "${ETCD_CLIENT_SERVICE_HOST}":"${ETCD_CLIENT_SERVICE_PORT}
+        sleep 2
+      done
+      echo "successfully connect to "${ETCD_CLIENT_SERVICE_HOST}":"${ETCD_CLIENT_SERVICE_PORT}
       EOF
-  volumeMounts:
-    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
 {{- end -}}
 
 {{- define "karmada.initContainer.waitStaticResource" -}}
@@ -631,9 +616,12 @@ kubectl config use-context karmada-host-context
     - -c
     - |
       bash <<'EOF'
-      {{- include "karmada.initContainer.build-kubeconfig" . | nindent 6 }}
-      kubectl wait --for=condition=complete job {{ include "karmada.name" . }}-static-resource -n {{ include "karmada.namespace" . }}
+      set -ex
+      while [[ $(kubectl --kubeconfig /etc/kubeconfig get configmap karmada-versions -n {{ .Values.systemNamespace }} -o jsonpath='{.data.karmadaImageVersion}') != {{ .Values.karmadaImageVersion }} ]]; do
+        echo "wait for karmada-static-resource-job finshed"; sleep 2
+      done
+      echo "karmada-static-resource-job successfully completed since expected configmap value was found"
       EOF
   volumeMounts:
-    {{- include "karmada.init-sa-secret.volumeMount" .| nindent 4 }}
+    {{- include "karmada.kubeconfig.volumeMount" .| nindent 4 }}
 {{- end -}}
diff --git a/charts/karmada/templates/karmada-aggregated-apiserver.yaml b/charts/karmada/templates/karmada-aggregated-apiserver.yaml
@@ -98,7 +98,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:

diff --git a/charts/karmada/templates/karmada-apiserver.yaml b/charts/karmada/templates/karmada-apiserver.yaml
@@ -137,7 +137,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:
             secretName: {{ $name }}-cert

diff --git a/charts/karmada/templates/karmada-controller-manager.yaml b/charts/karmada/templates/karmada-controller-manager.yaml
@@ -42,7 +42,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
       initContainers:
         {{- include "karmada.initContainer.waitStaticResource" . | nindent 8 }}

diff --git a/charts/karmada/templates/karmada-descheduler.yaml b/charts/karmada/templates/karmada-descheduler.yaml
@@ -77,7 +77,6 @@ spec:
           resources:
           {{- toYaml .Values.descheduler.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.descheduler.kubeconfig.volume" . | nindent 8 }}
       {{- include "karmada.scheduler.cert.volume" . | nindent 8 }}
 

diff --git a/charts/karmada/templates/karmada-metrics-adapter.yaml b/charts/karmada/templates/karmada-metrics-adapter.yaml
@@ -83,7 +83,6 @@ spec:
       {{- toYaml . | nindent 8 }}
       {{- end }}
       volumes:
-        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: apiserver-cert
           secret:

diff --git a/charts/karmada/templates/karmada-scheduler.yaml b/charts/karmada/templates/karmada-scheduler.yaml
@@ -76,7 +76,6 @@ spec:
           resources:
           {{- toYaml .Values.scheduler.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
       {{- include "karmada.scheduler.cert.volume" . | nindent 8 }}
 

diff --git a/charts/karmada/templates/karmada-search.yaml b/charts/karmada/templates/karmada-search.yaml
@@ -92,7 +92,6 @@ spec:
           resources:
           {{- toYaml .Values.apiServer.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.search.kubeconfig.volume" . | nindent 8 }}
       {{- include "karmada.search.etcd.cert.volume" . | nindent 8 }}
 ---

diff --git a/charts/karmada/templates/karmada-static-resource-job.yaml b/charts/karmada/templates/karmada-static-resource-job.yaml
@@ -42,6 +42,17 @@ spec:
           kubectl apply -k /crds --kubeconfig /etc/kubeconfig
           kubectl apply -f /static-resources/system-namespace.yaml --kubeconfig /etc/kubeconfig
           kubectl apply -f /static-resources/ --kubeconfig /etc/kubeconfig
+
+          kubectl --kubeconfig /etc/kubeconfig apply -f - <<InnerEOF
+          apiVersion: v1
+          kind: ConfigMap
+          metadata:
+            name: karmada-versions
+            namespace: {{ .Values.systemNamespace }}
+          data:
+            karmadaImageVersion: {{ .Values.karmadaImageVersion }}
+          InnerEOF
+
           EOF
         volumeMounts:
           - name: {{ $name }}-crds-kustomization

diff --git a/charts/karmada/templates/karmada-webhook.yaml b/charts/karmada/templates/karmada-webhook.yaml
@@ -71,7 +71,6 @@ spec:
           resources:
           {{- toYaml .Values.webhook.resources | nindent 12 }}
       volumes:
-      {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
       {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
         - name: {{ $name }}-webhook-cert-secret
           secret:

diff --git a/charts/karmada/templates/kube-controller-manager.yaml b/charts/karmada/templates/kube-controller-manager.yaml
@@ -89,7 +89,6 @@ spec:
         - name: apisever-cert
           secret:
             secretName: {{ $name }}-cert
-        {{- include "karmada.init-sa-secret.volume" . | nindent 8 }}
         {{- include "karmada.kubeconfig.volume" . | nindent 8 }}
 
 {{ if .Values.kubeControllerManager.podDisruptionBudget }}

diff --git a/charts/karmada/templates/post-install-job.yaml b/charts/karmada/templates/post-install-job.yaml
@@ -58,6 +58,5 @@ spec:
           done
 
           kubectl delete job {{ $name }}-static-resource -n {{ $namespace }}
-          kubectl delete secret {{ $name }}-hook-job -n {{ $namespace }}
           EOF
 {{- end }}
diff --git a/charts/karmada/templates/pre-install-job.yaml b/charts/karmada/templates/pre-install-job.yaml
@@ -459,21 +459,6 @@ metadata:
     {{- include "karmada.preInstallJob.labels" . | nindent 4 }}
   {{- end }}
 ---
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ $name }}-hook-job
-  namespace: {{ $namespace }}
-  annotations:
-    "kubernetes.io/service-account.name": {{ $name }}-hook-job
-    "helm.sh/hook": pre-install
-    "helm.sh/hook-weight": "1"
-    {{- if "karmada.preInstallJob.labels" }}
-    labels:
-      {{- include "karmada.preInstallJob.labels" . | nindent 4 }}
-    {{- end }}
-type: kubernetes.io/service-account-token
----
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata: