enhance Helm chart by startupProbe

* and more configuration options, e.g. several topologySpreadConstraints * also configure "ephemeral-storage" requests and limits
eclipse-ditto · Nov 15, 2024 · 16f8c06 · 16f8c06
1 parent 06ff4ad
commit 16f8c06
Show file tree

Hide file tree

Showing 11 changed files with 276 additions and 123 deletions.
diff --git a/deployment/helm/ditto/Chart.yaml b/deployment/helm/ditto/Chart.yaml
@@ -16,7 +16,7 @@ description: |
   A digital twin is a virtual, cloud based, representation of his real world counterpart
   (real world “Things”, e.g. devices like sensors, smart heating, connected cars, smart grids, EV charging stations etc).
 type: application
-version: 3.6.2  # chart version is effectively set by release-job
+version: 3.6.3  # chart version is effectively set by release-job
 appVersion: 3.6.2
 keywords:
   - iot-chart

diff --git a/deployment/helm/ditto/scripts/patch-pods-deletion-cost.sh b/deployment/helm/ditto/scripts/patch-pods-deletion-cost.sh
@@ -25,17 +25,19 @@ curl --fail --silent --cacert ${CACERT} -H "Authorization: Bearer ${TOKEN}" \
   "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/${NAMESPACE}/pods" \
   | jq '.items | map(select(.metadata.labels.actorSystemName == "ditto-cluster") | { pod: .metadata.name, ip: .status.podIP, cost: .metadata.annotations."controller.kubernetes.io/pod-deletion-cost"})' \
   > pod_ip_cost.json
-if [ $? -ne 0 ]; then
-  echo "Retrieving current pods curl failed [exit-code: $?]"
+curlExitCode=$?
+if [ $curlExitCode -ne 0 ]; then
+  echo "Retrieving current pods curl failed [exit-code: $curlExitCode]"
   exit 1
 fi
 
 # extract the first internal IP from `pod_ip_cost.json` in order to lookup Apache Pekko cluster membership data:
 somePekkoClusterIp=$(jq -r '.[0].ip' pod_ip_cost.json)
 echo "Accessing current Pekko Cluster members from internal ip: $somePekkoClusterIp ..."
 curl --fail --silent -o pekko_cluster_members.json http://$somePekkoClusterIp:7626/cluster/members
-if [ $? -ne 0 ]; then
-  echo "Accessing current Pekko Cluster members curl failed [exit-code: $?]"
+curlExitCode=$?
+if [ $curlExitCode -ne 0 ]; then
+  echo "Accessing current Pekko Cluster members curl failed [exit-code: $curlExitCode]"
   exit 1
 fi
 
@@ -64,8 +66,9 @@ while read pod; do
      curl -X PATCH --silent --output /dev/null --show-error --fail --cacert ${CACERT} -H "Authorization: Bearer ${TOKEN}" -H 'Content-Type: application/merge-patch+json' \
         "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/${NAMESPACE}/pods/${pod}" \
         --data '{"metadata": {"annotations": {"controller.kubernetes.io/pod-deletion-cost": null }}}'
-     if [ $? -ne 0 ]; then
-       echo "Clearing pod-deletion-cost curl failed [exit-code: $?]"
+     curlExitCode=$?
+     if [ $curlExitCode -ne 0 ]; then
+       echo "Clearing pod-deletion-cost curl failed [exit-code: $curlExitCode]"
        exit 1
      fi
   fi
@@ -78,8 +81,9 @@ jq -r '.[] | [.pod, .ip, .cost] | @tsv' new_cost_pod_and_ip.json |
     curl -X PATCH --silent --output /dev/null --show-error --fail --cacert ${CACERT} -H "Authorization: Bearer ${TOKEN}" -H 'Content-Type: application/merge-patch+json' \
        "https://$KUBERNETES_SERVICE_HOST:$KUBERNETES_SERVICE_PORT/api/v1/namespaces/${NAMESPACE}/pods/${pod}" \
        --data '{"metadata": {"annotations": {"controller.kubernetes.io/pod-deletion-cost": '\""$cost"\"' }}}'
-    if [ $? -ne 0 ]; then
-      echo "Patching pod-deletion-cost curl failed [exit-code: $?]"
+    curlExitCode=$?
+    if [ $curlExitCode -ne 0 ]; then
+      echo "Patching pod-deletion-cost curl failed [exit-code: $curlExitCode]"
       exit 1
     fi
   done

diff --git a/deployment/helm/ditto/templates/connectivity-deployment.yaml b/deployment/helm/ditto/templates/connectivity-deployment.yaml
@@ -8,14 +8,15 @@
 # http://www.eclipse.org/legal/epl-2.0
 #
 # SPDX-License-Identifier: EPL-2.0
+{{- $name := include "ditto.name" . -}}
 {{- if .Values.connectivity.enabled -}}
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "ditto.fullname" . }}-connectivity
   labels:
-    app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity
+    app.kubernetes.io/name: {{ $name }}-connectivity
 {{ include "ditto.labels" . | indent 4 }}
 spec:
   replicas: {{ .Values.connectivity.replicaCount }}
@@ -26,12 +27,12 @@ spec:
   minReadySeconds: {{ .Values.connectivity.minReadySeconds }}
   selector:
     matchLabels:
-      app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity
+      app.kubernetes.io/name: {{ $name }}-connectivity
       app.kubernetes.io/instance: {{ .Release.Name }}
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity
+        app.kubernetes.io/name: {{ $name }}-connectivity
         app.kubernetes.io/instance: {{ .Release.Name }}
         actorSystemName: {{ .Values.pekko.actorSystemName }}
         {{- with .Values.connectivity.additionalLabels }}
@@ -45,6 +46,9 @@ spec:
         {{- end }}
         checksum/mongodb-config: {{ include (print $.Template.BasePath "/mongodb-secret.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/connectivity-configmap.yaml") . | sha256sum }}
+        {{- with .Values.global.additionalAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
         {{- with .Values.connectivity.additionalAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -70,12 +74,14 @@ spec:
               mountPath: /var/log/ditto
         {{- end }}
       topologySpreadConstraints:
-        - maxSkew: {{ .Values.connectivity.topologySpreadConstraints.maxSkew }}
-          topologyKey: {{ .Values.connectivity.topologySpreadConstraints.topologyKey }}
-          whenUnsatisfiable: {{ .Values.connectivity.topologySpreadConstraints.whenUnsatisfiable }}
+      {{- range .Values.connectivity.topologySpreadConstraints }}
+        - maxSkew: {{ .maxSkew }}
+          topologyKey: {{ .topologyKey }}
+          whenUnsatisfiable: {{ .whenUnsatisfiable }}
           labelSelector:
             matchLabels:
-              app.kubernetes.io/name: {{ include "ditto.name" . }}-connectivity
+              app.kubernetes.io/name: {{ $name }}-connectivity
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}-connectivity
           image: {{ printf "%s:%s" .Values.connectivity.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.connectivity.image.tag ) ) }}
@@ -312,11 +318,19 @@ spec:
               protocol: TCP
               containerPort: {{ .Values.global.prometheus.port }}
             {{- end }}
+          startupProbe:
+            httpGet:
+              port: management
+              path: /ready
+            initialDelaySeconds: {{ .Values.connectivity.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.connectivity.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.connectivity.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.connectivity.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.connectivity.startupProbe.failureThreshold }}
           readinessProbe:
             httpGet:
               port: management
               path: /ready
-            initialDelaySeconds: {{ .Values.connectivity.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.connectivity.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.connectivity.readinessProbe.timeoutSeconds }}
             successThreshold: {{ .Values.connectivity.readinessProbe.successThreshold }}
@@ -352,11 +366,13 @@ spec:
             requests:
               cpu: {{ mulf .Values.connectivity.resources.cpu 1000 }}m
               memory: {{ .Values.connectivity.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.connectivity.resources.ephemeralStorageMi }}Mi
             limits:
               # ## no cpu limit to avoid CFS scheduler limits
               # ref: https://pekko.apache.org/docs/pekko/current/additional/deploying.html#deploying-to-kubernetes
               # cpu: ""
               memory: {{ .Values.connectivity.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.connectivity.resources.ephemeralStorageMi }}Mi
           {{- if .Values.openshift.enabled }}
           {{- with .Values.openshift.securityContext }}
           securityContext:

diff --git a/deployment/helm/ditto/templates/dittoui-deployment.yaml b/deployment/helm/ditto/templates/dittoui-deployment.yaml
@@ -8,14 +8,15 @@
 # http://www.eclipse.org/legal/epl-2.0
 #
 # SPDX-License-Identifier: EPL-2.0
+{{- $name := include "ditto.name" . -}}
 {{- if .Values.dittoui.enabled -}}
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "ditto.fullname" . }}-dittoui
   labels:
-    app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui
+    app.kubernetes.io/name: {{ $name }}-dittoui
 {{ include "ditto.labels" . | indent 4 }}
 spec:
   replicas: {{ .Values.dittoui.replicaCount }}
@@ -25,12 +26,12 @@ spec:
   {{- end }}
   selector:
     matchLabels:
-      app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui
+      app.kubernetes.io/name: {{ $name }}-dittoui
       app.kubernetes.io/instance: {{ .Release.Name }}
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui
+        app.kubernetes.io/name: {{ $name }}-dittoui
         app.kubernetes.io/instance: {{ .Release.Name }}
         {{- with .Values.dittoui.additionalLabels }}
         {{- toYaml . | nindent 8 }}
@@ -45,12 +46,14 @@ spec:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       topologySpreadConstraints:
-        - maxSkew: {{ .Values.dittoui.topologySpreadConstraints.maxSkew }}
-          topologyKey: {{ .Values.dittoui.topologySpreadConstraints.topologyKey }}
-          whenUnsatisfiable: {{ .Values.dittoui.topologySpreadConstraints.whenUnsatisfiable }}
+      {{- range .Values.dittoui.topologySpreadConstraints }}
+        - maxSkew: {{ .maxSkew }}
+          topologyKey: {{ .topologyKey }}
+          whenUnsatisfiable: {{ .whenUnsatisfiable }}
           labelSelector:
             matchLabels:
-              app.kubernetes.io/name: {{ include "ditto.name" . }}-dittoui
+              app.kubernetes.io/name: {{ $name }}-dittoui
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}-dittoui
           image: {{ printf "%s:%s" .Values.dittoui.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.dittoui.image.tag ) ) }}
@@ -67,9 +70,11 @@ spec:
             requests:
               cpu: {{ mulf .Values.dittoui.resources.cpu 1000 }}m
               memory: {{ .Values.dittoui.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.dittoui.resources.ephemeralStorageMi }}Mi
             limits:
               # cpu: # don't limit
               memory: {{ .Values.dittoui.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.dittoui.resources.ephemeralStorageMi }}Mi
           securityContext:
             runAsNonRoot: true
             runAsUser: 101

diff --git a/deployment/helm/ditto/templates/gateway-deployment.yaml b/deployment/helm/ditto/templates/gateway-deployment.yaml
@@ -8,14 +8,15 @@
 # http://www.eclipse.org/legal/epl-2.0
 #
 # SPDX-License-Identifier: EPL-2.0
+{{- $name := include "ditto.name" . -}}
 {{- if .Values.gateway.enabled -}}
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "ditto.fullname" . }}-gateway
   labels:
-    app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway
+    app.kubernetes.io/name: {{ $name }}-gateway
 {{ include "ditto.labels" . | indent 4 }}
 spec:
   replicas: {{ .Values.gateway.replicaCount }}
@@ -26,12 +27,12 @@ spec:
   minReadySeconds: {{ .Values.gateway.minReadySeconds }}
   selector:
     matchLabels:
-      app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway
+      app.kubernetes.io/name: {{ $name }}-gateway
       app.kubernetes.io/instance: {{ .Release.Name }}
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway
+        app.kubernetes.io/name: {{ $name }}-gateway
         app.kubernetes.io/instance: {{ .Release.Name }}
         actorSystemName: {{ .Values.pekko.actorSystemName }}
         {{- with .Values.gateway.additionalLabels }}
@@ -45,6 +46,9 @@ spec:
         {{- end }}
         checksum/password-config: {{ include (print $.Template.BasePath "/gateway-secret.yaml") . | sha256sum }}
         checksum/config: {{ include (print $.Template.BasePath "/gateway-configmap.yaml") . | sha256sum }}
+        {{- with .Values.global.additionalAnnotations }}
+        {{- toYaml . | nindent 8 }}
+        {{- end }}
         {{- with .Values.gateway.additionalAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -70,12 +74,14 @@ spec:
               mountPath: /var/log/ditto
         {{- end }}
       topologySpreadConstraints:
-        - maxSkew: {{ .Values.gateway.topologySpreadConstraints.maxSkew }}
-          topologyKey: {{ .Values.gateway.topologySpreadConstraints.topologyKey }}
-          whenUnsatisfiable: {{ .Values.gateway.topologySpreadConstraints.whenUnsatisfiable }}
+      {{- range .Values.gateway.topologySpreadConstraints }}
+        - maxSkew: {{ .maxSkew }}
+          topologyKey: {{ .topologyKey }}
+          whenUnsatisfiable: {{ .whenUnsatisfiable }}
           labelSelector:
             matchLabels:
-              app.kubernetes.io/name: {{ include "ditto.name" . }}-gateway
+              app.kubernetes.io/name: {{ $name }}-gateway
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}-gateway
           image: {{ printf "%s:%s" .Values.gateway.image.repository ( default .Chart.AppVersion ( default .Values.dittoTag .Values.gateway.image.tag ) ) }}
@@ -230,11 +236,19 @@ spec:
               protocol: TCP
               containerPort: {{ .Values.global.prometheus.port }}
             {{- end }}
+          startupProbe:
+            httpGet:
+              port: management
+              path: /ready
+            initialDelaySeconds: {{ .Values.gateway.startupProbe.initialDelaySeconds }}
+            periodSeconds: {{ .Values.gateway.startupProbe.periodSeconds }}
+            timeoutSeconds: {{ .Values.gateway.startupProbe.timeoutSeconds }}
+            successThreshold: {{ .Values.gateway.startupProbe.successThreshold }}
+            failureThreshold: {{ .Values.gateway.startupProbe.failureThreshold }}
           readinessProbe:
             httpGet:
               port: management
               path: /ready
-            initialDelaySeconds: {{ .Values.gateway.readinessProbe.initialDelaySeconds }}
             periodSeconds: {{ .Values.gateway.readinessProbe.periodSeconds }}
             timeoutSeconds: {{ .Values.gateway.readinessProbe.timeoutSeconds }}
             successThreshold: {{ .Values.gateway.readinessProbe.successThreshold }}
@@ -270,11 +284,13 @@ spec:
             requests:
               cpu: {{ mulf .Values.gateway.resources.cpu 1000 }}m
               memory: {{ .Values.gateway.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.gateway.resources.ephemeralStorageMi }}Mi
             limits:
               # ## no cpu limit to avoid CFS scheduler limits
               # ref: https://pekko.apache.org/docs/pekko/current/additional/deploying.html#deploying-to-kubernetes
               # cpu: ""
               memory: {{ .Values.gateway.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.gateway.resources.ephemeralStorageMi }}Mi
           {{- if .Values.openshift.enabled }}
           {{- with .Values.openshift.securityContext }}
           securityContext:

diff --git a/deployment/helm/ditto/templates/nginx-deployment.yaml b/deployment/helm/ditto/templates/nginx-deployment.yaml
@@ -8,14 +8,15 @@
 # http://www.eclipse.org/legal/epl-2.0
 #
 # SPDX-License-Identifier: EPL-2.0
+{{- $name := include "ditto.name" . -}}
 {{- if .Values.nginx.enabled -}}
 ---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   name: {{ include "ditto.fullname" . }}-nginx
   labels:
-    app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx
+    app.kubernetes.io/name: {{ $name }}-nginx
 {{ include "ditto.labels" . | indent 4 }}
 spec:
   replicas: {{ .Values.nginx.replicaCount }}
@@ -25,12 +26,12 @@ spec:
   {{- end }}
   selector:
     matchLabels:
-      app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx
+      app.kubernetes.io/name: {{ $name }}-nginx
       app.kubernetes.io/instance: {{ .Release.Name }}
   template:
     metadata:
       labels:
-        app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx
+        app.kubernetes.io/name: {{ $name }}-nginx
         app.kubernetes.io/instance: {{ .Release.Name }}
         {{- with .Values.nginx.additionalLabels }}
         {{- toYaml . | nindent 8 }}
@@ -62,12 +63,14 @@ spec:
               done
       {{- end }}
       topologySpreadConstraints:
-        - maxSkew: {{ .Values.nginx.topologySpreadConstraints.maxSkew }}
-          topologyKey: {{ .Values.nginx.topologySpreadConstraints.topologyKey }}
-          whenUnsatisfiable: {{ .Values.nginx.topologySpreadConstraints.whenUnsatisfiable }}
+      {{- range .Values.nginx.topologySpreadConstraints }}
+        - maxSkew: {{ .maxSkew }}
+          topologyKey: {{ .topologyKey }}
+          whenUnsatisfiable: {{ .whenUnsatisfiable }}
           labelSelector:
             matchLabels:
-              app.kubernetes.io/name: {{ include "ditto.name" . }}-nginx
+              app.kubernetes.io/name: {{ $name }}-nginx
+      {{- end }}
       containers:
         - name: {{ .Chart.Name }}-nginx
           image: "{{ .Values.nginx.image.repository }}:{{ .Values.nginx.image.tag }}"
@@ -92,9 +95,11 @@ spec:
             requests:
               cpu: {{ mulf .Values.nginx.resources.cpu 1000 }}m
               memory: {{ .Values.nginx.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.nginx.resources.ephemeralStorageMi }}Mi
             limits:
               # cpu: # don't limit
               memory: {{ .Values.nginx.resources.memoryMi }}Mi
+              ephemeral-storage: {{ .Values.nginx.resources.ephemeralStorageMi }}Mi
           volumeMounts:
             - name: nginx-conf
               mountPath: /etc/nginx/nginx.conf