🎉 Add variable to customize airbyte-pod-sweeper delete pod action

charts/airbyte-pod-sweeper/templates/configmap.yaml

@@ -8,4 +8,43 @@ metadata:
 
 data:
   sweep-pod.sh: |
-  {{- (.Files.Get "files/sweep-pod.sh") | nindent 4 }}
+    #!/bin/bash
+    get_worker_pods () {
+        kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \
+          --field-selector status.phase!=Running get pods \
+          -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}'
+    }
+    delete_worker_pod() {
+        printf "From status '%s' since '%s', " $2 $3
+        echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod
+    }
+    while :
+    do
+        # Shorter time window for completed pods
+        SUCCESS_DATE_STR=`date -d 'now - {{ .Values.timeToDeletePods.completed }} minutes' --utc -Ins`
+        SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s`
+        # Longer time window for pods in error (to debug)
+        NON_SUCCESS_DATE_STR=`date -d 'now - {{ .Values.timeToDeletePods.error }} minutes' --utc -Ins`
+        NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s`
+        (
+            IFS=$'\n'
+            for POD in `get_worker_pods`; do
+                IFS=' '
+                POD_NAME=`echo $POD | cut -d " " -f 1`
+                POD_STATUS=`echo $POD | cut -d " " -f 2`
+                POD_DATE_STR=`echo $POD | cut -d " " -f 3`
+                POD_START_DATE_STR=`echo $POD | cut -d " " -f 4`
+                POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'`
+                if [ "$POD_STATUS" = "Succeeded" ]; then
+                if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then
+                    delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+                fi
+                else
+                if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then
+                    delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"
+                fi
+                fi
+            done
+        )
+        sleep 60
+    done

charts/airbyte-pod-sweeper/templates/deployment.yaml

@@ -5,8 +5,6 @@ metadata:
   name: {{ include "common.names.fullname" . }}-pod-sweeper
   labels:
     {{- include "airbyte.labels" . | nindent 4 }}
-  annotations:
-    checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
 spec:
   replicas: 1
   selector:
@@ -16,8 +14,9 @@ spec:
     metadata:
       labels:
         {{- include "airbyte.selectorLabels" . | nindent 8 }}
-      {{- if .Values.podAnnotations }}
       annotations:
+        checksum/sweep-pod-script: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
+      {{- if .Values.podAnnotations }}
         {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }}
       {{- end }}
     spec:

charts/airbyte-pod-sweeper/values.yaml

@@ -213,3 +213,9 @@ extraVolumeMounts: []
 ##     emptyDir: {}
 ##
 extraVolumes: []
+
+##  podSweeper.timeToDeletePods.completed Time to remove pods on completed status (minutes).
+##  podSweeper.timeToDeletePods.error Time to remove pods on error status (minutes).
+timeToDeletePods:
+  completed: 120
+  error: 1440