From 19c54e6b72654e011c4399218a24098ef8e910b3 Mon Sep 17 00:00:00 2001 From: Caio Henrique Date: Thu, 13 Oct 2022 22:51:24 -0300 Subject: [PATCH 1/4] Add variable to customize airbyte-pod-sweeper delete pod --- charts/airbyte-pod-sweeper/files/sweep-pod.sh | 43 ------------------ .../templates/configmap.yaml | 44 ++++++++++++++++++- .../templates/deployment.yaml | 3 +- charts/airbyte-pod-sweeper/values.yaml | 6 +++ 4 files changed, 51 insertions(+), 45 deletions(-) delete mode 100644 charts/airbyte-pod-sweeper/files/sweep-pod.sh diff --git a/charts/airbyte-pod-sweeper/files/sweep-pod.sh b/charts/airbyte-pod-sweeper/files/sweep-pod.sh deleted file mode 100644 index a6b343c7bb9b..000000000000 --- a/charts/airbyte-pod-sweeper/files/sweep-pod.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -get_worker_pods () { - kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \ - --field-selector status.phase!=Running get pods \ - -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' -} - -delete_worker_pod() { - printf "From status '%s' since '%s', " $2 $3 - echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod -} - -while : -do - # Shorter time window for completed pods - SUCCESS_DATE_STR=`date -d 'now - 2 hours' --utc -Ins` - SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` - # Longer time window for pods in error (to debug) - NON_SUCCESS_DATE_STR=`date -d 'now - 24 hours' --utc -Ins` - NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` - ( - IFS=$'\n' - for POD in `get_worker_pods`; do - IFS=' ' - POD_NAME=`echo $POD | cut -d " " -f 1` - POD_STATUS=`echo $POD | cut -d " " -f 2` - POD_DATE_STR=`echo $POD | cut -d " " -f 3` - POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` - POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` - if [ "$POD_STATUS" = "Succeeded" ]; then - if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then - delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - else - if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then - delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - fi - done - ) - sleep 60 -done diff --git a/charts/airbyte-pod-sweeper/templates/configmap.yaml b/charts/airbyte-pod-sweeper/templates/configmap.yaml index 48d20e365cf5..afea58d23ec6 100644 --- a/charts/airbyte-pod-sweeper/templates/configmap.yaml +++ b/charts/airbyte-pod-sweeper/templates/configmap.yaml @@ -8,4 +8,46 @@ metadata: data: sweep-pod.sh: | - {{- (.Files.Get "files/sweep-pod.sh") | nindent 4 }} \ No newline at end of file + #!/bin/bash + + get_worker_pods () { + kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \ + --field-selector status.phase!=Running get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' + } + + delete_worker_pod() { + printf "From status '%s' since '%s', " $2 $3 + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod + } + + while : + do + # Shorter time window for completed pods + SUCCESS_DATE_STR=`date -d 'now - {{ .Values.timeToDeletePods.completed }} minutes' --utc -Ins` + SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` + # Longer time window for pods in error (to debug) + NON_SUCCESS_DATE_STR=`date -d 'now - {{ .Values.timeToDeletePods.error }} minutes' --utc -Ins` + NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` + ( + IFS=$'\n' + for POD in `get_worker_pods`; do + IFS=' ' + POD_NAME=`echo $POD | cut -d " " -f 1` + POD_STATUS=`echo $POD | cut -d " " -f 2` + POD_DATE_STR=`echo $POD | cut -d " " -f 3` + POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` + POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` + if [ "$POD_STATUS" = "Succeeded" ]; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + else + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + ) + sleep 60 + done diff --git a/charts/airbyte-pod-sweeper/templates/deployment.yaml b/charts/airbyte-pod-sweeper/templates/deployment.yaml index a02ac030553f..fff5e268fff3 100644 --- a/charts/airbyte-pod-sweeper/templates/deployment.yaml +++ b/charts/airbyte-pod-sweeper/templates/deployment.yaml @@ -16,8 +16,9 @@ spec: metadata: labels: {{- include "airbyte.selectorLabels" . | nindent 8 }} - {{- if .Values.podAnnotations }} annotations: + checksum/sweep-pod-script: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.podAnnotations }} {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} spec: diff --git a/charts/airbyte-pod-sweeper/values.yaml b/charts/airbyte-pod-sweeper/values.yaml index f3a3abfad72d..bd3ab9648964 100644 --- a/charts/airbyte-pod-sweeper/values.yaml +++ b/charts/airbyte-pod-sweeper/values.yaml @@ -213,3 +213,9 @@ extraVolumeMounts: [] ## emptyDir: {} ## extraVolumes: [] + +## podSweeper.timeToDeletePods.completed Time to remove pods on completed status in minutes +## podSweeper.timeToDeletePods.erro Time to remove pods on error status in minutes +timeToDeletePods: + completed: 120 + error: 1440 \ No newline at end of file From 1a0cfb50cb4b87a0b49a3d3692b14ad8509e2e63 Mon Sep 17 00:00:00 2001 From: Caio Henrique Date: Thu, 13 Oct 2022 22:57:12 -0300 Subject: [PATCH 2/4] Add variable to customize airbyte-pod-sweeper delete pod --- charts/airbyte-pod-sweeper/files/sweep-pod.sh | 43 ------------------ .../templates/configmap.yaml | 44 ++++++++++++++++++- .../templates/deployment.yaml | 5 +-- charts/airbyte-pod-sweeper/values.yaml | 6 +++ 4 files changed, 51 insertions(+), 47 deletions(-) delete mode 100644 charts/airbyte-pod-sweeper/files/sweep-pod.sh diff --git a/charts/airbyte-pod-sweeper/files/sweep-pod.sh b/charts/airbyte-pod-sweeper/files/sweep-pod.sh deleted file mode 100644 index a6b343c7bb9b..000000000000 --- a/charts/airbyte-pod-sweeper/files/sweep-pod.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash - -get_worker_pods () { - kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \ - --field-selector status.phase!=Running get pods \ - -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' -} - -delete_worker_pod() { - printf "From status '%s' since '%s', " $2 $3 - echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod -} - -while : -do - # Shorter time window for completed pods - SUCCESS_DATE_STR=`date -d 'now - 2 hours' --utc -Ins` - SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` - # Longer time window for pods in error (to debug) - NON_SUCCESS_DATE_STR=`date -d 'now - 24 hours' --utc -Ins` - NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` - ( - IFS=$'\n' - for POD in `get_worker_pods`; do - IFS=' ' - POD_NAME=`echo $POD | cut -d " " -f 1` - POD_STATUS=`echo $POD | cut -d " " -f 2` - POD_DATE_STR=`echo $POD | cut -d " " -f 3` - POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` - POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` - if [ "$POD_STATUS" = "Succeeded" ]; then - if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then - delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - else - if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then - delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" - fi - fi - done - ) - sleep 60 -done diff --git a/charts/airbyte-pod-sweeper/templates/configmap.yaml b/charts/airbyte-pod-sweeper/templates/configmap.yaml index 48d20e365cf5..afea58d23ec6 100644 --- a/charts/airbyte-pod-sweeper/templates/configmap.yaml +++ b/charts/airbyte-pod-sweeper/templates/configmap.yaml @@ -8,4 +8,46 @@ metadata: data: sweep-pod.sh: | - {{- (.Files.Get "files/sweep-pod.sh") | nindent 4 }} \ No newline at end of file + #!/bin/bash + + get_worker_pods () { + kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \ + --field-selector status.phase!=Running get pods \ + -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' + } + + delete_worker_pod() { + printf "From status '%s' since '%s', " $2 $3 + echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod + } + + while : + do + # Shorter time window for completed pods + SUCCESS_DATE_STR=`date -d 'now - {{ .Values.timeToDeletePods.completed }} minutes' --utc -Ins` + SUCCESS_DATE=`date -d $SUCCESS_DATE_STR +%s` + # Longer time window for pods in error (to debug) + NON_SUCCESS_DATE_STR=`date -d 'now - {{ .Values.timeToDeletePods.error }} minutes' --utc -Ins` + NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s` + ( + IFS=$'\n' + for POD in `get_worker_pods`; do + IFS=' ' + POD_NAME=`echo $POD | cut -d " " -f 1` + POD_STATUS=`echo $POD | cut -d " " -f 2` + POD_DATE_STR=`echo $POD | cut -d " " -f 3` + POD_START_DATE_STR=`echo $POD | cut -d " " -f 4` + POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'` + if [ "$POD_STATUS" = "Succeeded" ]; then + if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + else + if [ "$POD_DATE" -lt "$NON_SUCCESS_DATE" ]; then + delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR" + fi + fi + done + ) + sleep 60 + done diff --git a/charts/airbyte-pod-sweeper/templates/deployment.yaml b/charts/airbyte-pod-sweeper/templates/deployment.yaml index a02ac030553f..a566ef84c1c9 100644 --- a/charts/airbyte-pod-sweeper/templates/deployment.yaml +++ b/charts/airbyte-pod-sweeper/templates/deployment.yaml @@ -5,8 +5,6 @@ metadata: name: {{ include "common.names.fullname" . }}-pod-sweeper labels: {{- include "airbyte.labels" . | nindent 4 }} - annotations: - checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} spec: replicas: 1 selector: @@ -16,8 +14,9 @@ spec: metadata: labels: {{- include "airbyte.selectorLabels" . | nindent 8 }} - {{- if .Values.podAnnotations }} annotations: + checksum/sweep-pod-script: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }} + {{- if .Values.podAnnotations }} {{- include "common.tplvalues.render" (dict "value" .Values.podAnnotations "context" $) | nindent 8 }} {{- end }} spec: diff --git a/charts/airbyte-pod-sweeper/values.yaml b/charts/airbyte-pod-sweeper/values.yaml index f3a3abfad72d..bd3ab9648964 100644 --- a/charts/airbyte-pod-sweeper/values.yaml +++ b/charts/airbyte-pod-sweeper/values.yaml @@ -213,3 +213,9 @@ extraVolumeMounts: [] ## emptyDir: {} ## extraVolumes: [] + +## podSweeper.timeToDeletePods.completed Time to remove pods on completed status in minutes +## podSweeper.timeToDeletePods.erro Time to remove pods on error status in minutes +timeToDeletePods: + completed: 120 + error: 1440 \ No newline at end of file From 1ea98c9a786ed30ddfef360e05459a46540b5733 Mon Sep 17 00:00:00 2001 From: Caio Henrique Date: Thu, 13 Oct 2022 22:58:03 -0300 Subject: [PATCH 3/4] Add variable to customize airbyte-pod-sweeper delete pod --- charts/airbyte-pod-sweeper/templates/configmap.yaml | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/charts/airbyte-pod-sweeper/templates/configmap.yaml b/charts/airbyte-pod-sweeper/templates/configmap.yaml index afea58d23ec6..627fe89b5197 100644 --- a/charts/airbyte-pod-sweeper/templates/configmap.yaml +++ b/charts/airbyte-pod-sweeper/templates/configmap.yaml @@ -9,18 +9,15 @@ metadata: data: sweep-pod.sh: | #!/bin/bash - get_worker_pods () { kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \ --field-selector status.phase!=Running get pods \ -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}' } - delete_worker_pod() { printf "From status '%s' since '%s', " $2 $3 echo "$1" | grep -v "STATUS" | awk '{print $1}' | xargs --no-run-if-empty kubectl -n ${KUBE_NAMESPACE} delete pod } - while : do # Shorter time window for completed pods @@ -50,4 +47,4 @@ data: done ) sleep 60 - done + done \ No newline at end of file From 65601b1cafeb971e627add374dfcd465bb8f35df Mon Sep 17 00:00:00 2001 From: Caio Henrique Date: Thu, 13 Oct 2022 22:59:13 -0300 Subject: [PATCH 4/4] Add variable to customize airbyte-pod-sweeper delete pod --- charts/airbyte-pod-sweeper/values.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/charts/airbyte-pod-sweeper/values.yaml b/charts/airbyte-pod-sweeper/values.yaml index bd3ab9648964..f2e3dc204682 100644 --- a/charts/airbyte-pod-sweeper/values.yaml +++ b/charts/airbyte-pod-sweeper/values.yaml @@ -214,8 +214,8 @@ extraVolumeMounts: [] ## extraVolumes: [] -## podSweeper.timeToDeletePods.completed Time to remove pods on completed status in minutes -## podSweeper.timeToDeletePods.erro Time to remove pods on error status in minutes +## podSweeper.timeToDeletePods.completed Time to remove pods on completed status (minutes). +## podSweeper.timeToDeletePods.error Time to remove pods on error status (minutes). timeToDeletePods: completed: 120 error: 1440 \ No newline at end of file