Bugfix: sweep-pod.sh never deleting certain Failed pods (#14925) (#14931

)
airbytehq · Jul 25, 2022 · 9f80765 · 9f80765
1 parent 8ebed61
commit 9f80765
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 6 deletions.
diff --git a/charts/airbyte-pod-sweeper/files/sweep-pod.sh b/charts/airbyte-pod-sweeper/files/sweep-pod.sh
@@ -3,7 +3,7 @@
 get_worker_pods () {
     kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \
       --field-selector status.phase!=Running get pods \
-      -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime}{"\n"}{end}'
+      -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}'
 }
 
 delete_worker_pod() {
@@ -26,7 +26,8 @@ do
             POD_NAME=`echo $POD | cut -d " " -f 1`
             POD_STATUS=`echo $POD | cut -d " " -f 2`
             POD_DATE_STR=`echo $POD | cut -d " " -f 3`
-            POD_DATE=`date -d $POD_DATE_STR '+%s'`
+            POD_START_DATE_STR=`echo $POD | cut -d " " -f 4`
+            POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'`
             if [ "$POD_STATUS" = "Succeeded" ]; then
             if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then
                 delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"

diff --git a/kube/resources/pod-sweeper.yaml b/kube/resources/pod-sweeper.yaml
@@ -9,7 +9,7 @@ data:
     get_worker_pods () {
       kubectl -n ${KUBE_NAMESPACE} -L airbyte -l airbyte=worker-pod \
         --field-selector status.phase!=Running get pods \
-        -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime}{"\n"}{end}'
+        -o=jsonpath='{range .items[*]} {.metadata.name} {.status.phase} {.status.conditions[0].lastTransitionTime} {.status.startTime}{"\n"}{end}'
     }
 
     delete_worker_pod() {
@@ -25,16 +25,15 @@ data:
       # Longer time window for pods in error (to debug)
       NON_SUCCESS_DATE_STR=`date -d 'now - 24 hours' --utc -Ins`
       NON_SUCCESS_DATE=`date -d $NON_SUCCESS_DATE_STR +%s`
-      # default time to use in case its unavailable from kubectl
-      DEFAULT=`date --utc -Ins`
       (
           IFS=$'\n'
           for POD in `get_worker_pods`; do
               IFS=' '
               POD_NAME=`echo $POD | cut -d " " -f 1`
               POD_STATUS=`echo $POD | cut -d " " -f 2`
               POD_DATE_STR=`echo $POD | cut -d " " -f 3`
-              POD_DATE=`date -d ${POD_DATE_STR:-$DEFAULT} '+%s'`
+              POD_START_DATE_STR=`echo $POD | cut -d " " -f 4`
+              POD_DATE=`date -d ${POD_DATE_STR:-$POD_START_DATE_STR} '+%s'`
               if [ "$POD_STATUS" = "Succeeded" ]; then
                 if [ "$POD_DATE" -lt "$SUCCESS_DATE" ]; then
                   delete_worker_pod "$POD_NAME" "$POD_STATUS" "$POD_DATE_STR"