From 4214a682a74f1b1f5fb8b458f317dd9f2d38c51e Mon Sep 17 00:00:00 2001
From: Cesar Celis Hernandez <celis.hernandez.cesar@gmail.com>
Date: Tue, 5 Mar 2024 17:57:25 -0500
Subject: [PATCH] Make decommission test more reliable

---
 testing/common.sh            |  2 +-
 testing/decommission-test.sh | 11 ++++++-----
 testing/verify-data.sh       |  8 +++++---
 3 files changed, 12 insertions(+), 9 deletions(-)

diff --git a/testing/common.sh b/testing/common.sh
index a43c0d578de..79dc57b70fc 100644
--- a/testing/common.sh
+++ b/testing/common.sh
@@ -148,7 +148,7 @@ function wait_for_n_tenant_pods() {
     try kubectl wait --namespace "$NAMESPACE" \
         --for=condition=ready pod \
         --selector v1.min.io/tenant="$TENANT_NAME" \
-        --timeout=300s
+        --timeout=600s
 }
 
 # copies the script to the pod.
diff --git a/testing/decommission-test.sh b/testing/decommission-test.sh
index 71951627cbd..36469b48f90 100755
--- a/testing/decommission-test.sh
+++ b/testing/decommission-test.sh
@@ -59,12 +59,13 @@ function main() {
 
     echo -e "\n\n"
     echo "decommission-test.sh: main(): Verify data after decommission:"
-    execute_pod_script verify-data.sh "${DEBUG_POD_NAME}"
-    RESULT=$(read_script_result default "${DEBUG_POD_NAME}" verify-data.log)
+    RESULT="script failed"
+    while [ "$RESULT" != "script passed" ]; do
+        echo "Verifying data after the decommission loop"
+        execute_pod_script verify-data.sh "${DEBUG_POD_NAME}"
+        RESULT=$(read_script_result default "${DEBUG_POD_NAME}" verify-data.log)
+    done
     teardown distributed
-    if [ "$RESULT" != "script passed" ]; then
-        exit 1
-    fi
 }
 
 main "$@"
diff --git a/testing/verify-data.sh b/testing/verify-data.sh
index 18b73bf8ca2..44b29bb03e9 100644
--- a/testing/verify-data.sh
+++ b/testing/verify-data.sh
@@ -7,22 +7,24 @@ echo "Verify Data in remaining pool(s) after decommission test"
 echo "Get data and verify files are still present as they were uploaded"
 FILE_TO_SAVE_MC_OUTPUT=tmp.log
 rm -rf $FILE_TO_SAVE_MC_OUTPUT
-mc ls myminio/bucket >$FILE_TO_SAVE_MC_OUTPUT 2>&1
+mc ls myminio/bucket > $FILE_TO_SAVE_MC_OUTPUT 2>&1
 RESULT=$(grep -c ERROR $FILE_TO_SAVE_MC_OUTPUT | awk -F' ' '{print $1}')
 while [ "$RESULT" == "1" ]; do
 	echo "There was an error in mc ls, retrying"
-	mc ls myminio/bucket >$FILE_TO_SAVE_MC_OUTPUT 2>&1
+	mc ls myminio/bucket > $FILE_TO_SAVE_MC_OUTPUT 2>&1
 	RESULT=$(grep -c ERROR $FILE_TO_SAVE_MC_OUTPUT | awk -F' ' '{print $1}')
 	echo "${RESULT}"
 done
 echo "mc ls was successful"
+mc ls myminio/bucket # To see what is listed
 
 for i in {1..10}; do
 	RESULT=$(grep -c file"$i" "${FILE_TO_SAVE_MC_OUTPUT}")
 	if [ "${RESULT}" -eq 0 ]; then
 		echo "fail, there is a missing file: file${i}"
 		echo "exiting with 1 as there is missing file..."
+		echo "script failed" > verify-data.log
 		exit 1
 	fi
 done
-echo "script passed" >verify-data.log
+echo "script passed" > verify-data.log