diff --git a/testing/common.sh b/testing/common.sh index a43c0d578de..79dc57b70fc 100644 --- a/testing/common.sh +++ b/testing/common.sh @@ -148,7 +148,7 @@ function wait_for_n_tenant_pods() { try kubectl wait --namespace "$NAMESPACE" \ --for=condition=ready pod \ --selector v1.min.io/tenant="$TENANT_NAME" \ - --timeout=300s + --timeout=600s } # copies the script to the pod. diff --git a/testing/decommission-test.sh b/testing/decommission-test.sh index 71951627cbd..14534d2e542 100755 --- a/testing/decommission-test.sh +++ b/testing/decommission-test.sh @@ -59,12 +59,26 @@ function main() { echo -e "\n\n" echo "decommission-test.sh: main(): Verify data after decommission:" - execute_pod_script verify-data.sh "${DEBUG_POD_NAME}" - RESULT=$(read_script_result default "${DEBUG_POD_NAME}" verify-data.log) + RESULT="script failed" # The initial state is failure. + counter=0 + while [ $counter -lt 60 ]; do + echo "Decommission Check Attempt: $counter" + ((counter++)) + execute_pod_script verify-data.sh "${DEBUG_POD_NAME}" + RESULT=$(read_script_result default "${DEBUG_POD_NAME}" verify-data.log) + if [ "$RESULT" == "script passed" ]; then + break + fi + sleep 1 + done teardown distributed if [ "$RESULT" != "script passed" ]; then + echo "Decommission Test Failed" exit 1 fi + echo "Decomission Test Passed" + exit 0 + } main "$@" diff --git a/testing/verify-data.sh b/testing/verify-data.sh index 18b73bf8ca2..44b29bb03e9 100644 --- a/testing/verify-data.sh +++ b/testing/verify-data.sh @@ -7,22 +7,24 @@ echo "Verify Data in remaining pool(s) after decommission test" echo "Get data and verify files are still present as they were uploaded" FILE_TO_SAVE_MC_OUTPUT=tmp.log rm -rf $FILE_TO_SAVE_MC_OUTPUT -mc ls myminio/bucket >$FILE_TO_SAVE_MC_OUTPUT 2>&1 +mc ls myminio/bucket > $FILE_TO_SAVE_MC_OUTPUT 2>&1 RESULT=$(grep -c ERROR $FILE_TO_SAVE_MC_OUTPUT | awk -F' ' '{print $1}') while [ "$RESULT" == "1" ]; do echo "There was an error in mc ls, retrying" - mc ls myminio/bucket >$FILE_TO_SAVE_MC_OUTPUT 2>&1 + mc ls myminio/bucket > $FILE_TO_SAVE_MC_OUTPUT 2>&1 RESULT=$(grep -c ERROR $FILE_TO_SAVE_MC_OUTPUT | awk -F' ' '{print $1}') echo "${RESULT}" done echo "mc ls was successful" +mc ls myminio/bucket # To see what is listed for i in {1..10}; do RESULT=$(grep -c file"$i" "${FILE_TO_SAVE_MC_OUTPUT}") if [ "${RESULT}" -eq 0 ]; then echo "fail, there is a missing file: file${i}" echo "exiting with 1 as there is missing file..." + echo "script failed" > verify-data.log exit 1 fi done -echo "script passed" >verify-data.log +echo "script passed" > verify-data.log