K8SPXC-1398: Block SST script termination until it actually finishes (#…

…1063)

percona-xtradb-cluster-8.0-backup/Dockerfile

@@ -113,7 +113,7 @@ RUN set -ex; \
 	https://raw.githubusercontent.com/minio/mc/${MC_VERSION}/LICENSE
 
 COPY lib/pxc /usr/lib/pxc
-COPY recovery-*.sh run_backup.sh backup.sh post_backup.sh get-pxc-state /usr/bin/
+COPY recovery-*.sh run_backup.sh backup.sh get-pxc-state /usr/bin/
 
 VOLUME ["/backup"]
 USER 1001

percona-xtradb-cluster-8.0-backup/backup.sh

@@ -72,30 +72,24 @@ function request_streaming() {
     if [ -z "$NODE_NAME" ]; then
         /opt/percona/peer-list -on-start=/usr/bin/get-pxc-state -service=$PXC_SERVICE
         log 'ERROR' 'Cannot find node for backup'
-        log 'ERROR' 'Backup was finished unsuccessfull'
+        log 'ERROR' 'Backup was finished unsuccessful'
         exit 1
     fi
 
     set +o errexit
     log 'INFO' 'Garbd was started'
-        garbd \
-            --address "gcomm://$NODE_NAME.$PXC_SERVICE?gmcast.listen_addr=tcp://0.0.0.0:4567" \
-            --donor "$NODE_NAME" \
-            --group "$PXC_SERVICE" \
-            --options "$GARBD_OPTS" \
-            --sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \
-            --recv-script="/usr/bin/run_backup.sh" \
-            --post-recv-script="/usr/bin/post_backup.sh"
-        EXID_CODE=$?
+    garbd \
+        --address "gcomm://$NODE_NAME.$PXC_SERVICE?gmcast.listen_addr=tcp://0.0.0.0:4567" \
+        --donor "$NODE_NAME" \
+        --group "$PXC_SERVICE" \
+        --options "$GARBD_OPTS" \
+        --sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \
+        --recv-script="/usr/bin/run_backup.sh"
+    EXID_CODE=$?
 
     if [ -f '/tmp/backup-is-completed' ]; then
-        if [[ $EXID_CODE == 0 ]]; then
-            log 'INFO' 'Backup was finished successfully'
-            exit 0
-        else
-            log 'ERROR' 'Backup was finished successfully but post-recv-script has failed.'
-            exit $EXID_CODE
-        fi
+        log 'INFO' 'Backup was finished successfully'
+        exit 0
     fi
 
     log 'ERROR' 'Backup was finished unsuccessful'

percona-xtradb-cluster-8.0-backup/run_backup.sh

@@ -78,16 +78,22 @@ backup_volume() {
 	vault_store $BACKUP_DIR/${SST_INFO_NAME}
 
 	if (($SST_FAILED == 0)); then
-		FIRST_RECEIVED=0
 		socat -u "$SOCAT_OPTS" stdio >xtrabackup.stream
-		FIRST_RECEIVED=1
 		if [[ $? -ne 0 ]]; then
 			log 'ERROR' 'Socat(2) failed'
 			log 'ERROR' 'Backup was finished unsuccessfully'
 			exit 1
 		fi
 		log 'INFO' "Socat(2) returned $?"
 	fi
+
+	trap '' 15
+	if (($(stat -c%s xtrabackup.stream) < 5000000)); then
+		log 'ERROR' 'Backup is empty'
+		log 'ERROR' 'Backup was finished unsuccessfully'
+		exit 1
+	fi
+	md5sum xtrabackup.stream | tee md5sum.txt
 }
 
 backup_s3() {
@@ -110,11 +116,18 @@ backup_s3() {
 		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
 
 	if (($SST_FAILED == 0)); then
-		FIRST_RECEIVED=0
 		socat -u "$SOCAT_OPTS" stdio \
 			| xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 $XBCLOUD_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH" 2>&1 \
 			| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
-		FIRST_RECEIVED=1
+	fi
+
+	trap '' 15
+	mc -C /tmp/mc stat ${INSECURE_ARG} "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5"
+	md5_size=$(mc -C /tmp/mc stat ${INSECURE_ARG} --json "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5" | sed -e 's/.*"size":\([0-9]*\).*/\1/')
+	if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)); then
+		log 'ERROR' 'Backup is empty'
+		log 'ERROR' 'Backup was finished unsuccessfull'
+		exit 1
 	fi
 }
 
@@ -143,11 +156,9 @@ backup_azure() {
 		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
 
 	if (($SST_FAILED == 0)); then
-		FIRST_RECEIVED=0
 		socat -u "$SOCAT_OPTS" stdio \
 			| xbcloud put --parallel="$(grep -c processor /proc/cpuinfo)" $XBCLOUD_ARGS --storage=azure "$BACKUP_PATH" 2>&1 \
 			| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
-		FIRST_RECEIVED=1
 	fi
 }
 
@@ -166,4 +177,6 @@ fi
 if (($SST_FAILED == 0)); then
 	touch /tmp/backup-is-completed
 fi
+
+log 'INFO' 'Backup finished'
 exit $SST_FAILED

percona-xtradb-cluster-8.4-backup/Dockerfile

@@ -113,7 +113,7 @@ RUN set -ex; \
 	https://raw.githubusercontent.com/minio/mc/${MC_VERSION}/LICENSE
 
 COPY lib/pxc /usr/lib/pxc
-COPY recovery-*.sh run_backup.sh backup.sh post_backup.sh get-pxc-state /usr/bin/
+COPY recovery-*.sh run_backup.sh backup.sh get-pxc-state /usr/bin/
 
 VOLUME ["/backup"]
 USER 1001

percona-xtradb-cluster-8.4-backup/backup.sh

@@ -78,22 +78,21 @@ function request_streaming() {
 
     set +o errexit
     log 'INFO' 'Garbd was started'
-        garbd \
-            --address "gcomm://$NODE_NAME.$PXC_SERVICE?gmcast.listen_addr=tcp://0.0.0.0:4567" \
-            --donor "$NODE_NAME" \
-            --group "$PXC_SERVICE" \
-            --options "$GARBD_OPTS" \
-            --sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \
-            --recv-script="/usr/bin/run_backup.sh" \
-            --post-recv-script="/usr/bin/post_backup.sh"
-        EXID_CODE=$?
+    garbd \
+        --address "gcomm://$NODE_NAME.$PXC_SERVICE?gmcast.listen_addr=tcp://0.0.0.0:4567" \
+        --donor "$NODE_NAME" \
+        --group "$PXC_SERVICE" \
+        --options "$GARBD_OPTS" \
+        --sst "xtrabackup-v2:$LOCAL_IP:4444/xtrabackup_sst//1" \
+        --recv-script="/usr/bin/run_backup.sh"
+    EXID_CODE=$?
 
     if [ -f '/tmp/backup-is-completed' ]; then
         log 'INFO' 'Backup was finished successfully'
         exit 0
     fi
 
-    log 'ERROR' 'Backup was finished unsuccessfull'
+    log 'ERROR' 'Backup was finished unsuccessful'
 
     exit $EXID_CODE
 }

percona-xtradb-cluster-8.4-backup/run_backup.sh

@@ -78,16 +78,22 @@ backup_volume() {
 	vault_store $BACKUP_DIR/${SST_INFO_NAME}
 
 	if (($SST_FAILED == 0)); then
-		FIRST_RECEIVED=0
 		socat -u "$SOCAT_OPTS" stdio >xtrabackup.stream
-		FIRST_RECEIVED=1
 		if [[ $? -ne 0 ]]; then
 			log 'ERROR' 'Socat(2) failed'
 			log 'ERROR' 'Backup was finished unsuccessfully'
 			exit 1
 		fi
 		log 'INFO' "Socat(2) returned $?"
 	fi
+
+	trap '' 15
+	if (($(stat -c%s xtrabackup.stream) < 5000000)); then
+		log 'ERROR' 'Backup is empty'
+		log 'ERROR' 'Backup was finished unsuccessfully'
+		exit 1
+	fi
+	md5sum xtrabackup.stream | tee md5sum.txt
 }
 
 backup_s3() {
@@ -110,11 +116,18 @@ backup_s3() {
 		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
 
 	if (($SST_FAILED == 0)); then
-		FIRST_RECEIVED=0
 		socat -u "$SOCAT_OPTS" stdio \
 			| xbcloud put --storage=s3 --parallel="$(grep -c processor /proc/cpuinfo)" --md5 $XBCLOUD_ARGS --s3-bucket="$S3_BUCKET" "$S3_BUCKET_PATH" 2>&1 \
 			| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
-		FIRST_RECEIVED=1
+	fi
+
+	trap '' 15
+	mc -C /tmp/mc stat ${INSECURE_ARG} "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5"
+	md5_size=$(mc -C /tmp/mc stat ${INSECURE_ARG} --json "dest/$S3_BUCKET/$S3_BUCKET_PATH.md5" | sed -e 's/.*"size":\([0-9]*\).*/\1/')
+	if [[ $md5_size =~ "Object does not exist" ]] || ((md5_size < 23000)); then
+		log 'ERROR' 'Backup is empty'
+		log 'ERROR' 'Backup was finished unsuccessfull'
+		exit 1
 	fi
 }
 
@@ -143,11 +156,9 @@ backup_azure() {
 		| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
 
 	if (($SST_FAILED == 0)); then
-		FIRST_RECEIVED=0
 		socat -u "$SOCAT_OPTS" stdio \
 			| xbcloud put --parallel="$(grep -c processor /proc/cpuinfo)" $XBCLOUD_ARGS --storage=azure "$BACKUP_PATH" 2>&1 \
 			| (grep -v "error: http request failed: Couldn't resolve host name" || exit 1)
-		FIRST_RECEIVED=1
 	fi
 }
 
@@ -166,4 +177,6 @@ fi
 if (($SST_FAILED == 0)); then
 	touch /tmp/backup-is-completed
 fi
+
+log 'INFO' 'Backup finished'
 exit $SST_FAILED