br: fix br integration test (#53836)

br/pkg/utils/backoff.go

@@ -204,6 +204,11 @@
 			}
 		}
 	}
+	failpoint.Inject("set-import-attempt-to-one", func(_ failpoint.Value) {
+		if bo.attempt > 1 {
+			bo.attempt = 1
+		}
+	})
 	if bo.delayTime > bo.maxDelayTime {
 		return bo.maxDelayTime
 	}

br/tests/br_file_corruption/run.sh

@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Copyright 2024 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+DB="$TEST_NAME"
+TABLE="usertable"
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+run_sql "CREATE DATABASE $DB;"
+go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
+run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB"
+
+filename=$(find $TEST_DIR/$DB -regex ".*.sst" | head -n 1)
+filename_temp=$filename"_temp"
+filename_bak=$filename"_bak"
+echo "corruption" > $filename_temp
+cat $filename >> $filename_temp
+
+# file lost
+mv $filename $filename_bak
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'restore success' 
+    exit 1
+fi
+
+# file corruption
+mv $filename_temp $filename
+truncate --size=-11 $filename
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'restore success' 
+    exit 1
+fi

br/tests/br_file_corruption/workload

@@ -0,0 +1,12 @@
+recordcount=10000
+operationcount=0
+workload=core
+
+readallfields=true
+
+readproportion=0
+updateproportion=0
+scanproportion=0
+insertproportion=0
+
+requestdistribution=uniform

br/tests/br_full_ddl/run.sh

@@ -22,6 +22,7 @@ LOG=/$TEST_DIR/backup.log
 RESTORE_LOG=LOG=/$TEST_DIR/restore.log
 BACKUP_STAT=/$TEST_DIR/backup_stat
 RESOTRE_STAT=/$TEST_DIR/restore_stat
+res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
 
 run_sql "CREATE DATABASE $DB;"
 go-ycsb load mysql -P tests/$TEST_NAME/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
@@ -38,6 +39,23 @@ for i in $(seq $DDL_COUNT); do
     fi
 done
 
+# wait until the index creation/drop is done
+retry_cnt=0
+while true; do
+    run_sql "ADMIN SHOW DDL JOBS WHERE DB_NAME = '$DB' AND TABLE_NAME = '$TABLE' AND STATE != 'synced';"
+    if grep -Fq "1. row" $res_file; then
+        cat $res_file
+        retry_cnt=$((retry_cnt+1))
+        if [ "$retry_cnt" -gt 50 ]; then
+            echo 'the wait lag is too large'
+            exit 1
+        fi
+        continue
+    fi
+
+    break
+done
+
 # run analyze to generate stats
 run_sql "analyze table $DB.$TABLE;"
 # record field0's stats and remove last_update_version

br/tests/br_pitr/run.sh

@@ -114,3 +114,37 @@ expect_delete_range=$(($incremental_delete_range_count-$prepare_delete_range_cou
 check_contains "DELETE_RANGE_CNT: $expect_delete_range"
 ## check feature compatibility between PITR and accelerate indexing
 bash $CUR/check/check_ingest_repair.sh
+
+# start a new cluster for corruption
+echo "restart a services"
+restart_services
+
+echo "corrupt a log file"
+filename=$(find $TEST_DIR/$PREFIX/log -regex ".*\.log" | grep -v "schema-meta" | tail -n 1)
+filename_temp=$filename"_temp"
+filename_bak=$filename"_bak"
+echo "corruption" > $filename_temp
+cat $filename >> $filename_temp
+
+# file lost
+mv $filename $filename_bak
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success' 
+    exit 1
+fi
+
+# file corruption
+mv $filename_temp $filename
+truncate --size=-11 $filename
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success' 
+    exit 1
+fi

br/tests/br_txn/run.sh

@@ -97,12 +97,22 @@ run_test() {
     # delete data in range[start-key, end-key)
     clean "hello" "world" 
     # Ensure the data is deleted
-    checksum_new=$(checksum "hello" "world")
-
-    if [ "$checksum_new" != "$checksum_empty" ];then
-        echo "failed to delete data in range after backup"
-        fail_and_exit
-    fi
+    retry_cnt=0
+    while true; do
+        checksum_new=$(checksum "hello" "world")
+
+        if [ "$checksum_new" != "$checksum_empty" ]; then
+            echo "failed to delete data in range after backup; retry_cnt = $retry_cnt"
+            retry_cnt=$((retry_cnt+1))
+            if [ "$retry_cnt" -gt 50 ]; then
+                fail_and_exit
+            fi
+            sleep 1
+            continue
+        fi
+
+        break
+    done
 
     # restore rawkv
     echo "restore start..."

br/tests/run_group.sh

@@ -29,7 +29,7 @@ groups=(
 	["G05"]='br_range br_rawkv br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
 	["G06"]='br_skip_checksum br_small_batch_size br_split_region_fail br_systables br_table_filter br_txn'
 	["G07"]='br_clustered_index br_crypter br_table_partition br_tidb_placement_policy br_tiflash br_tikv_outage'
-	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom lightning_add_index lightning_alter_random lightning_auto_columns'
+	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption lightning_add_index lightning_alter_random lightning_auto_columns'
 	["G09"]='lightning_auto_random_default lightning_bom_file lightning_character_sets lightning_check_partial_imported lightning_checkpoint lightning_checkpoint_chunks lightning_checkpoint_columns lightning_checkpoint_dirty_tableid'
 	["G10"]='lightning_checkpoint_engines lightning_checkpoint_engines_order lightning_checkpoint_error_destroy lightning_checkpoint_parquet lightning_checkpoint_timestamp lightning_checksum_mismatch lightning_cmdline_override lightning_column_permutation lightning_common_handle'
 	["G11"]='lightning_compress lightning_concurrent-restore lightning_config_max_error lightning_config_skip_csv_header lightning_csv lightning_default-columns lightning_disable_scheduler_by_key_range lightning_disk_quota lightning_distributed_import'