pingcap · ti-chi-bot · Sep 13, 2024
diff --git a/br/pkg/utils/backoff.go b/br/pkg/utils/backoff.go
@@ -207,6 +207,11 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
 			}
 		}
 	}
+	failpoint.Inject("set-import-attempt-to-one", func(_ failpoint.Value) {
+		if bo.attempt > 1 {
+			bo.attempt = 1
+		}
+	})
 	if bo.delayTime > bo.maxDelayTime {
 		return bo.maxDelayTime
 	}

diff --git a/br/tests/br_file_corruption/run.sh b/br/tests/br_file_corruption/run.sh
@@ -0,0 +1,54 @@
+#!/bin/sh
+#
+# Copyright 2024 PingCAP, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -eux
+
+DB="$TEST_NAME"
+TABLE="usertable"
+CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+
+run_sql "CREATE DATABASE $DB;"
+go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
+run_br --pd $PD_ADDR backup full -s "local://$TEST_DIR/$DB"
+
+filename=$(find $TEST_DIR/$DB -regex ".*.sst" | head -n 1)
+filename_temp=$filename"_temp"
+filename_bak=$filename"_bak"
+echo "corruption" > $filename_temp
+cat $filename >> $filename_temp
+
+# file lost
+mv $filename $filename_bak
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'restore success' 
+    exit 1
+fi
+
+# file corruption
+mv $filename_temp $filename
+truncate --size=-11 $filename
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$DB" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'restore success' 
+    exit 1
+fi
diff --git a/br/tests/br_file_corruption/workload b/br/tests/br_file_corruption/workload
@@ -0,0 +1,12 @@
+recordcount=10000
+operationcount=0
+workload=core
+
+readallfields=true
+
+readproportion=0
+updateproportion=0
+scanproportion=0
+insertproportion=0
+
+requestdistribution=uniform
diff --git a/br/tests/br_full_ddl/run.sh b/br/tests/br_full_ddl/run.sh
@@ -23,6 +23,7 @@ RESTORE_LOG=LOG=/$TEST_DIR/restore.log
 BACKUP_STAT=/$TEST_DIR/backup_stat
 RESOTRE_STAT=/$TEST_DIR/restore_stat
 CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)
+res_file="$TEST_DIR/sql_res.$TEST_NAME.txt"
 
 run_sql "CREATE DATABASE $DB;"
 go-ycsb load mysql -P $CUR/workload -p mysql.host=$TIDB_IP -p mysql.port=$TIDB_PORT -p mysql.user=root -p mysql.db=$DB
@@ -39,6 +40,23 @@ for i in $(seq $DDL_COUNT); do
     fi
 done
 
+# wait until the index creation/drop is done
+retry_cnt=0
+while true; do
+    run_sql "ADMIN SHOW DDL JOBS WHERE DB_NAME = '$DB' AND TABLE_NAME = '$TABLE' AND STATE != 'synced';"
+    if grep -Fq "1. row" $res_file; then
+        cat $res_file
+        retry_cnt=$((retry_cnt+1))
+        if [ "$retry_cnt" -gt 50 ]; then
+            echo 'the wait lag is too large'
+            exit 1
+        fi
+        continue
+    fi
+
+    break
+done
+
 # run analyze to generate stats
 run_sql "analyze table $DB.$TABLE;"
 # record the stats and remove last_update_version

diff --git a/br/tests/br_pitr/run.sh b/br/tests/br_pitr/run.sh
@@ -100,6 +100,7 @@ restart_services
 echo "run pitr"
 run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" > $res_file 2>&1
 
+<<<<<<< HEAD
 # check something in downstream cluster
 echo "check br log"
 check_contains "restore log success summary"
@@ -112,3 +113,67 @@ expect_delete_range=$(($incremental_delete_range_count-$prepare_delete_range_cou
 check_contains "DELETE_RANGE_CNT: $expect_delete_range"
 ## check feature compatibility between PITR and accelerate indexing
 bash $CUR/check/check_ingest_repair.sh
+=======
+check_result
+
+# start a new cluster for incremental + log
+echo "restart a services"
+restart_services
+
+echo "run snapshot restore#2"
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
+
+echo "run incremental restore + log restore"
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/inc" > $res_file 2>&1
+
+check_result
+
+# start a new cluster for incremental + log
+echo "restart a services"
+restart_services
+
+echo "run snapshot restore#3"
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/full" 
+
+echo "run incremental restore but failed"
+restore_fail=0
+run_br --pd $PD_ADDR restore full -s "local://$TEST_DIR/$PREFIX/inc_fail" || restore_fail=1
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success' 
+    exit 1
+fi
+
+# start a new cluster for corruption
+echo "restart a services"
+restart_services
+
+echo "corrupt a log file"
+filename=$(find $TEST_DIR/$PREFIX/log -regex ".*\.log" | grep -v "schema-meta" | tail -n 1)
+filename_temp=$filename"_temp"
+filename_bak=$filename"_bak"
+echo "corruption" > $filename_temp
+cat $filename >> $filename_temp
+
+# file lost
+mv $filename $filename_bak
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success' 
+    exit 1
+fi
+
+# file corruption
+mv $filename_temp $filename
+truncate --size=-11 $filename
+export GO_FAILPOINTS="github.com/pingcap/tidb/br/pkg/utils/set-import-attempt-to-one=return(true)"
+restore_fail=0
+run_br --pd $PD_ADDR restore point -s "local://$TEST_DIR/$PREFIX/log" --full-backup-storage "local://$TEST_DIR/$PREFIX/full" || restore_fail=1
+export GO_FAILPOINTS=""
+if [ $restore_fail -ne 1 ]; then
+    echo 'pitr success' 
+    exit 1
+fi
+>>>>>>> 5399ca70da9 (br: fix br integration test (#53836))
diff --git a/br/tests/br_txn/run.sh b/br/tests/br_txn/run.sh
@@ -97,12 +97,22 @@ run_test() {
     # delete data in range[start-key, end-key)
     clean "hello" "world" 
     # Ensure the data is deleted
-    checksum_new=$(checksum "hello" "world")
-
-    if [ "$checksum_new" != "$checksum_empty" ];then
-        echo "failed to delete data in range after backup"
-        fail_and_exit
-    fi
+    retry_cnt=0
+    while true; do
+        checksum_new=$(checksum "hello" "world")
+
+        if [ "$checksum_new" != "$checksum_empty" ]; then
+            echo "failed to delete data in range after backup; retry_cnt = $retry_cnt"
+            retry_cnt=$((retry_cnt+1))
+            if [ "$retry_cnt" -gt 50 ]; then
+                fail_and_exit
+            fi
+            sleep 1
+            continue
+        fi
+
+        break
+    done
 
     # restore rawkv
     echo "restore start..."

diff --git a/br/tests/run_group_br_tests.sh b/br/tests/run_group_br_tests.sh
@@ -25,11 +25,18 @@ groups=(
 	["G02"]="br_full_cluster_restore br_full_index br_incremental_ddl br_pitr_failpoint"
 	["G03"]='br_incompatible_tidb_config br_incremental br_incremental_index br_incremental_only_ddl br_incremental_same_table br_insert_after_restore br_key_locked br_log_test br_move_backup br_mv_index br_other br_partition_add_index br_tidb_placement_policy br_tiflash'
 	["G04"]='br_range br_replica_read br_restore_TDE_enable br_restore_log_task_enable br_s3 br_shuffle_leader br_shuffle_region br_single_table'
+<<<<<<< HEAD
 	["G05"]='br_skip_checksum br_small_batch_size br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
 	["G06"]='br_tikv_outage'
 	["G07"]='br_tikv_outage3'
 	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom'
 	["G09"]='br_pitr'
+=======
+	["G05"]='br_skip_checksum br_split_region_fail br_systables br_table_filter br_txn br_stats br_clustered_index br_crypter'
+	["G06"]='br_tikv_outage br_tikv_outage3'
+	["G07"]='br_pitr'
+	["G08"]='br_tikv_outage2 br_ttl br_views_and_sequences br_z_gc_safepoint br_autorandom br_file_corruption'
+>>>>>>> 5399ca70da9 (br: fix br integration test (#53836))
 )
 
 # Get other cases not in groups, to avoid missing any case