[cherry-pick] restore: fix failed to retry grpc errors(tidb#27423) (#…

…1438)
pingcap · Dec 20, 2021 · a135dc5 · a135dc5
1 parent 809bb35
commit a135dc5
Show file tree

Hide file tree

Showing 3 changed files with 8 additions and 3 deletions.
diff --git a/pkg/restore/import.go b/pkg/restore/import.go
@@ -320,6 +320,10 @@ func (importer *FileImporter) Import(
 						log.Debug("failpoint download-sst-error injected.", zap.String("msg", msg))
 						e = errors.Annotate(e, msg)
 					})
+					failpoint.Inject("restore-gRPC-error", func(_ failpoint.Value) {
+						log.Warn("the connection to TiKV has been cut by a neko, meow :3")
+						e = status.Error(codes.Unavailable, "the connection to TiKV has been cut by a neko, meow :3")
+					})
 					if e != nil {
 						remainFiles = remainFiles[i:]
 						return errors.Trace(e)

diff --git a/pkg/utils/backoff.go b/pkg/utils/backoff.go
@@ -59,7 +59,8 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
 		bo.delayTime = 2 * bo.delayTime
 		bo.attempt--
 	} else {
-		switch errors.Cause(err) { // nolint:errorlint
+		e := errors.Cause(err)
+		switch e { // nolint:errorlint
 		case berrors.ErrKVEpochNotMatch, berrors.ErrKVDownloadFailed, berrors.ErrKVIngestFailed:
 			bo.delayTime = 2 * bo.delayTime
 			bo.attempt--
@@ -68,7 +69,7 @@ func (bo *importerBackoffer) NextBackoff(err error) time.Duration {
 			bo.delayTime = 0
 			bo.attempt = 0
 		default:
-			switch status.Code(err) {
+			switch status.Code(e) {
 			case codes.Unavailable, codes.Aborted:
 				bo.delayTime = 2 * bo.delayTime
 				bo.attempt--

diff --git a/tests/br_full/run.sh b/tests/br_full/run.sh
@@ -69,7 +69,7 @@ for ct in limit lz4 zstd; do
 
   # restore full
   echo "restore with $ct backup start..."
-  export GO_FAILPOINTS="github.com/pingcap/br/pkg/restore/restore-storage-error=1*return(\"connection refused\")"
+  export GO_FAILPOINTS="github.com/pingcap/br/pkg/restore/restore-storage-error=1*return(\"connection refused\");github.com/pingcap/br/pkg/restore/restore-gRPC-error=1*return(true)"
   run_br restore full -s "local://$TEST_DIR/$DB-$ct" --pd $PD_ADDR --ratelimit 1024
   export GO_FAILPOINTS=""