lightning: add retry for "write to tikv with no leader returned" (#43480

) ref #43055
pingcap · Apr 28, 2023 · c4eccee · c4eccee
1 parent 9183ddc
commit c4eccee
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 1 deletion.
diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go
@@ -1105,7 +1105,7 @@ func (local *local) WriteToTiKV(
 		}
 	}
 
-	// if there is not leader currently, we should directly return an error
+	// if there is no leader currently, we should directly return an error
 	if len(leaderPeerMetas) == 0 {
 		log.FromContext(ctx).Warn("write to tikv no leader", logutil.Region(region.Region), logutil.Leader(region.Leader),
 			zap.Uint64("leader_id", leaderID), logutil.SSTMeta(meta),

diff --git a/br/pkg/lightning/common/retry.go b/br/pkg/lightning/common/retry.go
@@ -39,6 +39,8 @@ var retryableErrorMsgList = []string{
 	// this error happens on when distsql.Checksum calls TiKV
 	// see https://github.com/pingcap/tidb/blob/2c3d4f1ae418881a95686e8b93d4237f2e76eec6/store/copr/coprocessor.go#L941
 	"coprocessor task terminated due to exceeding the deadline",
+	// this error happens when pd is transferring leader and we can retry it again
+	"write to tikv with no leader returned",
 }
 
 func isRetryableFromErrorMessage(err error) bool {

diff --git a/br/pkg/lightning/common/retry_test.go b/br/pkg/lightning/common/retry_test.go
@@ -97,6 +97,8 @@ func TestIsRetryableError(t *testing.T) {
 	require.True(t, IsRetryableError(status.Error(codes.Unavailable, "")))
 	require.True(t, IsRetryableError(status.Error(codes.DataLoss, "")))
 
+	require.True(t, IsRetryableError(errors.Errorf("write to tikv with no leader returned, region '%d', leader: %d", 1, 2)))
+
 	// sqlmock errors
 	require.False(t, IsRetryableError(fmt.Errorf("call to database Close was not expected")))
 	require.False(t, IsRetryableError(errors.New("call to database Close was not expected")))