From 52171adcd2d90851bef1fb3f55aa3db45e28e75b Mon Sep 17 00:00:00 2001
From: Ti Chi Robot <ti-community-prow-bot@tidb.io>
Date: Fri, 8 Nov 2024 23:42:48 +0800
Subject: [PATCH] br/pdutils: retry when encountered dns error (#53005)
 (#57245)

close pingcap/tidb#53029
---
 br/pkg/lightning/common/retry.go           |  4 +++
 br/pkg/lightning/common/retry_test.go      |  2 +-
 br/pkg/pdutil/pd.go                        |  4 +++
 tests/realtikvtest/brietest/BUILD.bazel    |  4 +++
 tests/realtikvtest/brietest/pdutil_test.go | 33 ++++++++++++++++++++++
 5 files changed, 46 insertions(+), 1 deletion(-)
 create mode 100644 tests/realtikvtest/brietest/pdutil_test.go

diff --git a/br/pkg/lightning/common/retry.go b/br/pkg/lightning/common/retry.go
index 47a030486c9e0..8fd20f8ac294f 100644
--- a/br/pkg/lightning/common/retry.go
+++ b/br/pkg/lightning/common/retry.go
@@ -108,6 +108,10 @@ func isSingleRetryableError(err error) bool {
 
 	switch nerr := err.(type) {
 	case net.Error:
+		var dErr *net.DNSError
+		if goerrors.As(nerr, &dErr) {
+			return true
+		}
 		if nerr.Timeout() {
 			return true
 		}
diff --git a/br/pkg/lightning/common/retry_test.go b/br/pkg/lightning/common/retry_test.go
index 5aa06237eeded..d62ec996fac22 100644
--- a/br/pkg/lightning/common/retry_test.go
+++ b/br/pkg/lightning/common/retry_test.go
@@ -40,7 +40,7 @@ func TestIsRetryableError(t *testing.T) {
 	require.True(t, IsRetryableError(ErrWriteTooSlow))
 	require.False(t, IsRetryableError(io.EOF))
 	require.False(t, IsRetryableError(&net.AddrError{}))
-	require.False(t, IsRetryableError(&net.DNSError{}))
+	require.True(t, IsRetryableError(&net.DNSError{}))
 	require.True(t, IsRetryableError(&net.DNSError{IsTimeout: true}))
 
 	// kv errors
diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go
index 60af51ac0cb74..208c9227daae8 100644
--- a/br/pkg/pdutil/pd.go
+++ b/br/pkg/pdutil/pd.go
@@ -177,6 +177,10 @@ func pdRequestWithCode(
 		if err != nil {
 			return 0, nil, errors.Trace(err)
 		}
+		failpoint.Inject("DNSError", func() {
+			req.Host = "nosuchhost"
+			req.URL.Host = "nosuchhost"
+		})
 		resp, err = cli.Do(req) //nolint:bodyclose
 		count++
 		failpoint.Inject("InjectClosed", func(v failpoint.Value) {
diff --git a/tests/realtikvtest/brietest/BUILD.bazel b/tests/realtikvtest/brietest/BUILD.bazel
index c3118c4d7a88a..aa010a323aa78 100644
--- a/tests/realtikvtest/brietest/BUILD.bazel
+++ b/tests/realtikvtest/brietest/BUILD.bazel
@@ -7,10 +7,12 @@ go_test(
         "backup_restore_test.go",
         "binlog_test.go",
         "main_test.go",
+        "pdutil_test.go",
     ],
     flaky = True,
     race = "on",
     deps = [
+        "//br/pkg/pdutil",
         "//config",
         "//parser/mysql",
         "//sessionctx/binloginfo",
@@ -18,8 +20,10 @@ go_test(
         "//testkit",
         "//testkit/testsetup",
         "//tests/realtikvtest",
+        "@com_github_pingcap_failpoint//:failpoint",
         "@com_github_pingcap_tipb//go-binlog",
         "@com_github_stretchr_testify//require",
+        "@com_github_tikv_pd_client//:client",
         "@org_golang_google_grpc//:grpc",
         "@org_uber_go_goleak//:goleak",
     ],
diff --git a/tests/realtikvtest/brietest/pdutil_test.go b/tests/realtikvtest/brietest/pdutil_test.go
new file mode 100644
index 0000000000000..298de04afa6f7
--- /dev/null
+++ b/tests/realtikvtest/brietest/pdutil_test.go
@@ -0,0 +1,33 @@
+// Copyright 2024 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package brietest
+
+import (
+	"context"
+	"testing"
+
+	"github.com/pingcap/failpoint"
+	"github.com/pingcap/tidb/br/pkg/pdutil"
+	"github.com/stretchr/testify/require"
+	pd "github.com/tikv/pd/client"
+)
+
+func TestCreateClient(t *testing.T) {
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/pdutil/DNSError", "119*return"))
+	require.NoError(t, failpoint.Enable("github.com/pingcap/tidb/br/pkg/pdutil/FastRetry", "return(true)"))
+	ctl, err := pdutil.NewPdController(context.Background(), "127.0.0.1:2379", nil, pd.SecurityOption{})
+	require.NoError(t, err)
+	ctl.Close()
+}