diff --git a/DEPS.bzl b/DEPS.bzl index 184e437c68233..cc0807019dffb 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -7032,13 +7032,13 @@ def go_deps(): name = "com_github_tikv_pd_client", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sha256 = "be8f0dd4f9cf5d652ff38400e0fb56988c90ce9051716c02c7ae16c3ac1d698a", - strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240108040922-c94810e110ce", + sha256 = "ea1ce959374afae8cbe31345e696f5731443c9ea6029de46fcf6844c09677c9c", + strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240109100024-dd8df25316e9", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip", - "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", + "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", ], ) go_repository( diff --git a/br/pkg/lightning/backend/local/BUILD.bazel b/br/pkg/lightning/backend/local/BUILD.bazel index 2a7fa31930e46..4777cc55dc981 100644 --- a/br/pkg/lightning/backend/local/BUILD.bazel +++ b/br/pkg/lightning/backend/local/BUILD.bazel @@ -83,6 +83,7 @@ go_library( "@com_github_tikv_client_go_v2//util", "@com_github_tikv_pd_client//:client", "@com_github_tikv_pd_client//http", + "@com_github_tikv_pd_client//retry", "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//backoff", "@org_golang_google_grpc//codes", diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go index d98a8a9b09bcd..79f3b4e1a96d4 100644 --- a/br/pkg/lightning/backend/local/local.go +++ b/br/pkg/lightning/backend/local/local.go @@ -60,6 +60,7 @@ import ( tikvclient "github.com/tikv/client-go/v2/tikv" pd "github.com/tikv/pd/client" pdhttp "github.com/tikv/pd/client/http" + "github.com/tikv/pd/client/retry" "go.uber.org/zap" "google.golang.org/grpc" "google.golang.org/grpc/backoff" @@ -556,7 +557,8 @@ func NewBackend( if err != nil { return nil, common.ErrCreateKVClient.Wrap(err).GenWithStackByArgs() } - pdHTTPCli := pdhttp.NewClient("lightning", pdAddrs, pdhttp.WithTLSConfig(tls.TLSConfig())) + pdHTTPCli := pdhttp.NewClient("lightning", pdAddrs, pdhttp.WithTLSConfig(tls.TLSConfig())). + WithBackoffer(retry.InitialBackoffer(time.Second, time.Second, pdutil.PDRequestRetryTime*time.Second)) splitCli := split.NewSplitClient(pdCli, pdHTTPCli, tls.TLSConfig(), false) importClientFactory := newImportClientFactoryImpl(splitCli, tls, config.MaxConnPerStore, config.ConnCompressType) var writeLimiter StoreWriteLimiter diff --git a/br/pkg/pdutil/BUILD.bazel b/br/pkg/pdutil/BUILD.bazel index b9b42e7d534b0..779a6959bfa08 100644 --- a/br/pkg/pdutil/BUILD.bazel +++ b/br/pkg/pdutil/BUILD.bazel @@ -24,6 +24,7 @@ go_library( "@com_github_pingcap_log//:log", "@com_github_tikv_pd_client//:client", "@com_github_tikv_pd_client//http", + "@com_github_tikv_pd_client//retry", "@org_golang_google_grpc//:grpc", "@org_uber_go_zap//:zap", ], diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go index 718407499e8cb..8903f8236a747 100644 --- a/br/pkg/pdutil/pd.go +++ b/br/pkg/pdutil/pd.go @@ -33,6 +33,7 @@ import ( "github.com/pingcap/tidb/pkg/util/codec" pd "github.com/tikv/pd/client" pdhttp "github.com/tikv/pd/client/http" + "github.com/tikv/pd/client/retry" "go.uber.org/zap" "google.golang.org/grpc" ) @@ -41,7 +42,7 @@ const ( maxMsgSize = int(128 * units.MiB) // pd.ScanRegion may return a large response pauseTimeout = 5 * time.Minute // pd request retry time when connection fail - pdRequestRetryTime = 120 + PDRequestRetryTime = 120 // set max-pending-peer-count to a large value to avoid scatter region failed. maxPendingPeerUnlimited uint64 = math.MaxInt32 ) @@ -167,7 +168,7 @@ func pdRequestWithCode( resp, err = cli.Do(req) //nolint:bodyclose count++ failpoint.Inject("InjectClosed", func(v failpoint.Value) { - if failType, ok := v.(int); ok && count <= pdRequestRetryTime-1 { + if failType, ok := v.(int); ok && count <= PDRequestRetryTime-1 { resp = nil switch failType { case 0: @@ -183,7 +184,7 @@ func pdRequestWithCode( } } }) - if count > pdRequestRetryTime || (resp != nil && resp.StatusCode < 500) || + if count > PDRequestRetryTime || (resp != nil && resp.StatusCode < 500) || (err != nil && !common.IsRetryableError(err)) { break } @@ -298,11 +299,13 @@ func NewPdController( if tlsConf != nil { pdHTTPCliConfig = append(pdHTTPCliConfig, pdhttp.WithTLSConfig(tlsConf)) } + pdHTTPCli := pdhttp.NewClient("br/lightning PD controller", addrs, pdHTTPCliConfig...). + WithBackoffer(retry.InitialBackoffer(time.Second, time.Second, PDRequestRetryTime*time.Second)) return &PdController{ addrs: processedAddrs, cli: cli, pdClient: pdClient, - pdHTTPCli: pdhttp.NewClient("br/lightning PD controller", addrs, pdHTTPCliConfig...), + pdHTTPCli: pdHTTPCli, version: version, // We should make a buffered channel here otherwise when context canceled, // gracefully shutdown will stick at resuming schedulers. diff --git a/br/pkg/pdutil/pd_serial_test.go b/br/pkg/pdutil/pd_serial_test.go index cc2498084c686..273db5a9f395c 100644 --- a/br/pkg/pdutil/pd_serial_test.go +++ b/br/pkg/pdutil/pd_serial_test.go @@ -182,7 +182,7 @@ func TestPDRequestRetry(t *testing.T) { bytes, err := io.ReadAll(r.Body) require.NoError(t, err) require.Equal(t, "test", string(bytes)) - if count <= pdRequestRetryTime-1 { + if count <= PDRequestRetryTime-1 { w.WriteHeader(http.StatusGatewayTimeout) return } @@ -203,7 +203,7 @@ func TestPDRequestRetry(t *testing.T) { count = 0 ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { count++ - if count <= pdRequestRetryTime+1 { + if count <= PDRequestRetryTime+1 { w.WriteHeader(http.StatusGatewayTimeout) return } diff --git a/go.mod b/go.mod index 4e3debf526574..dbcfa4ed46e17 100644 --- a/go.mod +++ b/go.mod @@ -105,7 +105,7 @@ require ( github.com/tdakkota/asciicheck v0.2.0 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 github.com/tikv/client-go/v2 v2.0.8-0.20231227070846-61c486af13a5 - github.com/tikv/pd/client v0.0.0-20240108040922-c94810e110ce + github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9 github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 github.com/twmb/murmur3 v1.1.6 github.com/uber/jaeger-client-go v2.22.1+incompatible diff --git a/go.sum b/go.sum index 9425dd038a8a8..c57573ad59f1f 100644 --- a/go.sum +++ b/go.sum @@ -862,8 +862,8 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= github.com/tikv/client-go/v2 v2.0.8-0.20231227070846-61c486af13a5 h1:UldBK/txpUdWwPlxfNJKLTrciZ8BUzhYtXj0RY0uliY= github.com/tikv/client-go/v2 v2.0.8-0.20231227070846-61c486af13a5/go.mod h1:byff6zglNXgereADRRJmKQnurwy1Z9hthX2I5ObKMNE= -github.com/tikv/pd/client v0.0.0-20240108040922-c94810e110ce h1:J9HrxW44J0OovyuPmQyFsaKRxG5eoui/DqTwb71Kp38= -github.com/tikv/pd/client v0.0.0-20240108040922-c94810e110ce/go.mod h1:ZilHJZR8wgqENRi26gtnPoKIXAB1EqytFweUhzxetx0= +github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9 h1:LnNWRdtxryzxl31GmxOJEFKUmwiG8nph9f5Wqdv8olY= +github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9/go.mod h1:ZilHJZR8wgqENRi26gtnPoKIXAB1EqytFweUhzxetx0= github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M= github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=