deps, lightning: update the PD HTTP client to refine the retry logic (#…

…50225)

close tikv/pd#7673, ref #35319

DEPS.bzl

@@ -7032,13 +7032,13 @@ def go_deps():
  name = "com_github_tikv_pd_client",
  build_file_proto_mode = "disable_global",
  importpath = "github.com/tikv/pd/client",
- sha256 = "be8f0dd4f9cf5d652ff38400e0fb56988c90ce9051716c02c7ae16c3ac1d698a",
- strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240108040922-c94810e110ce",
+ sha256 = "ea1ce959374afae8cbe31345e696f5731443c9ea6029de46fcf6844c09677c9c",
+ strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240109100024-dd8df25316e9",
  urls = [
- "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip",
- "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip",
- "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip",
- "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240108040922-c94810e110ce.zip",
+ "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip",
+ "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip",
+ "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip",
+ "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip",
  ],
  )
  go_repository(

br/pkg/lightning/backend/local/BUILD.bazel

@@ -83,6 +83,7 @@ go_library(
  "@com_github_tikv_client_go_v2//util",
  "@com_github_tikv_pd_client//:client",
  "@com_github_tikv_pd_client//http",
+ "@com_github_tikv_pd_client//retry",
  "@org_golang_google_grpc//:grpc",
  "@org_golang_google_grpc//backoff",
  "@org_golang_google_grpc//codes",

br/pkg/lightning/backend/local/local.go

@@ -60,6 +60,7 @@ import (
  tikvclient "github.com/tikv/client-go/v2/tikv"
  pd "github.com/tikv/pd/client"
  pdhttp "github.com/tikv/pd/client/http"
+ "github.com/tikv/pd/client/retry"
  "go.uber.org/zap"
  "google.golang.org/grpc"
  "google.golang.org/grpc/backoff"
@@ -556,7 +557,8 @@ func NewBackend(
  if err != nil {
  return nil, common.ErrCreateKVClient.Wrap(err).GenWithStackByArgs()
  }
- pdHTTPCli := pdhttp.NewClient("lightning", pdAddrs, pdhttp.WithTLSConfig(tls.TLSConfig()))
+ pdHTTPCli := pdhttp.NewClient("lightning", pdAddrs, pdhttp.WithTLSConfig(tls.TLSConfig())).
+ WithBackoffer(retry.InitialBackoffer(time.Second, time.Second, pdutil.PDRequestRetryTime*time.Second))
  splitCli := split.NewSplitClient(pdCli, pdHTTPCli, tls.TLSConfig(), false)
  importClientFactory := newImportClientFactoryImpl(splitCli, tls, config.MaxConnPerStore, config.ConnCompressType)
  var writeLimiter StoreWriteLimiter

br/pkg/pdutil/BUILD.bazel

@@ -24,6 +24,7 @@ go_library(
  "@com_github_pingcap_log//:log",
  "@com_github_tikv_pd_client//:client",
  "@com_github_tikv_pd_client//http",
+ "@com_github_tikv_pd_client//retry",
  "@org_golang_google_grpc//:grpc",
  "@org_uber_go_zap//:zap",
  ],

br/pkg/pdutil/pd.go

@@ -33,6 +33,7 @@ import (
  "github.com/pingcap/tidb/pkg/util/codec"
  pd "github.com/tikv/pd/client"
  pdhttp "github.com/tikv/pd/client/http"
+ "github.com/tikv/pd/client/retry"
  "go.uber.org/zap"
  "google.golang.org/grpc"
 )
@@ -41,7 +42,7 @@ const (
  maxMsgSize = int(128 * units.MiB) // pd.ScanRegion may return a large response
  pauseTimeout = 5 * time.Minute
  // pd request retry time when connection fail
- pdRequestRetryTime = 120
+ PDRequestRetryTime = 120
  // set max-pending-peer-count to a large value to avoid scatter region failed.
  maxPendingPeerUnlimited uint64 = math.MaxInt32
 )
@@ -167,7 +168,7 @@ func pdRequestWithCode(
  resp, err = cli.Do(req) //nolint:bodyclose
  count++
  failpoint.Inject("InjectClosed", func(v failpoint.Value) {
- if failType, ok := v.(int); ok && count <= pdRequestRetryTime-1 {
+ if failType, ok := v.(int); ok && count <= PDRequestRetryTime-1 {
  resp = nil
  switch failType {
  case 0:
@@ -183,7 +184,7 @@ func pdRequestWithCode(
  }
  }
  })
- if count > pdRequestRetryTime || (resp != nil && resp.StatusCode < 500) ||
+ if count > PDRequestRetryTime || (resp != nil && resp.StatusCode < 500) ||
  (err != nil && !common.IsRetryableError(err)) {
  break
  }
@@ -298,11 +299,13 @@ func NewPdController(
  if tlsConf != nil {
  pdHTTPCliConfig = append(pdHTTPCliConfig, pdhttp.WithTLSConfig(tlsConf))
  }
+ pdHTTPCli := pdhttp.NewClient("br/lightning PD controller", addrs, pdHTTPCliConfig...).
+ WithBackoffer(retry.InitialBackoffer(time.Second, time.Second, PDRequestRetryTime*time.Second))
  return &PdController{
  addrs: processedAddrs,
  cli: cli,
  pdClient: pdClient,
- pdHTTPCli: pdhttp.NewClient("br/lightning PD controller", addrs, pdHTTPCliConfig...),
+ pdHTTPCli: pdHTTPCli,
  version: version,
  // We should make a buffered channel here otherwise when context canceled,
  // gracefully shutdown will stick at resuming schedulers.

br/pkg/pdutil/pd_serial_test.go

@@ -182,7 +182,7 @@ func TestPDRequestRetry(t *testing.T) {
  bytes, err := io.ReadAll(r.Body)
  require.NoError(t, err)
  require.Equal(t, "test", string(bytes))
- if count <= pdRequestRetryTime-1 {
+ if count <= PDRequestRetryTime-1 {
  w.WriteHeader(http.StatusGatewayTimeout)
  return
  }
@@ -203,7 +203,7 @@ func TestPDRequestRetry(t *testing.T) {
  count = 0
  ts = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
  count++
- if count <= pdRequestRetryTime+1 {
+ if count <= PDRequestRetryTime+1 {
  w.WriteHeader(http.StatusGatewayTimeout)
  return
  }

go.mod

@@ -105,7 +105,7 @@ require (
  github.com/tdakkota/asciicheck v0.2.0
  github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2
  github.com/tikv/client-go/v2 v2.0.8-0.20231227070846-61c486af13a5
- github.com/tikv/pd/client v0.0.0-20240108040922-c94810e110ce
+ github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9
  github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966
  github.com/twmb/murmur3 v1.1.6
  github.com/uber/jaeger-client-go v2.22.1+incompatible

go.sum

@@ -862,8 +862,8 @@ github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW
 github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM=
 github.com/tikv/client-go/v2 v2.0.8-0.20231227070846-61c486af13a5 h1:UldBK/txpUdWwPlxfNJKLTrciZ8BUzhYtXj0RY0uliY=
 github.com/tikv/client-go/v2 v2.0.8-0.20231227070846-61c486af13a5/go.mod h1:byff6zglNXgereADRRJmKQnurwy1Z9hthX2I5ObKMNE=
-github.com/tikv/pd/client v0.0.0-20240108040922-c94810e110ce h1:J9HrxW44J0OovyuPmQyFsaKRxG5eoui/DqTwb71Kp38=
-github.com/tikv/pd/client v0.0.0-20240108040922-c94810e110ce/go.mod h1:ZilHJZR8wgqENRi26gtnPoKIXAB1EqytFweUhzxetx0=
+github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9 h1:LnNWRdtxryzxl31GmxOJEFKUmwiG8nph9f5Wqdv8olY=
+github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9/go.mod h1:ZilHJZR8wgqENRi26gtnPoKIXAB1EqytFweUhzxetx0=
 github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M=
 github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ=
 github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs=