From c072334a99525375f10ba2d44bcb822c370669ec Mon Sep 17 00:00:00 2001 From: Yongbo Jiang Date: Fri, 26 Jan 2024 11:39:21 +0800 Subject: [PATCH] *: impl mock PD service discovery and update pd client version (#50718) ref pingcap/tidb#35319 --- DEPS.bzl | 72 +++++----- br/pkg/backup/prepare_snap/prepare_test.go | 2 +- br/pkg/lightning/backend/local/local.go | 2 +- br/pkg/lightning/importer/import.go | 2 +- br/pkg/pdutil/BUILD.bazel | 1 + br/pkg/pdutil/pd.go | 5 +- br/pkg/pdutil/pd_serial_test.go | 3 +- go.mod | 11 +- go.sum | 26 ++-- pkg/ddl/tests/tiflash/ddl_tiflash_test.go | 4 +- pkg/executor/infoschema_cluster_table_test.go | 4 +- pkg/executor/tikv_regions_peers_table_test.go | 5 +- pkg/store/driver/main_test.go | 2 +- pkg/store/helper/helper_test.go | 1 + pkg/store/mockstore/mockstore.go | 8 ++ pkg/store/mockstore/unistore.go | 2 +- pkg/store/mockstore/unistore/BUILD.bazel | 5 +- pkg/store/mockstore/unistore/mock.go | 4 +- pkg/store/mockstore/unistore/pd.go | 123 +++++++++++++++++- pkg/store/mockstore/unistore/pd_test.go | 21 ++- 20 files changed, 234 insertions(+), 69 deletions(-) diff --git a/DEPS.bzl b/DEPS.bzl index bec9918e0b9bc..b626661084cda 100644 --- a/DEPS.bzl +++ b/DEPS.bzl @@ -1061,13 +1061,13 @@ def go_deps(): name = "com_github_cncf_xds_go", build_file_proto_mode = "disable_global", importpath = "github.com/cncf/xds/go", - sha256 = "a0c6e66eade357aeda4edaa9d09612085860dc4c0b44edf8226574939bdf6091", - strip_prefix = "github.com/cncf/xds/go@v0.0.0-20230607035331-e9ce68804cb4", + sha256 = "abb811f8a32a7b60656a02e2a3500455ae907e1e0c35a1012a61599795bb3b73", + strip_prefix = "github.com/cncf/xds/go@v0.0.0-20231109132714-523115ebc101", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20230607035331-e9ce68804cb4.zip", - "http://ats.apps.svc/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20230607035331-e9ce68804cb4.zip", - "https://cache.hawkingrei.com/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20230607035331-e9ce68804cb4.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20230607035331-e9ce68804cb4.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231109132714-523115ebc101.zip", + "http://ats.apps.svc/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231109132714-523115ebc101.zip", + "https://cache.hawkingrei.com/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231109132714-523115ebc101.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/cncf/xds/go/com_github_cncf_xds_go-v0.0.0-20231109132714-523115ebc101.zip", ], ) go_repository( @@ -7032,26 +7032,26 @@ def go_deps(): name = "com_github_tikv_client_go_v2", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/client-go/v2", - sha256 = "dfc92ddf47606d8e9d9fe165f4cb344f1ad631f04bcd7f372e11266d6a2206bc", - strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240123055405-3480b5ed7ce1", + sha256 = "11c26ac74edc44bad623ac96676af6015ebcc06da5495b4240f4f830238bf4eb", + strip_prefix = "github.com/tikv/client-go/v2@v2.0.8-0.20240125030910-e6f5a45b002e", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240123055405-3480b5ed7ce1.zip", - "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240123055405-3480b5ed7ce1.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240123055405-3480b5ed7ce1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240123055405-3480b5ed7ce1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240125030910-e6f5a45b002e.zip", + "http://ats.apps.svc/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240125030910-e6f5a45b002e.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240125030910-e6f5a45b002e.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/client-go/v2/com_github_tikv_client_go_v2-v2.0.8-0.20240125030910-e6f5a45b002e.zip", ], ) go_repository( name = "com_github_tikv_pd_client", build_file_proto_mode = "disable_global", importpath = "github.com/tikv/pd/client", - sha256 = "ea1ce959374afae8cbe31345e696f5731443c9ea6029de46fcf6844c09677c9c", - strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240109100024-dd8df25316e9", + sha256 = "d77235e2ea8b98a9d2d5877f90b4e6f67bc2d108db5d00873ae3c803ed4b7d23", + strip_prefix = "github.com/tikv/pd/client@v0.0.0-20240126020320-567c7d43a008", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", - "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", - "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240109100024-dd8df25316e9.zip", + "http://bazel-cache.pingcap.net:8080/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240126020320-567c7d43a008.zip", + "http://ats.apps.svc/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240126020320-567c7d43a008.zip", + "https://cache.hawkingrei.com/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240126020320-567c7d43a008.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/github.com/tikv/pd/client/com_github_tikv_pd_client-v0.0.0-20240126020320-567c7d43a008.zip", ], ) go_repository( @@ -10026,13 +10026,13 @@ def go_deps(): name = "org_golang_google_genproto", build_file_proto_mode = "disable_global", importpath = "google.golang.org/genproto", - sha256 = "5ed90251b73540d8576554928e989df1e0ab81f91a487960f0c93c77ff2e38c0", - strip_prefix = "google.golang.org/genproto@v0.0.0-20240108191215-35c7eff3a6b1", + sha256 = "e65fb3abc11c20b0910c40a3b269b79b06c2cdd6e7efab29a5ea50f18a62b7f9", + strip_prefix = "google.golang.org/genproto@v0.0.0-20240116215550-a9fa1716bcac", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240108191215-35c7eff3a6b1.zip", - "http://ats.apps.svc/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240108191215-35c7eff3a6b1.zip", - "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240108191215-35c7eff3a6b1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240108191215-35c7eff3a6b1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240116215550-a9fa1716bcac.zip", + "http://ats.apps.svc/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240116215550-a9fa1716bcac.zip", + "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240116215550-a9fa1716bcac.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/org_golang_google_genproto-v0.0.0-20240116215550-a9fa1716bcac.zip", ], ) go_repository( @@ -10065,26 +10065,26 @@ def go_deps(): name = "org_golang_google_genproto_googleapis_rpc", build_file_proto_mode = "disable_global", importpath = "google.golang.org/genproto/googleapis/rpc", - sha256 = "4e4c60b74b5bba370fbba795eab6313e42b6eabd668aea1ed87a5b25065ec144", - strip_prefix = "google.golang.org/genproto/googleapis/rpc@v0.0.0-20240108191215-35c7eff3a6b1", + sha256 = "1a8c10b3ca5a0aea26bb78d948edbcc126ce7e07e127f6f7a23af5e5d641b40f", + strip_prefix = "google.golang.org/genproto/googleapis/rpc@v0.0.0-20240123012728-ef4313101c80", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240108191215-35c7eff3a6b1.zip", - "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240108191215-35c7eff3a6b1.zip", - "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240108191215-35c7eff3a6b1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240108191215-35c7eff3a6b1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240123012728-ef4313101c80.zip", + "http://ats.apps.svc/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240123012728-ef4313101c80.zip", + "https://cache.hawkingrei.com/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240123012728-ef4313101c80.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/genproto/googleapis/rpc/org_golang_google_genproto_googleapis_rpc-v0.0.0-20240123012728-ef4313101c80.zip", ], ) go_repository( name = "org_golang_google_grpc", build_file_proto_mode = "disable_global", importpath = "google.golang.org/grpc", - sha256 = "c9088df910b74b8a815c2eabf7e7b458aa9a355312aee77994d5d8f6ca24928a", - strip_prefix = "google.golang.org/grpc@v1.60.1", + sha256 = "6971b322a68f83ae2055391e5bd57dfb359e67509961a75229b825d2bd15564d", + strip_prefix = "google.golang.org/grpc@v1.61.0", urls = [ - "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.60.1.zip", - "http://ats.apps.svc/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.60.1.zip", - "https://cache.hawkingrei.com/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.60.1.zip", - "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.60.1.zip", + "http://bazel-cache.pingcap.net:8080/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.61.0.zip", + "http://ats.apps.svc/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.61.0.zip", + "https://cache.hawkingrei.com/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.61.0.zip", + "https://storage.googleapis.com/pingcapmirror/gomod/google.golang.org/grpc/org_golang_google_grpc-v1.61.0.zip", ], ) go_repository( diff --git a/br/pkg/backup/prepare_snap/prepare_test.go b/br/pkg/backup/prepare_snap/prepare_test.go index 502af2249d9d9..163d446e7314b 100644 --- a/br/pkg/backup/prepare_snap/prepare_test.go +++ b/br/pkg/backup/prepare_snap/prepare_test.go @@ -224,7 +224,7 @@ func (m *mockStores) AssertIsNormalMode(t *testing.T) { func fakeCluster(t *testing.T, nodes int, keys ...[]byte) pd.Client { tmp := t.TempDir() - _, pdc, cluster, err := unistore.New(tmp) + _, pdc, cluster, err := unistore.New(tmp, nil) unistore.BootstrapWithMultiStores(cluster, nodes) require.NoError(t, err) cluster.SplitArbitrary(keys...) diff --git a/br/pkg/lightning/backend/local/local.go b/br/pkg/lightning/backend/local/local.go index 63904f6f9d255..b128efa07ecae 100644 --- a/br/pkg/lightning/backend/local/local.go +++ b/br/pkg/lightning/backend/local/local.go @@ -538,7 +538,7 @@ func NewBackend( if err != nil { return nil, common.ErrCreateKVClient.Wrap(err).GenWithStackByArgs() } - pdHTTPCli := pdhttp.NewClient("lightning", pdAddrs, pdhttp.WithTLSConfig(tls.TLSConfig())). + pdHTTPCli := pdhttp.NewClientWithServiceDiscovery("lightning", pdCli.GetServiceDiscovery(), pdhttp.WithTLSConfig(tls.TLSConfig())). WithBackoffer(retry.InitialBackoffer(time.Second, time.Second, pdutil.PDRequestRetryTime*time.Second)) splitCli := split.NewSplitClient(pdCli, pdHTTPCli, tls.TLSConfig(), false) importClientFactory := newImportClientFactoryImpl(splitCli, tls, config.MaxConnPerStore, config.ConnCompressType) diff --git a/br/pkg/lightning/importer/import.go b/br/pkg/lightning/importer/import.go index 57b80ae1f16d8..244833df31e69 100644 --- a/br/pkg/lightning/importer/import.go +++ b/br/pkg/lightning/importer/import.go @@ -366,7 +366,7 @@ func NewImportControllerWithPauser( if err != nil { return nil, errors.Trace(err) } - pdHTTPCli = pdhttp.NewClient("lightning", addrs, pdhttp.WithTLSConfig(tls.TLSConfig())) + pdHTTPCli = pdhttp.NewClientWithServiceDiscovery("lightning", pdCli.GetServiceDiscovery(), pdhttp.WithTLSConfig(tls.TLSConfig())) if cfg.TikvImporter.DuplicateResolution != config.DupeResAlgNone { if err := tikv.CheckTiKVVersion(ctx, pdHTTPCli, minTiKVVersionForDuplicateResolution, maxTiKVVersionForDuplicateResolution); err != nil { diff --git a/br/pkg/pdutil/BUILD.bazel b/br/pkg/pdutil/BUILD.bazel index 779a6959bfa08..a6e671586b515 100644 --- a/br/pkg/pdutil/BUILD.bazel +++ b/br/pkg/pdutil/BUILD.bazel @@ -41,6 +41,7 @@ go_test( flaky = True, shard_count = 8, deps = [ + "//pkg/store/mockstore/unistore", "//pkg/store/pdtypes", "//pkg/testkit/testsetup", "//pkg/util/codec", diff --git a/br/pkg/pdutil/pd.go b/br/pkg/pdutil/pd.go index 575fae954d09a..b5f6dd425497c 100644 --- a/br/pkg/pdutil/pd.go +++ b/br/pkg/pdutil/pd.go @@ -301,7 +301,10 @@ func NewPdController( if tlsConf != nil { pdHTTPCliConfig = append(pdHTTPCliConfig, pdhttp.WithTLSConfig(tlsConf)) } - pdHTTPCli := pdhttp.NewClient("br/lightning PD controller", addrs, pdHTTPCliConfig...). + pdHTTPCli := pdhttp.NewClientWithServiceDiscovery( + "br/lightning PD controller", + pdClient.GetServiceDiscovery(), + pdHTTPCliConfig...). WithBackoffer(retry.InitialBackoffer(time.Second, time.Second, PDRequestRetryTime*time.Second)) return &PdController{ addrs: processedAddrs, diff --git a/br/pkg/pdutil/pd_serial_test.go b/br/pkg/pdutil/pd_serial_test.go index 273db5a9f395c..f3328914191de 100644 --- a/br/pkg/pdutil/pd_serial_test.go +++ b/br/pkg/pdutil/pd_serial_test.go @@ -19,6 +19,7 @@ import ( "github.com/coreos/go-semver/semver" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" + "github.com/pingcap/tidb/pkg/store/mockstore/unistore" "github.com/pingcap/tidb/pkg/store/pdtypes" "github.com/pingcap/tidb/pkg/util/codec" "github.com/stretchr/testify/require" @@ -337,7 +338,7 @@ func TestPauseSchedulersByKeyRange(t *testing.T) { })) defer httpSrv.Close() - pdHTTPCli := pdhttp.NewClient("test", []string{httpSrv.URL}) + pdHTTPCli := pdhttp.NewClientWithServiceDiscovery("test", unistore.NewMockPDServiceDiscovery([]string{httpSrv.URL})) defer pdHTTPCli.Close() ctx, cancel := context.WithCancel(context.Background()) defer cancel() diff --git a/go.mod b/go.mod index 88fa3c56ef7dc..b839827e6f7b1 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/Shopify/sarama v1.29.0 github.com/aliyun/alibaba-cloud-sdk-go v1.61.1581 github.com/apache/skywalking-eyes v0.4.0 + github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 github.com/ashanbrown/makezero v1.1.1 github.com/aws/aws-sdk-go v1.45.25 github.com/bazelbuild/buildtools v0.0.0-20230926111657-7d855c59baeb @@ -106,8 +107,8 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tdakkota/asciicheck v0.2.0 github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 - github.com/tikv/client-go/v2 v2.0.8-0.20240123055405-3480b5ed7ce1 - github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9 + github.com/tikv/client-go/v2 v2.0.8-0.20240125030910-e6f5a45b002e + github.com/tikv/pd/client v0.0.0-20240126020320-567c7d43a008 github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 github.com/twmb/murmur3 v1.1.6 github.com/uber/jaeger-client-go v2.22.1+incompatible @@ -137,7 +138,7 @@ require ( golang.org/x/time v0.5.0 golang.org/x/tools v0.17.0 google.golang.org/api v0.156.0 - google.golang.org/grpc v1.60.1 + google.golang.org/grpc v1.61.0 gopkg.in/yaml.v2 v2.4.0 honnef.co/go/tools v0.4.6 k8s.io/api v0.28.2 @@ -166,7 +167,7 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect go.opentelemetry.io/otel/metric v1.21.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 // indirect k8s.io/utils v0.0.0-20230711102312-30195339c3c7 // indirect ) @@ -298,7 +299,7 @@ require ( golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect gonum.org/v1/gonum v0.8.2 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20240108191215-35c7eff3a6b1 // indirect + google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac // indirect google.golang.org/protobuf v1.32.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect diff --git a/go.sum b/go.sum index 7f2f4726a4050..44ae2ac8c0e72 100644 --- a/go.sum +++ b/go.sum @@ -109,6 +109,8 @@ github.com/apache/thrift v0.0.0-20181112125854-24918abba929/go.mod h1:cp2SuWMxlE github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714 h1:Jz3KVLYY5+JO7rDiX0sAuRGtuv2vG01r17Y9nLMWNUw= github.com/apache/thrift v0.13.1-0.20201008052519-daf620915714/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 h1:DklsrG3dyBCFEj5IhUbnKptjxatkF07cF2ak3yi77so= +github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2/go.mod h1:WaHUgvxTVq04UNunO+XhnAqY/wQc+bxr74GqbsZ/Jqw= github.com/ashanbrown/makezero v1.1.1 h1:iCQ87C0V0vSyO+M9E/FZYbu65auqH0lnsOkf5FcB28s= github.com/ashanbrown/makezero v1.1.1/go.mod h1:i1bJLCRSCHOcOa9Y6MyF2FTfMZMFdHvxKHxgO5Z1axI= github.com/aws/aws-sdk-go v1.30.19/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= @@ -157,8 +159,8 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cloudfoundry/gosigar v1.3.6 h1:gIc08FbB3QPb+nAQhINIK/qhf5REKkY0FTGgRGXkcVc= github.com/cloudfoundry/gosigar v1.3.6/go.mod h1:lNWstu5g5gw59O09Y+wsMNFzBSnU8a0u+Sfx4dq360E= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4 h1:/inchEIKaYC1Akx+H+gqO04wryn5h75LSazbRlnya1k= -github.com/cncf/xds/go v0.0.0-20230607035331-e9ce68804cb4/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101 h1:7To3pQ+pZo0i3dsWEbinPNFs5gPSBOsJtx3wTT94VBY= +github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v1.0.0/go.mod h1:5Ib8Meh+jk1RlHIXej6Pzevx/NLlNvQB9pmSBZErGA4= github.com/cockroachdb/datadriven v1.0.2 h1:H9MtNqVoVhvd9nCBwOyDjUEdZCREqbIdCJD93PBm/jA= github.com/cockroachdb/datadriven v1.0.2/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= @@ -861,10 +863,10 @@ github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2 h1:mbAskLJ0oJf github.com/tiancaiamao/appdash v0.0.0-20181126055449-889f96f722a2/go.mod h1:2PfKggNGDuadAa0LElHrByyrz4JPZ9fFx6Gs7nx7ZZU= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a h1:J/YdBZ46WKpXsxsW93SG+q0F8KI+yFrcIDT4c/RNoc4= github.com/tiancaiamao/gp v0.0.0-20221230034425-4025bc8a4d4a/go.mod h1:h4xBhSNtOeEosLJ4P7JyKXX7Cabg7AVkWCK5gV2vOrM= -github.com/tikv/client-go/v2 v2.0.8-0.20240123055405-3480b5ed7ce1 h1:y0of+EkoUDL1N1k6JItuA0ZMMrDU71vKhQ2p+pA0Fw4= -github.com/tikv/client-go/v2 v2.0.8-0.20240123055405-3480b5ed7ce1/go.mod h1:byff6zglNXgereADRRJmKQnurwy1Z9hthX2I5ObKMNE= -github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9 h1:LnNWRdtxryzxl31GmxOJEFKUmwiG8nph9f5Wqdv8olY= -github.com/tikv/pd/client v0.0.0-20240109100024-dd8df25316e9/go.mod h1:ZilHJZR8wgqENRi26gtnPoKIXAB1EqytFweUhzxetx0= +github.com/tikv/client-go/v2 v2.0.8-0.20240125030910-e6f5a45b002e h1:I/T4S2jY6yHQdwYQKDBXdDPt44xG/P9TkqUicraEjYk= +github.com/tikv/client-go/v2 v2.0.8-0.20240125030910-e6f5a45b002e/go.mod h1:jZLZhtui1Po+x616K7jxVAMe+aQfAuWqUZluRdO75Kc= +github.com/tikv/pd/client v0.0.0-20240126020320-567c7d43a008 h1:McTV/45piKWPinw0m7gFospPEGBW8AfK5J0RB/G9DP4= +github.com/tikv/pd/client v0.0.0-20240126020320-567c7d43a008/go.mod h1:Z/QAgOt29zvwBTd0H6pdx45VO6KRNc/O/DzGkVmSyZg= github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966 h1:quvGphlmUVU+nhpFa4gg4yJyTRJ13reZMDHrKwYw53M= github.com/timakin/bodyclose v0.0.0-20230421092635-574207250966/go.mod h1:27bSVNWSBOHm+qRp1T9qzaIpsWEP6TbUnei/43HK+PQ= github.com/tklauser/go-sysconf v0.3.9/go.mod h1:11DU/5sG7UexIrp/O6g35hrWzu0JxlwQ3LSFUzyeuhs= @@ -1375,12 +1377,12 @@ google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7Fc google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20240108191215-35c7eff3a6b1 h1:/IWabOtPziuXTEtI1KYCpM6Ss7vaAkeMxk+uXV/xvZs= -google.golang.org/genproto v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac h1:ZL/Teoy/ZGnzyrqK/Optxxp2pmVh+fmJ97slxSRyzUg= +google.golang.org/genproto v0.0.0-20240116215550-a9fa1716bcac/go.mod h1:+Rvu7ElI+aLzyDQhpHMFMMltsD6m7nqpuWDd2CwJw3k= google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1 h1:OPXtXn7fNMaXwO3JvOmF1QyTc00jsSFFz1vXXBOdCDo= google.golang.org/genproto/googleapis/api v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:B5xPO//w8qmBDjGReYLpR6UJPnkldGkCSMoH/2vxJeg= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1 h1:gphdwh0npgs8elJ4T6J+DQJHPVF7RsuJHCfwztUb4J4= -google.golang.org/genproto/googleapis/rpc v0.0.0-20240108191215-35c7eff3a6b1/go.mod h1:daQN87bsDqDoe316QbbvX60nMoJQa4r6Ds0ZuoAe5yA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80 h1:AjyfHzEPEFp/NpvfN5g+KDla3EMojjhRVZc1i7cj+oM= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240123012728-ef4313101c80/go.mod h1:PAREbraiVEVGVdTZsVWjSbbTtSyGbAgIIvni8a8CD5s= google.golang.org/grpc v0.0.0-20180607172857-7a6a684ca69e/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.12.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1398,8 +1400,8 @@ google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= -google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= -google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= +google.golang.org/grpc v1.61.0 h1:TOvOcuXn30kRao+gfcvsebNEa5iZIiLkisYEkf7R7o0= +google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9 h1:ATnmU8nL2NfIyTSiBvJVDIDIr3qBmeW+c7z7XU21eWs= google.golang.org/grpc/examples v0.0.0-20231221225426-4f03f3ff32c9/go.mod h1:j5uROIAAgi3YmtiETMt1LW0d/lHqQ7wwrIY4uGRXLQ4= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= diff --git a/pkg/ddl/tests/tiflash/ddl_tiflash_test.go b/pkg/ddl/tests/tiflash/ddl_tiflash_test.go index 06d850a8d9497..9e08fe327f9d1 100644 --- a/pkg/ddl/tests/tiflash/ddl_tiflash_test.go +++ b/pkg/ddl/tests/tiflash/ddl_tiflash_test.go @@ -187,7 +187,7 @@ func TestTiFlashNoRedundantPDRules(t *testing.T) { s, teardown := createTiFlashContext(t) defer teardown() - rpcClient, pdClient, cluster, err := unistore.New("") + rpcClient, pdClient, cluster, err := unistore.New("", nil) require.NoError(t, err) defer func() { rpcClient.Close() @@ -583,7 +583,7 @@ func TestSetPlacementRuleWithGCWorker(t *testing.T) { s, teardown := createTiFlashContext(t) defer teardown() - rpcClient, pdClient, cluster, err := unistore.New("") + rpcClient, pdClient, cluster, err := unistore.New("", nil) defer func() { rpcClient.Close() pdClient.Close() diff --git a/pkg/executor/infoschema_cluster_table_test.go b/pkg/executor/infoschema_cluster_table_test.go index aaed175f377b9..ea3fd5235e8a9 100644 --- a/pkg/executor/infoschema_cluster_table_test.go +++ b/pkg/executor/infoschema_cluster_table_test.go @@ -57,9 +57,11 @@ type infosSchemaClusterTableSuite struct { func createInfosSchemaClusterTableSuite(t *testing.T) *infosSchemaClusterTableSuite { s := new(infosSchemaClusterTableSuite) s.httpServer, s.mockAddr = s.setUpMockPDHTTPServer() + pdAddrs := []string{s.mockAddr} s.store, s.dom = testkit.CreateMockStoreAndDomain( t, - mockstore.WithTiKVOptions(tikv.WithPDHTTPClient("infoschema-cluster-table-test", []string{s.mockAddr})), + mockstore.WithTiKVOptions(tikv.WithPDHTTPClient("infoschema-cluster-table-test", pdAddrs)), + mockstore.WithPDAddr(pdAddrs), ) s.rpcServer, s.listenAddr = setUpRPCService(t, s.dom, "127.0.0.1:0") s.startTime = time.Now() diff --git a/pkg/executor/tikv_regions_peers_table_test.go b/pkg/executor/tikv_regions_peers_table_test.go index 5319960318b3f..53c7897e2a38b 100644 --- a/pkg/executor/tikv_regions_peers_table_test.go +++ b/pkg/executor/tikv_regions_peers_table_test.go @@ -111,8 +111,11 @@ func TestTikvRegionPeers(t *testing.T) { router.HandleFunc(pd.RegionByIDPrefix+"/"+"{id}", regionsInfoHandler) defer server.Close() + pdAddrs := []string{mockAddr} store := testkit.CreateMockStore(t, - mockstore.WithTiKVOptions(tikv.WithPDHTTPClient("tikv-regions-peers-table-test", []string{mockAddr}))) + mockstore.WithTiKVOptions(tikv.WithPDHTTPClient("tikv-regions-peers-table-test", pdAddrs)), + mockstore.WithPDAddr(pdAddrs), + ) store = &mockStore{ store.(helper.Storage), diff --git a/pkg/store/driver/main_test.go b/pkg/store/driver/main_test.go index f0624ab827f4e..9c09c0d02441b 100644 --- a/pkg/store/driver/main_test.go +++ b/pkg/store/driver/main_test.go @@ -87,7 +87,7 @@ func createTiKVStore(t *testing.T) (kv.Storage, *domain.Domain) { } func createUnistore(t *testing.T) (kv.Storage, *domain.Domain) { - client, pdClient, cluster, err := unistore.New("") + client, pdClient, cluster, err := unistore.New("", nil) require.NoError(t, err) unistore.BootstrapWithSingleStore(cluster) diff --git a/pkg/store/helper/helper_test.go b/pkg/store/helper/helper_test.go index 0702928091984..7f92a18db819a 100644 --- a/pkg/store/helper/helper_test.go +++ b/pkg/store/helper/helper_test.go @@ -153,6 +153,7 @@ func createMockStore(t *testing.T) (store helper.Storage) { mockstore.BootstrapWithMultiRegions(c, []byte("x")) }), mockstore.WithTiKVOptions(tikv.WithPDHTTPClient("store-helper-test", pdAddrs)), + mockstore.WithPDAddr(pdAddrs), ) require.NoError(t, err) diff --git a/pkg/store/mockstore/mockstore.go b/pkg/store/mockstore/mockstore.go index a3a87e6b86f7a..63d3734136270 100644 --- a/pkg/store/mockstore/mockstore.go +++ b/pkg/store/mockstore/mockstore.go @@ -96,6 +96,7 @@ type mockOptions struct { storeType StoreType ddlCheckerHijack bool tikvOptions []tikv.Option + pdAddrs []string } // MockTiKVStoreOption is used to control some behavior of mock tikv. @@ -110,6 +111,13 @@ func WithMultipleOptions(opts ...MockTiKVStoreOption) MockTiKVStoreOption { } } +// WithPDAddr set pd address for pd service discovery in mock PD client. +func WithPDAddr(addr []string) MockTiKVStoreOption { + return func(args *mockOptions) { + args.pdAddrs = addr + } +} + // WithTiKVOptions sets KV options. func WithTiKVOptions(opts ...tikv.Option) MockTiKVStoreOption { return func(args *mockOptions) { diff --git a/pkg/store/mockstore/unistore.go b/pkg/store/mockstore/unistore.go index 2526eb70677dd..36161729952a6 100644 --- a/pkg/store/mockstore/unistore.go +++ b/pkg/store/mockstore/unistore.go @@ -24,7 +24,7 @@ import ( ) func newUnistore(opts *mockOptions) (kv.Storage, error) { - client, pdClient, cluster, err := unistore.New(opts.path) + client, pdClient, cluster, err := unistore.New(opts.path, opts.pdAddrs) if err != nil { return nil, errors.Trace(err) } diff --git a/pkg/store/mockstore/unistore/BUILD.bazel b/pkg/store/mockstore/unistore/BUILD.bazel index 2280ed08334da..35e230fcdbbf8 100644 --- a/pkg/store/mockstore/unistore/BUILD.bazel +++ b/pkg/store/mockstore/unistore/BUILD.bazel @@ -22,6 +22,7 @@ go_library( "//pkg/tablecodec", "//pkg/util/codec", "//pkg/util/topsql/state", + "@com_github_asaskevich_govalidator//:govalidator", "@com_github_golang_protobuf//proto", "@com_github_pingcap_errors//:errors", "@com_github_pingcap_failpoint//:failpoint", @@ -39,6 +40,7 @@ go_library( "@com_github_tikv_client_go_v2//testutils", "@com_github_tikv_client_go_v2//tikvrpc", "@com_github_tikv_pd_client//:client", + "@org_golang_google_grpc//:grpc", "@org_golang_google_grpc//metadata", ], ) @@ -53,9 +55,10 @@ go_test( ], embed = [":unistore"], flaky = True, - shard_count = 4, + shard_count = 5, deps = [ "//pkg/testkit/testsetup", + "@com_github_asaskevich_govalidator//:govalidator", "@com_github_pingcap_kvproto//pkg/kvrpcpb", "@com_github_stretchr_testify//require", "@com_github_tikv_pd_client//:client", diff --git a/pkg/store/mockstore/unistore/mock.go b/pkg/store/mockstore/unistore/mock.go index 3cf178750ea0c..186ebd5f8f8e4 100644 --- a/pkg/store/mockstore/unistore/mock.go +++ b/pkg/store/mockstore/unistore/mock.go @@ -26,7 +26,7 @@ import ( ) // New creates an embed unistore client, pd client and cluster handler. -func New(path string) (*RPCClient, pd.Client, *Cluster, error) { +func New(path string, pdAddrs []string) (*RPCClient, pd.Client, *Cluster, error) { persistent := true if path == "" { var err error @@ -70,7 +70,7 @@ func New(path string) (*RPCClient, pd.Client, *Cluster, error) { rawHandler: newRawHandler(), } srv.RPCClient = client - pdClient := newPDClient(pd) + pdClient := newPDClient(pd, pdAddrs) return client, pdClient, cluster, nil } diff --git a/pkg/store/mockstore/unistore/pd.go b/pkg/store/mockstore/unistore/pd.go index f6325088a1032..ad3167fdf72e2 100644 --- a/pkg/store/mockstore/unistore/pd.go +++ b/pkg/store/mockstore/unistore/pd.go @@ -17,10 +17,13 @@ package unistore import ( "context" "errors" + "fmt" "math" + "strings" "sync" "sync/atomic" + "github.com/asaskevich/govalidator" "github.com/pingcap/kvproto/pkg/keyspacepb" "github.com/pingcap/kvproto/pkg/meta_storagepb" "github.com/pingcap/kvproto/pkg/pdpb" @@ -29,6 +32,7 @@ import ( us "github.com/pingcap/tidb/pkg/store/mockstore/unistore/tikv" "github.com/tikv/client-go/v2/oracle" pd "github.com/tikv/pd/client" + "google.golang.org/grpc" ) var _ pd.Client = new(pdClient) @@ -41,14 +45,19 @@ type pdClient struct { gcSafePointMu sync.Mutex globalConfig map[string]string externalTimestamp atomic.Uint64 + + // After using PD http client, we should impl mock PD service discovery + // which needs PD server HTTP address. + addrs []string } -func newPDClient(pd *us.MockPD) *pdClient { +func newPDClient(pd *us.MockPD, addrs []string) *pdClient { return &pdClient{ MockPD: pd, ResourceManagerClient: infosync.NewMockResourceManagerClient(), serviceSafePoints: make(map[string]uint64), globalConfig: make(map[string]string), + addrs: addrs, } } @@ -100,6 +109,118 @@ func (c *pdClient) GetLocalTSAsync(ctx context.Context, dcLocation string) pd.TS return &mockTSFuture{c, ctx, false} } +func (c *pdClient) GetServiceDiscovery() pd.ServiceDiscovery { + return NewMockPDServiceDiscovery(c.addrs) +} + +var _ pd.ServiceDiscovery = (*mockPDServiceDiscovery)(nil) +var _ pd.ServiceClient = (*mockPDServiceClient)(nil) + +type mockPDServiceClient struct { + addr string +} + +func newMockPDServiceClient(addr string) pd.ServiceClient { + if !strings.HasPrefix(addr, "http") { + addr = fmt.Sprintf("%s://%s", "http", addr) + } + return &mockPDServiceClient{addr: addr} +} + +func (c *mockPDServiceClient) GetAddress() string { + return c.addr +} + +func (c *mockPDServiceClient) GetHTTPAddress() string { + return c.addr +} + +func (c *mockPDServiceClient) GetClientConn() *grpc.ClientConn { + return nil +} + +func (c *mockPDServiceClient) BuildGRPCTargetContext(ctx context.Context, _ bool) context.Context { + return ctx +} + +func (c *mockPDServiceClient) Available() bool { + return true +} + +func (c *mockPDServiceClient) NeedRetry(*pdpb.Error, error) bool { + return false +} + +func (c *mockPDServiceClient) IsConnectedToLeader() bool { + return true +} + +type mockPDServiceDiscovery struct { + addrs []string + clis []pd.ServiceClient +} + +// NewMockPDServiceDiscovery returns a mock PD ServiceDiscovery +func NewMockPDServiceDiscovery(addrs []string) pd.ServiceDiscovery { + addresses := make([]string, 0) + clis := make([]pd.ServiceClient, 0) + for _, addr := range addrs { + if check := govalidator.IsURL(addr); !check { + continue + } + addresses = append(addresses, addr) + clis = append(clis, newMockPDServiceClient(addr)) + } + return &mockPDServiceDiscovery{addrs: addresses, clis: clis} +} + +func (c *mockPDServiceDiscovery) Init() error { + return nil +} + +func (c *mockPDServiceDiscovery) Close() {} + +func (c *mockPDServiceDiscovery) GetClusterID() uint64 { return 0 } + +func (c *mockPDServiceDiscovery) GetKeyspaceID() uint32 { return 0 } + +func (c *mockPDServiceDiscovery) GetKeyspaceGroupID() uint32 { return 0 } + +func (c *mockPDServiceDiscovery) GetServiceURLs() []string { + return c.addrs +} + +func (c *mockPDServiceDiscovery) GetServingEndpointClientConn() *grpc.ClientConn { return nil } + +func (c *mockPDServiceDiscovery) GetClientConns() *sync.Map { return nil } + +func (c *mockPDServiceDiscovery) GetServingAddr() string { return "" } + +func (c *mockPDServiceDiscovery) GetBackupAddrs() []string { return nil } + +func (c *mockPDServiceDiscovery) GetServiceClient() pd.ServiceClient { + if len(c.clis) > 0 { + return c.clis[0] + } + return nil +} + +func (c *mockPDServiceDiscovery) GetAllServiceClients() []pd.ServiceClient { + return c.clis +} + +func (c *mockPDServiceDiscovery) GetOrCreateGRPCConn(addr string) (*grpc.ClientConn, error) { + return nil, nil +} + +func (c *mockPDServiceDiscovery) ScheduleCheckMemberChanged() {} + +func (c *mockPDServiceDiscovery) CheckMemberChanged() error { return nil } + +func (c *mockPDServiceDiscovery) AddServingAddrSwitchedCallback(callbacks ...func()) {} + +func (c *mockPDServiceDiscovery) AddServiceAddrsSwitchedCallback(callbacks ...func()) {} + type mockTSFuture struct { pdc *pdClient ctx context.Context diff --git a/pkg/store/mockstore/unistore/pd_test.go b/pkg/store/mockstore/unistore/pd_test.go index 4cfbfd094e396..6a79a7f38f5cf 100644 --- a/pkg/store/mockstore/unistore/pd_test.go +++ b/pkg/store/mockstore/unistore/pd_test.go @@ -18,6 +18,7 @@ import ( "context" "testing" + "github.com/asaskevich/govalidator" "github.com/stretchr/testify/require" pd "github.com/tikv/pd/client" ) @@ -30,7 +31,7 @@ type GlobalConfigTestSuite struct { func SetUpSuite() *GlobalConfigTestSuite { s := &GlobalConfigTestSuite{} - s.rpc, s.client, s.cluster, _ = New("") + s.rpc, s.client, s.cluster, _ = New("", nil) return s } @@ -93,3 +94,21 @@ func (s *GlobalConfigTestSuite) TearDownSuite() { s.rpc.Close() s.cluster.Close() } + +func TestMockPDServiceDiscovery(t *testing.T) { + re := require.New(t) + pdAddrs := []string{"invalid_pd_address", "127.0.0.1:2379", "http://172.32.21.32:2379"} + for i, addr := range pdAddrs { + check := govalidator.IsURL(addr) + if i > 0 { + re.True(check) + } else { + re.False(check) + } + } + sd := NewMockPDServiceDiscovery(pdAddrs) + clis := sd.GetAllServiceClients() + re.Len(clis, 2) + re.Equal(clis[0].GetHTTPAddress(), "http://127.0.0.1:2379") + re.Equal(clis[1].GetHTTPAddress(), "http://172.32.21.32:2379") +}