Merge remote-tracking branch 'origin/master' into optimize-heartbeat/…

…async
nolouch · Apr 11, 2024 · 0985676 · 0985676
2 parents 03e2945 + 67461dd
commit 0985676
Show file tree

Hide file tree

Showing 51 changed files with 1,050 additions and 154 deletions.
diff --git a/Makefile b/Makefile
@@ -80,7 +80,7 @@ BUILD_BIN_PATH := $(ROOT_PATH)/bin
 
 build: pd-server pd-ctl pd-recover
 
-tools: pd-tso-bench pd-heartbeat-bench regions-dump stores-dump pd-api-bench
+tools: pd-tso-bench pd-heartbeat-bench regions-dump stores-dump pd-api-bench pd-ut
 
 PD_SERVER_DEP :=
 ifeq ($(SWAGGER), 1)
@@ -108,7 +108,6 @@ pd-server-basic:
 .PHONY: pre-build build tools pd-server pd-server-basic
 
 # Tools
-
 pd-ctl:
  cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ctl pd-ctl/main.go
 pd-tso-bench:
@@ -127,8 +126,12 @@ regions-dump:
  cd tools && CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/regions-dump regions-dump/main.go
 stores-dump:
  cd tools && CGO_ENABLED=0 go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/stores-dump stores-dump/main.go
+pd-ut: pd-xprog
+ cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/pd-ut pd-ut/ut.go
+pd-xprog:
+ cd tools && GOEXPERIMENT=$(BUILD_GOEXPERIMENT) CGO_ENABLED=$(BUILD_TOOL_CGO_ENABLED) go build -tags xprog -gcflags '$(GCFLAGS)' -ldflags '$(LDFLAGS)' -o $(BUILD_BIN_PATH)/xprog pd-ut/xprog.go
 
-.PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench
+.PHONY: pd-ctl pd-tso-bench pd-recover pd-analysis pd-heartbeat-bench simulator regions-dump stores-dump pd-api-bench pd-ut
 
 #### Docker image ####
 
@@ -225,6 +228,12 @@ failpoint-disable: install-tools
 
 #### Test ####
 
+ut: pd-ut
+ @$(FAILPOINT_ENABLE)
+ ./bin/pd-ut run --race
+ @$(CLEAN_UT_BINARY)
+ @$(FAILPOINT_DISABLE)
+
 PACKAGE_DIRECTORIES := $(subst $(PD_PKG)/,,$(PACKAGES))
 TEST_PKGS := $(filter $(shell find . -iname "*_test.go" -exec dirname {} \; | \
  sort -u | sed -e "s/^\./github.com\/tikv\/pd/"),$(PACKAGES))
@@ -303,13 +312,16 @@ split:
 
 clean: failpoint-disable clean-test clean-build
 
+CLEAN_UT_BINARY := find . -name '*.test.bin'| xargs rm -f
+
 clean-test:
  # Cleaning test tmp...
  rm -rf /tmp/test_pd*
  rm -rf /tmp/pd-tests*
  rm -rf /tmp/test_etcd*
  rm -f $(REAL_CLUSTER_TEST_PATH)/playground.log
  go clean -testcache
+ @$(CLEAN_UT_BINARY)
 
 clean-build:
  # Cleaning building files...

diff --git a/client/client.go b/client/client.go
@@ -69,16 +69,10 @@ type GlobalConfigItem struct {
  PayLoad []byte
 }
 
-// Client is a PD (Placement Driver) RPC client.
-// It should not be used after calling Close().
-type Client interface {
- // GetClusterID gets the cluster ID from PD.
- GetClusterID(ctx context.Context) uint64
+// RPCClient is a PD (Placement Driver) RPC and related mcs client which can only call RPC.
+type RPCClient interface {
  // GetAllMembers gets the members Info from PD
  GetAllMembers(ctx context.Context) ([]*pdpb.Member, error)
- // GetLeaderURL returns current leader's URL. It returns "" before
- // syncing leader from server.
- GetLeaderURL() string
  // GetRegion gets a region and its leader Peer from PD by key.
  // The region may expire after split. Caller is responsible for caching and
  // taking care of region change.
@@ -133,17 +127,12 @@ type Client interface {
  StoreGlobalConfig(ctx context.Context, configPath string, items []GlobalConfigItem) error
  // WatchGlobalConfig returns a stream with all global config and updates
  WatchGlobalConfig(ctx context.Context, configPath string, revision int64) (chan []GlobalConfigItem, error)
- // UpdateOption updates the client option.
- UpdateOption(option DynamicOption, value any) error
 
  // GetExternalTimestamp returns external timestamp
  GetExternalTimestamp(ctx context.Context) (uint64, error)
  // SetExternalTimestamp sets external timestamp
  SetExternalTimestamp(ctx context.Context, timestamp uint64) error
 
- // GetServiceDiscovery returns ServiceDiscovery
- GetServiceDiscovery() ServiceDiscovery
-
  // TSOClient is the TSO client.
  TSOClient
  // MetaStorageClient is the meta storage client.
@@ -154,6 +143,24 @@ type Client interface {
  GCClient
  // ResourceManagerClient manages resource group metadata and token assignment.
  ResourceManagerClient
+}
+
+// Client is a PD (Placement Driver) RPC client.
+// It should not be used after calling Close().
+type Client interface {
+ RPCClient
+
+ // GetClusterID gets the cluster ID from PD.
+ GetClusterID(ctx context.Context) uint64
+ // GetLeaderURL returns current leader's URL. It returns "" before
+ // syncing leader from server.
+ GetLeaderURL() string
+ // GetServiceDiscovery returns ServiceDiscovery
+ GetServiceDiscovery() ServiceDiscovery
+
+ // UpdateOption updates the client option.
+ UpdateOption(option DynamicOption, value any) error
+
  // Close closes the client.
  Close()
 }

diff --git a/client/resource_group/controller/util_test.go b/client/resource_group/controller/util_test.go
@@ -27,7 +27,6 @@ type example struct {
 }
 
 func TestDurationJSON(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  example := &example{}
 
@@ -41,7 +40,6 @@ func TestDurationJSON(t *testing.T) {
 }
 
 func TestDurationTOML(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  example := &example{}
 

diff --git a/pkg/audit/audit_test.go b/pkg/audit/audit_test.go
@@ -32,7 +32,6 @@ import (
 )
 
 func TestLabelMatcher(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  matcher := &LabelMatcher{"testSuccess"}
  labels1 := &BackendLabels{Labels: []string{"testFail", "testSuccess"}}
@@ -42,7 +41,6 @@ func TestLabelMatcher(t *testing.T) {
 }
 
 func TestPrometheusHistogramBackend(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  serviceAuditHistogramTest := prometheus.NewHistogramVec(
  prometheus.HistogramOpts{
@@ -90,7 +88,6 @@ func TestPrometheusHistogramBackend(t *testing.T) {
 }
 
 func TestLocalLogBackendUsingFile(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  backend := NewLocalLogBackend(true)
  fname := testutil.InitTempFileLogger("info")

diff --git a/pkg/autoscaling/calculation_test.go b/pkg/autoscaling/calculation_test.go
@@ -29,7 +29,6 @@ import (
 )
 
 func TestGetScaledTiKVGroups(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  ctx, cancel := context.WithCancel(context.Background())
  defer cancel()
@@ -214,7 +213,6 @@ func (q *mockQuerier) Query(options *QueryOptions) (QueryResult, error) {
 }
 
 func TestGetTotalCPUUseTime(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  querier := &mockQuerier{}
  instances := []instance{
@@ -237,7 +235,6 @@ func TestGetTotalCPUUseTime(t *testing.T) {
 }
 
 func TestGetTotalCPUQuota(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  querier := &mockQuerier{}
  instances := []instance{
@@ -260,7 +257,6 @@ func TestGetTotalCPUQuota(t *testing.T) {
 }
 
 func TestScaleOutGroupLabel(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  var jsonStr = []byte(`
 {
@@ -303,7 +299,6 @@ func TestScaleOutGroupLabel(t *testing.T) {
 }
 
 func TestStrategyChangeCount(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  var count uint64 = 2
  strategy := &Strategy{

diff --git a/pkg/autoscaling/prometheus_test.go b/pkg/autoscaling/prometheus_test.go
@@ -180,7 +180,6 @@ func (c *normalClient) Do(_ context.Context, req *http.Request) (response *http.
 }
 
 func TestRetrieveCPUMetrics(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  client := &normalClient{
  mockData: make(map[string]*response),
@@ -225,7 +224,6 @@ func (c *emptyResponseClient) Do(_ context.Context, req *http.Request) (r *http.
 }
 
 func TestEmptyResponse(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  client := &emptyResponseClient{}
  querier := NewPrometheusQuerier(client)
@@ -253,7 +251,6 @@ func (c *errorHTTPStatusClient) Do(_ context.Context, req *http.Request) (r *htt
 }
 
 func TestErrorHTTPStatus(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  client := &errorHTTPStatusClient{}
  querier := NewPrometheusQuerier(client)
@@ -279,7 +276,6 @@ func (c *errorPrometheusStatusClient) Do(_ context.Context, req *http.Request) (
 }
 
 func TestErrorPrometheusStatus(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  client := &errorPrometheusStatusClient{}
  querier := NewPrometheusQuerier(client)
@@ -290,7 +286,6 @@ func TestErrorPrometheusStatus(t *testing.T) {
 }
 
 func TestGetInstanceNameFromAddress(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  testCases := []struct {
  address string
@@ -328,7 +323,6 @@ func TestGetInstanceNameFromAddress(t *testing.T) {
 }
 
 func TestGetDurationExpression(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  testCases := []struct {
  duration time.Duration

diff --git a/pkg/balancer/balancer_test.go b/pkg/balancer/balancer_test.go
@@ -22,7 +22,6 @@ import (
 )
 
 func TestBalancerPutAndDelete(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  balancers := []Balancer[uint32]{
  NewRoundRobin[uint32](),
@@ -56,7 +55,6 @@ func TestBalancerPutAndDelete(t *testing.T) {
 }
 
 func TestBalancerDuplicate(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  balancers := []Balancer[uint32]{
  NewRoundRobin[uint32](),
@@ -77,7 +75,6 @@ func TestBalancerDuplicate(t *testing.T) {
 }
 
 func TestRoundRobin(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  balancer := NewRoundRobin[uint32]()
  for i := 0; i < 100; i++ {

diff --git a/pkg/cache/cache_test.go b/pkg/cache/cache_test.go
@@ -25,7 +25,6 @@ import (
 )
 
 func TestExpireRegionCache(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  ctx, cancel := context.WithCancel(context.Background())
  defer cancel()
@@ -121,7 +120,6 @@ func sortIDs(ids []uint64) []uint64 {
 }
 
 func TestLRUCache(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  cache := newLRU(3)
 
@@ -199,7 +197,6 @@ func TestLRUCache(t *testing.T) {
 }
 
 func TestFifoCache(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  cache := NewFIFO(3)
  cache.Put(1, "1")
@@ -227,7 +224,6 @@ func TestFifoCache(t *testing.T) {
 }
 
 func TestFifoFromLastSameElems(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  type testStruct struct {
  value string
@@ -260,7 +256,6 @@ func TestFifoFromLastSameElems(t *testing.T) {
 }
 
 func TestTwoQueueCache(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  cache := newTwoQueue(3)
  cache.Put(1, "1")
@@ -345,7 +340,6 @@ func (pq PriorityQueueItemTest) ID() uint64 {
 }
 
 func TestPriorityQueue(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  testData := []PriorityQueueItemTest{0, 1, 2, 3, 4, 5}
  pq := NewPriorityQueue(0)

diff --git a/pkg/codec/codec_test.go b/pkg/codec/codec_test.go
@@ -21,7 +21,6 @@ import (
 )
 
 func TestDecodeBytes(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  key := "abcdefghijklmnopqrstuvwxyz"
  for i := 0; i < len(key); i++ {
@@ -32,7 +31,6 @@ func TestDecodeBytes(t *testing.T) {
 }
 
 func TestTableID(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  key := EncodeBytes([]byte("t\x80\x00\x00\x00\x00\x00\x00\xff"))
  re.Equal(int64(0xff), key.TableID())

diff --git a/pkg/core/rangetree/range_tree_test.go b/pkg/core/rangetree/range_tree_test.go
@@ -85,7 +85,6 @@ func bucketDebrisFactory(startKey, endKey []byte, item RangeItem) []RangeItem {
 }
 
 func TestRingPutItem(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  bucketTree := NewRangeTree(2, bucketDebrisFactory)
  bucketTree.Update(newSimpleBucketItem([]byte("002"), []byte("100")))
@@ -120,7 +119,6 @@ func TestRingPutItem(t *testing.T) {
 }
 
 func TestDebris(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  ringItem := newSimpleBucketItem([]byte("010"), []byte("090"))
  var overlaps []RangeItem

diff --git a/pkg/core/storelimit/limit_test.go b/pkg/core/storelimit/limit_test.go
@@ -45,7 +45,6 @@ func TestStoreLimit(t *testing.T) {
 }
 
 func TestSlidingWindow(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  capacity := int64(defaultWindowSize)
  s := NewSlidingWindows()
@@ -92,7 +91,6 @@ func TestSlidingWindow(t *testing.T) {
 }
 
 func TestWindow(t *testing.T) {
- t.Parallel()
  re := require.New(t)
  capacity := int64(100 * 10)
  s := newWindow(capacity)