Merge branch 'master' into replace_ctl_test

client/http/interface.go

@@ -79,6 +79,7 @@ type Client interface {
 	AccelerateScheduleInBatch(context.Context, []*KeyRange) error
 	/* Other interfaces */
 	GetMinResolvedTSByStoresIDs(context.Context, []uint64) (uint64, map[uint64]uint64, error)
+	GetPDVersion(context.Context) (string, error)
 	/* Micro Service interfaces */
 	GetMicroServiceMembers(context.Context, string) ([]string, error)
 
@@ -754,3 +755,16 @@ func (c *client) GetMicroServiceMembers(ctx context.Context, service string) ([]
 	}
 	return members, nil
 }
+
+// GetPDVersion gets the release version of the PD binary.
+func (c *client) GetPDVersion(ctx context.Context) (string, error) {
+	var ver struct {
+		Version string `json:"version"`
+	}
+	err := c.request(ctx, newRequestInfo().
+		WithName(getPDVersionName).
+		WithURI(Version).
+		WithMethod(http.MethodGet).
+		WithResp(&ver))
+	return ver.Version, err
+}

client/http/request_info.go

@@ -63,6 +63,7 @@ const (
 	accelerateScheduleInBatchName           = "AccelerateScheduleInBatch"
 	getMinResolvedTSByStoresIDsName         = "GetMinResolvedTSByStoresIDs"
 	getMicroServiceMembersName              = "GetMicroServiceMembers"
+	getPDVersionName                        = "GetPDVersion"
 )
 
 type requestInfo struct {

go.mod

@@ -34,7 +34,7 @@ require (
 	github.com/pingcap/kvproto v0.0.0-20231222062942-c0c73f41d0b2
 	github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3
 	github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21
-	github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c
+	github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2
 	github.com/prometheus/client_golang v1.11.1
 	github.com/prometheus/common v0.26.0
 	github.com/sasha-s/go-deadlock v0.2.0

go.sum

@@ -460,8 +460,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I=
 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM=
-github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c h1:iEZwsxxOxXaH0zEfzVAn6fjveOlPh3v3DsYlhWJAVi0=
-github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c=
+github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2 h1:5GB9XSo9tDdjxEzoiibA9rPgrE0YUSKrPmPYZym8dY0=
+github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c=
 github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4=
 github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

pkg/mcs/resourcemanager/server/manager.go

@@ -63,7 +63,12 @@ type Manager struct {
 		isTiFlash    bool
 	}
 	// record update time of each resource group
-	consumptionRecord map[string]time.Time
+	consumptionRecord map[consumptionRecordKey]time.Time
+}
+
+type consumptionRecordKey struct {
+	name   string
+	ruType string
 }
 
 // ConfigProvider is used to get resource manager config from the given
@@ -84,7 +89,7 @@ func NewManager[T ConfigProvider](srv bs.Server) *Manager {
 			isBackground bool
 			isTiFlash    bool
 		}, defaultConsumptionChanSize),
-		consumptionRecord: make(map[string]time.Time),
+		consumptionRecord: make(map[consumptionRecordKey]time.Time),
 	}
 	// The first initialization after the server is started.
 	srv.AddStartCallback(func() {
@@ -411,27 +416,27 @@ func (m *Manager) backgroundMetricsFlush(ctx context.Context) {
 				writeRequestCountMetrics.Add(consumption.KvWriteRpcCount)
 			}
 
-			m.consumptionRecord[name] = time.Now()
+			m.consumptionRecord[consumptionRecordKey{name: name, ruType: ruLabelType}] = time.Now()
 
 			// TODO: maybe we need to distinguish background ru.
 			if rg := m.GetMutableResourceGroup(name); rg != nil {
 				rg.UpdateRUConsumption(consumptionInfo.Consumption)
 			}
 		case <-cleanUpTicker.C:
 			// Clean up the metrics that have not been updated for a long time.
-			for name, lastTime := range m.consumptionRecord {
+			for r, lastTime := range m.consumptionRecord {
 				if time.Since(lastTime) > metricsCleanupTimeout {
-					readRequestUnitCost.DeleteLabelValues(name, name)
-					writeRequestUnitCost.DeleteLabelValues(name, name)
-					sqlLayerRequestUnitCost.DeleteLabelValues(name, name)
-					readByteCost.DeleteLabelValues(name, name)
-					writeByteCost.DeleteLabelValues(name, name)
-					kvCPUCost.DeleteLabelValues(name, name)
-					sqlCPUCost.DeleteLabelValues(name, name)
-					requestCount.DeleteLabelValues(name, name, readTypeLabel)
-					requestCount.DeleteLabelValues(name, name, writeTypeLabel)
-					availableRUCounter.DeleteLabelValues(name, name)
-					delete(m.consumptionRecord, name)
+					readRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
+					writeRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
+					sqlLayerRequestUnitCost.DeleteLabelValues(r.name, r.name, r.ruType)
+					readByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
+					writeByteCost.DeleteLabelValues(r.name, r.name, r.ruType)
+					kvCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
+					sqlCPUCost.DeleteLabelValues(r.name, r.name, r.ruType)
+					requestCount.DeleteLabelValues(r.name, r.name, readTypeLabel)
+					requestCount.DeleteLabelValues(r.name, r.name, writeTypeLabel)
+					availableRUCounter.DeleteLabelValues(r.name, r.name, r.ruType)
+					delete(m.consumptionRecord, r)
 				}
 			}
 		case <-availableRUTicker.C:

scripts/dashboard-version

@@ -1,4 +1,3 @@
 # This file is updated by running scripts/update-dashboard.sh
 # Don't edit it manullay
-# the version will become X.Y.Z-<commit-short-hash> format later
-2023.12.18.1
+7.6.0-473bf80a

scripts/update-dashboard.sh

@@ -45,3 +45,6 @@ go mod tidy
 
 cd ./tests/integrations
 go mod tidy
+
+cd ../../tools
+go mod tidy

server/api/region.go

@@ -256,7 +256,13 @@ func (h *regionHandler) GetRegionByID(w http.ResponseWriter, r *http.Request) {
 	}
 
 	regionInfo := rc.GetRegion(regionID)
-	h.rd.JSON(w, http.StatusOK, NewAPIRegionInfo(regionInfo))
+	b, err := marshalRegionInfoJSON(r.Context(), regionInfo)
+	if err != nil {
+		h.rd.JSON(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	h.rd.Data(w, http.StatusOK, b)
 }
 
 // @Tags     region
@@ -286,7 +292,13 @@ func (h *regionHandler) GetRegion(w http.ResponseWriter, r *http.Request) {
 	}
 
 	regionInfo := rc.GetRegionByKey([]byte(key))
-	h.rd.JSON(w, http.StatusOK, NewAPIRegionInfo(regionInfo))
+	b, err := marshalRegionInfoJSON(r.Context(), regionInfo)
+	if err != nil {
+		h.rd.JSON(w, http.StatusInternalServerError, err.Error())
+		return
+	}
+
+	h.rd.Data(w, http.StatusOK, b)
 }
 
 // @Tags     region
@@ -323,6 +335,24 @@ func newRegionsHandler(svr *server.Server, rd *render.Render) *regionsHandler {
 	}
 }
 
+// marshalRegionInfoJSON marshals region to bytes in `RegionInfo`'s JSON format.
+// It is used to reduce the cost of JSON serialization.
+func marshalRegionInfoJSON(ctx context.Context, r *core.RegionInfo) ([]byte, error) {
+	out := &jwriter.Writer{}
+
+	region := &RegionInfo{}
+	select {
+	case <-ctx.Done():
+		// Return early, avoid the unnecessary computation.
+		// See more details in https://github.com/tikv/pd/issues/6835
+		return nil, ctx.Err()
+	default:
+	}
+
+	covertAPIRegionInfo(r, region, out)
+	return out.Buffer.BuildBytes(), out.Error
+}
+
 // marshalRegionsInfoJSON marshals regions to bytes in `RegionsInfo`'s JSON format.
 // It is used to reduce the cost of JSON serialization.
 func marshalRegionsInfoJSON(ctx context.Context, regions []*core.RegionInfo) ([]byte, error) {
@@ -346,27 +376,31 @@ func marshalRegionsInfoJSON(ctx context.Context, regions []*core.RegionInfo) ([]
 		if i > 0 {
 			out.RawByte(',')
 		}
-		InitRegion(r, region)
-		// EasyJSON will not check anonymous struct pointer field and will panic if the field is nil.
-		// So we need to set the field to default value explicitly when the anonymous struct pointer is nil.
-		region.Leader.setDefaultIfNil()
-		for i := range region.Peers {
-			region.Peers[i].setDefaultIfNil()
-		}
-		for i := range region.PendingPeers {
-			region.PendingPeers[i].setDefaultIfNil()
-		}
-		for i := range region.DownPeers {
-			region.DownPeers[i].setDefaultIfNil()
-		}
-		region.MarshalEasyJSON(out)
+		covertAPIRegionInfo(r, region, out)
 	}
 	out.RawByte(']')
 
 	out.RawByte('}')
 	return out.Buffer.BuildBytes(), out.Error
 }
 
+func covertAPIRegionInfo(r *core.RegionInfo, region *RegionInfo, out *jwriter.Writer) {
+	InitRegion(r, region)
+	// EasyJSON will not check anonymous struct pointer field and will panic if the field is nil.
+	// So we need to set the field to default value explicitly when the anonymous struct pointer is nil.
+	region.Leader.setDefaultIfNil()
+	for i := range region.Peers {
+		region.Peers[i].setDefaultIfNil()
+	}
+	for i := range region.PendingPeers {
+		region.PendingPeers[i].setDefaultIfNil()
+	}
+	for i := range region.DownPeers {
+		region.DownPeers[i].setDefaultIfNil()
+	}
+	region.MarshalEasyJSON(out)
+}
+
 // @Tags     region
 // @Summary  List all regions in the cluster.
 // @Produce  json

tests/integrations/client/http_client_test.go

@@ -30,6 +30,7 @@ import (
 	"github.com/tikv/pd/pkg/schedule/placement"
 	"github.com/tikv/pd/pkg/utils/testutil"
 	"github.com/tikv/pd/pkg/utils/tsoutil"
+	"github.com/tikv/pd/pkg/versioninfo"
 	"github.com/tikv/pd/tests"
 )
 
@@ -478,3 +479,10 @@ func (suite *httpClientTestSuite) TestTransferLeader() {
 	re.Len(members.Members, 2)
 	re.Equal(leader.GetName(), members.Leader.GetName())
 }
+
+func (suite *httpClientTestSuite) TestVersion() {
+	re := suite.Require()
+	ver, err := suite.client.GetPDVersion(suite.ctx)
+	re.NoError(err)
+	re.Equal(versioninfo.PDReleaseVersion, ver)
+}

tests/integrations/go.mod

@@ -127,7 +127,7 @@ require (
 	github.com/pingcap/errcode v0.3.0 // indirect
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c // indirect
 	github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect
-	github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c // indirect
+	github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2 // indirect
 	github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

tests/integrations/go.sum

@@ -437,8 +437,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I=
 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM=
-github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c h1:iEZwsxxOxXaH0zEfzVAn6fjveOlPh3v3DsYlhWJAVi0=
-github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c=
+github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2 h1:5GB9XSo9tDdjxEzoiibA9rPgrE0YUSKrPmPYZym8dY0=
+github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c=
 github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4=
 github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

tools/go.mod

@@ -129,7 +129,7 @@ require (
 	github.com/phf/go-queue v0.0.0-20170504031614-9abe38d0371d // indirect
 	github.com/pingcap/errcode v0.3.0 // indirect
 	github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 // indirect
-	github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c // indirect
+	github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2 // indirect
 	github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect

tools/go.sum

@@ -438,8 +438,8 @@ github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3 h1:HR/ylkkLmGdSSDaD8
 github.com/pingcap/log v1.1.1-0.20221110025148-ca232912c9f3/go.mod h1:DWQW5jICDR7UJh4HtxXSM20Churx4CQL0fwL/SoOSA4=
 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21 h1:QV6jqlfOkh8hqvEAgwBZa+4bSgO0EeKC7s5c6Luam2I=
 github.com/pingcap/sysutil v1.0.1-0.20230407040306-fb007c5aff21/go.mod h1:QYnjfA95ZaMefyl1NO8oPtKeb8pYUdnDVhQgf+qdpjM=
-github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c h1:iEZwsxxOxXaH0zEfzVAn6fjveOlPh3v3DsYlhWJAVi0=
-github.com/pingcap/tidb-dashboard v0.0.0-20231218095437-aa621ed4de2c/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c=
+github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2 h1:5GB9XSo9tDdjxEzoiibA9rPgrE0YUSKrPmPYZym8dY0=
+github.com/pingcap/tidb-dashboard v0.0.0-20231229113139-473bf80ad4f2/go.mod h1:ucZBRz52icb23T/5Z4CsuUHmarYiin7p2MeiVBe+o8c=
 github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e h1:FBaTXU8C3xgt/drM58VHxojHo/QoG1oPsgWTGvaSpO4=
 github.com/pingcap/tipb v0.0.0-20220718022156-3e2483c20a9e/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs=
 github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA=

tools/pd-ctl/tests/region/region_test.go

@@ -208,3 +208,56 @@ func TestRegion(t *testing.T) {
 		{core.HexRegionKeyStr(r5.GetEndKey()), ""},
 	}, *rangeHoles)
 }
+
+func TestRegionNoLeader(t *testing.T) {
+	re := require.New(t)
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	cluster, err := pdTests.NewTestCluster(ctx, 1)
+	re.NoError(err)
+	err = cluster.RunInitialServers()
+	re.NoError(err)
+	cluster.WaitLeader()
+	url := cluster.GetConfig().GetClientURL()
+	stores := []*metapb.Store{
+		{
+			Id:            1,
+			State:         metapb.StoreState_Up,
+			LastHeartbeat: time.Now().UnixNano(),
+		},
+		{
+			Id:            2,
+			State:         metapb.StoreState_Up,
+			LastHeartbeat: time.Now().UnixNano(),
+		},
+		{
+			Id:            3,
+			State:         metapb.StoreState_Up,
+			LastHeartbeat: time.Now().UnixNano(),
+		},
+	}
+
+	leaderServer := cluster.GetLeaderServer()
+	re.NoError(leaderServer.BootstrapCluster())
+	for i := 0; i < len(stores); i++ {
+		pdTests.MustPutStore(re, cluster, stores[i])
+	}
+
+	metaRegion := &metapb.Region{
+		Id:       100,
+		StartKey: []byte(""),
+		EndKey:   []byte(""),
+		Peers: []*metapb.Peer{
+			{Id: 1, StoreId: 1},
+			{Id: 5, StoreId: 2},
+			{Id: 6, StoreId: 3}},
+		RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1},
+	}
+	r := core.NewRegionInfo(metaRegion, nil)
+
+	cluster.GetLeaderServer().GetRaftCluster().GetBasicCluster().SetRegion(r)
+
+	cmd := ctl.GetRootCmd()
+	_, err = tests.ExecuteCommand(cmd, "-u", url, "region", "100")
+	re.NoError(err)
+}