From f9f2805726cf9ed6a1390b29029772a357f0957a Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Thu, 27 Jul 2023 15:43:35 +0800 Subject: [PATCH 1/4] This is an automated cherry-pick of #6838 close tikv/pd#6835 Signed-off-by: ti-chi-bot --- go.mod | 6 + server/api/region.go | 221 +++++++++++-- server/api/region_easyjson.go | 567 ++++++++++++++++++++++++++++++++++ server/api/region_test.go | 174 ++++++++--- 4 files changed, 901 insertions(+), 67 deletions(-) create mode 100644 server/api/region_easyjson.go diff --git a/go.mod b/go.mod index f79572dcbfc..45ea6d79f73 100644 --- a/go.mod +++ b/go.mod @@ -114,8 +114,14 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect +<<<<<<< HEAD github.com/leodido/go-urn v1.2.0 // indirect github.com/mailru/easyjson v0.7.6 // indirect +======= + github.com/leodido/go-urn v1.2.1 // indirect + github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect + github.com/mailru/easyjson v0.7.6 +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.8 // indirect diff --git a/server/api/region.go b/server/api/region.go index 48e8176435f..726ef715a84 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -16,6 +16,7 @@ package api import ( "container/heap" + "context" "encoding/hex" "fmt" "net/http" @@ -25,6 +26,7 @@ import ( "strings" "github.com/gorilla/mux" + jwriter "github.com/mailru/easyjson/jwriter" "github.com/pingcap/failpoint" "github.com/pingcap/kvproto/pkg/metapb" "github.com/pingcap/kvproto/pkg/pdpb" @@ -53,6 +55,17 @@ type MetaPeer struct { IsLearner bool `json:"is_learner,omitempty"` } +func (m *MetaPeer) setDefaultIfNil() { + if m.Peer == nil { + m.Peer = &metapb.Peer{ + Id: m.GetId(), + StoreId: m.GetStoreId(), + Role: m.GetRole(), + IsWitness: m.GetIsWitness(), + } + } +} + // PDPeerStats is api compatible with *pdpb.PeerStats. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. type PDPeerStats struct { @@ -60,6 +73,16 @@ type PDPeerStats struct { Peer MetaPeer `json:"peer"` } +func (s *PDPeerStats) setDefaultIfNil() { + if s.PeerStats == nil { + s.PeerStats = &pdpb.PeerStats{ + Peer: s.GetPeer(), + DownSeconds: s.GetDownSeconds(), + } + } + s.Peer.setDefaultIfNil() +} + func fromPeer(peer *metapb.Peer) MetaPeer { if peer == nil { return MetaPeer{} @@ -102,6 +125,7 @@ func fromPeerStatsSlice(peers []*pdpb.PeerStats) []PDPeerStats { // RegionInfo records detail region info for api usage. // NOTE: This type is exported by HTTP API. Please pay more attention when modifying it. +// easyjson:json type RegionInfo struct { ID uint64 `json:"id"` StartKey string `json:"start_key"` @@ -168,9 +192,9 @@ func InitRegion(r *core.RegionInfo, s *RegionInfo) *RegionInfo { s.ApproximateSize = r.GetApproximateSize() s.ApproximateKeys = r.GetApproximateKeys() s.ReplicationStatus = fromPBReplicationStatus(r.GetReplicationStatus()) + s.Buckets = nil keys := r.GetBuckets().GetKeys() - if len(keys) > 0 { s.Buckets = make([]string, len(keys)) for i, key := range keys { @@ -312,15 +336,48 @@ func newRegionsHandler(svr *server.Server, rd *render.Render) *regionsHandler { } } -func convertToAPIRegions(regions []*core.RegionInfo) *RegionsInfo { - regionInfos := make([]RegionInfo, len(regions)) +// marshalRegionsInfoJSON marshals regions to bytes in `RegionsInfo`'s JSON format. +// It is used to reduce the cost of JSON serialization. +func marshalRegionsInfoJSON(ctx context.Context, regions []*core.RegionInfo) ([]byte, error) { + out := &jwriter.Writer{} + out.RawByte('{') + + out.RawString("\"count\":") + out.Int(len(regions)) + + out.RawString(",\"regions\":") + out.RawByte('[') + region := &RegionInfo{} for i, r := range regions { - InitRegion(r, ®ionInfos[i]) - } - return &RegionsInfo{ - Count: len(regions), - Regions: regionInfos, + select { + case <-ctx.Done(): + // Return early, avoid the unnecessary computation. + // See more details in https://github.com/tikv/pd/issues/6835 + return nil, ctx.Err() + default: + } + if i > 0 { + out.RawByte(',') + } + InitRegion(r, region) + // EasyJSON will not check anonymous struct pointer field and will panic if the field is nil. + // So we need to set the field to default value explicitly when the anonymous struct pointer is nil. + region.Leader.setDefaultIfNil() + for i := range region.Peers { + region.Peers[i].setDefaultIfNil() + } + for i := range region.PendingPeers { + region.PendingPeers[i].setDefaultIfNil() + } + for i := range region.DownPeers { + region.DownPeers[i].setDefaultIfNil() + } + region.MarshalEasyJSON(out) } + out.RawByte(']') + + out.RawByte('}') + return out.Buffer.BuildBytes(), out.Error } // @Tags region @@ -331,8 +388,12 @@ func convertToAPIRegions(regions []*core.RegionInfo) *RegionsInfo { func (h *regionsHandler) GetRegions(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) regions := rc.GetRegions() - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -362,8 +423,12 @@ func (h *regionsHandler) ScanRegions(w http.ResponseWriter, r *http.Request) { limit = maxRegionLimit } regions := rc.ScanRegions([]byte(startKey), []byte(endKey), limit) - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -394,25 +459,101 @@ func (h *regionsHandler) GetStoreRegions(w http.ResponseWriter, r *http.Request) return } regions := rc.GetStoreRegions(uint64(id)) - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) +} + +// @Tags region +<<<<<<< HEAD +======= +// @Summary List regions belongs to the given keyspace ID. +// @Param keyspace_id query string true "Keyspace ID" +// @Param limit query integer false "Limit count" default(16) +// @Produce json +// @Success 200 {object} RegionsInfo +// @Failure 400 {string} string "The input is invalid." +// @Router /regions/keyspace/id/{id} [get] +func (h *regionsHandler) GetKeyspaceRegions(w http.ResponseWriter, r *http.Request) { + rc := getCluster(r) + vars := mux.Vars(r) + keyspaceIDStr := vars["id"] + if keyspaceIDStr == "" { + h.rd.JSON(w, http.StatusBadRequest, "keyspace id is empty") + return + } + + keyspaceID64, err := strconv.ParseUint(keyspaceIDStr, 10, 32) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + keyspaceID := uint32(keyspaceID64) + keyspaceManager := h.svr.GetKeyspaceManager() + if _, err := keyspaceManager.LoadKeyspaceByID(keyspaceID); err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + + limit := defaultRegionLimit + if limitStr := r.URL.Query().Get("limit"); limitStr != "" { + limit, err = strconv.Atoi(limitStr) + if err != nil { + h.rd.JSON(w, http.StatusBadRequest, err.Error()) + return + } + } + if limit > maxRegionLimit { + limit = maxRegionLimit + } + regionBound := keyspace.MakeRegionBound(keyspaceID) + regions := rc.ScanRegions(regionBound.RawLeftBound, regionBound.RawRightBound, limit) + if limit <= 0 || limit > len(regions) { + txnRegion := rc.ScanRegions(regionBound.TxnLeftBound, regionBound.TxnRightBound, limit-len(regions)) + regions = append(regions, txnRegion...) + } + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) // @Summary List all regions that miss peer. // @Produce json // @Success 200 {object} RegionsInfo // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/miss-peer [get] func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD +======= + h.getRegionsByType(w, statistics.MissPeer, r) +} + +func (h *regionsHandler) getRegionsByType( + w http.ResponseWriter, + typ statistics.RegionStatisticType, + r *http.Request, +) { +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.MissPeer) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region @@ -422,6 +563,7 @@ func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, r *http.Reque // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/extra-peer [get] func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.ExtraPeer) if err != nil { @@ -430,6 +572,9 @@ func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Requ } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.ExtraPeer, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -439,6 +584,7 @@ func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Requ // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/pending-peer [get] func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.PendingPeer) if err != nil { @@ -447,6 +593,9 @@ func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Re } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.PendingPeer, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -456,6 +605,7 @@ func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/down-peer [get] func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.DownPeer) if err != nil { @@ -464,6 +614,9 @@ func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Reque } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.DownPeer, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -473,6 +626,7 @@ func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Reque // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/learner-peer [get] func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.LearnerPeer) if err != nil { @@ -481,6 +635,9 @@ func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Re } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.LearnerPeer, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -490,6 +647,7 @@ func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/offline-peer [get] func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetOfflinePeer(statistics.OfflinePeer) if err != nil { @@ -498,6 +656,9 @@ func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Re } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.OfflinePeer, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -507,6 +668,7 @@ func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/oversized-region [get] func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.OversizedRegion) if err != nil { @@ -515,6 +677,9 @@ func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Requ } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.OversizedRegion, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -524,6 +689,7 @@ func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Requ // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/undersized-region [get] func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.UndersizedRegion) if err != nil { @@ -532,6 +698,9 @@ func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Req } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.UndersizedRegion, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -541,6 +710,7 @@ func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Req // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/empty-region [get] func (h *regionsHandler) GetEmptyRegions(w http.ResponseWriter, r *http.Request) { +<<<<<<< HEAD handler := h.svr.GetHandler() regions, err := handler.GetRegionsByType(statistics.EmptyRegion) if err != nil { @@ -549,6 +719,9 @@ func (h *regionsHandler) GetEmptyRegions(w http.ResponseWriter, r *http.Request) } regionsInfo := convertToAPIRegions(regions) h.rd.JSON(w, http.StatusOK, regionsInfo) +======= + h.getRegionsByType(w, statistics.EmptyRegion, r) +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } type histItem struct { @@ -688,8 +861,12 @@ func (h *regionsHandler) GetRegionSiblings(w http.ResponseWriter, r *http.Reques } left, right := rc.GetAdjacentRegions(region) - regionsInfo := convertToAPIRegions([]*core.RegionInfo{left, right}) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), []*core.RegionInfo{left, right}) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } const ( @@ -910,8 +1087,12 @@ func (h *regionsHandler) GetTopNRegions(w http.ResponseWriter, r *http.Request, limit = maxRegionLimit } regions := TopNRegions(rc.GetRegions(), less, limit) - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + h.rd.Data(w, http.StatusOK, b) } // @Tags region diff --git a/server/api/region_easyjson.go b/server/api/region_easyjson.go new file mode 100644 index 00000000000..4bd9fe69e42 --- /dev/null +++ b/server/api/region_easyjson.go @@ -0,0 +1,567 @@ +// Code generated by easyjson for marshaling/unmarshaling. DO NOT EDIT. + +package api + +import ( + json "encoding/json" + easyjson "github.com/mailru/easyjson" + jlexer "github.com/mailru/easyjson/jlexer" + jwriter "github.com/mailru/easyjson/jwriter" + metapb "github.com/pingcap/kvproto/pkg/metapb" + pdpb "github.com/pingcap/kvproto/pkg/pdpb" +) + +// suppress unused package warning +var ( + _ *json.RawMessage + _ *jlexer.Lexer + _ *jwriter.Writer + _ easyjson.Marshaler +) + +func easyjson75d7afa0DecodeGithubComTikvPdServerApi(in *jlexer.Lexer, out *RegionInfo) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "id": + out.ID = uint64(in.Uint64()) + case "start_key": + out.StartKey = string(in.String()) + case "end_key": + out.EndKey = string(in.String()) + case "epoch": + if in.IsNull() { + in.Skip() + out.RegionEpoch = nil + } else { + if out.RegionEpoch == nil { + out.RegionEpoch = new(metapb.RegionEpoch) + } + easyjson75d7afa0DecodeGithubComPingcapKvprotoPkgMetapb(in, out.RegionEpoch) + } + case "peers": + if in.IsNull() { + in.Skip() + out.Peers = nil + } else { + in.Delim('[') + if out.Peers == nil { + if !in.IsDelim(']') { + out.Peers = make([]MetaPeer, 0, 2) + } else { + out.Peers = []MetaPeer{} + } + } else { + out.Peers = (out.Peers)[:0] + } + for !in.IsDelim(']') { + var v1 MetaPeer + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &v1) + out.Peers = append(out.Peers, v1) + in.WantComma() + } + in.Delim(']') + } + case "leader": + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &out.Leader) + case "down_peers": + if in.IsNull() { + in.Skip() + out.DownPeers = nil + } else { + in.Delim('[') + if out.DownPeers == nil { + if !in.IsDelim(']') { + out.DownPeers = make([]PDPeerStats, 0, 1) + } else { + out.DownPeers = []PDPeerStats{} + } + } else { + out.DownPeers = (out.DownPeers)[:0] + } + for !in.IsDelim(']') { + var v2 PDPeerStats + easyjson75d7afa0DecodeGithubComTikvPdServerApi2(in, &v2) + out.DownPeers = append(out.DownPeers, v2) + in.WantComma() + } + in.Delim(']') + } + case "pending_peers": + if in.IsNull() { + in.Skip() + out.PendingPeers = nil + } else { + in.Delim('[') + if out.PendingPeers == nil { + if !in.IsDelim(']') { + out.PendingPeers = make([]MetaPeer, 0, 2) + } else { + out.PendingPeers = []MetaPeer{} + } + } else { + out.PendingPeers = (out.PendingPeers)[:0] + } + for !in.IsDelim(']') { + var v3 MetaPeer + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &v3) + out.PendingPeers = append(out.PendingPeers, v3) + in.WantComma() + } + in.Delim(']') + } + case "cpu_usage": + out.CPUUsage = uint64(in.Uint64()) + case "written_bytes": + out.WrittenBytes = uint64(in.Uint64()) + case "read_bytes": + out.ReadBytes = uint64(in.Uint64()) + case "written_keys": + out.WrittenKeys = uint64(in.Uint64()) + case "read_keys": + out.ReadKeys = uint64(in.Uint64()) + case "approximate_size": + out.ApproximateSize = int64(in.Int64()) + case "approximate_keys": + out.ApproximateKeys = int64(in.Int64()) + case "buckets": + if in.IsNull() { + in.Skip() + out.Buckets = nil + } else { + in.Delim('[') + if out.Buckets == nil { + if !in.IsDelim(']') { + out.Buckets = make([]string, 0, 4) + } else { + out.Buckets = []string{} + } + } else { + out.Buckets = (out.Buckets)[:0] + } + for !in.IsDelim(']') { + var v4 string + v4 = string(in.String()) + out.Buckets = append(out.Buckets, v4) + in.WantComma() + } + in.Delim(']') + } + case "replication_status": + if in.IsNull() { + in.Skip() + out.ReplicationStatus = nil + } else { + if out.ReplicationStatus == nil { + out.ReplicationStatus = new(ReplicationStatus) + } + easyjson75d7afa0DecodeGithubComTikvPdServerApi3(in, out.ReplicationStatus) + } + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi(out *jwriter.Writer, in RegionInfo) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"id\":" + out.RawString(prefix[1:]) + out.Uint64(uint64(in.ID)) + } + { + const prefix string = ",\"start_key\":" + out.RawString(prefix) + out.String(string(in.StartKey)) + } + { + const prefix string = ",\"end_key\":" + out.RawString(prefix) + out.String(string(in.EndKey)) + } + if in.RegionEpoch != nil { + const prefix string = ",\"epoch\":" + out.RawString(prefix) + easyjson75d7afa0EncodeGithubComPingcapKvprotoPkgMetapb(out, *in.RegionEpoch) + } + if len(in.Peers) != 0 { + const prefix string = ",\"peers\":" + out.RawString(prefix) + { + out.RawByte('[') + for v5, v6 := range in.Peers { + if v5 > 0 { + out.RawByte(',') + } + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, v6) + } + out.RawByte(']') + } + } + if true { + const prefix string = ",\"leader\":" + out.RawString(prefix) + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, in.Leader) + } + if len(in.DownPeers) != 0 { + const prefix string = ",\"down_peers\":" + out.RawString(prefix) + { + out.RawByte('[') + for v7, v8 := range in.DownPeers { + if v7 > 0 { + out.RawByte(',') + } + easyjson75d7afa0EncodeGithubComTikvPdServerApi2(out, v8) + } + out.RawByte(']') + } + } + if len(in.PendingPeers) != 0 { + const prefix string = ",\"pending_peers\":" + out.RawString(prefix) + { + out.RawByte('[') + for v9, v10 := range in.PendingPeers { + if v9 > 0 { + out.RawByte(',') + } + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, v10) + } + out.RawByte(']') + } + } + { + const prefix string = ",\"cpu_usage\":" + out.RawString(prefix) + out.Uint64(uint64(in.CPUUsage)) + } + { + const prefix string = ",\"written_bytes\":" + out.RawString(prefix) + out.Uint64(uint64(in.WrittenBytes)) + } + { + const prefix string = ",\"read_bytes\":" + out.RawString(prefix) + out.Uint64(uint64(in.ReadBytes)) + } + { + const prefix string = ",\"written_keys\":" + out.RawString(prefix) + out.Uint64(uint64(in.WrittenKeys)) + } + { + const prefix string = ",\"read_keys\":" + out.RawString(prefix) + out.Uint64(uint64(in.ReadKeys)) + } + { + const prefix string = ",\"approximate_size\":" + out.RawString(prefix) + out.Int64(int64(in.ApproximateSize)) + } + { + const prefix string = ",\"approximate_keys\":" + out.RawString(prefix) + out.Int64(int64(in.ApproximateKeys)) + } + if len(in.Buckets) != 0 { + const prefix string = ",\"buckets\":" + out.RawString(prefix) + { + out.RawByte('[') + for v11, v12 := range in.Buckets { + if v11 > 0 { + out.RawByte(',') + } + out.String(string(v12)) + } + out.RawByte(']') + } + } + if in.ReplicationStatus != nil { + const prefix string = ",\"replication_status\":" + out.RawString(prefix) + easyjson75d7afa0EncodeGithubComTikvPdServerApi3(out, *in.ReplicationStatus) + } + out.RawByte('}') +} + +// MarshalJSON supports json.Marshaler interface +func (v RegionInfo) MarshalJSON() ([]byte, error) { + w := jwriter.Writer{} + easyjson75d7afa0EncodeGithubComTikvPdServerApi(&w, v) + return w.Buffer.BuildBytes(), w.Error +} + +// MarshalEasyJSON supports easyjson.Marshaler interface +func (v RegionInfo) MarshalEasyJSON(w *jwriter.Writer) { + easyjson75d7afa0EncodeGithubComTikvPdServerApi(w, v) +} + +// UnmarshalJSON supports json.Unmarshaler interface +func (v *RegionInfo) UnmarshalJSON(data []byte) error { + r := jlexer.Lexer{Data: data} + easyjson75d7afa0DecodeGithubComTikvPdServerApi(&r, v) + return r.Error() +} + +// UnmarshalEasyJSON supports easyjson.Unmarshaler interface +func (v *RegionInfo) UnmarshalEasyJSON(l *jlexer.Lexer) { + easyjson75d7afa0DecodeGithubComTikvPdServerApi(l, v) +} +func easyjson75d7afa0DecodeGithubComTikvPdServerApi3(in *jlexer.Lexer, out *ReplicationStatus) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "state": + out.State = string(in.String()) + case "state_id": + out.StateID = uint64(in.Uint64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi3(out *jwriter.Writer, in ReplicationStatus) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"state\":" + out.RawString(prefix[1:]) + out.String(string(in.State)) + } + { + const prefix string = ",\"state_id\":" + out.RawString(prefix) + out.Uint64(uint64(in.StateID)) + } + out.RawByte('}') +} +func easyjson75d7afa0DecodeGithubComTikvPdServerApi2(in *jlexer.Lexer, out *PDPeerStats) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + out.PeerStats = new(pdpb.PeerStats) + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "peer": + easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in, &out.Peer) + case "down_seconds": + out.DownSeconds = uint64(in.Uint64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi2(out *jwriter.Writer, in PDPeerStats) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"peer\":" + out.RawString(prefix[1:]) + easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out, in.Peer) + } + if in.DownSeconds != 0 { + const prefix string = ",\"down_seconds\":" + out.RawString(prefix) + out.Uint64(uint64(in.DownSeconds)) + } + out.RawByte('}') +} +func easyjson75d7afa0DecodeGithubComTikvPdServerApi1(in *jlexer.Lexer, out *MetaPeer) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + out.Peer = new(metapb.Peer) + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "role_name": + out.RoleName = string(in.String()) + case "is_learner": + out.IsLearner = bool(in.Bool()) + case "id": + out.Id = uint64(in.Uint64()) + case "store_id": + out.StoreId = uint64(in.Uint64()) + case "role": + out.Role = metapb.PeerRole(in.Int32()) + case "is_witness": + out.IsWitness = bool(in.Bool()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComTikvPdServerApi1(out *jwriter.Writer, in MetaPeer) { + out.RawByte('{') + first := true + _ = first + { + const prefix string = ",\"role_name\":" + out.RawString(prefix[1:]) + out.String(string(in.RoleName)) + } + if in.IsLearner { + const prefix string = ",\"is_learner\":" + out.RawString(prefix) + out.Bool(bool(in.IsLearner)) + } + if in.Id != 0 { + const prefix string = ",\"id\":" + out.RawString(prefix) + out.Uint64(uint64(in.Id)) + } + if in.StoreId != 0 { + const prefix string = ",\"store_id\":" + out.RawString(prefix) + out.Uint64(uint64(in.StoreId)) + } + if in.Role != 0 { + const prefix string = ",\"role\":" + out.RawString(prefix) + out.Int32(int32(in.Role)) + } + if in.IsWitness { + const prefix string = ",\"is_witness\":" + out.RawString(prefix) + out.Bool(bool(in.IsWitness)) + } + out.RawByte('}') +} +func easyjson75d7afa0DecodeGithubComPingcapKvprotoPkgMetapb(in *jlexer.Lexer, out *metapb.RegionEpoch) { + isTopLevel := in.IsStart() + if in.IsNull() { + if isTopLevel { + in.Consumed() + } + in.Skip() + return + } + in.Delim('{') + for !in.IsDelim('}') { + key := in.UnsafeFieldName(false) + in.WantColon() + if in.IsNull() { + in.Skip() + in.WantComma() + continue + } + switch key { + case "conf_ver": + out.ConfVer = uint64(in.Uint64()) + case "version": + out.Version = uint64(in.Uint64()) + default: + in.SkipRecursive() + } + in.WantComma() + } + in.Delim('}') + if isTopLevel { + in.Consumed() + } +} +func easyjson75d7afa0EncodeGithubComPingcapKvprotoPkgMetapb(out *jwriter.Writer, in metapb.RegionEpoch) { + out.RawByte('{') + first := true + _ = first + if in.ConfVer != 0 { + const prefix string = ",\"conf_ver\":" + first = false + out.RawString(prefix[1:]) + out.Uint64(uint64(in.ConfVer)) + } + if in.Version != 0 { + const prefix string = ",\"version\":" + if first { + first = false + out.RawString(prefix[1:]) + } else { + out.RawString(prefix) + } + out.Uint64(uint64(in.Version)) + } + out.RawByte('}') +} diff --git a/server/api/region_test.go b/server/api/region_test.go index 63d54b2b04b..1b3f743a6ea 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -16,6 +16,7 @@ package api import ( "bytes" + "context" "encoding/hex" "encoding/json" "fmt" @@ -24,6 +25,7 @@ import ( "net/url" "sort" "testing" + "time" "github.com/docker/go-units" "github.com/pingcap/failpoint" @@ -31,7 +33,14 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" +<<<<<<< HEAD tu "github.com/tikv/pd/pkg/testutil" +======= + "github.com/tikv/pd/pkg/core" + "github.com/tikv/pd/pkg/schedule/placement" + "github.com/tikv/pd/pkg/utils/apiutil" + tu "github.com/tikv/pd/pkg/utils/testutil" +>>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) "github.com/tikv/pd/server" "github.com/tikv/pd/server/core" "github.com/tikv/pd/server/schedule/placement" @@ -459,6 +468,40 @@ func (suite *regionTestSuite) TestTopN() { } } +func TestRegionsWithKillRequest(t *testing.T) { + re := require.New(t) + svr, cleanup := mustNewServer(re) + defer cleanup() + server.MustWaitLeader(re, []*server.Server{svr}) + + addr := svr.GetAddr() + url := fmt.Sprintf("%s%s/api/v1/regions", addr, apiPrefix) + mustBootstrapCluster(re, svr) + regionCount := 100000 + for i := 0; i < regionCount; i++ { + r := core.NewTestRegionInfo(uint64(i+2), 1, + []byte(fmt.Sprintf("%09d", i)), + []byte(fmt.Sprintf("%09d", i+1)), + core.SetApproximateKeys(10), core.SetApproximateSize(10)) + mustRegionHeartbeat(re, svr, r) + } + + ctx, cancel := context.WithCancel(context.Background()) + req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, bytes.NewBuffer(nil)) + re.NoError(err) + respCh := make(chan *http.Response) + go func() { + resp, err := testDialClient.Do(req) // nolint:bodyclose + re.Error(err) + re.Contains(err.Error(), "context canceled") + respCh <- resp + }() + time.Sleep(100 * time.Millisecond) // wait for the request to be sent + cancel() // close the request + resp := <-respCh + re.Nil(resp) +} + type getRegionTestSuite struct { suite.Suite svr *server.Server @@ -741,54 +784,60 @@ func (suite *regionsReplicatedTestSuite) TestCheckRegionsReplicated() { suite.Equal("REPLICATED", status) } -// Create n regions (0..n) of n stores (0..n). -// Each region contains np peers, the first peer is the leader. -// (copied from server/cluster_test.go) -func newTestRegions() []*core.RegionInfo { - n := uint64(10000) - np := uint64(3) - - regions := make([]*core.RegionInfo, 0, n) - for i := uint64(0); i < n; i++ { - peers := make([]*metapb.Peer, 0, np) - for j := uint64(0); j < np; j++ { - peer := &metapb.Peer{ - Id: i*np + j, - } - peer.StoreId = (i + j) % n - peers = append(peers, peer) - } - region := &metapb.Region{ - Id: i, - Peers: peers, - StartKey: []byte(fmt.Sprintf("%d", i)), - EndKey: []byte(fmt.Sprintf("%d", i+1)), - RegionEpoch: &metapb.RegionEpoch{ConfVer: 2, Version: 2}, - } - regions = append(regions, core.NewRegionInfo(region, peers[0])) - } - return regions -} - -func BenchmarkRenderJSON(b *testing.B) { - regionInfos := newTestRegions() - rd := createStreamingRender() - regions := convertToAPIRegions(regionInfos) - - b.ResetTimer() - for i := 0; i < b.N; i++ { - var buffer bytes.Buffer - rd.JSON(&buffer, 200, regions) +func TestRegionsInfoMarshal(t *testing.T) { + re := require.New(t) + regionWithNilPeer := core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}) + core.SetPeers([]*metapb.Peer{{Id: 2}, nil})(regionWithNilPeer) + cases := [][]*core.RegionInfo{ + {}, + { + // leader is nil + core.NewRegionInfo(&metapb.Region{Id: 1}, nil), + // Peers is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.SetPeers([]*metapb.Peer{})), + // There is nil peer in peers. + regionWithNilPeer, + }, + { + // PendingPeers is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithPendingPeers([]*metapb.Peer{})), + // There is nil peer in peers. + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithPendingPeers([]*metapb.Peer{nil})), + }, + { + // DownPeers is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithDownPeers([]*pdpb.PeerStats{})), + // There is nil peer in peers. + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.WithDownPeers([]*pdpb.PeerStats{{Peer: nil}})), + }, + { + // Buckets is nil + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.SetBuckets(nil)), + // Buckets is empty + core.NewRegionInfo(&metapb.Region{Id: 1}, &metapb.Peer{Id: 1}, + core.SetBuckets(&metapb.Buckets{})), + }, + { + core.NewRegionInfo(&metapb.Region{Id: 1, StartKey: []byte{}, EndKey: []byte{}, + RegionEpoch: &metapb.RegionEpoch{Version: 1, ConfVer: 1}}, + &metapb.Peer{Id: 1}, core.SetCPUUsage(10), + core.SetApproximateKeys(10), core.SetApproximateSize(10), + core.SetWrittenBytes(10), core.SetReadBytes(10), + core.SetReadKeys(10), core.SetWrittenKeys(10)), + }, } -} - -func BenchmarkConvertToAPIRegions(b *testing.B) { - regionInfos := newTestRegions() - - b.ResetTimer() - for i := 0; i < b.N; i++ { - regions := convertToAPIRegions(regionInfos) - _ = regions.Count + regionsInfo := &RegionsInfo{} + for _, regions := range cases { + b, err := marshalRegionsInfoJSON(context.Background(), regions) + re.NoError(err) + err = json.Unmarshal(b, regionsInfo) + re.NoError(err) } } @@ -807,3 +856,34 @@ func BenchmarkHexRegionKeyStr(b *testing.B) { _ = core.HexRegionKeyStr(key) } } + +func BenchmarkGetRegions(b *testing.B) { + re := require.New(b) + svr, cleanup := mustNewServer(re) + defer cleanup() + server.MustWaitLeader(re, []*server.Server{svr}) + + addr := svr.GetAddr() + url := fmt.Sprintf("%s%s/api/v1/regions", addr, apiPrefix) + mustBootstrapCluster(re, svr) + regionCount := 1000000 + for i := 0; i < regionCount; i++ { + r := core.NewTestRegionInfo(uint64(i+2), 1, + []byte(fmt.Sprintf("%09d", i)), + []byte(fmt.Sprintf("%09d", i+1)), + core.SetApproximateKeys(10), core.SetApproximateSize(10)) + mustRegionHeartbeat(re, svr, r) + } + resp, _ := apiutil.GetJSON(testDialClient, url, nil) + regions := &RegionsInfo{} + err := json.NewDecoder(resp.Body).Decode(regions) + re.NoError(err) + re.Equal(regionCount, regions.Count) + resp.Body.Close() + + b.ResetTimer() + for i := 0; i < b.N; i++ { + resp, _ := apiutil.GetJSON(testDialClient, url, nil) + resp.Body.Close() + } +} From c1ecc9d8c03c557e0fc5adcf0824e5e7860a0622 Mon Sep 17 00:00:00 2001 From: AndreMouche Date: Thu, 27 Jul 2023 14:24:41 -0700 Subject: [PATCH 2/4] fix conflicts Signed-off-by: AndreMouche --- go.mod | 5 - go.sum | 7 +- server/api/region.go | 200 ++++---------------------------------- server/api/region_test.go | 4 - 4 files changed, 23 insertions(+), 193 deletions(-) diff --git a/go.mod b/go.mod index 45ea6d79f73..1f014c349a8 100644 --- a/go.mod +++ b/go.mod @@ -114,14 +114,9 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect -<<<<<<< HEAD github.com/leodido/go-urn v1.2.0 // indirect - github.com/mailru/easyjson v0.7.6 // indirect -======= - github.com/leodido/go-urn v1.2.1 // indirect github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.6 ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect github.com/mattn/go-runewidth v0.0.8 // indirect diff --git a/go.sum b/go.sum index a0587f98dcd..940c431ad6b 100644 --- a/go.sum +++ b/go.sum @@ -216,8 +216,9 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY= github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= @@ -306,6 +307,9 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= +github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -715,7 +719,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508= diff --git a/server/api/region.go b/server/api/region.go index 726ef715a84..5d03f404e81 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -388,12 +388,7 @@ func marshalRegionsInfoJSON(ctx context.Context, regions []*core.RegionInfo) ([] func (h *regionsHandler) GetRegions(w http.ResponseWriter, r *http.Request) { rc := getCluster(r) regions := rc.GetRegions() - b, err := marshalRegionsInfoJSON(r.Context(), regions) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - h.rd.Data(w, http.StatusOK, b) + h.returnWithRegions(w, r, regions) } // @Tags region @@ -423,12 +418,7 @@ func (h *regionsHandler) ScanRegions(w http.ResponseWriter, r *http.Request) { limit = maxRegionLimit } regions := rc.ScanRegions([]byte(startKey), []byte(endKey), limit) - b, err := marshalRegionsInfoJSON(r.Context(), regions) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - h.rd.Data(w, http.StatusOK, b) + h.returnWithRegions(w, r, regions) } // @Tags region @@ -459,103 +449,19 @@ func (h *regionsHandler) GetStoreRegions(w http.ResponseWriter, r *http.Request) return } regions := rc.GetStoreRegions(uint64(id)) - b, err := marshalRegionsInfoJSON(r.Context(), regions) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - h.rd.Data(w, http.StatusOK, b) + h.returnWithRegions(w, r, regions) } // @Tags region -<<<<<<< HEAD -======= -// @Summary List regions belongs to the given keyspace ID. -// @Param keyspace_id query string true "Keyspace ID" -// @Param limit query integer false "Limit count" default(16) -// @Produce json -// @Success 200 {object} RegionsInfo -// @Failure 400 {string} string "The input is invalid." -// @Router /regions/keyspace/id/{id} [get] -func (h *regionsHandler) GetKeyspaceRegions(w http.ResponseWriter, r *http.Request) { - rc := getCluster(r) - vars := mux.Vars(r) - keyspaceIDStr := vars["id"] - if keyspaceIDStr == "" { - h.rd.JSON(w, http.StatusBadRequest, "keyspace id is empty") - return - } - - keyspaceID64, err := strconv.ParseUint(keyspaceIDStr, 10, 32) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - keyspaceID := uint32(keyspaceID64) - keyspaceManager := h.svr.GetKeyspaceManager() - if _, err := keyspaceManager.LoadKeyspaceByID(keyspaceID); err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - - limit := defaultRegionLimit - if limitStr := r.URL.Query().Get("limit"); limitStr != "" { - limit, err = strconv.Atoi(limitStr) - if err != nil { - h.rd.JSON(w, http.StatusBadRequest, err.Error()) - return - } - } - if limit > maxRegionLimit { - limit = maxRegionLimit - } - regionBound := keyspace.MakeRegionBound(keyspaceID) - regions := rc.ScanRegions(regionBound.RawLeftBound, regionBound.RawRightBound, limit) - if limit <= 0 || limit > len(regions) { - txnRegion := rc.ScanRegions(regionBound.TxnLeftBound, regionBound.TxnRightBound, limit-len(regions)) - regions = append(regions, txnRegion...) - } - b, err := marshalRegionsInfoJSON(r.Context(), regions) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - h.rd.Data(w, http.StatusOK, b) -} - -// @Tags region ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) // @Summary List all regions that miss peer. // @Produce json // @Success 200 {object} RegionsInfo // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/miss-peer [get] func (h *regionsHandler) GetMissPeerRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD -======= h.getRegionsByType(w, statistics.MissPeer, r) } -func (h *regionsHandler) getRegionsByType( - w http.ResponseWriter, - typ statistics.RegionStatisticType, - r *http.Request, -) { ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.MissPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - b, err := marshalRegionsInfoJSON(r.Context(), regions) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - h.rd.Data(w, http.StatusOK, b) -} - // @Tags region // @Summary List all regions that has extra peer. // @Produce json @@ -563,18 +469,7 @@ func (h *regionsHandler) getRegionsByType( // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/extra-peer [get] func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.ExtraPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= h.getRegionsByType(w, statistics.ExtraPeer, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -584,18 +479,7 @@ func (h *regionsHandler) GetExtraPeerRegions(w http.ResponseWriter, r *http.Requ // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/pending-peer [get] func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.PendingPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= h.getRegionsByType(w, statistics.PendingPeer, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -605,18 +489,7 @@ func (h *regionsHandler) GetPendingPeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/down-peer [get] func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.DownPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= h.getRegionsByType(w, statistics.DownPeer, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -626,18 +499,7 @@ func (h *regionsHandler) GetDownPeerRegions(w http.ResponseWriter, r *http.Reque // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/learner-peer [get] func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.LearnerPeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= h.getRegionsByType(w, statistics.LearnerPeer, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -647,18 +509,7 @@ func (h *regionsHandler) GetLearnerPeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/offline-peer [get] func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetOfflinePeer(statistics.OfflinePeer) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= h.getRegionsByType(w, statistics.OfflinePeer, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -668,18 +519,7 @@ func (h *regionsHandler) GetOfflinePeerRegions(w http.ResponseWriter, r *http.Re // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/oversized-region [get] func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.OversizedRegion) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= h.getRegionsByType(w, statistics.OversizedRegion, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -689,18 +529,7 @@ func (h *regionsHandler) GetOverSizedRegions(w http.ResponseWriter, r *http.Requ // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/undersized-region [get] func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.UndersizedRegion) - if err != nil { - h.rd.JSON(w, http.StatusInternalServerError, err.Error()) - return - } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= h.getRegionsByType(w, statistics.UndersizedRegion, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) } // @Tags region @@ -710,18 +539,25 @@ func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Req // @Failure 500 {string} string "PD server failed to proceed the request." // @Router /regions/check/empty-region [get] func (h *regionsHandler) GetEmptyRegions(w http.ResponseWriter, r *http.Request) { -<<<<<<< HEAD - handler := h.svr.GetHandler() - regions, err := handler.GetRegionsByType(statistics.EmptyRegion) + h.getRegionsByType(w, statistics.EmptyRegion, r) + +} + +func (h *regionsHandler) getRegionsByType(w http.ResponseWriter, t statistics.RegionStatisticType, r *http.Request) { + regions, err := h.svr.GetHandler().GetRegionsByType(t) if err != nil { h.rd.JSON(w, http.StatusInternalServerError, err.Error()) return } - regionsInfo := convertToAPIRegions(regions) - h.rd.JSON(w, http.StatusOK, regionsInfo) -======= - h.getRegionsByType(w, statistics.EmptyRegion, r) ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) + h.returnWithRegions(w, r, regions) +} + +func (h *regionsHandler) returnWithRegions(w http.ResponseWriter, r *http.Request, regions []*core.RegionInfo) { + b, err := marshalRegionsInfoJSON(r.Context(), regions) + if err != nil { + h.rd.JSON(w, http.StatusInternalServerError, err.Error()) + } + h.rd.Data(w, http.StatusOK, b) } type histItem struct { diff --git a/server/api/region_test.go b/server/api/region_test.go index 1b3f743a6ea..9cf804c7d74 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -33,14 +33,10 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" -<<<<<<< HEAD - tu "github.com/tikv/pd/pkg/testutil" -======= "github.com/tikv/pd/pkg/core" "github.com/tikv/pd/pkg/schedule/placement" "github.com/tikv/pd/pkg/utils/apiutil" tu "github.com/tikv/pd/pkg/utils/testutil" ->>>>>>> 4db173597 (api: use easyjson and context in regions interface (#6838)) "github.com/tikv/pd/server" "github.com/tikv/pd/server/core" "github.com/tikv/pd/server/schedule/placement" From b00aef27d29bb959c22aa47c33663689891a15b9 Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 9 Aug 2023 11:44:23 +0800 Subject: [PATCH 3/4] fix conflict Signed-off-by: lhy1024 --- go.mod | 2 +- go.sum | 3 --- server/api/region_test.go | 10 ++++------ server/core/test_util.go | 16 ++++++++++++++++ 4 files changed, 21 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 1f014c349a8..c4a56de1ce5 100644 --- a/go.mod +++ b/go.mod @@ -51,6 +51,7 @@ require ( ) require ( + github.com/google/go-cmp v0.5.9 // indirect github.com/samber/lo v1.37.0 // indirect gorm.io/datatypes v1.1.0 // indirect ) @@ -115,7 +116,6 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/leodido/go-urn v1.2.0 // indirect - github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a // indirect github.com/mailru/easyjson v0.7.6 github.com/mattn/go-colorable v0.1.8 // indirect github.com/mattn/go-isatty v0.0.12 // indirect diff --git a/go.sum b/go.sum index 940c431ad6b..3be16d3d55b 100644 --- a/go.sum +++ b/go.sum @@ -307,9 +307,6 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= -github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= -github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= -github.com/lufia/plan9stats v0.0.0-20230326075908-cb1d2100619a/go.mod h1:JKx41uQRwqlTZabZc+kILPrO/3jlKnQ2Z8b7YiVw5cE= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= diff --git a/server/api/region_test.go b/server/api/region_test.go index 9cf804c7d74..8527acf41ec 100644 --- a/server/api/region_test.go +++ b/server/api/region_test.go @@ -33,10 +33,8 @@ import ( "github.com/pingcap/kvproto/pkg/pdpb" "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" - "github.com/tikv/pd/pkg/core" - "github.com/tikv/pd/pkg/schedule/placement" - "github.com/tikv/pd/pkg/utils/apiutil" - tu "github.com/tikv/pd/pkg/utils/testutil" + "github.com/tikv/pd/pkg/apiutil" + tu "github.com/tikv/pd/pkg/testutil" "github.com/tikv/pd/server" "github.com/tikv/pd/server/core" "github.com/tikv/pd/server/schedule/placement" @@ -475,7 +473,7 @@ func TestRegionsWithKillRequest(t *testing.T) { mustBootstrapCluster(re, svr) regionCount := 100000 for i := 0; i < regionCount; i++ { - r := core.NewTestRegionInfo(uint64(i+2), 1, + r := core.NewTestRegionInfoWithID(uint64(i+2), 1, []byte(fmt.Sprintf("%09d", i)), []byte(fmt.Sprintf("%09d", i+1)), core.SetApproximateKeys(10), core.SetApproximateSize(10)) @@ -864,7 +862,7 @@ func BenchmarkGetRegions(b *testing.B) { mustBootstrapCluster(re, svr) regionCount := 1000000 for i := 0; i < regionCount; i++ { - r := core.NewTestRegionInfo(uint64(i+2), 1, + r := core.NewTestRegionInfoWithID(uint64(i+2), 1, []byte(fmt.Sprintf("%09d", i)), []byte(fmt.Sprintf("%09d", i+1)), core.SetApproximateKeys(10), core.SetApproximateSize(10)) diff --git a/server/core/test_util.go b/server/core/test_util.go index 055623edca4..9a72702c294 100644 --- a/server/core/test_util.go +++ b/server/core/test_util.go @@ -85,6 +85,22 @@ func NewTestRegionInfo(start, end []byte) *RegionInfo { }} } +// NewTestRegionInfoWithID creates a new RegionInfo for test purpose. +func NewTestRegionInfoWithID(regionID, storeID uint64, start, end []byte, opts ...RegionCreateOption) *RegionInfo { + leader := &metapb.Peer{ + Id: regionID, + StoreId: storeID, + } + metaRegion := &metapb.Region{ + Id: regionID, + StartKey: start, + EndKey: end, + Peers: []*metapb.Peer{leader}, + RegionEpoch: &metapb.RegionEpoch{ConfVer: 1, Version: 1}, + } + return NewRegionInfo(metaRegion, leader, opts...) +} + // NewStoreInfoWithDisk is created with all disk infos. func NewStoreInfoWithDisk(id, used, available, capacity, regionSize uint64) *StoreInfo { stats := &pdpb.StoreStats{} From d3d7af54d02dd7c7e3896f5c67e258368a45f8af Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Wed, 9 Aug 2023 12:05:02 +0800 Subject: [PATCH 4/4] fix lint Signed-off-by: lhy1024 --- server/api/region.go | 1 - server/schedule/waiting_operator.go | 4 +--- tests/client/go.sum | 3 +-- 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/server/api/region.go b/server/api/region.go index 5d03f404e81..06a002a561d 100644 --- a/server/api/region.go +++ b/server/api/region.go @@ -540,7 +540,6 @@ func (h *regionsHandler) GetUndersizedRegions(w http.ResponseWriter, r *http.Req // @Router /regions/check/empty-region [get] func (h *regionsHandler) GetEmptyRegions(w http.ResponseWriter, r *http.Request) { h.getRegionsByType(w, statistics.EmptyRegion, r) - } func (h *regionsHandler) getRegionsByType(w http.ResponseWriter, t statistics.RegionStatisticType, r *http.Request) { diff --git a/server/schedule/waiting_operator.go b/server/schedule/waiting_operator.go index 513f8edea1a..9097e4fa45e 100644 --- a/server/schedule/waiting_operator.go +++ b/server/schedule/waiting_operator.go @@ -69,9 +69,7 @@ func (b *RandBuckets) ListOperator() []*operator.Operator { var ops []*operator.Operator for i := range b.buckets { bucket := b.buckets[i] - for j := range bucket.ops { - ops = append(ops, bucket.ops[j]) - } + ops = append(ops, bucket.ops...) } return ops } diff --git a/tests/client/go.sum b/tests/client/go.sum index f6d4f13dafa..eeed9bf5b59 100644 --- a/tests/client/go.sum +++ b/tests/client/go.sum @@ -184,8 +184,8 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20211122183932-1daafda22083 h1:c8EUapQFi+kjzedr4c6WqbwMdmB95+oDBWZ5XFHFYxY= github.com/google/pprof v0.0.0-20211122183932-1daafda22083/go.mod h1:KgnwoLYCZ8IQu3XUZ8Nc/bM9CCZFOyjUNOSygVozoDg= @@ -624,7 +624,6 @@ golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0 h1:/wp5JvzpHIxhs/dumFmF7BXTf3Z+dd4uXta4kVyO508=