From 071b116eb27d8c9a93f49bf9ed6099b02544842a Mon Sep 17 00:00:00 2001 From: lhy1024 Date: Fri, 13 Sep 2024 16:15:04 +0800 Subject: [PATCH] scheduler: add conflict detect for grant hot leader scheduler Signed-off-by: lhy1024 --- server/api/scheduler.go | 66 +++++++++++++++++++ tools/pd-ctl/pdctl/command/scheduler.go | 7 +- .../pd-ctl/tests/scheduler/scheduler_test.go | 34 +++++++++- 3 files changed, 102 insertions(+), 5 deletions(-) diff --git a/server/api/scheduler.go b/server/api/scheduler.go index c96e4c123de..a25aefd76cf 100644 --- a/server/api/scheduler.go +++ b/server/api/scheduler.go @@ -15,6 +15,9 @@ package api import ( + "encoding/json" + "fmt" + "io" "net/http" "net/url" "strconv" @@ -24,7 +27,9 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/log" "github.com/tikv/pd/pkg/errs" + "github.com/tikv/pd/pkg/mcs/utils/constant" "github.com/tikv/pd/pkg/schedule/types" + "github.com/tikv/pd/pkg/slice" "github.com/tikv/pd/pkg/utils/apiutil" "github.com/tikv/pd/server" "github.com/unrolled/render" @@ -143,6 +148,15 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques } collector(strconv.FormatUint(limit, 10)) case types.GrantHotRegionScheduler: + schedulers, err := h.getSchedulers() + if err != nil { + h.r.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + if slice.Contains(schedulers, types.BalanceHotRegionScheduler.String()) { + h.r.JSON(w, http.StatusBadRequest, "balance-hot-region-scheduler is running, please remove it first") + return + } leaderID, ok := input["store-leader-id"].(string) if !ok { h.r.JSON(w, http.StatusBadRequest, "missing leader id") @@ -155,6 +169,16 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques } collector(leaderID) collector(peerIDs) + case types.BalanceHotRegionScheduler: + schedulers, err := h.getSchedulers() + if err != nil { + h.r.JSON(w, http.StatusInternalServerError, err.Error()) + return + } + if slice.Contains(schedulers, types.GrantHotRegionScheduler.String()) { + h.r.JSON(w, http.StatusBadRequest, "grant-hot-region-scheduler is running, please remove it first") + return + } } if err := h.AddScheduler(tp, args...); err != nil { @@ -246,6 +270,48 @@ func (h *schedulerHandler) PauseOrResumeScheduler(w http.ResponseWriter, r *http h.r.JSON(w, http.StatusOK, "Pause or resume the scheduler successfully.") } +func (h *schedulerHandler) getSchedulers() ([]string, error) { + rc, err := h.GetRaftCluster() + if err != nil { + return nil, err + } + if !rc.IsServiceIndependent(constant.SchedulingServiceName) { + schedulers, err := h.GetSchedulerByStatus("", false) + if err != nil { + return nil, err + } + return schedulers.([]string), nil + } + + addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), constant.SchedulingServiceName) + if !ok { + return nil, errs.ErrNotFoundSchedulingAddr.FastGenByArgs() + } + url := fmt.Sprintf("%s/scheduling/api/v1/schedulers", addr) + req, err := http.NewRequest(http.MethodGet, url, http.NoBody) + if err != nil { + return nil, err + } + resp, err := h.svr.GetHTTPClient().Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, errs.ErrSchedulingServer.FastGenByArgs(resp.StatusCode) + } + bs, err := io.ReadAll(resp.Body) + if err != nil { + return nil, err + } + var schedulers []string + err = json.Unmarshal(bs, &schedulers) + if err != nil { + return nil, err + } + return schedulers, nil +} + type schedulerConfigHandler struct { svr *server.Server rd *render.Render diff --git a/tools/pd-ctl/pdctl/command/scheduler.go b/tools/pd-ctl/pdctl/command/scheduler.go index 43c6429cd4f..16a1a92f2b1 100644 --- a/tools/pd-ctl/pdctl/command/scheduler.go +++ b/tools/pd-ctl/pdctl/command/scheduler.go @@ -354,9 +354,10 @@ func NewSplitBucketSchedulerCommand() *cobra.Command { // NewGrantHotRegionSchedulerCommand returns a command to add a grant-hot-region-scheduler. func NewGrantHotRegionSchedulerCommand() *cobra.Command { c := &cobra.Command{ - Use: "grant-hot-region-scheduler ", - Short: "add a scheduler to grant hot region to fixed stores", - Run: addSchedulerForGrantHotRegionCommandFunc, + Use: "grant-hot-region-scheduler ", + Short: `add a scheduler to grant hot region to fixed stores. + Note: If there is balance-hot-region-scheduler running, please remove it first, otherwise grant-hot-region-scheduler will not work.`, + Run: addSchedulerForGrantHotRegionCommandFunc, } return c } diff --git a/tools/pd-ctl/tests/scheduler/scheduler_test.go b/tools/pd-ctl/tests/scheduler/scheduler_test.go index f680a4bd2e7..4cad309d42c 100644 --- a/tools/pd-ctl/tests/scheduler/scheduler_test.go +++ b/tools/pd-ctl/tests/scheduler/scheduler_test.go @@ -488,12 +488,32 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust }) // test grant hot region scheduler config - checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{ + // case 1: add grant-hot-region-scheduler when balance-hot-region-scheduler is running + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, nil) + re.Contains(echo, "balance-hot-region-scheduler is running, please remove it first") + + // case 2: add grant-hot-region-scheduler when balance-hot-region-scheduler is paused + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "pause", "balance-hot-region-scheduler", "60"}, map[string]bool{ "balance-region-scheduler": true, "balance-leader-scheduler": true, "balance-hot-region-scheduler": true, - "grant-hot-region-scheduler": true, }) + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, nil) + re.Contains(echo, "balance-hot-region-scheduler is running, please remove it first") + + // case 3: add grant-hot-region-scheduler when balance-hot-region-scheduler is disabled + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "balance-hot-region-scheduler"}, map[string]bool{ + "balance-region-scheduler": true, + "balance-leader-scheduler": true, + }) + + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{ + "balance-region-scheduler": true, + "balance-leader-scheduler": true, + "grant-hot-region-scheduler": true, + }) + + // case 4: test grant-hot-region-scheduler config var conf3 map[string]any expected3 := map[string]any{ "store-id": []any{float64(1), float64(2), float64(3)}, @@ -509,7 +529,17 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler"}, &conf3) return reflect.DeepEqual(expected3, conf3) }) + + // case 5: remove grant-hot-region-scheduler + echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, nil) + re.Contains(echo, "grant-hot-region-scheduler is running, please remove it first") + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "grant-hot-region-scheduler"}, map[string]bool{ + "balance-region-scheduler": true, + "balance-leader-scheduler": true, + }) + + checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, map[string]bool{ "balance-region-scheduler": true, "balance-leader-scheduler": true, "balance-hot-region-scheduler": true,