scheduler: add conflict detect for grant hot leader scheduler

Signed-off-by: lhy1024 <admin@liudos.us>

server/api/scheduler.go

@@ -15,6 +15,9 @@
 package api
 
 import (
+ "encoding/json"
+ "fmt"
+ "io"
  "net/http"
  "net/url"
  "strconv"
@@ -24,7 +27,9 @@ import (
  "github.com/pingcap/errors"
  "github.com/pingcap/log"
  "github.com/tikv/pd/pkg/errs"
+ "github.com/tikv/pd/pkg/mcs/utils/constant"
  "github.com/tikv/pd/pkg/schedule/types"
+ "github.com/tikv/pd/pkg/slice"
  "github.com/tikv/pd/pkg/utils/apiutil"
  "github.com/tikv/pd/server"
  "github.com/unrolled/render"
@@ -143,6 +148,15 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques
  }
  collector(strconv.FormatUint(limit, 10))
  case types.GrantHotRegionScheduler:
+ schedulers, err := h.getSchedulers()
+ if err != nil {
+ h.r.JSON(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+ if slice.Contains(schedulers, types.BalanceHotRegionScheduler.String()) {
+ h.r.JSON(w, http.StatusBadRequest, "balance-hot-region-scheduler is running, please remove it first")
+ return
+ }
  leaderID, ok := input["store-leader-id"].(string)
  if !ok {
  h.r.JSON(w, http.StatusBadRequest, "missing leader id")
@@ -155,6 +169,16 @@ func (h *schedulerHandler) CreateScheduler(w http.ResponseWriter, r *http.Reques
  }
  collector(leaderID)
  collector(peerIDs)
+ case types.BalanceHotRegionScheduler:
+ schedulers, err := h.getSchedulers()
+ if err != nil {
+ h.r.JSON(w, http.StatusInternalServerError, err.Error())
+ return
+ }
+ if slice.Contains(schedulers, types.GrantHotRegionScheduler.String()) {
+ h.r.JSON(w, http.StatusBadRequest, "grant-hot-region-scheduler is running, please remove it first")
+ return
+ }
  }
 
  if err := h.AddScheduler(tp, args...); err != nil {
@@ -246,6 +270,48 @@ func (h *schedulerHandler) PauseOrResumeScheduler(w http.ResponseWriter, r *http
  h.r.JSON(w, http.StatusOK, "Pause or resume the scheduler successfully.")
 }
 
+func (h *schedulerHandler) getSchedulers() ([]string, error) {
+ rc, err := h.GetRaftCluster()
+ if err != nil {
+ return nil, err
+ }
+ if !rc.IsServiceIndependent(constant.SchedulingServiceName) {
+ schedulers, err := h.GetSchedulerByStatus("", false)
+ if err != nil {
+ return nil, err
+ }
+ return schedulers.([]string), nil
+ }
+
+ addr, ok := h.svr.GetServicePrimaryAddr(h.svr.Context(), constant.SchedulingServiceName)
+ if !ok {
+ return nil, errs.ErrNotFoundSchedulingAddr.FastGenByArgs()
+ }
+ url := fmt.Sprintf("%s/scheduling/api/v1/schedulers", addr)
+ req, err := http.NewRequest(http.MethodGet, url, http.NoBody)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := h.svr.GetHTTPClient().Do(req)
+ if err != nil {
+ return nil, err
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != http.StatusOK {
+ return nil, errs.ErrSchedulingServer.FastGenByArgs(resp.StatusCode)
+ }
+ bs, err := io.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ var schedulers []string
+ err = json.Unmarshal(bs, &schedulers)
+ if err != nil {
+ return nil, err
+ }
+ return schedulers, nil
+}
+
 type schedulerConfigHandler struct {
  svr *server.Server
  rd *render.Render

tools/pd-ctl/pdctl/command/scheduler.go

@@ -354,9 +354,10 @@ func NewSplitBucketSchedulerCommand() *cobra.Command {
 // NewGrantHotRegionSchedulerCommand returns a command to add a grant-hot-region-scheduler.
 func NewGrantHotRegionSchedulerCommand() *cobra.Command {
  c := &cobra.Command{
- Use: "grant-hot-region-scheduler <store_leader_id> <store_leader_id,store_peer_id_1,store_peer_id_2>",
- Short: "add a scheduler to grant hot region to fixed stores",
- Run: addSchedulerForGrantHotRegionCommandFunc,
+ Use: "grant-hot-region-scheduler <store_leader_id> <store_leader_id,store_peer_id_1,store_peer_id_2>",
+ Short: `add a scheduler to grant hot region to fixed stores. 
+ Note: If there is balance-hot-region-scheduler running, please remove it first, otherwise grant-hot-region-scheduler will not work.`,
+ Run: addSchedulerForGrantHotRegionCommandFunc,
  }
  return c
 }

tools/pd-ctl/tests/scheduler/scheduler_test.go

@@ -488,12 +488,32 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust
  })
 
  // test grant hot region scheduler config
- checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{
+ // case 1: add grant-hot-region-scheduler when balance-hot-region-scheduler is running
+ echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, nil)
+ re.Contains(echo, "balance-hot-region-scheduler is running, please remove it first")
+
+ // case 2: add grant-hot-region-scheduler when balance-hot-region-scheduler is paused
+ checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "pause", "balance-hot-region-scheduler", "60"}, map[string]bool{
  "balance-region-scheduler": true,
  "balance-leader-scheduler": true,
  "balance-hot-region-scheduler": true,
- "grant-hot-region-scheduler": true,
  })
+ echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, nil)
+ re.Contains(echo, "balance-hot-region-scheduler is running, please remove it first")
+
+ // case 3: add grant-hot-region-scheduler when balance-hot-region-scheduler is disabled
+ checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "balance-hot-region-scheduler"}, map[string]bool{
+ "balance-region-scheduler": true,
+ "balance-leader-scheduler": true,
+ })
+
+ checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "grant-hot-region-scheduler", "1", "1,2,3"}, map[string]bool{
+ "balance-region-scheduler": true,
+ "balance-leader-scheduler": true,
+ "grant-hot-region-scheduler": true,
+ })
+
+ // case 4: test grant-hot-region-scheduler config
  var conf3 map[string]any
  expected3 := map[string]any{
  "store-id": []any{float64(1), float64(2), float64(3)},
@@ -509,7 +529,17 @@ func (suite *schedulerTestSuite) checkSchedulerConfig(cluster *pdTests.TestClust
  mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler"}, &conf3)
  return reflect.DeepEqual(expected3, conf3)
  })
+
+ // case 5: remove grant-hot-region-scheduler
+ echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, nil)
+ re.Contains(echo, "grant-hot-region-scheduler is running, please remove it first")
+
  checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "remove", "grant-hot-region-scheduler"}, map[string]bool{
+ "balance-region-scheduler": true,
+ "balance-leader-scheduler": true,
+ })
+
+ checkSchedulerCommand(re, cmd, pdAddr, []string{"-u", pdAddr, "scheduler", "add", "balance-hot-region-scheduler"}, map[string]bool{
  "balance-region-scheduler": true,
  "balance-leader-scheduler": true,
  "balance-hot-region-scheduler": true,