Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

mcs: fix scheduler memory sync in api server #7389

Merged
merged 8 commits into from
Nov 20, 2023
Merged

mcs: fix scheduler memory sync in api server #7389

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
b4ce36c
add tests
lhy1024 Nov 17, 2023
607e7dd
fix prepare and cleanup
lhy1024 Nov 17, 2023
9664137
fix lint
lhy1024 Nov 17, 2023
ba04e4c
rename
lhy1024 Nov 17, 2023
840a44d
address comments
lhy1024 Nov 20, 2023
d548a11
address comments
lhy1024 Nov 20, 2023
5961cce
Merge branch 'master' into fix-sync
lhy1024 Nov 20, 2023
1c7b769
Merge branch 'master' into fix-sync
ti-chi-bot[bot] Nov 20, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion pkg/mcs/scheduling/server/config/watcher.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,8 @@ func (cw *Watcher) initializeSchedulerConfigWatcher() error {
prefixToTrim := cw.schedulerConfigPathPrefix + "/"
putFn := func(kv *mvccpb.KeyValue) error {
name := strings.TrimPrefix(string(kv.Key), prefixToTrim)
log.Info("update scheduler config", zap.String("name", string(kv.Value)))
log.Info("update scheduler config", zap.String("name", name),
zap.String("value", string(kv.Value)))
err := cw.storage.SaveSchedulerConfig(name, kv.Value)
if err != nil {
log.Warn("failed to save scheduler config",
Expand Down
4 changes: 3 additions & 1 deletion pkg/schedule/schedulers/scheduler_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,7 +161,8 @@ func (c *Controller) AddSchedulerHandler(scheduler Scheduler, args ...string) er
return err
}
c.cluster.GetSchedulerConfig().AddSchedulerCfg(scheduler.GetType(), args)
return nil
err := scheduler.Prepare(c.cluster)
return err
}

// RemoveSchedulerHandler removes the HTTP handler for a scheduler.
Expand All @@ -188,6 +189,7 @@ func (c *Controller) RemoveSchedulerHandler(name string) error {
return err
}

s.(Scheduler).Cleanup(c.cluster)
delete(c.schedulerHandlers, name)

return nil
Expand Down
126 changes: 94 additions & 32 deletions tests/pdctl/scheduler/scheduler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ package scheduler_test
import (
"context"
"encoding/json"
"fmt"
"reflect"
"strings"
"testing"
Expand All @@ -28,6 +29,7 @@ import (
"github.com/stretchr/testify/suite"
"github.com/tikv/pd/pkg/core"
sc "github.com/tikv/pd/pkg/schedule/config"
"github.com/tikv/pd/pkg/slice"
"github.com/tikv/pd/pkg/utils/testutil"
"github.com/tikv/pd/pkg/versioninfo"
"github.com/tikv/pd/tests"
Expand Down Expand Up @@ -84,7 +86,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {

checkSchedulerCommand := func(args []string, expected map[string]bool) {
if args != nil {
mustExec(re, cmd, args, nil)
echo := mustExec(re, cmd, args, nil)
re.Contains(echo, "Success!")
}
testutil.Eventually(re, func() bool {
var schedulers []string
Expand Down Expand Up @@ -137,9 +140,40 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
}
checkSchedulerCommand(args, expected)

schedulers := []string{"evict-leader-scheduler", "grant-leader-scheduler"}
// avoid the influence of the scheduler order
schedulers := []string{"evict-leader-scheduler", "grant-leader-scheduler", "evict-leader-scheduler", "grant-leader-scheduler"}

checkStorePause := func(changedStores []uint64, schedulerName string) {
status := func() string {
switch schedulerName {
case "evict-leader-scheduler":
return "paused"
case "grant-leader-scheduler":
return "resumed"
default:
re.Fail(fmt.Sprintf("unknown scheduler %s", schedulerName))
return ""
}
}()
for _, store := range stores {
isStorePaused := !cluster.GetLeaderServer().GetRaftCluster().GetStore(store.GetId()).AllowLeaderTransfer()
if slice.AnyOf(changedStores, func(i int) bool {
return store.GetId() == changedStores[i]
}) {
re.True(isStorePaused,
fmt.Sprintf("store %d should be %s with %s", store.GetId(), status, schedulerName))
} else {
re.False(isStorePaused,
fmt.Sprintf("store %d should not be %s with %s", store.GetId(), status, schedulerName))
}
if sche := cluster.GetSchedulingPrimaryServer(); sche != nil {
re.Equal(isStorePaused, !sche.GetCluster().GetStore(store.GetId()).AllowLeaderTransfer())
}
}
}

for idx := range schedulers {
checkStorePause([]uint64{}, schedulers[idx])
// scheduler add command
args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "2"}
expected = map[string]bool{
Expand All @@ -155,6 +189,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
expectedConfig := make(map[string]interface{})
expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}}
checkSchedulerConfigCommand(expectedConfig, schedulers[idx])
checkStorePause([]uint64{2}, schedulers[idx])

// scheduler config update command
args = []string{"-u", pdAddr, "scheduler", "config", schedulers[idx], "add-store", "3"}
Expand All @@ -165,14 +200,12 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
}
checkSchedulerCommand(args, expected)

// check update success
// FIXME: remove this check after scheduler config is updated
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It is removed.

if cluster.GetSchedulingPrimaryServer() == nil && schedulers[idx] == "grant-leader-scheduler" {
expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}, "3": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}}
checkSchedulerConfigCommand(expectedConfig, schedulers[idx])
}
checkSchedulerCommand(args, expected)
expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}, "3": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}}
checkSchedulerConfigCommand(expectedConfig, schedulers[idx])
checkStorePause([]uint64{2, 3}, schedulers[idx])

// scheduler delete command
args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx]}
Expand All @@ -183,6 +216,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"balance-witness-scheduler": true,
}
checkSchedulerCommand(args, expected)
checkStorePause([]uint64{}, schedulers[idx])

// scheduler add command
args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "2"}
Expand All @@ -194,6 +228,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"balance-witness-scheduler": true,
}
checkSchedulerCommand(args, expected)
checkStorePause([]uint64{2}, schedulers[idx])

// scheduler add command twice
args = []string{"-u", pdAddr, "scheduler", "add", schedulers[idx], "4"}
Expand All @@ -209,6 +244,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
// check add success
expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}, "4": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}}
checkSchedulerConfigCommand(expectedConfig, schedulers[idx])
checkStorePause([]uint64{2, 4}, schedulers[idx])

// scheduler remove command [old]
args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx] + "-4"}
Expand All @@ -224,6 +260,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
// check remove success
expectedConfig["store-id-ranges"] = map[string]interface{}{"2": []interface{}{map[string]interface{}{"end-key": "", "start-key": ""}}}
checkSchedulerConfigCommand(expectedConfig, schedulers[idx])
checkStorePause([]uint64{2}, schedulers[idx])

// scheduler remove command, when remove the last store, it should remove whole scheduler
args = []string{"-u", pdAddr, "scheduler", "remove", schedulers[idx] + "-2"}
Expand All @@ -234,6 +271,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"balance-witness-scheduler": true,
}
checkSchedulerCommand(args, expected)
checkStorePause([]uint64{}, schedulers[idx])
}

// test shuffle region config
Expand All @@ -247,7 +285,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
var roles []string
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles)
re.Equal([]string{"leader", "follower", "learner"}, roles)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "set-roles", "learner"}, nil)
echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "set-roles", "learner"}, nil) // todo:add check output
re.Contains(echo, "Success!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles)
re.Equal([]string{"learner"}, roles)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler"}, &roles)
Expand All @@ -270,7 +309,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler"}, &conf3)
re.Equal(expected3, conf3)

mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler", "set", "2", "1,2,3"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "grant-hot-region-scheduler", "set", "2", "1,2,3"}, nil)
re.Contains(echo, "Success!")
expected3["store-leader-id"] = float64(2)
// FIXME: remove this check after scheduler config is updated
if cluster.GetSchedulingPrimaryServer() == nil { // "grant-hot-region-scheduler"
Expand All @@ -279,7 +319,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
}

// test remove and add scheduler
echo := mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "balance-region-scheduler"}, nil)
re.Contains(echo, "Success!")
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil)
re.Contains(echo, "Success!")
Expand Down Expand Up @@ -326,60 +366,75 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
re.Equal(expected1, conf)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "show"}, &conf)
re.Equal(expected1, conf)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "src-tolerance-ratio", "1.02"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "src-tolerance-ratio", "1.02"}, nil)
re.Contains(echo, "Success!")
expected1["src-tolerance-ratio"] = 1.02
var conf1 map[string]interface{}
// FIXME: remove this check after scheduler config is updated
if cluster.GetSchedulingPrimaryServer() == nil { // "balance-hot-region-scheduler"
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)

mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,key"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,key"}, nil)
re.Contains(echo, "Success!")
expected1["read-priorities"] = []interface{}{"byte", "key"}
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)

mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key"}, nil)
re.Contains(echo, "Failed!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,byte"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,byte"}, nil)
re.Contains(echo, "Success!")
expected1["read-priorities"] = []interface{}{"key", "byte"}
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "foo,bar"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "foo,bar"}, nil)
re.Contains(echo, "Failed!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", ""}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", ""}, nil)
re.Contains(echo, "Failed!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key"}, nil)
re.Contains(echo, "Failed!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,byte"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "byte,byte"}, nil)
re.Contains(echo, "Failed!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key,byte"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "read-priorities", "key,key,byte"}, nil)
re.Contains(echo, "Failed!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)

// write-priorities is divided into write-leader-priorities and write-peer-priorities
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-priorities", "key,byte"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "write-priorities", "key,byte"}, nil)
re.Contains(echo, "Failed!")
re.Contains(echo, "Config item is not found.")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)

mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v0"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v0"}, nil)
re.Contains(echo, "Failed!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
expected1["rank-formula-version"] = "v2"
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v2"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v2"}, nil)
re.Contains(echo, "Success!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)
expected1["rank-formula-version"] = "v1"
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v1"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "rank-formula-version", "v1"}, nil)
re.Contains(echo, "Success!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)

expected1["forbid-rw-type"] = "read"
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "forbid-rw-type", "read"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler", "set", "forbid-rw-type", "read"}, nil)
re.Contains(echo, "Success!")
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-hot-region-scheduler"}, &conf1)
re.Equal(expected1, conf1)

Expand Down Expand Up @@ -412,7 +467,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
conf1 = make(map[string]interface{})
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "show"}, &conf)
re.Equal(4., conf["batch"])
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "set", "batch", "3"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler", "set", "batch", "3"}, nil)
re.Contains(echo, "Success!")
testutil.Eventually(re, func() bool {
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "balance-leader-scheduler"}, &conf1)
return conf1["batch"] == 3.
Expand Down Expand Up @@ -465,7 +521,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
}

mustUsage([]string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler"})
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil)
re.Contains(echo, "Success!")
checkSchedulerWithStatusCommand("paused", []string{
"balance-leader-scheduler",
})
Expand All @@ -476,7 +533,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
}, testutil.WithWaitFor(30*time.Second))

mustUsage([]string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler", "60"})
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil)
re.Contains(echo, "Success!")
checkSchedulerWithStatusCommand("paused", nil)

// set label scheduler to disabled manually.
Expand Down Expand Up @@ -547,11 +605,14 @@ func (suite *schedulerTestSuite) checkSchedulerDiagnostic(cluster *tests.TestClu
checkSchedulerDescribeCommand("balance-region-scheduler", "pending", "1 store(s) RegionNotMatchRule; ")

// scheduler delete command
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "remove", "balance-region-scheduler"}, nil)
re.Contains(echo, "Success!")
checkSchedulerDescribeCommand("balance-region-scheduler", "disabled", "")

mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil)
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil)
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil)
re.Contains(echo, "Success!")
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "resume", "balance-leader-scheduler"}, nil)
re.Contains(echo, "Success!")
checkSchedulerDescribeCommand("balance-leader-scheduler", "normal", "")
}

Expand Down Expand Up @@ -604,7 +665,8 @@ func TestForwardSchedulerRequest(t *testing.T) {
re.Contains(string(output), "Usage")
}
mustUsage([]string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler"})
mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil)
echo := mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "pause", "balance-leader-scheduler", "60"}, nil)
re.Contains(echo, "Success!")
checkSchedulerWithStatusCommand := func(status string, expected []string) {
var schedulers []string
mustExec(re, cmd, []string{"-u", backendEndpoints, "scheduler", "show", "--status", status}, &schedulers)
Expand Down
Loading