Skip to content

Commit

Permalink
scheduler: enable evict-slow-store by default. (#7505)
Browse files Browse the repository at this point in the history 
close #7564, ref tikv/tikv#15909

Enable `evict-slow-store` scheduler by default.

Signed-off-by: lucasliang <nkcs_lykx@hotmail.com>
  • Loading branch information
@LykxSassinator
LykxSassinator authored Dec 18, 2023
1 parent 40c2523 commit cfb1d8f
Show file tree
Hide file tree
Showing 7 changed files with 45 additions and 15 deletions.
1 change: 1 addition & 0 deletions pkg/schedule/config/config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -556,6 +556,7 @@ var DefaultSchedulers = SchedulerConfigs{
{Type: "balance-witness"},
{Type: "hot-region"},
{Type: "transfer-witness-leader"},
{Type: "evict-slow-store"},
}

// IsDefaultScheduler checks whether the scheduler is enable by default.
Expand Down
18 changes: 11 additions & 7 deletions server/cluster/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3017,6 +3017,7 @@ func TestAddScheduler(t *testing.T) {
re.NoError(controller.RemoveScheduler(schedulers.HotRegionName))
re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName))
re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName))
re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName))
re.Empty(controller.GetSchedulerNames())

stream := mockhbstream.NewHeartbeatStream()
Expand Down Expand Up @@ -3107,13 +3108,15 @@ func TestPersistScheduler(t *testing.T) {
re.NoError(err)
re.Len(sches, defaultCount+2)

// remove 5 schedulers
// remove all default schedulers
re.NoError(controller.RemoveScheduler(schedulers.BalanceLeaderName))
re.NoError(controller.RemoveScheduler(schedulers.BalanceRegionName))
re.NoError(controller.RemoveScheduler(schedulers.HotRegionName))
re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName))
re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName))
re.Len(controller.GetSchedulerNames(), defaultCount-3)
re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName))
// only remains 2 items with independent config.
re.Len(controller.GetSchedulerNames(), 2)
re.NoError(co.GetCluster().GetSchedulerConfig().Persist(storage))
co.Stop()
co.GetSchedulersController().Wait()
Expand All @@ -3137,7 +3140,7 @@ func TestPersistScheduler(t *testing.T) {
re.NoError(err)
re.Len(sches, 3)

// option have 6 items because the default scheduler do not remove.
// option have 9 items because the default scheduler do not remove.
re.Len(newOpt.GetSchedulers(), defaultCount+3)
re.NoError(newOpt.Persist(storage))
tc.RaftCluster.SetScheduleConfig(newOpt.GetScheduleConfig())
Expand All @@ -3164,9 +3167,9 @@ func TestPersistScheduler(t *testing.T) {
brs, err := schedulers.CreateScheduler(schedulers.BalanceRegionType, oc, storage, schedulers.ConfigSliceDecoder(schedulers.BalanceRegionType, []string{"", ""}))
re.NoError(err)
re.NoError(controller.AddScheduler(brs))
re.Len(controller.GetSchedulerNames(), defaultCount)
re.Len(controller.GetSchedulerNames(), 5)

// the scheduler option should contain 6 items
// the scheduler option should contain 9 items
// the `hot scheduler` are disabled
re.Len(co.GetCluster().GetSchedulerConfig().(*config.PersistOptions).GetSchedulers(), defaultCount+3)
re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName))
Expand All @@ -3185,9 +3188,9 @@ func TestPersistScheduler(t *testing.T) {

co.Run()
controller = co.GetSchedulersController()
re.Len(controller.GetSchedulerNames(), defaultCount-1)
re.Len(controller.GetSchedulerNames(), 4)
re.NoError(controller.RemoveScheduler(schedulers.EvictLeaderName))
re.Len(controller.GetSchedulerNames(), defaultCount-2)
re.Len(controller.GetSchedulerNames(), 3)
}

func TestRemoveScheduler(t *testing.T) {
Expand Down Expand Up @@ -3225,6 +3228,7 @@ func TestRemoveScheduler(t *testing.T) {
re.NoError(controller.RemoveScheduler(schedulers.GrantLeaderName))
re.NoError(controller.RemoveScheduler(schedulers.BalanceWitnessName))
re.NoError(controller.RemoveScheduler(schedulers.TransferWitnessLeaderName))
re.NoError(controller.RemoveScheduler(schedulers.EvictSlowStoreName))
// all removed
sches, _, err = storage.LoadAllSchedulerConfigs()
re.NoError(err)
Expand Down
5 changes: 3 additions & 2 deletions tests/integrations/mcs/scheduling/api_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -382,13 +382,14 @@ func (suite *apiTestSuite) checkConfig(cluster *tests.TestCluster) {
suite.Equal(cfg.DataDir, s.GetConfig().DataDir)
testutil.Eventually(re, func() bool {
// wait for all schedulers to be loaded in scheduling server.
return len(cfg.Schedule.SchedulersPayload) == 5
return len(cfg.Schedule.SchedulersPayload) == 6
})
suite.Contains(cfg.Schedule.SchedulersPayload, "balance-leader-scheduler")
suite.Contains(cfg.Schedule.SchedulersPayload, "balance-region-scheduler")
suite.Contains(cfg.Schedule.SchedulersPayload, "balance-hot-region-scheduler")
suite.Contains(cfg.Schedule.SchedulersPayload, "balance-witness-scheduler")
suite.Contains(cfg.Schedule.SchedulersPayload, "transfer-witness-leader-scheduler")
suite.Contains(cfg.Schedule.SchedulersPayload, "evict-slow-store-scheduler")
}

func (suite *apiTestSuite) TestConfigForward() {
Expand All @@ -412,7 +413,7 @@ func (suite *apiTestSuite) checkConfigForward(cluster *tests.TestCluster) {
re.Equal(cfg["replication"].(map[string]interface{})["max-replicas"],
float64(opts.GetReplicationConfig().MaxReplicas))
schedulers := cfg["schedule"].(map[string]interface{})["schedulers-payload"].(map[string]interface{})
return len(schedulers) == 5
return len(schedulers) == 6
})

// Test to change config in api server
Expand Down
4 changes: 2 additions & 2 deletions tests/integrations/mcs/scheduling/server_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ func (suite *serverTestSuite) TestPrimaryChange() {
testutil.Eventually(re, func() bool {
watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName)
return ok && oldPrimaryAddr == watchedAddr &&
len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 5
len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 6
})
// change primary
primary.Close()
Expand All @@ -147,7 +147,7 @@ func (suite *serverTestSuite) TestPrimaryChange() {
testutil.Eventually(re, func() bool {
watchedAddr, ok := suite.pdLeader.GetServicePrimaryAddr(suite.ctx, mcs.SchedulingServiceName)
return ok && newPrimaryAddr == watchedAddr &&
len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 5
len(primary.GetCluster().GetCoordinator().GetSchedulersController().GetSchedulerNames()) == 6
})
}

Expand Down
20 changes: 19 additions & 1 deletion tests/pdctl/scheduler/scheduler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@ func (suite *schedulerTestSuite) SetupSuite() {
"balance-hot-region-scheduler",
"balance-witness-scheduler",
"transfer-witness-leader-scheduler",
"evict-slow-store-scheduler",
}
}

Expand Down Expand Up @@ -173,6 +174,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"balance-hot-region-scheduler": true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(nil, expected)

Expand All @@ -183,6 +185,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"balance-hot-region-scheduler": true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(args, expected)

Expand Down Expand Up @@ -228,6 +231,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
schedulers[idx]: true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(args, expected)

Expand All @@ -245,6 +249,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
schedulers[idx]: true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}

// check update success
Expand All @@ -260,6 +265,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"balance-hot-region-scheduler": true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(args, expected)
checkStorePause([]uint64{}, schedulers[idx])
Expand All @@ -272,6 +278,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
schedulers[idx]: true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(args, expected)
checkStorePause([]uint64{2}, schedulers[idx])
Expand All @@ -284,6 +291,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
schedulers[idx]: true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(args, expected)

Expand All @@ -300,6 +308,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
schedulers[idx]: true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(args, expected)

Expand All @@ -315,6 +324,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"balance-hot-region-scheduler": true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
}
checkSchedulerCommand(args, expected)
checkStorePause([]uint64{}, schedulers[idx])
Expand All @@ -327,6 +337,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"shuffle-region-scheduler": true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
})
var roles []string
mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "config", "shuffle-region-scheduler", "show-roles"}, &roles)
Expand All @@ -348,6 +359,7 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
"grant-hot-region-scheduler": true,
"transfer-witness-leader-scheduler": true,
"balance-witness-scheduler": true,
"evict-slow-store-scheduler": true,
})
var conf3 map[string]interface{}
expected3 := map[string]interface{}{
Expand Down Expand Up @@ -527,7 +539,11 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
evictSlownessSchedulers := []string{"evict-slow-store-scheduler", "evict-slow-trend-scheduler"}
for _, schedulerName := range evictSlownessSchedulers {
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", schedulerName}, nil)
re.Contains(echo, "Success!")
if strings.Contains(echo, "Success!") {
re.Contains(echo, "Success!")
} else {
re.Contains(echo, "scheduler existed")
}
testutil.Eventually(re, func() bool {
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "show"}, nil)
return strings.Contains(echo, schedulerName)
Expand All @@ -546,6 +562,8 @@ func (suite *schedulerTestSuite) checkScheduler(cluster *tests.TestCluster) {
return !strings.Contains(echo, schedulerName)
})
}
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "evict-slow-store-scheduler"}, nil)
re.Contains(echo, "Success!")

// test shuffle hot region scheduler
echo = mustExec(re, cmd, []string{"-u", pdAddr, "scheduler", "add", "shuffle-hot-region-scheduler"}, nil)
Expand Down
4 changes: 4 additions & 0 deletions tests/server/api/scheduler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -477,6 +477,10 @@ func (suite *scheduleTestSuite) checkAPI(cluster *tests.TestCluster) {
suite.NoError(err)
},
},
{
name: "evict-slow-store-scheduler",
createdName: "evict-slow-store-scheduler",
},
}
for _, testCase := range testCases {
input := make(map[string]interface{})
Expand Down
8 changes: 5 additions & 3 deletions tests/server/cluster/cluster_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1310,6 +1310,7 @@ func TestTransferLeaderForScheduler(t *testing.T) {

time.Sleep(time.Second)
re.True(leaderServer.GetRaftCluster().IsPrepared())
schedsNum := len(rc.GetCoordinator().GetSchedulersController().GetSchedulerNames())
// Add evict leader scheduler
api.MustAddScheduler(re, leaderServer.GetAddr(), schedulers.EvictLeaderName, map[string]interface{}{
"store_id": 1,
Expand All @@ -1318,8 +1319,9 @@ func TestTransferLeaderForScheduler(t *testing.T) {
"store_id": 2,
})
// Check scheduler updated.
schedsNum += 1
schedulersController := rc.GetCoordinator().GetSchedulersController()
re.Len(schedulersController.GetSchedulerNames(), 6)
re.Len(schedulersController.GetSchedulerNames(), schedsNum)
checkEvictLeaderSchedulerExist(re, schedulersController, true)
checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2})

Expand All @@ -1339,7 +1341,7 @@ func TestTransferLeaderForScheduler(t *testing.T) {
re.True(leaderServer.GetRaftCluster().IsPrepared())
// Check scheduler updated.
schedulersController = rc1.GetCoordinator().GetSchedulersController()
re.Len(schedulersController.GetSchedulerNames(), 6)
re.Len(schedulersController.GetSchedulerNames(), schedsNum)
checkEvictLeaderSchedulerExist(re, schedulersController, true)
checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2})

Expand All @@ -1358,7 +1360,7 @@ func TestTransferLeaderForScheduler(t *testing.T) {
re.True(leaderServer.GetRaftCluster().IsPrepared())
// Check scheduler updated
schedulersController = rc.GetCoordinator().GetSchedulersController()
re.Len(schedulersController.GetSchedulerNames(), 6)
re.Len(schedulersController.GetSchedulerNames(), schedsNum)
checkEvictLeaderSchedulerExist(re, schedulersController, true)
checkEvictLeaderStoreIDs(re, schedulersController, []uint64{1, 2})

Expand Down

0 comments on commit cfb1d8f

Please sign in to comment.