slow node detection: enable evict-slow-trend on raft-kv2. (#6945)

close #6868, close tikv/tikv#15267, ref tikv/tikv#15271

This pr contains:
+ Enable `evict-slow-trend` scheduler on `raft-kv2` as default.
+ Optimize the detection strategy on the perception of Network I/O delays on TiKV nodes.

Signed-off-by: lucasliang <nkcs_lykx@hotmail.com>

pkg/mcs/scheduling/server/config/config.go

@@ -606,11 +606,21 @@ func (o *PersistConfig) GetRegionMaxKeys() uint64 {
 	return o.GetStoreConfig().GetRegionMaxKeys()
 }
 
+// IsSynced returns true if the cluster config is synced.
+func (o *PersistConfig) IsSynced() bool {
+	return o.GetStoreConfig().IsSynced()
+}
+
 // IsEnableRegionBucket return true if the region bucket is enabled.
 func (o *PersistConfig) IsEnableRegionBucket() bool {
 	return o.GetStoreConfig().IsEnableRegionBucket()
 }
 
+// IsRaftKV2 returns the whether the cluster use `raft-kv2` engine.
+func (o *PersistConfig) IsRaftKV2() bool {
+	return o.GetStoreConfig().IsRaftKV2()
+}
+
 // TODO: implement the following methods
 
 // AddSchedulerCfg adds the scheduler configurations.

pkg/schedule/config/config_provider.go

@@ -139,5 +139,7 @@ type StoreConfigProvider interface {
 	GetRegionMaxKeys() uint64
 	CheckRegionSize(uint64, uint64) error
 	CheckRegionKeys(uint64, uint64) error
+	IsSynced() bool
 	IsEnableRegionBucket() bool
+	IsRaftKV2() bool
 }

pkg/schedule/config/store_config.go

@@ -48,6 +48,8 @@ type StoreConfig struct {
 	RegionMaxSizeMB    uint64 `json:"-"`
 	RegionSplitSizeMB  uint64 `json:"-"`
 	RegionBucketSizeMB uint64 `json:"-"`
+
+	Sync bool `json:"sync"`
 }
 
 // Storage is the config for the tikv storage.
@@ -73,6 +75,7 @@ func (c *StoreConfig) Adjust() {
 	if c == nil {
 		return
 	}
+
 	c.RegionMaxSizeMB = typeutil.ParseMBFromText(c.RegionMaxSize, defaultRegionMaxSize)
 	c.RegionSplitSizeMB = typeutil.ParseMBFromText(c.RegionSplitSize, defaultRegionSplitSize)
 	c.RegionBucketSizeMB = typeutil.ParseMBFromText(c.RegionBucketSize, defaultBucketSize)
@@ -124,6 +127,21 @@ func (c *StoreConfig) GetRegionMaxKeys() uint64 {
 	return uint64(c.RegionMaxKeys)
 }
 
+// SetSynced marks StoreConfig has been synced.
+func (c *StoreConfig) SetSynced() {
+	if c != nil {
+		c.Sync = true
+	}
+}
+
+// IsSynced returns whether the StoreConfig is synced or not.
+func (c *StoreConfig) IsSynced() bool {
+	if c == nil {
+		return false
+	}
+	return c.Sync
+}
+
 // IsEnableRegionBucket return true if the region bucket is enabled.
 func (c *StoreConfig) IsEnableRegionBucket() bool {
 	if c == nil {

pkg/schedule/coordinator.go

@@ -19,6 +19,7 @@ import (
 	"context"
 	"strconv"
 	"sync"
+	"testing"
 	"time"
 
 	"github.com/pingcap/errors"
@@ -375,7 +376,6 @@ func (c *Coordinator) initSchedulers() {
 	if err != nil {
 		log.Fatal("cannot load schedulers' config", errs.ZapError(err))
 	}
-
 	scheduleCfg := c.cluster.GetSchedulerConfig().GetScheduleConfig().Clone()
 	// The new way to create scheduler with the independent configuration.
 	for i, name := range scheduleNames {
@@ -439,6 +439,20 @@ func (c *Coordinator) initSchedulers() {
 	if err := c.cluster.GetSchedulerConfig().Persist(c.cluster.GetStorage()); err != nil {
 		log.Error("cannot persist schedule config", errs.ZapError(err))
 	}
+
+	// If the cluster was set up with `raft-kv2` engine, this cluster should
+	// enable `evict-slow-trend` scheduler as default.
+	if c.GetCluster().GetStoreConfig().IsRaftKV2() {
+		name := schedulers.EvictSlowTrendType
+		args := []string{}
+
+		s, err := schedulers.CreateScheduler(name, c.opController, c.cluster.GetStorage(), schedulers.ConfigSliceDecoder(name, args), c.schedulers.RemoveScheduler)
+		if err != nil {
+			log.Warn("initializing evict-slow-trend scheduler failed", errs.ZapError(err))
+		} else if err = c.schedulers.AddScheduler(s, args...); err != nil {
+			log.Error("can not add scheduler", zap.String("scheduler-name", s.GetName()), zap.Strings("scheduler-args", args), errs.ZapError(err))
+		}
+	}
 }
 
 // LoadPlugin load user plugin
@@ -625,7 +639,11 @@ func (c *Coordinator) ResetHotSpotMetrics() {
 
 // ShouldRun returns true if the coordinator should run.
 func (c *Coordinator) ShouldRun() bool {
-	return c.prepareChecker.check(c.cluster.GetBasicCluster())
+	isSynced := c.cluster.GetStoreConfig().IsSynced()
+	if testing.Testing() {
+		isSynced = true
+	}
+	return c.prepareChecker.check(c.cluster.GetBasicCluster()) && isSynced
 }
 
 // GetSchedulersController returns the schedulers controller.