Modify placement rule to support witness (#5292)

close #5568

Modify placement rule to support witness

Signed-off-by: Wenbo Zhang <ethercflow@gmail.com>

go.mod

@@ -25,7 +25,7 @@ require (
 	github.com/pingcap/errcode v0.3.0
 	github.com/pingcap/errors v0.11.5-0.20211224045212-9687c2b0f87c
 	github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce
-	github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad
+	github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a
 	github.com/pingcap/log v1.1.1-0.20221015072633-39906604fb81
 	github.com/pingcap/sysutil v0.0.0-20211208032423-041a72e5860d
 	github.com/pingcap/tidb-dashboard v0.0.0-20220728104842-3743e533b594

go.sum

@@ -417,8 +417,8 @@ github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce h1:Y1kCxlCtlPTMt
 github.com/pingcap/failpoint v0.0.0-20200702092429-9f69995143ce/go.mod h1:w4PEZ5y16LeofeeGwdgZB4ddv9bLyDuIX+ljstgKZyk=
 github.com/pingcap/kvproto v0.0.0-20191211054548-3c6b38ea5107/go.mod h1:WWLmULLO7l8IOcQG+t+ItJ3fEcrL5FxF0Wu+HrMy26w=
 github.com/pingcap/kvproto v0.0.0-20200411081810-b85805c9476c/go.mod h1:IOdRDPLyda8GX2hE/jO7gqaCV/PNFh8BZQCQZXfIOqI=
-github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad h1:lGKxsEwdE0pVXzHYD1SQ1vfa3t/bFVU/latrQz8b/w0=
-github.com/pingcap/kvproto v0.0.0-20220818063303-5c20f55db5ad/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
+github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a h1:McYxPhA8SHqfUtLfQHHN0fQl4dy93IkhlX4Pp2MKIFA=
+github.com/pingcap/kvproto v0.0.0-20221014081430-26e28e6a281a/go.mod h1:OYtxs0786qojVTmkVeufx93xe+jUgm56GUYRIKnmaGI=
 github.com/pingcap/log v0.0.0-20191012051959-b742a5d432e9/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20200511115504-543df19646ad/go.mod h1:4rbK1p9ILyIfb6hU7OG2CiWSqMXnp3JMbiaVJ6mvoY8=
 github.com/pingcap/log v0.0.0-20210625125904-98ed8e2eb1c7/go.mod h1:8AanEdAHATuRurdGxZXBz0At+9avep+ub7U1AGYLIMM=

server/api/trend_test.go

@@ -39,9 +39,9 @@ func TestTrend(t *testing.T) {
 	}
 
 	// Create 3 regions, all peers on store1 and store2, and the leaders are all on store1.
-	region4 := newRegionInfo(4, "", "a", 2, 2, []uint64{1, 2}, nil, 1)
-	region5 := newRegionInfo(5, "a", "b", 2, 2, []uint64{1, 2}, nil, 1)
-	region6 := newRegionInfo(6, "b", "", 2, 2, []uint64{1, 2}, nil, 1)
+	region4 := newRegionInfo(4, "", "a", 2, 2, []uint64{1, 2}, nil, nil, 1)
+	region5 := newRegionInfo(5, "a", "b", 2, 2, []uint64{1, 2}, nil, []uint64{2}, 1)
+	region6 := newRegionInfo(6, "b", "", 2, 2, []uint64{1, 2}, nil, nil, 1)
 	mustRegionHeartbeat(re, svr, region4)
 	mustRegionHeartbeat(re, svr, region5)
 	mustRegionHeartbeat(re, svr, region6)
@@ -57,6 +57,8 @@ func TestTrend(t *testing.T) {
 	op, err := svr.GetHandler().GetOperator(5)
 	re.NoError(err)
 	re.NotNil(op)
+	re.True(op.Step(0).(operator.AddLearner).IsWitness)
+
 	newPeerID := op.Step(0).(operator.AddLearner).PeerID
 	region5 = region5.Clone(core.WithAddPeer(&metapb.Peer{Id: newPeerID, StoreId: 3, Role: metapb.PeerRole_Learner}), core.WithIncConfVer())
 	mustRegionHeartbeat(re, svr, region5)
@@ -97,20 +99,34 @@ func TestTrend(t *testing.T) {
 	}
 }
 
-func newRegionInfo(id uint64, startKey, endKey string, confVer, ver uint64, voters []uint64, learners []uint64, leaderStore uint64) *core.RegionInfo {
+func newRegionInfo(id uint64, startKey, endKey string, confVer, ver uint64, voters []uint64, learners []uint64, witnesses []uint64, leaderStore uint64) *core.RegionInfo {
 	var (
 		peers  = make([]*metapb.Peer, 0, len(voters)+len(learners))
 		leader *metapb.Peer
 	)
 	for _, id := range voters {
-		p := &metapb.Peer{Id: 10 + id, StoreId: id}
+		witness := false
+		for _, wid := range witnesses {
+			if id == wid {
+				witness = true
+				break
+			}
+		}
+		p := &metapb.Peer{Id: 10 + id, StoreId: id, IsWitness: witness}
 		if id == leaderStore {
 			leader = p
 		}
 		peers = append(peers, p)
 	}
 	for _, id := range learners {
-		p := &metapb.Peer{Id: 10 + id, StoreId: id, Role: metapb.PeerRole_Learner}
+		witness := false
+		for _, wid := range witnesses {
+			if id == wid {
+				witness = true
+				break
+			}
+		}
+		p := &metapb.Peer{Id: 10 + id, StoreId: id, Role: metapb.PeerRole_Learner, IsWitness: witness}
 		peers = append(peers, p)
 	}
 	return core.NewRegionInfo(

server/core/peer.go

@@ -36,9 +36,6 @@ func IsVoter(peer *metapb.Peer) bool {
 // IsVoterOrIncomingVoter judges whether peer role will become Voter.
 // The peer is not nil and the role is equal to IncomingVoter or Voter.
 func IsVoterOrIncomingVoter(peer *metapb.Peer) bool {
-	if peer == nil {
-		return false
-	}
 	switch peer.GetRole() {
 	case metapb.PeerRole_IncomingVoter, metapb.PeerRole_Voter:
 		return true
@@ -49,9 +46,6 @@ func IsVoterOrIncomingVoter(peer *metapb.Peer) bool {
 // IsLearnerOrDemotingVoter judges whether peer role will become Learner.
 // The peer is not nil and the role is equal to DemotingVoter or Learner.
 func IsLearnerOrDemotingVoter(peer *metapb.Peer) bool {
-	if peer == nil {
-		return false
-	}
 	switch peer.GetRole() {
 	case metapb.PeerRole_DemotingVoter, metapb.PeerRole_Learner:
 		return true

server/handler.go

@@ -629,7 +629,7 @@ func (h *Handler) AddTransferPeerOperator(regionID uint64, fromStoreID, toStoreI
 		return err
 	}
 
-	newPeer := &metapb.Peer{StoreId: toStoreID, Role: oldPeer.GetRole()}
+	newPeer := &metapb.Peer{StoreId: toStoreID, Role: oldPeer.GetRole(), IsWitness: oldPeer.GetIsWitness()}
 	op, err := operator.CreateMovePeerOperator("admin-move-peer", c, region, operator.OpAdmin, fromStoreID, newPeer)
 	if err != nil {
 		log.Debug("fail to create move peer operator", errs.ZapError(err))

server/schedule/checker/rule_checker.go

@@ -33,11 +33,12 @@ import (
 )
 
 var (
-	errNoStoreToAdd       = errors.New("no store to add peer")
-	errNoStoreToReplace   = errors.New("no store to replace peer")
-	errPeerCannotBeLeader = errors.New("peer cannot be leader")
-	errNoNewLeader        = errors.New("no new leader")
-	errRegionNoLeader     = errors.New("region no leader")
+	errNoStoreToAdd        = errors.New("no store to add peer")
+	errNoStoreToReplace    = errors.New("no store to replace peer")
+	errPeerCannotBeLeader  = errors.New("peer cannot be leader")
+	errPeerCannotBeWitness = errors.New("peer cannot be witness")
+	errNoNewLeader         = errors.New("no new leader")
+	errRegionNoLeader      = errors.New("region no leader")
 )
 
 const maxPendingListLen = 100000
@@ -181,7 +182,7 @@ func (c *RuleChecker) addRulePeer(region *core.RegionInfo, rf *placement.RuleFit
 		c.handleFilterState(region, filterByTempState)
 		return nil, errNoStoreToAdd
 	}
-	peer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole()}
+	peer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole(), IsWitness: rf.Rule.IsWitness}
 	op, err := operator.CreateAddPeerOperator("add-rule-peer", c.cluster, region, peer, operator.OpReplica)
 	if err != nil {
 		return nil, err
@@ -199,7 +200,7 @@ func (c *RuleChecker) replaceUnexpectRulePeer(region *core.RegionInfo, rf *place
 		c.handleFilterState(region, filterByTempState)
 		return nil, errNoStoreToReplace
 	}
-	newPeer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole()}
+	newPeer := &metapb.Peer{StoreId: store, Role: rf.Rule.Role.MetaPeerRole(), IsWitness: rf.Rule.IsWitness}
 	//  pick the smallest leader store to avoid the Offline store be snapshot generator bottleneck.
 	var newLeader *metapb.Peer
 	if region.GetLeader().GetId() == peer.GetId() {
@@ -266,6 +267,24 @@ func (c *RuleChecker) fixLooseMatchPeer(region *core.RegionInfo, fit *placement.
 		checkerCounter.WithLabelValues("rule_checker", "demote-voter-role").Inc()
 		return operator.CreateDemoteVoterOperator("fix-demote-voter", c.cluster, region, peer)
 	}
+	if region.GetLeader().GetId() == peer.GetId() && rf.Rule.IsWitness {
+		return nil, errPeerCannotBeWitness
+	}
+	if !core.IsWitness(peer) && rf.Rule.IsWitness {
+		lv := "set-voter-witness"
+		if core.IsLearner(peer) {
+			lv = "set-learner-witness"
+		}
+		checkerCounter.WithLabelValues("rule_checker", lv).Inc()
+		return operator.CreateWitnessPeerOperator("fix-witness-peer", c.cluster, region, peer)
+	} else if core.IsWitness(peer) && !rf.Rule.IsWitness {
+		lv := "set-voter-non-witness"
+		if core.IsLearner(peer) {
+			lv = "set-learner-non-witness"
+		}
+		checkerCounter.WithLabelValues("rule_checker", lv).Inc()
+		return operator.CreateNonWitnessPeerOperator("fix-non-witness-peer", c.cluster, region, peer)
+	}
 	return nil, nil
 }
 
@@ -308,7 +327,7 @@ func (c *RuleChecker) fixBetterLocation(region *core.RegionInfo, rf *placement.R
 		return nil, nil
 	}
 	checkerCounter.WithLabelValues("rule_checker", "move-to-better-location").Inc()
-	newPeer := &metapb.Peer{StoreId: newStore, Role: rf.Rule.Role.MetaPeerRole()}
+	newPeer := &metapb.Peer{StoreId: newStore, Role: rf.Rule.Role.MetaPeerRole(), IsWitness: rf.Rule.IsWitness}
 	return operator.CreateMovePeerOperator("move-to-better-location", c.cluster, region, operator.OpReplica, oldStore, newPeer)
 }
 

server/schedule/checker/rule_checker_test.go

@@ -302,6 +302,93 @@ func (suite *ruleCheckerTestSuite) TestFixLeaderRoleWithUnhealthyRegion() {
 	suite.Nil(op)
 }
 
+func (suite *ruleCheckerTestSuite) TestFixRuleWitness() {
+	suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"})
+	suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "follower"})
+	suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"})
+	suite.cluster.AddLeaderRegion(1, 1, 2)
+
+	suite.ruleManager.SetRule(&placement.Rule{
+		GroupID:   "pd",
+		ID:        "r1",
+		Index:     100,
+		Override:  true,
+		Role:      placement.Voter,
+		Count:     1,
+		IsWitness: true,
+		LabelConstraints: []placement.LabelConstraint{
+			{Key: "C", Op: "in", Values: []string{"voter"}},
+		},
+	})
+	op := suite.rc.Check(suite.cluster.GetRegion(1))
+	suite.NotNil(op)
+	suite.Equal("add-rule-peer", op.Desc())
+	suite.Equal(uint64(3), op.Step(0).(operator.AddLearner).ToStore)
+	suite.True(op.Step(0).(operator.AddLearner).IsWitness)
+}
+
+func (suite *ruleCheckerTestSuite) TestFixRuleWitness2() {
+	suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"})
+	suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"})
+	suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"})
+	suite.cluster.AddLeaderRegion(1, 1, 2, 3)
+
+	suite.ruleManager.SetRule(&placement.Rule{
+		GroupID:   "pd",
+		ID:        "r1",
+		Index:     100,
+		Override:  true,
+		Role:      placement.Voter,
+		Count:     1,
+		IsWitness: true,
+		LabelConstraints: []placement.LabelConstraint{
+			{Key: "C", Op: "in", Values: []string{"voter"}},
+		},
+	})
+	op := suite.rc.Check(suite.cluster.GetRegion(1))
+	suite.NotNil(op)
+	suite.Equal("fix-witness-peer", op.Desc())
+	suite.Equal(uint64(3), op.Step(0).(operator.BecomeWitness).StoreID)
+}
+
+func (suite *ruleCheckerTestSuite) TestFixRuleWitness3() {
+	suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"})
+	suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"})
+	suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"})
+	suite.cluster.AddLeaderRegion(1, 1, 2, 3)
+
+	r := suite.cluster.GetRegion(1)
+	// set peer3 to witness
+	r = r.Clone(core.WithWitnesses([]*metapb.Peer{r.GetPeer(3)}))
+
+	op := suite.rc.Check(r)
+	suite.NotNil(op)
+	suite.Equal("fix-non-witness-peer", op.Desc())
+	suite.Equal(uint64(3), op.Step(0).(operator.BecomeNonWitness).StoreID)
+}
+
+func (suite *ruleCheckerTestSuite) TestFixRuleWitness4() {
+	suite.cluster.AddLabelsStore(1, 1, map[string]string{"A": "leader"})
+	suite.cluster.AddLabelsStore(2, 1, map[string]string{"B": "voter"})
+	suite.cluster.AddLabelsStore(3, 1, map[string]string{"C": "voter"})
+	suite.cluster.AddLeaderRegion(1, 1, 2, 3)
+
+	suite.ruleManager.SetRule(&placement.Rule{
+		GroupID:   "pd",
+		ID:        "r1",
+		Index:     100,
+		Override:  true,
+		Role:      placement.Voter,
+		Count:     2,
+		IsWitness: true,
+		LabelConstraints: []placement.LabelConstraint{
+			{Key: "A", Op: "In", Values: []string{"leader"}},
+		},
+	})
+	op := suite.rc.Check(suite.cluster.GetRegion(1))
+	suite.Nil(op)
+}
+
 func (suite *ruleCheckerTestSuite) TestBetterReplacement() {
 	suite.cluster.AddLabelsStore(1, 1, map[string]string{"host": "host1"})
 	suite.cluster.AddLabelsStore(2, 1, map[string]string{"host": "host1"})