-
Notifications
You must be signed in to change notification settings - Fork 4.4k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
try to perform a leadership transfer when leaving #11376
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,3 @@ | ||
```release-note:enhancement | ||
try to transfer leadership to another server when leaving. | ||
``` |
Original file line number | Diff line number | Diff line change | ||||
---|---|---|---|---|---|---|
|
@@ -17,6 +17,8 @@ import ( | |||||
"sync/atomic" | ||||||
"time" | ||||||
|
||||||
"github.com/hashicorp/go-version" | ||||||
|
||||||
"github.com/armon/go-metrics" | ||||||
connlimit "github.com/hashicorp/go-connlimit" | ||||||
"github.com/hashicorp/go-hclog" | ||||||
|
@@ -92,6 +94,8 @@ const ( | |||||
// from Serf with the Catalog. If this is exhausted we will drop updates, | ||||||
// and wait for a periodic reconcile. | ||||||
reconcileChSize = 256 | ||||||
|
||||||
LeaderTransferMinVersion = "1.6.0" | ||||||
) | ||||||
|
||||||
const ( | ||||||
|
@@ -992,8 +996,28 @@ func (s *Server) Leave() error { | |||||
// removed for some reasonable period of time. | ||||||
isLeader := s.IsLeader() | ||||||
if isLeader && numPeers > 1 { | ||||||
if err := s.autopilot.RemoveServer(raft.ServerID(s.config.NodeID)); err != nil { | ||||||
s.logger.Error("failed to remove ourself as a Raft peer", "error", err) | ||||||
leadershipTransferVersion := version.Must(version.NewVersion(LeaderTransferMinVersion)) | ||||||
removeServer := false | ||||||
if ok, _ := ServersInDCMeetMinimumVersion(s, s.config.Datacenter, leadershipTransferVersion); !ok { | ||||||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Shouldn't this be
Suggested change
|
||||||
// Transfer leadership to another node then leave the cluster | ||||||
future := s.raft.LeadershipTransfer() | ||||||
if err := future.Error(); err != nil { | ||||||
s.logger.Error("failed to transfer leadership, removing the server", "error", err) | ||||||
// leadership transfer failed, fallback to removing the server from raft | ||||||
removeServer = true | ||||||
} else { | ||||||
// we are not leader anymore, continue the flow to leave as follower | ||||||
isLeader = false | ||||||
} | ||||||
} else { | ||||||
// Leadership transfer is not available in the current version, fallback to removing the server from raft | ||||||
removeServer = true | ||||||
} | ||||||
if removeServer { | ||||||
future := s.raft.RemoveServer(raft.ServerID(s.config.NodeID), 0, 0) | ||||||
if err := future.Error(); err != nil { | ||||||
s.logger.Error("failed to remove ourself as raft peer", "error", err) | ||||||
} | ||||||
Comment on lines
+999
to
+1020
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Two high level questions/suggestions:
func (s *Server) leadershipTransfer() (success bool) {
leadershipTransferVersion := version.Must(version.NewVersion(LeaderTransferMinVersion))
ok, _ := ServersInDCMeetMinimumVersion(s, s.config.Datacenter, leadershipTransferVersion)
if !ok {
return false
}
future := s.raft.LeadershipTransfer()
if err := future.Error(); err != nil {
s.logger.Error("failed to transfer leadership, removing the server", "error", err)
return false
}
return true
} Then in the caller: if !s.leadershipTransfer() {
future := s.raft.RemoveServer(raft.ServerID(s.config.NodeID), 0, 0)
if err := future.Error(); err != nil {
s.logger.Error("failed to remove ourself as raft peer", "error", err)
}
} There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. The main reason of the comments is because I did not find a good way of removing the nested if else, I will attempt the extract in the new PR. |
||||||
} | ||||||
} | ||||||
|
||||||
|
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
What do you think about something like this?