Decrease rate of Raft heartbeat messages. (#3708) (#3714)

Before we used to have 20ms ticks, but they would overload the
Raft tick channel, causing "A tick missed to fire. Node blocks
too long!" logs. Etcd uses 100ms and they haven't seen those
issues. Additionally, using 100ms for ticks does not cause
proposals to slow down, because they get sent out asap and don't
rely on ticks. So, setting this to 100ms instead of 20ms is a
NOOP.

(cherry picked from commit 4b41d9c)

conn/node.go

@@ -103,8 +103,8 @@ func NewNode(rc *pb.RaftContext, store *raftwal.DiskStorage) *Node {
 		Store:  store,
 		Cfg: &raft.Config{
 			ID:                       rc.Id,
-			ElectionTick:             100, // 2s if we call Tick() every 20 ms.
-			HeartbeatTick:            1,   // 20ms if we call Tick() every 20 ms.
+			ElectionTick:             20, // 2s if we call Tick() every 100 ms.
+			HeartbeatTick:            1,  // 100ms if we call Tick() every 100 ms.
 			Storage:                  store,
 			MaxInflightMsgs:          256,
 			MaxSizePerMsg:            256 << 10, // 256 KB should allow more batching.

worker/draft.go

@@ -702,7 +702,11 @@ func (n *node) Run() {
 	firstRun := true
 	var leader bool
 	// See also our configuration of HeartbeatTick and ElectionTick.
-	ticker := time.NewTicker(20 * time.Millisecond)
+	// Before we used to have 20ms ticks, but they would overload the Raft tick channel, causing
+	// "tick missed to fire" logs. Etcd uses 100ms and they haven't seen those issues.
+	// Additionally, using 100ms for ticks does not cause proposals to slow down, because they get
+	// sent out asap and don't rely on ticks. So, setting this to 100ms instead of 20ms is a NOOP.
+	ticker := time.NewTicker(100 * time.Millisecond)
 	defer ticker.Stop()
 
 	done := make(chan struct{})