raft: Add a Cancel method that interrupts proposals

Shutting down the manager can deadlock if a component is waiting for
store updates to go through. This Cancel method allows current and
future proposals to be interrupted to avoid those deadlocks. Once
everything using raft has shut down, the Stop method can be used to
complete raft shutdown.

Signed-off-by: Aaron Lehmann <aaron.lehmann@docker.com>

manager/manager.go

@@ -467,6 +467,8 @@ func (m *Manager) Stop(ctx context.Context, clearData bool) {
 		close(localSrvDone)
 	}()
 
+	m.raftNode.Cancel()
+
 	m.dispatcher.Stop()
 	m.logbroker.Stop()
 	m.caserver.Stop()

manager/state/raft/raft.go

@@ -115,6 +115,7 @@ type Node struct {
 	// RemovedFromRaft notifies about node deletion from raft cluster
 	RemovedFromRaft     chan struct{}
 	removeRaftFunc      func()
+	cancelFunc          func()
 	leadershipBroadcast *watch.Queue
 
 	// used to coordinate shutdown
@@ -227,6 +228,15 @@ func NewNode(opts NodeOptions) *Node {
 		}
 	}(n)
 
+	n.cancelFunc = func(n *Node) func() {
+		var cancelOnce sync.Once
+		return func() {
+			cancelOnce.Do(func() {
+				close(n.stopped)
+			})
+		}
+	}(n)
+
 	return n
 }
 
@@ -604,6 +614,15 @@ func (n *Node) getCurrentRaftConfig() api.RaftConfig {
 	return raftConfig
 }
 
+// Cancel interrupts all ongoing proposals, and prevents new ones from
+// starting. This is useful for the shutdown sequence because it allows
+// the manager to shut down raft-dependent services that might otherwise
+// block on shutdown if quorum isn't met. Then the raft node can be completely
+// shut down once no more code is using it.
+func (n *Node) Cancel() {
+	n.cancelFunc()
+}
+
 // Done returns channel which is closed when raft node is fully stopped.
 func (n *Node) Done() <-chan struct{} {
 	return n.doneCh
@@ -613,8 +632,7 @@ func (n *Node) stop(ctx context.Context) {
 	n.stopMu.Lock()
 	defer n.stopMu.Unlock()
 
-	close(n.stopped)
-
+	n.Cancel()
 	n.waitProp.Wait()
 	n.asyncTasks.Wait()