Merge pull request #88255 from tbg/backport22.1-86645

pkg/kv/kvserver/raft_transport.go

@@ -605,6 +605,9 @@ func (t *RaftTransport) SendAsync(
  }
  return true
  default:
+ if logRaftSendQueueFullEvery.ShouldLog() {
+ log.Warningf(t.AnnotateCtx(context.Background()), "raft send queue to n%d is full", toNodeID)
+ }
  releaseRaftMessageRequest(req)
  return false
  }

pkg/kv/kvserver/store_raft.go

@@ -28,6 +28,11 @@ import (
  "go.etcd.io/etcd/raft/v3/raftpb"
 )
 
+var (
+ logRaftRecvQueueFullEvery = log.Every(1 * time.Second)
+ logRaftSendQueueFullEvery = log.Every(1 * time.Second)
+)
+
 type raftRequestInfo struct {
  req *kvserverpb.RaftMessageRequest
  respStream RaftMessageResponseStream
@@ -177,6 +182,9 @@ func (s *Store) HandleRaftUncoalescedRequest(
  // TODO(peter): Return an error indicating the request was dropped. Note
  // that dropping the request is safe. Raft will retry.
  s.metrics.RaftRcvdMsgDropped.Inc(1)
+ if logRaftRecvQueueFullEvery.ShouldLog() {
+ log.Warningf(ctx, "raft receive queue for r%d is full", req.RangeID)
+ }
  return false
  }
  q.infos = append(q.infos, raftRequestInfo{
@@ -540,9 +548,9 @@ func (s *Store) processTick(_ context.Context, rangeID roachpb.RangeID) bool {
 // See the comment in shouldFollowerQuiesceOnNotify for details on how these two
 // functions combine to provide the guarantee that:
 //
-// If a quorum of replica in a Raft group is alive and at least
-// one of these replicas is up-to-date, the Raft group will catch
-// up any of the live, lagging replicas.
+// If a quorum of replica in a Raft group is alive and at least
+// one of these replicas is up-to-date, the Raft group will catch
+// up any of the live, lagging replicas.
 //
 // Note that this mechanism can race with concurrent invocations of processTick,
 // which may have a copy of the previous livenessMap where the now-live node is