kv: Don't treat context cancellation as a SendError

SendErrors cause range descriptors to be evicted from the range cache,
but happen innocuously all the time as requests like HeartbeatTxns are
cancelled because they're no longer needed.

This was part of the cause of range 1 getting thousands of qps on 30
node TPC-C testing, as initially called out in the comments on cockroachdb#26608.

Release note: None

pkg/kv/dist_sender.go

@@ -34,6 +34,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/tracing"
+	"github.com/pkg/errors"
 )
 
 const (
@@ -1294,6 +1295,20 @@ func (ds *DistSender) sendToReplicas(
 			log.VErrEventf(ctx, 1, "application error: %s", br.Error)
 		}
 
+		// Has the caller given up?
+		if ctx.Err() != nil {
+			errMsg := fmt.Sprintf("context done during DistSender.Send: %s", ctx.Err())
+			log.Eventf(ctx, errMsg)
+			if ambiguousError != nil {
+				return nil, roachpb.NewAmbiguousResultError(errMsg)
+			}
+			// Don't consider this a SendError, because SendErrors indicate that we
+			// were unable to reach a replica that could serve the request, and they
+			// cause range cache evictions. Context cancellations just mean the
+			// sender changed its mind or the request timed out.
+			return nil, errors.New(errMsg)
+		}
+
 		if transport.IsExhausted() {
 			if ambiguousError != nil {
 				return nil, roachpb.NewAmbiguousResultError(fmt.Sprintf("error=%s [exhausted]", ambiguousError))
@@ -1308,16 +1323,6 @@ func (ds *DistSender) sendToReplicas(
 			)
 		}
 
-		// Has the caller given up?
-		if ctx.Err() != nil {
-			errMsg := fmt.Sprintf("context done during DistSender.Send: %s", ctx.Err())
-			log.Eventf(ctx, errMsg)
-			if ambiguousError != nil {
-				return nil, roachpb.NewAmbiguousResultError(errMsg)
-			}
-			return nil, roachpb.NewSendError(errMsg)
-		}
-
 		ds.metrics.NextReplicaErrCount.Inc(1)
 		log.VEventf(ctx, 2, "error: %v %v; trying next peer %s", br, err, transport.NextReplica())
 		br, err = transport.SendNext(ctx)