-
Notifications
You must be signed in to change notification settings - Fork 3.8k
/
replica.go
7212 lines (6611 loc) · 274 KB
/
replica.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
// Copyright 2014 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
package storage
import (
"bytes"
"context"
"fmt"
"math"
"math/rand"
"os"
"reflect"
"sort"
"strings"
"sync/atomic"
"time"
"unsafe"
"github.com/google/btree"
"github.com/kr/pretty"
opentracing "github.com/opentracing/opentracing-go"
"github.com/pkg/errors"
"go.etcd.io/etcd/raft"
"go.etcd.io/etcd/raft/raftpb"
"github.com/cockroachdb/cockroach/pkg/base"
"github.com/cockroachdb/cockroach/pkg/config"
"github.com/cockroachdb/cockroach/pkg/gossip"
"github.com/cockroachdb/cockroach/pkg/internal/client"
"github.com/cockroachdb/cockroach/pkg/keys"
"github.com/cockroachdb/cockroach/pkg/roachpb"
"github.com/cockroachdb/cockroach/pkg/settings"
"github.com/cockroachdb/cockroach/pkg/settings/cluster"
"github.com/cockroachdb/cockroach/pkg/storage/abortspan"
"github.com/cockroachdb/cockroach/pkg/storage/batcheval"
"github.com/cockroachdb/cockroach/pkg/storage/batcheval/result"
"github.com/cockroachdb/cockroach/pkg/storage/closedts/ctpb"
ctstorage "github.com/cockroachdb/cockroach/pkg/storage/closedts/storage"
"github.com/cockroachdb/cockroach/pkg/storage/engine"
"github.com/cockroachdb/cockroach/pkg/storage/engine/enginepb"
"github.com/cockroachdb/cockroach/pkg/storage/rangefeed"
"github.com/cockroachdb/cockroach/pkg/storage/rditer"
"github.com/cockroachdb/cockroach/pkg/storage/spanset"
"github.com/cockroachdb/cockroach/pkg/storage/split"
"github.com/cockroachdb/cockroach/pkg/storage/stateloader"
"github.com/cockroachdb/cockroach/pkg/storage/storagebase"
"github.com/cockroachdb/cockroach/pkg/storage/storagepb"
"github.com/cockroachdb/cockroach/pkg/storage/txnwait"
"github.com/cockroachdb/cockroach/pkg/util"
"github.com/cockroachdb/cockroach/pkg/util/encoding"
"github.com/cockroachdb/cockroach/pkg/util/envutil"
"github.com/cockroachdb/cockroach/pkg/util/hlc"
"github.com/cockroachdb/cockroach/pkg/util/humanizeutil"
"github.com/cockroachdb/cockroach/pkg/util/log"
"github.com/cockroachdb/cockroach/pkg/util/protoutil"
"github.com/cockroachdb/cockroach/pkg/util/retry"
"github.com/cockroachdb/cockroach/pkg/util/stop"
"github.com/cockroachdb/cockroach/pkg/util/syncutil"
"github.com/cockroachdb/cockroach/pkg/util/timeutil"
"github.com/cockroachdb/cockroach/pkg/util/tracing"
"github.com/cockroachdb/cockroach/pkg/util/uuid"
)
const (
// configGossipTTL is the time-to-live for configuration maps.
configGossipTTL = 0 // does not expire
// optimizePutThreshold is the minimum length of a contiguous run
// of batched puts or conditional puts, after which the constituent
// put operations will possibly be optimized by determining whether
// the key space being written is starting out empty.
optimizePutThreshold = 10
replicaChangeTxnName = "change-replica"
splitTxnName = "split"
mergeTxnName = "merge"
// MaxQuotaReplicaLivenessDuration is the maximum duration that a replica
// can remain inactive while still being counting against the range's
// available proposal quota.
MaxQuotaReplicaLivenessDuration = 10 * time.Second
defaultReplicaRaftMuWarnThreshold = 500 * time.Millisecond
)
var testingDisableQuiescence = envutil.EnvOrDefaultBool("COCKROACH_DISABLE_QUIESCENCE", false)
var syncRaftLog = settings.RegisterBoolSetting(
"kv.raft_log.synchronize",
"set to true to synchronize on Raft log writes to persistent storage ('false' risks data loss)",
true,
)
// MaxCommandSizeFloor is the minimum allowed value for the MaxCommandSize
// cluster setting.
const MaxCommandSizeFloor = 4 << 20 // 4MB
// MaxCommandSize wraps "kv.raft.command.max_size".
var MaxCommandSize = settings.RegisterValidatedByteSizeSetting(
"kv.raft.command.max_size",
"maximum size of a raft command",
64<<20,
func(size int64) error {
if size < MaxCommandSizeFloor {
return fmt.Errorf("max_size must be greater than %s", humanizeutil.IBytes(MaxCommandSizeFloor))
}
return nil
},
)
// FollowerReadsEnabled controls whether replicas attempt to serve follower
// reads. The closed timestamp machinery is unaffected by this, i.e. the same
// information is collected and passed around, regardless of the value of this
// setting.
var FollowerReadsEnabled = settings.RegisterBoolSetting(
"kv.closed_timestamp.follower_reads_enabled",
"allow (all) replicas to serve consistent historical reads based on closed timestamp information",
false,
)
type proposalRetryReason int
const (
proposalNoRetry proposalRetryReason = iota
// proposalIllegalLeaseIndex indicates the proposal failed to apply at
// a Lease index it was not legal for. The command should be retried.
proposalIllegalLeaseIndex
// proposalAmbiguousShouldBeReevaluated indicates that it's ambiguous whether
// the command was committed (and possibly even applied) or not. The command
// should be retried. However, the original proposal may have succeeded, so if
// the retry does not succeed, care must be taken to correctly inform the
// caller via an AmbiguousResultError.
proposalAmbiguousShouldBeReevaluated
// proposalErrorReproposing indicates that re-proposal
// failed. Because the original proposal may have succeeded, an
// AmbiguousResultError must be returned. The command should not be
// retried.
proposalErrorReproposing
// proposalRangeNoLongerExists indicates the proposal was for a
// range that no longer exists. Because the original proposal may
// have succeeded, an AmbiguousResultError must be returned. The
// command should not be retried.
proposalRangeNoLongerExists
)
// proposalResult indicates the result of a proposal. Exactly one of
// Reply, Err and ProposalRetry is set, and it represents the result of
// the proposal.
type proposalResult struct {
Reply *roachpb.BatchResponse
Err *roachpb.Error
ProposalRetry proposalRetryReason
Intents []result.IntentsWithArg
EndTxns []result.EndTxnIntents
}
// ReplicaChecksum contains progress on a replica checksum computation.
type ReplicaChecksum struct {
CollectChecksumResponse
// started is true if the checksum computation has started.
started bool
// If gcTimestamp is nonzero, GC this checksum after gcTimestamp. gcTimestamp
// is zero if and only if the checksum computation is in progress.
gcTimestamp time.Time
// This channel is closed after the checksum is computed, and is used
// as a notification.
notify chan struct{}
}
type atomicDescString struct {
strPtr unsafe.Pointer
}
// store atomically updates d.strPtr with the string representation of desc.
func (d *atomicDescString) store(replicaID roachpb.ReplicaID, desc *roachpb.RangeDescriptor) {
var buf bytes.Buffer
fmt.Fprintf(&buf, "%d/", desc.RangeID)
if replicaID == 0 {
fmt.Fprintf(&buf, "?:")
} else {
fmt.Fprintf(&buf, "%d:", replicaID)
}
if !desc.IsInitialized() {
buf.WriteString("{-}")
} else {
const maxRangeChars = 30
rngStr := keys.PrettyPrintRange(roachpb.Key(desc.StartKey), roachpb.Key(desc.EndKey), maxRangeChars)
buf.WriteString(rngStr)
}
str := buf.String()
atomic.StorePointer(&d.strPtr, unsafe.Pointer(&str))
}
// String returns the string representation of the range; since we are not
// using a lock, the copy might be inconsistent.
func (d *atomicDescString) String() string {
return *(*string)(atomic.LoadPointer(&d.strPtr))
}
// DestroyReason indicates if a replica is alive, destroyed, corrupted or pending destruction.
type DestroyReason int
const (
// The replica is alive.
destroyReasonAlive DestroyReason = iota
// The replica has been corrupted.
destroyReasonCorrupted
// The replica has been marked for GC, but hasn't been GCed yet.
destroyReasonRemovalPending
// The replica has been GCed.
destroyReasonRemoved
// The replica has been merged into its left-hand neighbor, but its left-hand
// neighbor hasn't yet subsumed it.
destroyReasonMergePending
)
type destroyStatus struct {
reason DestroyReason
err error
}
// IsAlive returns true when a replica is alive.
func (s destroyStatus) IsAlive() bool {
return s.reason == destroyReasonAlive
}
// RemovedOrCorrupt returns true if a replica has either been removed or is corrupted.
func (s destroyStatus) RemovedOrCorrupt() bool {
return (s.reason == destroyReasonCorrupted) || (s.reason == destroyReasonRemoved)
}
func (s *destroyStatus) Set(err error, reason DestroyReason) {
s.err = err
s.reason = reason
}
func (s *destroyStatus) Reset() {
s.Set(nil, destroyReasonAlive)
}
// A Replica is a contiguous keyspace with writes managed via an
// instance of the Raft consensus algorithm. Many ranges may exist
// in a store and they are unlikely to be contiguous. Ranges are
// independent units and are responsible for maintaining their own
// integrity by replacing failed replicas, splitting and merging
// as appropriate.
type Replica struct {
log.AmbientContext
// TODO(tschottdorf): Duplicates r.mu.state.desc.RangeID; revisit that.
RangeID roachpb.RangeID // Only set by the constructor
store *Store
abortSpan *abortspan.AbortSpan // Avoids anomalous reads after abort
txnWaitQueue *txnwait.Queue // Queues push txn attempts by txn ID
// leaseholderStats tracks all incoming BatchRequests to the replica and which
// localities they come from in order to aid in lease rebalancing decisions.
leaseholderStats *replicaStats
// writeStats tracks the number of keys written by applied raft commands
// in order to aid in replica rebalancing decisions.
writeStats *replicaStats
// creatingReplica is set when a replica is created as uninitialized
// via a raft message.
creatingReplica *roachpb.ReplicaDescriptor
// Held in read mode during read-only commands. Held in exclusive mode to
// prevent read-only commands from executing. Acquired before the embedded
// RWMutex.
readOnlyCmdMu syncutil.RWMutex
// rangeStr is a string representation of a RangeDescriptor that can be
// atomically read and updated without needing to acquire the replica.mu lock.
// All updates to state.Desc should be duplicated here.
rangeStr atomicDescString
// raftMu protects Raft processing the replica.
//
// Locking notes: Replica.raftMu < Replica.mu
raftMu struct {
syncutil.Mutex
// Note that there are two StateLoaders, in raftMu and mu,
// depending on which lock is being held.
stateLoader stateloader.StateLoader
// on-disk storage for sideloaded SSTables. nil when there's no ReplicaID.
sideloaded sideloadStorage
// rangefeed is an instance of a rangefeed Processor that is capable of
// routing rangefeed events to a set of subscribers. Will be nil if no
// subscribers are registered.
rangefeed *rangefeed.Processor
}
// Contains the lease history when enabled.
leaseHistory *leaseHistory
cmdQMu struct {
// Protects all fields in the cmdQMu struct.
//
// Locking notes: Replica.mu < Replica.cmdQMu
syncutil.Mutex
// Enforces at most one command is running per key(s) within each span
// scope. The globally-scoped component tracks user writes (i.e. all
// keys for which keys.Addr is the identity), the locally-scoped component
// the rest (e.g. RangeDescriptor, transaction record, Lease, ...).
// Commands with different accesses but the same scope are stored in the
// same component.
queues [spanset.NumSpanScope]*CommandQueue
}
mu struct {
// Protects all fields in the mu struct.
syncutil.RWMutex
// The destroyed status of a replica indicating if it's alive, corrupt,
// scheduled for destruction or has been GCed.
destroyStatus
// Is the range quiescent? Quiescent ranges are not Tick()'d and unquiesce
// whenever a Raft operation is performed.
quiescent bool
// mergeComplete is non-nil if a merge is in-progress, in which case any
// requests should be held until the completion of the merge is signaled by
// the closing of the channel.
mergeComplete chan struct{}
// The state of the Raft state machine.
state storagepb.ReplicaState
// Counter used for assigning lease indexes for proposals.
lastAssignedLeaseIndex uint64
// Last index/term persisted to the raft log (not necessarily
// committed). Note that lastTerm may be 0 (and thus invalid) even when
// lastIndex is known, in which case the term will have to be retrieved
// from the Raft log entry. Use the invalidLastTerm constant for this
// case.
lastIndex, lastTerm uint64
// The raft log index of a pending preemptive snapshot. Used to prohibit
// raft log truncation while a preemptive snapshot is in flight. A value of
// 0 indicates that there is no pending snapshot.
pendingSnapshotIndex uint64
// raftLogSize is the approximate size in bytes of the persisted raft log.
// On server restart, this value is assumed to be zero to avoid costly scans
// of the raft log. This will be correct when all log entries predating this
// process have been truncated.
raftLogSize int64
// raftLogLastCheckSize is the value of raftLogSize the last time the Raft
// log was checked for truncation or at the time of the last Raft log
// truncation.
raftLogLastCheckSize int64
// pendingLeaseRequest is used to coalesce RequestLease requests.
pendingLeaseRequest pendingLeaseRequest
// minLeaseProposedTS is the minimum acceptable lease.ProposedTS; only
// leases proposed after this timestamp can be used for proposing commands.
// This is used to protect against several hazards:
// - leases held (or even proposed) before a restart cannot be used after a
// restart. This is because:
// a) the command queue is wiped during the restart; there might be
// writes in flight that are not reflected in the new command queue. So,
// we need to synchronize all new reads with those old in-flight writes.
// Forcing acquisition of a new lease essentially flushes all the
// previous raft commands.
// b) a lease transfer might have been in progress at the time of the
// restart. Using the existing lease after the restart would break the
// transfer proposer's promise to not use the existing lease.
// - a lease cannot be used after a transfer is initiated. Moreover, even
// lease extension that were in flight at the time of the transfer cannot be
// used, if they eventually apply.
minLeaseProposedTS hlc.Timestamp
// A pointer to the zone config for this replica.
zone *config.ZoneConfig
// proposals stores the Raft in-flight commands which originated at
// this Replica, i.e. all commands for which propose has been called,
// but which have not yet applied.
//
// The *ProposalData in the map are "owned" by it. Elements from the
// map must only be referenced while Replica.mu is held, except if the
// element is removed from the map first. The notable exception is the
// contained RaftCommand, which we treat as immutable.
proposals map[storagebase.CmdIDKey]*ProposalData
internalRaftGroup *raft.RawNode
// The ID of the replica within the Raft group. May be 0 if the replica has
// been created from a preemptive snapshot (i.e. before being added to the
// Raft group). The replica ID will be non-zero whenever the replica is
// part of a Raft group.
replicaID roachpb.ReplicaID
// The minimum allowed ID for this replica. Initialized from
// RaftTombstone.NextReplicaID.
minReplicaID roachpb.ReplicaID
// The ID of the leader replica within the Raft group. Used to determine
// when the leadership changes.
leaderID roachpb.ReplicaID
// The most recently added replica for the range and when it was added.
// Used to determine whether a replica is new enough that we shouldn't
// penalize it for being slightly behind. These field gets cleared out once
// we know that the replica has caught up.
lastReplicaAdded roachpb.ReplicaID
lastReplicaAddedTime time.Time
// The most recently updated time for each follower of this range.
lastUpdateTimes map[roachpb.ReplicaID]time.Time
// The last seen replica descriptors from incoming Raft messages. These are
// stored so that the replica still knows the replica descriptors for itself
// and for its message recipients in the circumstances when its RangeDescriptor
// is out of date.
//
// Normally, a replica knows about the other replica descriptors for a
// range via the RangeDescriptor stored in Replica.mu.state.Desc. But that
// descriptor is only updated during a Split or ChangeReplicas operation.
// There are periods during a Replica's lifetime when that information is
// out of date:
//
// 1. When a replica is being newly created as the result of an incoming
// Raft message for it. This is the common case for ChangeReplicas and an
// uncommon case for Splits. The leader will be sending the replica
// messages and the replica needs to be able to respond before it can
// receive an updated range descriptor (via a snapshot,
// changeReplicasTrigger, or splitTrigger).
//
// 2. If the node containing a replica is partitioned or down while the
// replicas for the range are updated. When the node comes back up, other
// replicas may begin communicating with it and it needs to be able to
// respond. Unlike 1 where there is no range descriptor, in this situation
// the replica has a range descriptor but it is out of date. Note that a
// replica being removed from a node and then quickly re-added before the
// replica has been GC'd will also use the last seen descriptors. In
// effect, this is another path for which the replica's local range
// descriptor is out of date.
//
// The last seen replica descriptors are updated on receipt of every raft
// message via Replica.setLastReplicaDescriptors (see
// Store.HandleRaftRequest). These last seen descriptors are used when
// the replica's RangeDescriptor contains missing or out of date descriptors
// for a replica (see Replica.sendRaftMessage).
//
// Removing a replica from Store.mu.replicas is not a problem because
// when a replica is completely removed, it won't be recreated until
// there is another event that will repopulate the replicas map in the
// range descriptor. When it is temporarily dropped and recreated, the
// newly recreated replica will have a complete range descriptor.
lastToReplica, lastFromReplica roachpb.ReplicaDescriptor
// submitProposalFn can be set to mock out the propose operation.
submitProposalFn func(*ProposalData) error
// Computed checksum at a snapshot UUID.
checksums map[uuid.UUID]ReplicaChecksum
// proposalQuota is the quota pool maintained by the lease holder where
// incoming writes acquire quota from a fixed quota pool before going
// through. If there is no quota available, the write is throttled
// until quota is made available to the pool.
// Acquired quota for a given command is only released when all the
// replicas have persisted the corresponding entry into their logs.
proposalQuota *quotaPool
proposalQuotaBaseIndex uint64
// For command size based allocations we keep track of the sizes of all
// in-flight commands.
commandSizes map[storagebase.CmdIDKey]int
// Once the leader observes a proposal come 'out of Raft', we consult
// the 'commandSizes' map to determine the size of the associated
// command and add it to a queue of quotas we have yet to release back
// to the quota pool. We only do so when all replicas have persisted
// the corresponding entry into their logs.
quotaReleaseQueue []int
// Counts calls to Replica.tick()
ticks int
// Counts Raft messages refused due to queue congestion.
droppedMessages int
// Note that there are two replicaStateLoaders, in raftMu and mu,
// depending on which lock is being held.
stateLoader stateloader.StateLoader
// draining specifies whether this replica is draining. Raft leadership
// transfers due to a lease change will be attempted even if the target does
// not have all the log entries.
draining bool
}
// Split keeps information for load-based splitting.
splitMu struct {
syncutil.Mutex
lastReqTime time.Time // most recent time recorded by requests.
count int64 // reqs since last nanos
qps float64 // last reqs/s rate
splitFinder *split.Finder
}
unreachablesMu struct {
syncutil.Mutex
remotes map[roachpb.ReplicaID]struct{}
}
}
var _ batcheval.EvalContext = &Replica{}
// KeyRange is an interface type for the replicasByKey BTree, to compare
// Replica and ReplicaPlaceholder.
type KeyRange interface {
Desc() *roachpb.RangeDescriptor
rangeKeyItem
btree.Item
fmt.Stringer
}
var _ KeyRange = &Replica{}
// withRaftGroupLocked calls the supplied function with the (lazily
// initialized) Raft group. The supplied function should return true for the
// unquiesceAndWakeLeader argument if the replica should be unquiesced (and the
// leader awoken). See handleRaftReady for an instance of where this value
// varies.
//
// Requires that Replica.mu is held. Also requires that Replica.raftMu is held
// if either the caller can't guarantee that r.mu.internalRaftGroup != nil or
// the provided function requires Replica.raftMu.
func (r *Replica) withRaftGroupLocked(
mayCampaignOnWake bool, f func(r *raft.RawNode) (unquiesceAndWakeLeader bool, _ error),
) error {
if r.mu.destroyStatus.RemovedOrCorrupt() {
// Silently ignore all operations on destroyed replicas. We can't return an
// error here as all errors returned from this method are considered fatal.
return nil
}
if r.mu.replicaID == 0 {
// The replica's raft group has not yet been configured (i.e. the replica
// was created from a preemptive snapshot).
return nil
}
if r.mu.internalRaftGroup == nil {
r.raftMu.Mutex.AssertHeld()
ctx := r.AnnotateCtx(context.TODO())
raftGroup, err := raft.NewRawNode(newRaftConfig(
raft.Storage((*replicaRaftStorage)(r)),
uint64(r.mu.replicaID),
r.mu.state.RaftAppliedIndex,
r.store.cfg,
&raftLogger{ctx: ctx},
), nil)
if err != nil {
return err
}
r.mu.internalRaftGroup = raftGroup
if mayCampaignOnWake {
r.maybeCampaignOnWakeLocked(ctx)
}
}
// This wrapper function is a hack to add range IDs to stack traces
// using the same pattern as Replica.sendWithRangeID.
unquiesce, err := func(rangeID roachpb.RangeID, raftGroup *raft.RawNode) (bool, error) {
return f(raftGroup)
}(r.RangeID, r.mu.internalRaftGroup)
if unquiesce {
r.unquiesceAndWakeLeaderLocked()
}
return err
}
// withRaftGroup calls the supplied function with the (lazily initialized)
// Raft group. It acquires and releases the Replica lock, so r.mu must not be
// held (or acquired by the supplied function).
//
// If mayCampaignOnWake is true, the replica may initiate a raft
// election if it was previously in a dormant state. Most callers
// should set this to true, because the prevote feature minimizes the
// disruption from unnecessary elections. The exception is that we
// should not initiate an election while handling incoming raft
// messages (which may include MsgVotes from an election in progress,
// and this election would be disrupted if we started our own).
//
// Has the same requirement for Replica.raftMu as withRaftGroupLocked.
func (r *Replica) withRaftGroup(
mayCampaignOnWake bool, f func(r *raft.RawNode) (unquiesceAndWakeLeader bool, _ error),
) error {
r.mu.Lock()
defer r.mu.Unlock()
return r.withRaftGroupLocked(mayCampaignOnWake, f)
}
func shouldCampaignOnWake(
leaseStatus storagepb.LeaseStatus,
lease roachpb.Lease,
storeID roachpb.StoreID,
raftStatus raft.Status,
) bool {
// When waking up a range, campaign unless we know that another
// node holds a valid lease (this is most important after a split,
// when all replicas create their raft groups at about the same
// time, with a lease pre-assigned to one of them). Note that
// thanks to PreVote, unnecessary campaigns are not disruptive so
// we should err on the side of campaigining here.
anotherOwnsLease := leaseStatus.State == storagepb.LeaseState_VALID && !lease.OwnedBy(storeID)
// If we're already campaigning or know who the leader is, don't
// start a new term.
noLeader := raftStatus.RaftState == raft.StateFollower && raftStatus.Lead == 0
return !anotherOwnsLease && noLeader
}
// maybeCampaignOnWakeLocked is called when the range wakes from a
// dormant state (either the initial "raftGroup == nil" state or after
// being quiescent) and campaigns for raft leadership if appropriate.
func (r *Replica) maybeCampaignOnWakeLocked(ctx context.Context) {
// Raft panics if a node that is not currently a member of the
// group tries to campaign. That happens primarily when we apply
// preemptive snapshots.
if _, currentMember := r.mu.state.Desc.GetReplicaDescriptorByID(r.mu.replicaID); !currentMember {
return
}
leaseStatus := r.leaseStatus(*r.mu.state.Lease, r.store.Clock().Now(), r.mu.minLeaseProposedTS)
raftStatus := r.mu.internalRaftGroup.Status()
if shouldCampaignOnWake(leaseStatus, *r.mu.state.Lease, r.store.StoreID(), *raftStatus) {
log.VEventf(ctx, 3, "campaigning")
if err := r.mu.internalRaftGroup.Campaign(); err != nil {
log.VEventf(ctx, 1, "failed to campaign: %s", err)
}
}
}
var _ client.Sender = &Replica{}
func newReplica(rangeID roachpb.RangeID, store *Store) *Replica {
r := &Replica{
AmbientContext: store.cfg.AmbientCtx,
RangeID: rangeID,
store: store,
abortSpan: abortspan.New(rangeID),
txnWaitQueue: txnwait.NewQueue(store),
}
r.mu.pendingLeaseRequest = makePendingLeaseRequest(r)
r.mu.stateLoader = stateloader.Make(r.store.cfg.Settings, rangeID)
r.mu.quiescent = true
r.mu.zone = config.DefaultZoneConfigRef()
if leaseHistoryMaxEntries > 0 {
r.leaseHistory = newLeaseHistory()
}
if store.cfg.StorePool != nil {
r.leaseholderStats = newReplicaStats(store.Clock(), store.cfg.StorePool.getNodeLocalityString)
}
// Pass nil for the localityOracle because we intentionally don't track the
// origin locality of write load.
r.writeStats = newReplicaStats(store.Clock(), nil)
// Init rangeStr with the range ID.
r.rangeStr.store(0, &roachpb.RangeDescriptor{RangeID: rangeID})
// Add replica log tag - the value is rangeStr.String().
r.AmbientContext.AddLogTag("r", &r.rangeStr)
// Add replica pointer value. NB: this was historically useful for debugging
// replica GC issues, but is a distraction at the moment.
// r.AmbientContext.AddLogTagStr("@", fmt.Sprintf("%x", unsafe.Pointer(r)))
r.raftMu.stateLoader = stateloader.Make(r.store.cfg.Settings, rangeID)
return r
}
// NewReplica initializes the replica using the given metadata. If the
// replica is initialized (i.e. desc contains more than a RangeID),
// replicaID should be 0 and the replicaID will be discovered from the
// descriptor.
func NewReplica(
desc *roachpb.RangeDescriptor, store *Store, replicaID roachpb.ReplicaID,
) (*Replica, error) {
r := newReplica(desc.RangeID, store)
return r, r.init(desc, store.Clock(), replicaID)
}
func (r *Replica) init(
desc *roachpb.RangeDescriptor, clock *hlc.Clock, replicaID roachpb.ReplicaID,
) error {
r.raftMu.Lock()
defer r.raftMu.Unlock()
r.mu.Lock()
defer r.mu.Unlock()
return r.initRaftMuLockedReplicaMuLocked(desc, clock, replicaID)
}
func (r *Replica) initRaftMuLockedReplicaMuLocked(
desc *roachpb.RangeDescriptor, clock *hlc.Clock, replicaID roachpb.ReplicaID,
) error {
ctx := r.AnnotateCtx(context.TODO())
if r.mu.state.Desc != nil && r.isInitializedRLocked() {
log.Fatalf(ctx, "r%d: cannot reinitialize an initialized replica", desc.RangeID)
}
if desc.IsInitialized() && replicaID != 0 {
return errors.Errorf("replicaID must be 0 when creating an initialized replica")
}
r.cmdQMu.Lock()
r.cmdQMu.queues[spanset.SpanGlobal] = NewCommandQueue(true /* optimizeOverlap */)
r.cmdQMu.queues[spanset.SpanLocal] = NewCommandQueue(false /* optimizeOverlap */)
r.cmdQMu.Unlock()
r.mu.proposals = map[storagebase.CmdIDKey]*ProposalData{}
r.mu.checksums = map[uuid.UUID]ReplicaChecksum{}
// Clear the internal raft group in case we're being reset. Since we're
// reloading the raft state below, it isn't safe to use the existing raft
// group.
r.mu.internalRaftGroup = nil
var err error
if r.mu.state, err = r.mu.stateLoader.Load(ctx, r.store.Engine(), desc); err != nil {
return err
}
// Init the minLeaseProposedTS such that we won't use an existing lease (if
// any). This is so that, after a restart, we don't propose under old leases.
// If the replica is being created through a split, this value will be
// overridden.
if !r.store.cfg.TestingKnobs.DontPreventUseOfOldLeaseOnStart {
// Only do this if there was a previous lease. This shouldn't be important
// to do but consider that the first lease which is obtained is back-dated
// to a zero start timestamp (and this de-flakes some tests). If we set the
// min proposed TS here, this lease could not be renewed (by the semantics
// of minLeaseProposedTS); and since minLeaseProposedTS is copied on splits,
// this problem would multiply to a number of replicas at cluster bootstrap.
// Instead, we make the first lease special (which is OK) and the problem
// disappears.
if r.mu.state.Lease.Sequence > 0 {
r.mu.minLeaseProposedTS = clock.Now()
}
}
r.rangeStr.store(0, r.mu.state.Desc)
r.mu.lastIndex, err = r.mu.stateLoader.LoadLastIndex(ctx, r.store.Engine())
if err != nil {
return err
}
r.mu.lastTerm = invalidLastTerm
pErr, err := r.mu.stateLoader.LoadReplicaDestroyedError(ctx, r.store.Engine())
if err != nil {
return err
}
if r.mu.destroyStatus.RemovedOrCorrupt() {
if err := pErr.GetDetail(); err != nil {
r.mu.destroyStatus.Set(err, destroyReasonRemoved)
}
}
if replicaID == 0 {
repDesc, ok := desc.GetReplicaDescriptor(r.store.StoreID())
if !ok {
// This is intentionally not an error and is the code path exercised
// during preemptive snapshots. The replica ID will be sent when the
// actual raft replica change occurs.
return nil
}
replicaID = repDesc.ReplicaID
}
r.rangeStr.store(replicaID, r.mu.state.Desc)
if err := r.setReplicaIDRaftMuLockedMuLocked(replicaID); err != nil {
return err
}
r.assertStateLocked(ctx, r.store.Engine())
return nil
}
// String returns the string representation of the replica using an
// inconsistent copy of the range descriptor. Therefore, String does not
// require a lock and its output may not be atomic with other ongoing work in
// the replica. This is done to prevent deadlocks in logging sites.
func (r *Replica) String() string {
return fmt.Sprintf("[n%d,s%d,r%s]", r.store.Ident.NodeID, r.store.Ident.StoreID, &r.rangeStr)
}
func (r *Replica) preDestroyRaftMuLocked(
ctx context.Context,
reader engine.Reader,
batch engine.Batch,
nextReplicaID roachpb.ReplicaID,
destroyData bool,
) error {
desc := r.Desc()
err := clearRangeData(ctx, desc, reader, batch, destroyData)
if err != nil {
return err
}
// Save a tombstone to ensure that replica IDs never get reused.
//
// NB: Legacy tombstones (which are in the replicated key space) are wiped
// in clearRangeData, but that's OK since we're writing a new one in the same
// batch (and in particular, sequenced *after* the wipe).
return r.setTombstoneKey(ctx, batch, nextReplicaID)
}
func (r *Replica) postDestroyRaftMuLocked(ctx context.Context, ms enginepb.MVCCStats) error {
// Suggest the cleared range to the compactor queue.
//
// TODO(benesch): we would ideally atomically suggest the compaction with
// the deletion of the data itself.
desc := r.Desc()
r.store.compactor.Suggest(ctx, storagepb.SuggestedCompaction{
StartKey: roachpb.Key(desc.StartKey),
EndKey: roachpb.Key(desc.EndKey),
Compaction: storagepb.Compaction{
Bytes: ms.Total(),
SuggestedAtNanos: timeutil.Now().UnixNano(),
},
})
// NB: we need the nil check below because it's possible that we're GC'ing a
// Replica without a replicaID, in which case it does not have a sideloaded
// storage.
//
// TODO(tschottdorf): at node startup, we should remove all on-disk
// directories belonging to replicas which aren't present. A crash before a
// call to postDestroyRaftMuLocked will currently leave the files around
// forever.
if r.raftMu.sideloaded != nil {
return r.raftMu.sideloaded.Clear(ctx)
}
return nil
}
// destroyRaftMuLocked deletes data associated with a replica, leaving a
// tombstone. If `destroyData` is true, data in all of the range's keyspaces
// will be deleted. Otherwise, only data in the range-ID local keyspace will be
// deleted. Requires that Replica.raftMu is held.
func (r *Replica) destroyRaftMuLocked(ctx context.Context, nextReplicaID roachpb.ReplicaID) error {
startTime := timeutil.Now()
ms := r.GetMVCCStats()
const destroyData = true
batch := r.Engine().NewWriteOnlyBatch()
defer batch.Close()
if err := r.preDestroyRaftMuLocked(ctx, r.Engine(), batch, nextReplicaID, destroyData); err != nil {
return err
}
preTime := timeutil.Now()
// We need to sync here because we are potentially deleting sideloaded
// proposals from the file system next. We could write the tombstone only in
// a synchronous batch first and then delete the data alternatively, but
// then need to handle the case in which there is both the tombstone and
// leftover replica data.
if err := batch.Commit(true); err != nil {
return err
}
commitTime := timeutil.Now()
if err := r.postDestroyRaftMuLocked(ctx, ms); err != nil {
return err
}
log.Infof(ctx, "removed %d (%d+%d) keys in %0.0fms [clear=%0.0fms commit=%0.0fms]",
ms.KeyCount+ms.SysCount, ms.KeyCount, ms.SysCount,
commitTime.Sub(startTime).Seconds()*1000,
preTime.Sub(startTime).Seconds()*1000,
commitTime.Sub(preTime).Seconds()*1000)
return nil
}
func (r *Replica) cancelPendingCommandsLocked() {
r.mu.AssertHeld()
for _, p := range r.mu.proposals {
r.cleanupFailedProposalLocked(p)
// NB: each proposal needs its own version of the error (i.e. don't try to
// share the error across proposals).
p.finishApplication(proposalResult{
Err: roachpb.NewError(roachpb.NewAmbiguousResultError("removing replica")),
ProposalRetry: proposalRangeNoLongerExists,
})
}
}
// cleanupFailedProposalLocked cleans up after a proposal that has failed. It
// clears any references to the proposal and releases associated quota.
func (r *Replica) cleanupFailedProposalLocked(p *ProposalData) {
// Clear the proposal from the proposals map. May be a no-op if the
// proposal has not yet been inserted into the map.
delete(r.mu.proposals, p.idKey)
// Release associated quota pool resources if we have been tracking
// this command.
//
// NB: We may be double free-ing here in cases where proposals are
// duplicated. To counter this our quota pool is capped at the initial
// quota size.
if cmdSize, ok := r.mu.commandSizes[p.idKey]; ok {
r.mu.proposalQuota.add(int64(cmdSize))
delete(r.mu.commandSizes, p.idKey)
}
}
// setTombstoneKey writes a tombstone to disk to ensure that replica IDs never
// get reused. It determines what the minimum next replica ID can be using
// the provided nextReplicaID and the Replica's own ID.
//
// We have to be careful to set the right key, since a replica can be using an
// ID that it hasn't yet received a RangeDescriptor for if it receives raft
// requests for that replica ID (as seen in #14231).
func (r *Replica) setTombstoneKey(
ctx context.Context, eng engine.ReadWriter, externalNextReplicaID roachpb.ReplicaID,
) error {
r.mu.Lock()
nextReplicaID := r.mu.state.Desc.NextReplicaID
if nextReplicaID < externalNextReplicaID {
nextReplicaID = externalNextReplicaID
}
if nextReplicaID > r.mu.minReplicaID {
r.mu.minReplicaID = nextReplicaID
}
r.mu.Unlock()
tombstoneKey := keys.RaftTombstoneKey(r.RangeID)
tombstone := &roachpb.RaftTombstone{
NextReplicaID: nextReplicaID,
}
return engine.MVCCPutProto(ctx, eng, nil, tombstoneKey,
hlc.Timestamp{}, nil, tombstone)
}
func (r *Replica) setReplicaID(replicaID roachpb.ReplicaID) error {
r.raftMu.Lock()
defer r.raftMu.Unlock()
r.mu.Lock()
defer r.mu.Unlock()
return r.setReplicaIDRaftMuLockedMuLocked(replicaID)
}
func (r *Replica) setReplicaIDRaftMuLockedMuLocked(replicaID roachpb.ReplicaID) error {
if r.mu.replicaID == replicaID {
// The common case: the replica ID is unchanged.
return nil
}
if replicaID == 0 {
// If the incoming message does not have a new replica ID it is a
// preemptive snapshot. We'll update minReplicaID if the snapshot is
// accepted.
return nil
}
if replicaID < r.mu.minReplicaID {
return &roachpb.RaftGroupDeletedError{}
}
if r.mu.replicaID > replicaID {
return errors.Errorf("replicaID cannot move backwards from %d to %d", r.mu.replicaID, replicaID)
}
if r.mu.destroyStatus.reason == destroyReasonRemovalPending {
// An earlier incarnation of this replica was removed, but apparently it has been re-added
// now, so reset the status.
r.mu.destroyStatus.Reset()
}
// if r.mu.replicaID != 0 {
// // TODO(bdarnell): clean up previous raftGroup (update peers)
// }
// Initialize or update the sideloaded storage. If the sideloaded storage
// already exists (which is iff the previous replicaID was non-zero), then
// we have to move the contained files over (this corresponds to the case in
// which our replica is removed and re-added to the range, without having
// the replica GC'ed in the meantime).
//
// Note that we can't race with a concurrent replicaGC here because both that
// and this is under raftMu.
var prevSideloadedDir string
if ss := r.raftMu.sideloaded; ss != nil {
prevSideloadedDir = ss.Dir()
}
var err error
if r.raftMu.sideloaded, err = newDiskSideloadStorage(
r.store.cfg.Settings,
r.mu.state.Desc.RangeID,
replicaID,
r.store.Engine().GetAuxiliaryDir(),
r.store.limiters.BulkIOWriteRate,
r.store.engine,
); err != nil {
return errors.Wrap(err, "while initializing sideloaded storage")
}
if prevSideloadedDir != "" {
if _, err := os.Stat(prevSideloadedDir); err != nil {