@@ -44,34 +44,7 @@ const B: usize = 6;
44
44
pub const MIN_LEN : usize = B - 1 ;
45
45
pub const CAPACITY : usize = 2 * B - 1 ;
46
46
47
- /// The underlying representation of leaf nodes. Note that it is often unsafe to actually store
48
- /// these, since only the first `len` keys and values are assumed to be initialized. As such,
49
- /// these should always be put behind pointers, and specifically behind `BoxedNode` in the owned
50
- /// case.
51
- ///
52
- /// We have a separate type for the header and rely on it matching the prefix of `LeafNode`, in
53
- /// order to statically allocate a single dummy node to avoid allocations. This struct is
54
- /// `repr(C)` to prevent them from being reordered. `LeafNode` does not just contain a
55
- /// `NodeHeader` because we do not want unnecessary padding between `len` and the keys.
56
- /// Crucially, `NodeHeader` can be safely transmuted to different K and V. (This is exploited
57
- /// by `as_header`.)
58
- #[ repr( C ) ]
59
- struct NodeHeader < K , V > {
60
- /// We use `*const` as opposed to `*mut` so as to be covariant in `K` and `V`.
61
- /// This either points to an actual node or is null.
62
- parent : * const InternalNode < K , V > ,
63
-
64
- /// This node's index into the parent node's `edges` array.
65
- /// `*node.parent.edges[node.parent_idx]` should be the same thing as `node`.
66
- /// This is only guaranteed to be initialized when `parent` is non-null.
67
- parent_idx : MaybeUninit < u16 > ,
68
-
69
- /// The number of keys and values this node stores.
70
- ///
71
- /// This next to `parent_idx` to encourage the compiler to join `len` and
72
- /// `parent_idx` into the same 32-bit word, reducing space overhead.
73
- len : u16 ,
74
- }
47
+ /// The underlying representation of leaf nodes.
75
48
#[ repr( C ) ]
76
49
struct LeafNode < K , V > {
77
50
/// We use `*const` as opposed to `*mut` so as to be covariant in `K` and `V`.
@@ -111,21 +84,6 @@ impl<K, V> LeafNode<K, V> {
111
84
}
112
85
}
113
86
114
- impl < K , V > NodeHeader < K , V > {
115
- fn is_shared_root ( & self ) -> bool {
116
- ptr:: eq ( self , & EMPTY_ROOT_NODE as * const _ as * const _ )
117
- }
118
- }
119
-
120
- // We need to implement Sync here in order to make a static instance.
121
- unsafe impl Sync for NodeHeader < ( ) , ( ) > { }
122
-
123
- // An empty node used as a placeholder for the root node, to avoid allocations.
124
- // We use just a header in order to save space, since no operation on an empty tree will
125
- // ever take a pointer past the first key.
126
- static EMPTY_ROOT_NODE : NodeHeader < ( ) , ( ) > =
127
- NodeHeader { parent : ptr:: null ( ) , parent_idx : MaybeUninit :: uninit ( ) , len : 0 } ;
128
-
129
87
/// The underlying representation of internal nodes. As with `LeafNode`s, these should be hidden
130
88
/// behind `BoxedNode`s to prevent dropping uninitialized keys and values. Any pointer to an
131
89
/// `InternalNode` can be directly casted to a pointer to the underlying `LeafNode` portion of the
@@ -154,12 +112,9 @@ impl<K, V> InternalNode<K, V> {
154
112
}
155
113
156
114
/// A managed, non-null pointer to a node. This is either an owned pointer to
157
- /// `LeafNode<K, V>`, an owned pointer to `InternalNode<K, V>`, or a (not owned)
158
- /// pointer to `NodeHeader<(), ()` (more specifically, the pointer to EMPTY_ROOT_NODE).
159
- /// All of these types have a `NodeHeader<K, V>` prefix, meaning that they have at
160
- /// least the same size as `NodeHeader<K, V>` and store the same kinds of data at the same
161
- /// offsets; and they have a pointer alignment at least as large as `NodeHeader<K, V>`'s.
162
- /// However, `BoxedNode` contains no information as to which of the three types
115
+ /// `LeafNode<K, V>` or an owned pointer to `InternalNode<K, V>`.
116
+ ///
117
+ /// However, `BoxedNode` contains no information as to which of the two types
163
118
/// of nodes it actually contains, and, partially due to this lack of information,
164
119
/// has no destructor.
165
120
struct BoxedNode < K , V > {
@@ -184,8 +139,9 @@ impl<K, V> BoxedNode<K, V> {
184
139
}
185
140
}
186
141
187
- /// Either an owned tree or a shared, empty tree. Note that this does not have a destructor,
188
- /// and must be cleaned up manually if it is an owned tree.
142
+ /// An owned tree.
143
+ ///
144
+ /// Note that this does not have a destructor, and must be cleaned up manually.
189
145
pub struct Root < K , V > {
190
146
node : BoxedNode < K , V > ,
191
147
/// The number of levels below the root node.
@@ -196,20 +152,6 @@ unsafe impl<K: Sync, V: Sync> Sync for Root<K, V> {}
196
152
unsafe impl < K : Send , V : Send > Send for Root < K , V > { }
197
153
198
154
impl < K , V > Root < K , V > {
199
- /// Whether the instance of `Root` wraps a shared, empty root node. If not,
200
- /// the entire tree is uniquely owned by the owner of the `Root` instance.
201
- pub fn is_shared_root ( & self ) -> bool {
202
- self . as_ref ( ) . is_shared_root ( )
203
- }
204
-
205
- /// Returns a shared tree, wrapping a shared root node that is eternally empty.
206
- pub fn shared_empty_root ( ) -> Self {
207
- Root {
208
- node : unsafe { BoxedNode :: from_ptr ( NonNull :: from ( & EMPTY_ROOT_NODE ) . cast ( ) ) } ,
209
- height : 0 ,
210
- }
211
- }
212
-
213
155
/// Returns a new owned tree, with its own root node that is initially empty.
214
156
pub fn new_leaf ( ) -> Self {
215
157
Root { node : BoxedNode :: from_leaf ( Box :: new ( unsafe { LeafNode :: new ( ) } ) ) , height : 0 }
@@ -245,7 +187,6 @@ impl<K, V> Root<K, V> {
245
187
/// Adds a new internal node with a single edge, pointing to the previous root, and make that
246
188
/// new node the root. This increases the height by 1 and is the opposite of `pop_level`.
247
189
pub fn push_level ( & mut self ) -> NodeRef < marker:: Mut < ' _ > , K , V , marker:: Internal > {
248
- debug_assert ! ( !self . is_shared_root( ) ) ;
249
190
let mut new_node = Box :: new ( unsafe { InternalNode :: new ( ) } ) ;
250
191
new_node. edges [ 0 ] . write ( unsafe { BoxedNode :: from_ptr ( self . node . as_ptr ( ) ) } ) ;
251
192
@@ -308,11 +249,6 @@ impl<K, V> Root<K, V> {
308
249
/// `Leaf`, the `NodeRef` points to a leaf node, when this is `Internal` the
309
250
/// `NodeRef` points to an internal node, and when this is `LeafOrInternal` the
310
251
/// `NodeRef` could be pointing to either type of node.
311
- /// Note that in case of a leaf node, this might still be the shared root!
312
- /// Only turn this into a `LeafNode` reference if you know it is not the shared root!
313
- /// Shared references must be dereferenceable *for the entire size of their pointee*,
314
- /// so '&LeafNode` or `&InternalNode` pointing to the shared root is undefined behavior.
315
- /// Turning this into a `NodeHeader` reference is always safe.
316
252
pub struct NodeRef < BorrowType , K , V , Type > {
317
253
/// The number of levels below the node.
318
254
height : usize ,
@@ -354,7 +290,7 @@ impl<BorrowType, K, V, Type> NodeRef<BorrowType, K, V, Type> {
354
290
/// Note that, despite being safe, calling this function can have the side effect
355
291
/// of invalidating mutable references that unsafe code has created.
356
292
pub fn len ( & self ) -> usize {
357
- self . as_header ( ) . len as usize
293
+ self . as_leaf ( ) . len as usize
358
294
}
359
295
360
296
/// Returns the height of this node in the whole tree. Zero height denotes the
@@ -374,35 +310,24 @@ impl<BorrowType, K, V, Type> NodeRef<BorrowType, K, V, Type> {
374
310
NodeRef { height : self . height , node : self . node , root : self . root , _marker : PhantomData }
375
311
}
376
312
377
- /// Exposes the leaf "portion" of any leaf or internal node that is not the shared root .
313
+ /// Exposes the leaf "portion" of any leaf or internal node.
378
314
/// If the node is a leaf, this function simply opens up its data.
379
315
/// If the node is an internal node, so not a leaf, it does have all the data a leaf has
380
316
/// (header, keys and values), and this function exposes that.
381
- /// Unsafe because the node must not be the shared root. For more information,
382
- /// see the `NodeRef` comments.
383
- unsafe fn as_leaf ( & self ) -> & LeafNode < K , V > {
384
- debug_assert ! ( !self . is_shared_root( ) ) ;
385
- self . node . as_ref ( )
386
- }
387
-
388
- fn as_header ( & self ) -> & NodeHeader < K , V > {
389
- unsafe { & * ( self . node . as_ptr ( ) as * const NodeHeader < K , V > ) }
390
- }
391
-
392
- /// Returns whether the node is the shared, empty root.
393
- pub fn is_shared_root ( & self ) -> bool {
394
- self . as_header ( ) . is_shared_root ( )
317
+ fn as_leaf ( & self ) -> & LeafNode < K , V > {
318
+ // The node must be valid for at least the LeafNode portion.
319
+ // This is not a reference in the NodeRef type because we don't know if
320
+ // it should be unique or shared.
321
+ unsafe { self . node . as_ref ( ) }
395
322
}
396
323
397
324
/// Borrows a view into the keys stored in the node.
398
- /// Unsafe because the caller must ensure that the node is not the shared root.
399
- pub unsafe fn keys ( & self ) -> & [ K ] {
325
+ pub fn keys ( & self ) -> & [ K ] {
400
326
self . reborrow ( ) . into_key_slice ( )
401
327
}
402
328
403
329
/// Borrows a view into the values stored in the node.
404
- /// Unsafe because the caller must ensure that the node is not the shared root.
405
- unsafe fn vals ( & self ) -> & [ V ] {
330
+ fn vals ( & self ) -> & [ V ] {
406
331
self . reborrow ( ) . into_val_slice ( )
407
332
}
408
333
@@ -416,7 +341,7 @@ impl<BorrowType, K, V, Type> NodeRef<BorrowType, K, V, Type> {
416
341
pub fn ascend (
417
342
self ,
418
343
) -> Result < Handle < NodeRef < BorrowType , K , V , marker:: Internal > , marker:: Edge > , Self > {
419
- let parent_as_leaf = self . as_header ( ) . parent as * const LeafNode < K , V > ;
344
+ let parent_as_leaf = self . as_leaf ( ) . parent as * const LeafNode < K , V > ;
420
345
if let Some ( non_zero) = NonNull :: new ( parent_as_leaf as * mut _ ) {
421
346
Ok ( Handle {
422
347
node : NodeRef {
@@ -425,7 +350,7 @@ impl<BorrowType, K, V, Type> NodeRef<BorrowType, K, V, Type> {
425
350
root : self . root ,
426
351
_marker : PhantomData ,
427
352
} ,
428
- idx : unsafe { usize:: from ( * self . as_header ( ) . parent_idx . as_ptr ( ) ) } ,
353
+ idx : unsafe { usize:: from ( * self . as_leaf ( ) . parent_idx . as_ptr ( ) ) } ,
429
354
_marker : PhantomData ,
430
355
} )
431
356
} else {
@@ -464,7 +389,6 @@ impl<K, V> NodeRef<marker::Owned, K, V, marker::LeafOrInternal> {
464
389
pub unsafe fn deallocate_and_ascend (
465
390
self ,
466
391
) -> Option < Handle < NodeRef < marker:: Owned , K , V , marker:: Internal > , marker:: Edge > > {
467
- assert ! ( !self . is_shared_root( ) ) ;
468
392
let height = self . height ;
469
393
let node = self . node ;
470
394
let ret = self . ascend ( ) . ok ( ) ;
@@ -507,41 +431,37 @@ impl<'a, K, V, Type> NodeRef<marker::Mut<'a>, K, V, Type> {
507
431
/// (header, keys and values), and this function exposes that.
508
432
///
509
433
/// Returns a raw ptr to avoid asserting exclusive access to the entire node.
510
- /// This also implies you can invoke this member on the shared root, but the resulting pointer
511
- /// might not be properly aligned and definitely would not allow accessing keys and values.
512
434
fn as_leaf_mut ( & mut self ) -> * mut LeafNode < K , V > {
513
435
self . node . as_ptr ( )
514
436
}
515
437
516
- /// Unsafe because the caller must ensure that the node is not the shared root.
517
- unsafe fn keys_mut ( & mut self ) -> & mut [ K ] {
518
- self . reborrow_mut ( ) . into_key_slice_mut ( )
438
+ fn keys_mut ( & mut self ) -> & mut [ K ] {
439
+ // SAFETY: the caller will not be able to call further methods on self
440
+ // until the key slice reference is dropped, as we have unique access
441
+ // for the lifetime of the borrow.
442
+ unsafe { self . reborrow_mut ( ) . into_key_slice_mut ( ) }
519
443
}
520
444
521
- /// Unsafe because the caller must ensure that the node is not the shared root.
522
- unsafe fn vals_mut ( & mut self ) -> & mut [ V ] {
523
- self . reborrow_mut ( ) . into_val_slice_mut ( )
445
+ fn vals_mut ( & mut self ) -> & mut [ V ] {
446
+ // SAFETY: the caller will not be able to call further methods on self
447
+ // until the value slice reference is dropped, as we have unique access
448
+ // for the lifetime of the borrow.
449
+ unsafe { self . reborrow_mut ( ) . into_val_slice_mut ( ) }
524
450
}
525
451
}
526
452
527
453
impl < ' a , K : ' a , V : ' a , Type > NodeRef < marker:: Immut < ' a > , K , V , Type > {
528
- /// Unsafe because the caller must ensure that the node is not the shared root.
529
- unsafe fn into_key_slice ( self ) -> & ' a [ K ] {
530
- debug_assert ! ( !self . is_shared_root( ) ) ;
531
- // We cannot be the shared root, so `as_leaf` is okay.
532
- slice:: from_raw_parts ( MaybeUninit :: first_ptr ( & self . as_leaf ( ) . keys ) , self . len ( ) )
454
+ fn into_key_slice ( self ) -> & ' a [ K ] {
455
+ unsafe { slice:: from_raw_parts ( MaybeUninit :: first_ptr ( & self . as_leaf ( ) . keys ) , self . len ( ) ) }
533
456
}
534
457
535
- /// Unsafe because the caller must ensure that the node is not the shared root.
536
- unsafe fn into_val_slice ( self ) -> & ' a [ V ] {
537
- debug_assert ! ( !self . is_shared_root( ) ) ;
538
- // We cannot be the shared root, so `as_leaf` is okay.
539
- slice:: from_raw_parts ( MaybeUninit :: first_ptr ( & self . as_leaf ( ) . vals ) , self . len ( ) )
458
+ fn into_val_slice ( self ) -> & ' a [ V ] {
459
+ unsafe { slice:: from_raw_parts ( MaybeUninit :: first_ptr ( & self . as_leaf ( ) . vals ) , self . len ( ) ) }
540
460
}
541
461
542
- /// Unsafe because the caller must ensure that the node is not the shared root.
543
- unsafe fn into_slices ( self ) -> ( & ' a [ K ] , & ' a [ V ] ) {
544
- let k = ptr:: read ( & self ) ;
462
+ fn into_slices ( self ) -> ( & ' a [ K ] , & ' a [ V ] ) {
463
+ // SAFETY: equivalent to reborrow() except not requiring Type: 'a
464
+ let k = unsafe { ptr:: read ( & self ) } ;
545
465
( k. into_key_slice ( ) , self . into_val_slice ( ) )
546
466
}
547
467
}
@@ -553,37 +473,41 @@ impl<'a, K: 'a, V: 'a, Type> NodeRef<marker::Mut<'a>, K, V, Type> {
553
473
unsafe { & mut * ( self . root as * mut Root < K , V > ) }
554
474
}
555
475
556
- /// Unsafe because the caller must ensure that the node is not the shared root.
557
- unsafe fn into_key_slice_mut ( mut self ) -> & ' a mut [ K ] {
558
- debug_assert ! ( ! self . is_shared_root ( ) ) ;
559
- // We cannot be the shared root, so `as_leaf_mut` is okay.
560
- slice :: from_raw_parts_mut (
561
- MaybeUninit :: first_ptr_mut ( & mut ( * self . as_leaf_mut ( ) ) . keys ) ,
562
- self . len ( ) ,
563
- )
476
+ fn into_key_slice_mut ( mut self ) -> & ' a mut [ K ] {
477
+ // SAFETY: The keys of a node must always be initialized up to length.
478
+ unsafe {
479
+ slice :: from_raw_parts_mut (
480
+ MaybeUninit :: first_ptr_mut ( & mut ( * self . as_leaf_mut ( ) ) . keys ) ,
481
+ self . len ( ) ,
482
+ )
483
+ }
564
484
}
565
485
566
- /// Unsafe because the caller must ensure that the node is not the shared root.
567
- unsafe fn into_val_slice_mut ( mut self ) -> & ' a mut [ V ] {
568
- debug_assert ! ( !self . is_shared_root( ) ) ;
569
- slice:: from_raw_parts_mut (
570
- MaybeUninit :: first_ptr_mut ( & mut ( * self . as_leaf_mut ( ) ) . vals ) ,
571
- self . len ( ) ,
572
- )
486
+ fn into_val_slice_mut ( mut self ) -> & ' a mut [ V ] {
487
+ // SAFETY: The values of a node must always be initialized up to length.
488
+ unsafe {
489
+ slice:: from_raw_parts_mut (
490
+ MaybeUninit :: first_ptr_mut ( & mut ( * self . as_leaf_mut ( ) ) . vals ) ,
491
+ self . len ( ) ,
492
+ )
493
+ }
573
494
}
574
495
575
- /// Unsafe because the caller must ensure that the node is not the shared root.
576
- unsafe fn into_slices_mut ( mut self ) -> ( & ' a mut [ K ] , & ' a mut [ V ] ) {
577
- debug_assert ! ( !self . is_shared_root( ) ) ;
496
+ fn into_slices_mut ( mut self ) -> ( & ' a mut [ K ] , & ' a mut [ V ] ) {
578
497
// We cannot use the getters here, because calling the second one
579
498
// invalidates the reference returned by the first.
580
499
// More precisely, it is the call to `len` that is the culprit,
581
500
// because that creates a shared reference to the header, which *can*
582
501
// overlap with the keys (and even the values, for ZST keys).
583
502
let len = self . len ( ) ;
584
503
let leaf = self . as_leaf_mut ( ) ;
585
- let keys = slice:: from_raw_parts_mut ( MaybeUninit :: first_ptr_mut ( & mut ( * leaf) . keys ) , len) ;
586
- let vals = slice:: from_raw_parts_mut ( MaybeUninit :: first_ptr_mut ( & mut ( * leaf) . vals ) , len) ;
504
+ // SAFETY: The keys and values of a node must always be initialized up to length.
505
+ let keys = unsafe {
506
+ slice:: from_raw_parts_mut ( MaybeUninit :: first_ptr_mut ( & mut ( * leaf) . keys ) , len)
507
+ } ;
508
+ let vals = unsafe {
509
+ slice:: from_raw_parts_mut ( MaybeUninit :: first_ptr_mut ( & mut ( * leaf) . vals ) , len)
510
+ } ;
587
511
( keys, vals)
588
512
}
589
513
}
@@ -592,7 +516,6 @@ impl<'a, K, V> NodeRef<marker::Mut<'a>, K, V, marker::Leaf> {
592
516
/// Adds a key/value pair the end of the node.
593
517
pub fn push ( & mut self , key : K , val : V ) {
594
518
assert ! ( self . len( ) < CAPACITY ) ;
595
- debug_assert ! ( !self . is_shared_root( ) ) ;
596
519
597
520
let idx = self . len ( ) ;
598
521
@@ -607,7 +530,6 @@ impl<'a, K, V> NodeRef<marker::Mut<'a>, K, V, marker::Leaf> {
607
530
/// Adds a key/value pair to the beginning of the node.
608
531
pub fn push_front ( & mut self , key : K , val : V ) {
609
532
assert ! ( self . len( ) < CAPACITY ) ;
610
- debug_assert ! ( !self . is_shared_root( ) ) ;
611
533
612
534
unsafe {
613
535
slice_insert ( self . keys_mut ( ) , 0 , key) ;
@@ -624,7 +546,6 @@ impl<'a, K, V> NodeRef<marker::Mut<'a>, K, V, marker::Internal> {
624
546
pub fn push ( & mut self , key : K , val : V , edge : Root < K , V > ) {
625
547
assert ! ( edge. height == self . height - 1 ) ;
626
548
assert ! ( self . len( ) < CAPACITY ) ;
627
- debug_assert ! ( !self . is_shared_root( ) ) ;
628
549
629
550
let idx = self . len ( ) ;
630
551
@@ -658,7 +579,6 @@ impl<'a, K, V> NodeRef<marker::Mut<'a>, K, V, marker::Internal> {
658
579
pub fn push_front ( & mut self , key : K , val : V , edge : Root < K , V > ) {
659
580
assert ! ( edge. height == self . height - 1 ) ;
660
581
assert ! ( self . len( ) < CAPACITY ) ;
661
- debug_assert ! ( !self . is_shared_root( ) ) ;
662
582
663
583
unsafe {
664
584
slice_insert ( self . keys_mut ( ) , 0 , key) ;
@@ -744,8 +664,7 @@ impl<'a, K, V> NodeRef<marker::Mut<'a>, K, V, marker::LeafOrInternal> {
744
664
}
745
665
}
746
666
747
- /// Unsafe because the caller must ensure that the node is not the shared root.
748
- unsafe fn into_kv_pointers_mut ( mut self ) -> ( * mut K , * mut V ) {
667
+ fn into_kv_pointers_mut ( mut self ) -> ( * mut K , * mut V ) {
749
668
( self . keys_mut ( ) . as_mut_ptr ( ) , self . vals_mut ( ) . as_mut_ptr ( ) )
750
669
}
751
670
}
@@ -904,7 +823,6 @@ impl<'a, K, V> Handle<NodeRef<marker::Mut<'a>, K, V, marker::Leaf>, marker::Edge
904
823
fn insert_fit ( & mut self , key : K , val : V ) -> * mut V {
905
824
// Necessary for correctness, but in a private module
906
825
debug_assert ! ( self . node. len( ) < CAPACITY ) ;
907
- debug_assert ! ( !self . node. is_shared_root( ) ) ;
908
826
909
827
unsafe {
910
828
slice_insert ( self . node . keys_mut ( ) , self . idx , key) ;
@@ -1081,7 +999,6 @@ impl<'a, K, V> Handle<NodeRef<marker::Mut<'a>, K, V, marker::Leaf>, marker::KV>
1081
999
/// - All the key/value pairs to the right of this handle are put into a newly
1082
1000
/// allocated node.
1083
1001
pub fn split ( mut self ) -> ( NodeRef < marker:: Mut < ' a > , K , V , marker:: Leaf > , K , V , Root < K , V > ) {
1084
- assert ! ( !self . node. is_shared_root( ) ) ;
1085
1002
unsafe {
1086
1003
let mut new_node = Box :: new ( LeafNode :: new ( ) ) ;
1087
1004
@@ -1113,7 +1030,6 @@ impl<'a, K, V> Handle<NodeRef<marker::Mut<'a>, K, V, marker::Leaf>, marker::KV>
1113
1030
pub fn remove (
1114
1031
mut self ,
1115
1032
) -> ( Handle < NodeRef < marker:: Mut < ' a > , K , V , marker:: Leaf > , marker:: Edge > , K , V ) {
1116
- assert ! ( !self . node. is_shared_root( ) ) ;
1117
1033
unsafe {
1118
1034
let k = slice_remove ( self . node . keys_mut ( ) , self . idx ) ;
1119
1035
let v = slice_remove ( self . node . vals_mut ( ) , self . idx ) ;
0 commit comments