From da612213e622996d8391df9cddea20d0e7a4aac2 Mon Sep 17 00:00:00 2001 From: Stein Somers Date: Mon, 19 Oct 2020 10:23:11 +0200 Subject: [PATCH 1/3] BTreeMap: reuse NodeRef as Root, keep BoxedNode confined to edges --- library/alloc/src/collections/btree/map.rs | 2 +- library/alloc/src/collections/btree/mem.rs | 34 ++++++ library/alloc/src/collections/btree/mod.rs | 1 + .../alloc/src/collections/btree/navigate.rs | 47 ++------ library/alloc/src/collections/btree/node.rs | 112 +++++++++--------- .../alloc/src/collections/btree/node/tests.rs | 4 +- src/etc/gdb_providers.py | 15 ++- 7 files changed, 110 insertions(+), 105 deletions(-) create mode 100644 library/alloc/src/collections/btree/mem.rs diff --git a/library/alloc/src/collections/btree/map.rs b/library/alloc/src/collections/btree/map.rs index 20c6ebd22928b..882f37d19cb6c 100644 --- a/library/alloc/src/collections/btree/map.rs +++ b/library/alloc/src/collections/btree/map.rs @@ -1414,7 +1414,7 @@ impl IntoIterator for BTreeMap { fn into_iter(self) -> IntoIter { let mut me = ManuallyDrop::new(self); if let Some(root) = me.root.take() { - let (f, b) = root.into_ref().full_range(); + let (f, b) = root.full_range(); IntoIter { front: Some(f), back: Some(b), length: me.length } } else { diff --git a/library/alloc/src/collections/btree/mem.rs b/library/alloc/src/collections/btree/mem.rs new file mode 100644 index 0000000000000..5e7d9fa3f91ba --- /dev/null +++ b/library/alloc/src/collections/btree/mem.rs @@ -0,0 +1,34 @@ +use core::intrinsics; +use core::mem; +use core::ptr; + +/// This replaces the value behind the `v` unique reference by calling the +/// relevant function. +/// +/// If a panic occurs in the `change` closure, the entire process will be aborted. +#[inline] +pub fn take_mut(v: &mut T, change: impl FnOnce(T) -> T) { + replace(v, |value| (change(value), ())) +} + +/// This replaces the value behind the `v` unique reference by calling the +/// relevant function, and returns a result obtained along the way. +/// +/// If a panic occurs in the `change` closure, the entire process will be aborted. +#[inline] +pub fn replace(v: &mut T, change: impl FnOnce(T) -> (T, R)) -> R { + struct PanicGuard; + impl Drop for PanicGuard { + fn drop(&mut self) { + intrinsics::abort() + } + } + let guard = PanicGuard; + let value = unsafe { ptr::read(v) }; + let (new_value, ret) = change(value); + unsafe { + ptr::write(v, new_value); + } + mem::forget(guard); + ret +} diff --git a/library/alloc/src/collections/btree/mod.rs b/library/alloc/src/collections/btree/mod.rs index bcc50ed561587..8e1ba175fba03 100644 --- a/library/alloc/src/collections/btree/mod.rs +++ b/library/alloc/src/collections/btree/mod.rs @@ -1,5 +1,6 @@ mod borrow; pub mod map; +mod mem; mod navigate; mod node; mod remove; diff --git a/library/alloc/src/collections/btree/navigate.rs b/library/alloc/src/collections/btree/navigate.rs index 55ce7d275464e..de78148fc82be 100644 --- a/library/alloc/src/collections/btree/navigate.rs +++ b/library/alloc/src/collections/btree/navigate.rs @@ -1,7 +1,5 @@ use core::borrow::Borrow; use core::cmp::Ordering; -use core::intrinsics; -use core::mem; use core::ops::Bound::{Excluded, Included, Unbounded}; use core::ops::RangeBounds; use core::ptr; @@ -304,37 +302,6 @@ macro_rules! def_next_kv_uncheched_dealloc { def_next_kv_uncheched_dealloc! {unsafe fn next_kv_unchecked_dealloc: right_kv} def_next_kv_uncheched_dealloc! {unsafe fn next_back_kv_unchecked_dealloc: left_kv} -/// This replaces the value behind the `v` unique reference by calling the -/// relevant function. -/// -/// If a panic occurs in the `change` closure, the entire process will be aborted. -#[inline] -fn take_mut(v: &mut T, change: impl FnOnce(T) -> T) { - replace(v, |value| (change(value), ())) -} - -/// This replaces the value behind the `v` unique reference by calling the -/// relevant function, and returns a result obtained along the way. -/// -/// If a panic occurs in the `change` closure, the entire process will be aborted. -#[inline] -fn replace(v: &mut T, change: impl FnOnce(T) -> (T, R)) -> R { - struct PanicGuard; - impl Drop for PanicGuard { - fn drop(&mut self) { - intrinsics::abort() - } - } - let guard = PanicGuard; - let value = unsafe { ptr::read(v) }; - let (new_value, ret) = change(value); - unsafe { - ptr::write(v, new_value); - } - mem::forget(guard); - ret -} - impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge> { /// Moves the leaf edge handle to the next leaf edge and returns references to the /// key and value in between. @@ -342,7 +309,7 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Ed /// # Safety /// There must be another KV in the direction travelled. pub unsafe fn next_unchecked(&mut self) -> (&'a K, &'a V) { - replace(self, |leaf_edge| { + super::mem::replace(self, |leaf_edge| { let kv = leaf_edge.next_kv(); let kv = unsafe { unwrap_unchecked(kv.ok()) }; (kv.next_leaf_edge(), kv.into_kv()) @@ -355,7 +322,7 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Ed /// # Safety /// There must be another KV in the direction travelled. pub unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a V) { - replace(self, |leaf_edge| { + super::mem::replace(self, |leaf_edge| { let kv = leaf_edge.next_back_kv(); let kv = unsafe { unwrap_unchecked(kv.ok()) }; (kv.next_back_leaf_edge(), kv.into_kv()) @@ -370,7 +337,7 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::E /// # Safety /// There must be another KV in the direction travelled. pub unsafe fn next_unchecked(&mut self) -> (&'a K, &'a mut V) { - let kv = replace(self, |leaf_edge| { + let kv = super::mem::replace(self, |leaf_edge| { let kv = leaf_edge.next_kv(); let kv = unsafe { unwrap_unchecked(kv.ok()) }; (unsafe { ptr::read(&kv) }.next_leaf_edge(), kv) @@ -385,7 +352,7 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::E /// # Safety /// There must be another KV in the direction travelled. pub unsafe fn next_back_unchecked(&mut self) -> (&'a K, &'a mut V) { - let kv = replace(self, |leaf_edge| { + let kv = super::mem::replace(self, |leaf_edge| { let kv = leaf_edge.next_back_kv(); let kv = unsafe { unwrap_unchecked(kv.ok()) }; (unsafe { ptr::read(&kv) }.next_back_leaf_edge(), kv) @@ -401,7 +368,7 @@ impl<'a, K, V> Handle, K, V, marker::Leaf>, marker::Edge /// # Safety /// There must be another KV in the direction travelled. pub unsafe fn move_next_unchecked(&mut self) { - take_mut(self, |leaf_edge| { + super::mem::take_mut(self, |leaf_edge| { let kv = leaf_edge.next_kv(); let kv = unsafe { unwrap_unchecked(kv.ok()) }; kv.next_leaf_edge() @@ -423,7 +390,7 @@ impl Handle, marker::Edge> { /// call this method again subject to its safety conditions, or call counterpart /// `next_back_unchecked` subject to its safety conditions. pub unsafe fn next_unchecked(&mut self) -> (K, V) { - replace(self, |leaf_edge| { + super::mem::replace(self, |leaf_edge| { let kv = unsafe { next_kv_unchecked_dealloc(leaf_edge) }; let k = unsafe { ptr::read(kv.reborrow().into_kv().0) }; let v = unsafe { ptr::read(kv.reborrow().into_kv().1) }; @@ -444,7 +411,7 @@ impl Handle, marker::Edge> { /// call this method again subject to its safety conditions, or call counterpart /// `next_unchecked` subject to its safety conditions. pub unsafe fn next_back_unchecked(&mut self) -> (K, V) { - replace(self, |leaf_edge| { + super::mem::replace(self, |leaf_edge| { let kv = unsafe { next_back_kv_unchecked_dealloc(leaf_edge) }; let k = unsafe { ptr::read(kv.reborrow().into_kv().0) }; let v = unsafe { ptr::read(kv.reborrow().into_kv().1) }; diff --git a/library/alloc/src/collections/btree/node.rs b/library/alloc/src/collections/btree/node.rs index 886d8abd030bf..dd3d27fd01d20 100644 --- a/library/alloc/src/collections/btree/node.rs +++ b/library/alloc/src/collections/btree/node.rs @@ -119,15 +119,11 @@ struct BoxedNode { } impl BoxedNode { - fn from_leaf(node: Box>) -> Self { - BoxedNode { ptr: Box::into_unique(node) } + fn from_owned(mut node: NonNull>) -> Self { + BoxedNode { ptr: unsafe { Unique::new_unchecked(node.as_mut()) } } } - fn from_internal(node: Box>) -> Self { - BoxedNode { ptr: Unique::from(&mut Box::leak(node).data) } - } - - fn as_ptr(&self) -> NonNull> { + fn as_nonnull(&self) -> NonNull> { NonNull::from(self.ptr) } } @@ -135,58 +131,59 @@ impl BoxedNode { /// An owned tree. /// /// Note that this does not have a destructor, and must be cleaned up manually. -pub struct Root { - node: BoxedNode, - /// The number of levels below the root node. - height: usize, -} +pub type Root = NodeRef; unsafe impl Sync for Root {} unsafe impl Send for Root {} impl Root { - /// Returns the number of levels below the root. - pub fn height(&self) -> usize { - self.height - } - /// Returns a new owned tree, with its own root node that is initially empty. pub fn new_leaf() -> Self { - Root { node: BoxedNode::from_leaf(Box::new(unsafe { LeafNode::new() })), height: 0 } + Self::from_leaf(Box::new(unsafe { LeafNode::new() })) } - /// Borrows and returns an immutable reference to the node owned by the root. + fn from_leaf(leaf: Box>) -> Self { + NodeRef { height: 0, node: NonNull::from(Box::leak(leaf)), _marker: PhantomData } + } + + fn from_internal(internal: Box>, height: usize) -> Self { + NodeRef { height, node: NonNull::from(&mut Box::leak(internal).data), _marker: PhantomData } + } + + /// Reborrows the owned node as an immutable reference. pub fn node_as_ref(&self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { height: self.height, node: self.node.as_ptr(), _marker: PhantomData } + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - /// Borrows and returns a mutable reference to the node owned by the root. + /// Reborrows the owned node as a mutable reference. pub fn node_as_mut(&mut self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { height: self.height, node: self.node.as_ptr(), _marker: PhantomData } + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - /// Borrows and returns a mutable reference to the leaf node owned by the root. + /// Reborrows the owned leaf node as a mutable reference. /// # Safety /// The root node is a leaf. unsafe fn leaf_node_as_mut(&mut self) -> NodeRef, K, V, marker::Leaf> { debug_assert!(self.height == 0); - NodeRef { height: self.height, node: self.node.as_ptr(), _marker: PhantomData } + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - /// Borrows and returns a mutable reference to the internal node owned by the root. + /// Reborrows the owned internal node as a mutable reference. /// # Safety /// The root node is not a leaf. unsafe fn internal_node_as_mut(&mut self) -> NodeRef, K, V, marker::Internal> { debug_assert!(self.height > 0); - NodeRef { height: self.height, node: self.node.as_ptr(), _marker: PhantomData } + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } + /// Reborrows the owned internal node as a slightly mutable reference. pub fn node_as_valmut(&mut self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { height: self.height, node: self.node.as_ptr(), _marker: PhantomData } + NodeRef { height: self.height, node: self.node, _marker: PhantomData } } - pub fn into_ref(self) -> NodeRef { - NodeRef { height: self.height, node: self.node.as_ptr(), _marker: PhantomData } + /// Narrows the type-specific node reference to a type-agnostic pointer. + fn into_boxed_node(self) -> BoxedNode { + BoxedNode::from_owned(self.node) } /// Adds a new internal node with a single edge pointing to the previous root node, @@ -194,10 +191,11 @@ impl Root { /// and is the opposite of `pop_internal_level`. pub fn push_internal_level(&mut self) -> NodeRef, K, V, marker::Internal> { let mut new_node = Box::new(unsafe { InternalNode::new() }); - new_node.edges[0].write(unsafe { ptr::read(&mut self.node) }); - - self.node = BoxedNode::from_internal(new_node); - self.height += 1; + let new_height = self.height + 1; + super::mem::take_mut(self, |root| { + new_node.edges[0].write(root.into_boxed_node()); + Root::from_internal(new_node, new_height) + }); unsafe { let mut ret = self.internal_node_as_mut(); @@ -218,15 +216,14 @@ impl Root { pub fn pop_internal_level(&mut self) { assert!(self.height > 0); - let top = self.node.ptr; + let top = self.node; - let mut internal_node = unsafe { self.internal_node_as_mut() }; - self.node = unsafe { internal_node.as_internal_mut().edges[0].assume_init_read() }; - self.height -= 1; + let internal_node = NodeRef { height: self.height, node: self.node, _marker: PhantomData }; + *self = internal_node.first_edge().descend(); self.node_as_mut().as_leaf_mut().parent = None; unsafe { - Global.dealloc(NonNull::from(top).cast(), Layout::new::>()); + Global.dealloc(top.cast(), Layout::new::>()); } } } @@ -368,6 +365,10 @@ impl NodeRef { } impl NodeRef { + fn from_boxed_node(edge: BoxedNode, height: usize) -> Self { + NodeRef { height, node: edge.as_nonnull(), _marker: PhantomData } + } + /// Finds the parent of the current node. Returns `Ok(handle)` if the current /// node actually has a parent, where `handle` points to the edge of the parent /// that points to the current node. Returns `Err(self)` if the current node has @@ -650,7 +651,7 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::Internal> { unsafe { ptr::write(self.key_mut_at(idx), key); ptr::write(self.val_mut_at(idx), val); - self.as_internal_mut().edges.get_unchecked_mut(idx + 1).write(edge.node); + self.as_internal_mut().edges.get_unchecked_mut(idx + 1).write(edge.into_boxed_node()); Handle::new_edge(self.reborrow_mut(), idx + 1).correct_parent_link(); } } @@ -664,7 +665,7 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::Internal> { unsafe { slice_insert(self.keys_mut(), 0, key); slice_insert(self.vals_mut(), 0, val); - slice_insert(self.edges_mut(), 0, edge.node); + slice_insert(self.edges_mut(), 0, edge.into_boxed_node()); } self.as_leaf_mut().len += 1; @@ -688,10 +689,10 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { let edge = match self.reborrow_mut().force() { ForceResult::Leaf(_) => None, ForceResult::Internal(internal) => { - let edge = ptr::read(internal.edge_at(idx + 1)); - let mut new_root = Root { node: edge, height: internal.height - 1 }; - new_root.node_as_mut().as_leaf_mut().parent = None; - Some(new_root) + let boxed_node = ptr::read(internal.edge_at(idx + 1)); + let mut edge = Root::from_boxed_node(boxed_node, internal.height - 1); + edge.node_as_mut().as_leaf_mut().parent = None; + Some(edge) } }; @@ -714,13 +715,13 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { let edge = match self.reborrow_mut().force() { ForceResult::Leaf(_) => None, ForceResult::Internal(mut internal) => { - let edge = slice_remove(internal.edges_mut(), 0); - let mut new_root = Root { node: edge, height: internal.height - 1 }; - new_root.node_as_mut().as_leaf_mut().parent = None; + let boxed_node = slice_remove(internal.edges_mut(), 0); + let mut edge = Root::from_boxed_node(boxed_node, internal.height - 1); + edge.node_as_mut().as_leaf_mut().parent = None; internal.correct_childrens_parent_links(0..old_len); - Some(new_root) + Some(edge) } }; @@ -984,7 +985,7 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, unsafe { slice_insert(self.node.keys_mut(), self.idx, key); slice_insert(self.node.vals_mut(), self.idx, val); - slice_insert(self.node.edges_mut(), self.idx + 1, edge.node); + slice_insert(self.node.edges_mut(), self.idx + 1, edge.into_boxed_node()); self.node.as_leaf_mut().len += 1; self.node.correct_childrens_parent_links((self.idx + 1)..=self.node.len()); @@ -1074,11 +1075,10 @@ impl Handle, marke // reference (Rust issue #73987) and invalidate any other references // to or inside the array, should any be around. let internal_node = self.node.as_internal_ptr(); - NodeRef { - height: self.node.height - 1, - node: unsafe { (&*(*internal_node).edges.get_unchecked(self.idx).as_ptr()).as_ptr() }, - _marker: PhantomData, - } + NodeRef::from_boxed_node( + unsafe { (*internal_node).edges.get_unchecked(self.idx).assume_init_read() }, + self.node.height - 1, + ) } } @@ -1163,7 +1163,7 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Leaf>, mark let (k, v) = self.split_leaf_data(&mut new_node); - let right = Root { node: BoxedNode::from_leaf(new_node), height: 0 }; + let right = Root::from_leaf(new_node); (self.node, k, v, right) } } @@ -1215,7 +1215,7 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, let (k, v) = self.split_leaf_data(&mut new_node.data); let height = self.node.height; - let mut right = Root { node: BoxedNode::from_internal(new_node), height }; + let mut right = Root::from_internal(new_node, height); right.internal_node_as_mut().correct_childrens_parent_links(0..=new_len); diff --git a/library/alloc/src/collections/btree/node/tests.rs b/library/alloc/src/collections/btree/node/tests.rs index e56fc2aa51e7c..4461625e7f53b 100644 --- a/library/alloc/src/collections/btree/node/tests.rs +++ b/library/alloc/src/collections/btree/node/tests.rs @@ -158,8 +158,8 @@ fn test_partial_cmp_eq() { assert_eq!(top_edge_1.partial_cmp(&top_edge_2), None); root1.pop_internal_level(); - unsafe { root1.into_ref().deallocate_and_ascend() }; - unsafe { root2.into_ref().deallocate_and_ascend() }; + unsafe { root1.deallocate_and_ascend() }; + unsafe { root2.deallocate_and_ascend() }; } #[test] diff --git a/src/etc/gdb_providers.py b/src/etc/gdb_providers.py index eec3027085c91..71f6a3b404b8c 100644 --- a/src/etc/gdb_providers.py +++ b/src/etc/gdb_providers.py @@ -207,15 +207,14 @@ def children(self): yield "borrow", self.borrow -# Yields children (in a provider's sense of the word) for a tree headed by a BoxedNode. +# Yields children (in a provider's sense of the word) for a tree headed by a node. # In particular, yields each key/value pair in the node and in any child nodes. -def children_of_node(boxed_node, height): +def children_of_node(node_ptr, height): def cast_to_internal(node): internal_type_name = node.type.target().name.replace("LeafNode", "InternalNode", 1) internal_type = lookup_type(internal_type_name) return node.cast(internal_type.pointer()) - node_ptr = unwrap_unique_or_non_null(boxed_node["ptr"]) leaf = node_ptr.dereference() keys = leaf["keys"] vals = leaf["vals"] @@ -225,7 +224,8 @@ def cast_to_internal(node): for i in xrange(0, length + 1): if height > 0: boxed_child_node = edges[i]["value"]["value"] - for child in children_of_node(boxed_child_node, height - 1): + child_node = unwrap_unique_or_non_null(boxed_child_node["ptr"]) + for child in children_of_node(child_node, height - 1): yield child if i < length: # Avoid "Cannot perform pointer math on incomplete type" on zero-sized arrays. @@ -240,9 +240,12 @@ def children_of_map(map): root = map["root"] if root.type.name.startswith("core::option::Option<"): root = root.cast(gdb.lookup_type(root.type.name[21:-1])) - boxed_root_node = root["node"] + node_ptr = root["node"] + if node_ptr.type.name.startswith("alloc::collections::btree::node::BoxedNode<"): + node_ptr = node_ptr["ptr"] + node_ptr = unwrap_unique_or_non_null(node_ptr) height = root["height"] - for child in children_of_node(boxed_root_node, height): + for child in children_of_node(node_ptr, height): yield child From 39221722fb7d0b7608303b27d70dd64a87f3fb50 Mon Sep 17 00:00:00 2001 From: Stein Somers Date: Thu, 22 Oct 2020 13:41:59 +0200 Subject: [PATCH 2/3] tweak gdb --- src/etc/gdb_providers.py | 2 +- src/test/debuginfo/pretty-std-collections.rs | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/etc/gdb_providers.py b/src/etc/gdb_providers.py index 71f6a3b404b8c..fa21305424de7 100644 --- a/src/etc/gdb_providers.py +++ b/src/etc/gdb_providers.py @@ -219,7 +219,7 @@ def cast_to_internal(node): keys = leaf["keys"] vals = leaf["vals"] edges = cast_to_internal(node_ptr)["edges"] if height > 0 else None - length = int(leaf["len"]) + length = leaf["len"] for i in xrange(0, length + 1): if height > 0: diff --git a/src/test/debuginfo/pretty-std-collections.rs b/src/test/debuginfo/pretty-std-collections.rs index cc2a3a345102a..b79f00a9d0438 100644 --- a/src/test/debuginfo/pretty-std-collections.rs +++ b/src/test/debuginfo/pretty-std-collections.rs @@ -101,7 +101,7 @@ fn main() { btree_set.insert(i); } - let mut empty_btree_set: BTreeSet = BTreeSet::new(); + let empty_btree_set: BTreeSet = BTreeSet::new(); // BTreeMap let mut btree_map = BTreeMap::new(); @@ -109,7 +109,7 @@ fn main() { btree_map.insert(i, i); } - let mut empty_btree_map: BTreeMap = BTreeMap::new(); + let empty_btree_map: BTreeMap = BTreeMap::new(); let mut option_btree_map: BTreeMap> = BTreeMap::new(); option_btree_map.insert(false, None); From f897d16727a442b09e6867de76bbc66760724527 Mon Sep 17 00:00:00 2001 From: Stein Somers Date: Mon, 19 Oct 2020 10:23:11 +0200 Subject: [PATCH 3/3] BTreeMap: let NodeRef point to the actual type of a node --- library/alloc/src/collections/btree/node.rs | 233 +++++++++++++------- src/etc/gdb_providers.py | 48 +++- 2 files changed, 184 insertions(+), 97 deletions(-) diff --git a/library/alloc/src/collections/btree/node.rs b/library/alloc/src/collections/btree/node.rs index dd3d27fd01d20..062efe332b908 100644 --- a/library/alloc/src/collections/btree/node.rs +++ b/library/alloc/src/collections/btree/node.rs @@ -31,6 +31,7 @@ use core::cmp::Ordering; use core::marker::PhantomData; use core::mem::{self, MaybeUninit}; +use core::num::NonZeroUsize; use core::ptr::{self, NonNull, Unique}; use core::slice; @@ -119,13 +120,21 @@ struct BoxedNode { } impl BoxedNode { - fn from_owned(mut node: NonNull>) -> Self { + fn from_owned_leaf(mut node: NonNull>) -> Self { BoxedNode { ptr: unsafe { Unique::new_unchecked(node.as_mut()) } } } - fn as_nonnull(&self) -> NonNull> { + fn from_owned_internal(node: NonNull>) -> Self { + BoxedNode { ptr: unsafe { Unique::new_unchecked(&mut (*node.as_ptr()).data) } } + } + + fn as_leaf(&self) -> NonNull> { NonNull::from(self.ptr) } + + fn as_internal(&self) -> NonNull> { + NonNull::from(self.ptr).cast() + } } /// An owned tree. @@ -143,47 +152,56 @@ impl Root { } fn from_leaf(leaf: Box>) -> Self { - NodeRef { height: 0, node: NonNull::from(Box::leak(leaf)), _marker: PhantomData } + NodeRef { + inner: NodeRefInner::Leaf { node: NonNull::from(Box::leak(leaf)) }, + _marker: PhantomData, + } } - fn from_internal(internal: Box>, height: usize) -> Self { - NodeRef { height, node: NonNull::from(&mut Box::leak(internal).data), _marker: PhantomData } + fn from_internal(internal: Box>, height: NonZeroUsize) -> Self { + NodeRef { + inner: NodeRefInner::Internal { height, node: NonNull::from(Box::leak(internal)) }, + _marker: PhantomData, + } } /// Reborrows the owned node as an immutable reference. pub fn node_as_ref(&self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + NodeRef { inner: self.inner, _marker: PhantomData } } /// Reborrows the owned node as a mutable reference. pub fn node_as_mut(&mut self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + NodeRef { inner: self.inner, _marker: PhantomData } } /// Reborrows the owned leaf node as a mutable reference. /// # Safety /// The root node is a leaf. unsafe fn leaf_node_as_mut(&mut self) -> NodeRef, K, V, marker::Leaf> { - debug_assert!(self.height == 0); - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + debug_assert!(matches!(self.inner, NodeRefInner::Leaf {..})); + NodeRef { inner: self.inner, _marker: PhantomData } } /// Reborrows the owned internal node as a mutable reference. /// # Safety /// The root node is not a leaf. unsafe fn internal_node_as_mut(&mut self) -> NodeRef, K, V, marker::Internal> { - debug_assert!(self.height > 0); - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + debug_assert!(matches!(self.inner, NodeRefInner::Internal {..})); + NodeRef { inner: self.inner, _marker: PhantomData } } /// Reborrows the owned internal node as a slightly mutable reference. pub fn node_as_valmut(&mut self) -> NodeRef, K, V, marker::LeafOrInternal> { - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + NodeRef { inner: self.inner, _marker: PhantomData } } /// Narrows the type-specific node reference to a type-agnostic pointer. fn into_boxed_node(self) -> BoxedNode { - BoxedNode::from_owned(self.node) + match self.inner { + NodeRefInner::Leaf { node } => BoxedNode::from_owned_leaf(node), + NodeRefInner::Internal { node, .. } => BoxedNode::from_owned_internal(node), + } } /// Adds a new internal node with a single edge pointing to the previous root node, @@ -191,7 +209,7 @@ impl Root { /// and is the opposite of `pop_internal_level`. pub fn push_internal_level(&mut self) -> NodeRef, K, V, marker::Internal> { let mut new_node = Box::new(unsafe { InternalNode::new() }); - let new_height = self.height + 1; + let new_height = unsafe { NonZeroUsize::new_unchecked(self.height() + 1) }; super::mem::take_mut(self, |root| { new_node.edges[0].write(root.into_boxed_node()); Root::from_internal(new_node, new_height) @@ -214,16 +232,16 @@ impl Root { /// /// Panics if there is no internal level, i.e., if the root node is a leaf. pub fn pop_internal_level(&mut self) { - assert!(self.height > 0); + debug_assert!(matches!(self.inner, NodeRefInner::Internal {..})); - let top = self.node; + let top = self.inner; - let internal_node = NodeRef { height: self.height, node: self.node, _marker: PhantomData }; + let internal_node = NodeRef { inner: self.inner, _marker: PhantomData }; *self = internal_node.first_edge().descend(); self.node_as_mut().as_leaf_mut().parent = None; unsafe { - Global.dealloc(top.cast(), Layout::new::>()); + top.deallocate(); } } } @@ -249,17 +267,24 @@ impl Root { /// `NodeRef` points to an internal node, and when this is `LeafOrInternal` the /// `NodeRef` could be pointing to either type of node. pub struct NodeRef { - /// The number of levels below the node, a property of the node that cannot be - /// entirely described by `Type` and that the node does not store itself either. - /// Unconstrained if `Type` is `LeafOrInternal`, must be zero if `Type` is `Leaf`, - /// and must be non-zero if `Type` is `Internal`. - height: usize, - /// The pointer to the leaf or internal node. The definition of `InternalNode` - /// ensures that the pointer is valid either way. - node: NonNull>, + inner: NodeRefInner, _marker: PhantomData<(BorrowType, Type)>, } +enum NodeRefInner { + Leaf { + /// Pointer type `Node` written out to be covariant. + node: NonNull>, + }, + Internal { + /// Pointer type `Node` written out to be covariant. + node: NonNull>, + /// The number of levels below the node, a property of the node that cannot be + /// entirely described by its type and that the node does not store itself either. + height: NonZeroUsize, + }, +} + impl<'a, K: 'a, V: 'a, Type> Copy for NodeRef, K, V, Type> {} impl<'a, K: 'a, V: 'a, Type> Clone for NodeRef, K, V, Type> { fn clone(&self) -> Self { @@ -267,6 +292,13 @@ impl<'a, K: 'a, V: 'a, Type> Clone for NodeRef, K, V, Type> { } } +impl Copy for NodeRefInner {} +impl Clone for NodeRefInner { + fn clone(&self) -> Self { + *self + } +} + unsafe impl Sync for NodeRef {} unsafe impl<'a, K: Sync + 'a, V: Sync + 'a, Type> Send for NodeRef, K, V, Type> {} @@ -280,7 +312,10 @@ impl NodeRef { /// Returns a raw ptr to avoid invalidating other references to this node, /// which is possible when BorrowType is marker::ValMut. fn as_internal_ptr(&self) -> *const InternalNode { - self.node.as_ptr() as *const InternalNode + match &self.inner { + NodeRefInner::Leaf { .. } => unreachable!(), + NodeRefInner::Internal { node, .. } => node.as_ptr(), + } } } @@ -288,7 +323,10 @@ impl<'a, K, V> NodeRef, K, V, marker::Internal> { /// Exposes the data of an internal node for reading, /// when we know we have exclusive access. fn as_internal(&mut self) -> &InternalNode { - unsafe { &*self.as_internal_ptr() } + match &mut self.inner { + NodeRefInner::Leaf { .. } => unreachable!(), + NodeRefInner::Internal { node, .. } => unsafe { node.as_ref() }, + } } } @@ -297,7 +335,10 @@ impl<'a, K, V> NodeRef, K, V, marker::Internal> { /// /// We don't need to return a raw ptr because we have unique access to the entire node. fn as_internal_mut(&mut self) -> &mut InternalNode { - unsafe { &mut *(self.node.as_ptr() as *mut InternalNode) } + match &mut self.inner { + NodeRefInner::Leaf { .. } => unreachable!(), + NodeRefInner::Internal { node, .. } => unsafe { node.as_mut() }, + } } } @@ -315,23 +356,29 @@ impl NodeRef { /// Returns the height of this node with respect to the leaf level. Zero height means the /// node is a leaf itself. pub fn height(&self) -> usize { - self.height + match self.inner { + NodeRefInner::Leaf { .. } => 0, + NodeRefInner::Internal { height, .. } => height.get(), + } } /// Temporarily takes out another, immutable reference to the same node. fn reborrow(&self) -> NodeRef, K, V, Type> { - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + NodeRef { inner: self.inner, _marker: PhantomData } } /// Exposes the leaf portion of any leaf or internal node. /// /// Returns a raw ptr to avoid invalidating other references to this node, /// which is possible when BorrowType is marker::ValMut. - fn as_leaf_ptr(&self) -> *const LeafNode { + fn as_leaf_ptr(&self) -> *mut LeafNode { // The node must be valid for at least the LeafNode portion. // This is not a reference in the NodeRef type because we don't know if // it should be unique or shared. - self.node.as_ptr() + match &self.inner { + NodeRefInner::Leaf { node } => node.as_ptr(), + NodeRefInner::Internal { node, .. } => unsafe { &raw mut (*node.as_ptr()).data }, + } } /// Borrows a reference to one of the keys stored in the node. @@ -366,7 +413,13 @@ impl NodeRef { impl NodeRef { fn from_boxed_node(edge: BoxedNode, height: usize) -> Self { - NodeRef { height, node: edge.as_nonnull(), _marker: PhantomData } + NodeRef { + inner: match NonZeroUsize::new(height) { + None => NodeRefInner::Leaf { node: edge.as_leaf() }, + Some(height) => NodeRefInner::Internal { height, node: edge.as_internal() }, + }, + _marker: PhantomData, + } } /// Finds the parent of the current node. Returns `Ok(handle)` if the current @@ -386,8 +439,10 @@ impl NodeRef { .as_ref() .map(|parent| Handle { node: NodeRef { - height: self.height + 1, - node: parent.cast(), + inner: NodeRefInner::Internal { + height: unsafe { NonZeroUsize::new_unchecked(self.height() + 1) }, + node: *parent, + }, _marker: PhantomData, }, idx: unsafe { usize::from((*leaf_ptr).parent_idx.assume_init()) }, @@ -436,28 +491,31 @@ impl NodeRef { pub unsafe fn deallocate_and_ascend( self, ) -> Option, marker::Edge>> { - let height = self.height; - let node = self.node; + let node = self.inner; let ret = self.ascend().ok(); - unsafe { - Global.dealloc( - node.cast(), - if height > 0 { - Layout::new::>() - } else { - Layout::new::>() - }, - ); - } + unsafe { node.deallocate() }; ret } } +impl NodeRefInner { + unsafe fn deallocate(self) { + match self { + NodeRefInner::Leaf { node } => unsafe { + Global.dealloc(node.cast(), Layout::new::>()) + }, + NodeRefInner::Internal { node, .. } => unsafe { + Global.dealloc(node.cast(), Layout::new::>()) + }, + } + } +} + impl<'a, K, V, Type> NodeRef, K, V, Type> { /// Unsafely asserts to the compiler the static information that this node is an `Internal`. unsafe fn cast_to_internal_unchecked(self) -> NodeRef, K, V, marker::Internal> { - debug_assert!(self.height > 0); - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + debug_assert!(matches!(self.inner, NodeRefInner::Internal {..})); + NodeRef { inner: self.inner, _marker: PhantomData } } /// Temporarily takes out another, mutable reference to the same node. Beware, as @@ -471,14 +529,17 @@ impl<'a, K, V, Type> NodeRef, K, V, Type> { // that restricts the use of navigation methods on reborrowed pointers, // preventing this unsafety. unsafe fn reborrow_mut(&mut self) -> NodeRef, K, V, Type> { - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + NodeRef { inner: self.inner, _marker: PhantomData } } /// Exposes the leaf portion of any leaf or internal node for writing. /// /// We don't need to return a raw ptr because we have unique access to the entire node. fn as_leaf_mut(&mut self) -> &'a mut LeafNode { - unsafe { &mut (*self.node.as_ptr()) } + match &mut self.inner { + NodeRefInner::Leaf { ref mut node } => unsafe { &mut *node.as_ptr() }, + NodeRefInner::Internal { ref mut node, .. } => unsafe { &mut (*node.as_ptr()).data }, + } } /// Borrows a mutable reference to one of the keys stored in the node. @@ -584,7 +645,7 @@ impl<'a, K, V, Type> NodeRef, K, V, Type> { // We only create a reference to the one element we are interested in, // to avoid aliasing with outstanding references to other elements, // in particular, those returned to the caller in earlier iterations. - let leaf = self.node.as_ptr(); + let leaf = self.as_leaf_ptr(); let keys = unsafe { &raw const (*leaf).keys }; let vals = unsafe { &raw mut (*leaf).vals }; // We must coerce to unsized array pointers because of Rust issue #74679. @@ -642,7 +703,7 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::Internal> { /// Adds a key/value pair, and an edge to go to the right of that pair, /// to the end of the node. pub fn push(&mut self, key: K, val: V, edge: Root) { - assert!(edge.height == self.height - 1); + assert!(edge.height() == self.height() - 1); let len = &mut self.as_leaf_mut().len; let idx = usize::from(*len); @@ -659,7 +720,7 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::Internal> { /// Adds a key/value pair, and an edge to go to the left of that pair, /// to the beginning of the node. fn push_front(&mut self, key: K, val: V, edge: Root) { - assert!(edge.height == self.height - 1); + assert!(edge.height() == self.height() - 1); assert!(self.len() < CAPACITY); unsafe { @@ -690,7 +751,7 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { ForceResult::Leaf(_) => None, ForceResult::Internal(internal) => { let boxed_node = ptr::read(internal.edge_at(idx + 1)); - let mut edge = Root::from_boxed_node(boxed_node, internal.height - 1); + let mut edge = Root::from_boxed_node(boxed_node, internal.height() - 1); edge.node_as_mut().as_leaf_mut().parent = None; Some(edge) } @@ -716,7 +777,7 @@ impl<'a, K: 'a, V: 'a> NodeRef, K, V, marker::LeafOrInternal> { ForceResult::Leaf(_) => None, ForceResult::Internal(mut internal) => { let boxed_node = slice_remove(internal.edges_mut(), 0); - let mut edge = Root::from_boxed_node(boxed_node, internal.height - 1); + let mut edge = Root::from_boxed_node(boxed_node, internal.height() - 1); edge.node_as_mut().as_leaf_mut().parent = None; internal.correct_childrens_parent_links(0..old_len); @@ -744,18 +805,13 @@ impl NodeRef { NodeRef, NodeRef, > { - if self.height == 0 { - ForceResult::Leaf(NodeRef { - height: self.height, - node: self.node, - _marker: PhantomData, - }) - } else { - ForceResult::Internal(NodeRef { - height: self.height, - node: self.node, - _marker: PhantomData, - }) + match self.inner { + NodeRefInner::Leaf { .. } => { + ForceResult::Leaf(NodeRef { inner: self.inner, _marker: PhantomData }) + } + NodeRefInner::Internal { .. } => { + ForceResult::Internal(NodeRef { inner: self.inner, _marker: PhantomData }) + } } } } @@ -816,12 +872,16 @@ impl Handle, mar impl NodeRef { /// Could be a public implementation of PartialEq, but only used in this module. fn eq(&self, other: &Self) -> bool { - let Self { node, height, _marker: _ } = self; - if *node == other.node { - debug_assert_eq!(*height, other.height); - true - } else { - false + match (self.inner, other.inner) { + (NodeRefInner::Leaf { node: a }, NodeRefInner::Leaf { node: b }) if a == b => true, + ( + NodeRefInner::Internal { node: node_a, height: height_a }, + NodeRefInner::Internal { node: node_b, height: height_b }, + ) if node_a == node_b => { + debug_assert_eq!(height_a, height_b); + true + } + _ => false, } } } @@ -980,7 +1040,7 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, /// that there is enough space in the node for the new pair to fit. fn insert_fit(&mut self, key: K, val: V, edge: Root) { debug_assert!(self.node.len() < CAPACITY); - debug_assert!(edge.height == self.node.height - 1); + debug_assert!(edge.height() == self.node.height() - 1); unsafe { slice_insert(self.node.keys_mut(), self.idx, key); @@ -1001,7 +1061,7 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, val: V, edge: Root, ) -> InsertResult<'a, K, V, marker::Internal> { - assert!(edge.height == self.node.height - 1); + assert!(edge.height() == self.node.height() - 1); if self.node.len() < CAPACITY { self.insert_fit(key, val, edge); @@ -1077,7 +1137,7 @@ impl Handle, marke let internal_node = self.node.as_internal_ptr(); NodeRef::from_boxed_node( unsafe { (*internal_node).edges.get_unchecked(self.idx).assume_init_read() }, - self.node.height - 1, + self.node.height() - 1, ) } } @@ -1214,7 +1274,10 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, ); let (k, v) = self.split_leaf_data(&mut new_node.data); - let height = self.node.height; + let height = match self.node.inner { + NodeRefInner::Leaf { .. } => unreachable!(), + NodeRefInner::Internal { height, .. } => height, + }; let mut right = Root::from_internal(new_node, height); right.internal_node_as_mut().correct_childrens_parent_links(0..=new_len); @@ -1267,7 +1330,7 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, left_node.as_leaf_mut().len += right_len as u16 + 1; - if self.node.height > 1 { + if self.node.height() > 1 { // SAFETY: the height of the nodes being merged is one below the height // of the node of this edge, thus above zero, so they are internal. let mut left_node = left_node.cast_to_internal_unchecked(); @@ -1280,9 +1343,9 @@ impl<'a, K: 'a, V: 'a> Handle, K, V, marker::Internal>, left_node.correct_childrens_parent_links(left_len + 1..=left_len + 1 + right_len); - Global.dealloc(right_node.node.cast(), Layout::new::>()); + right_node.inner.deallocate(); } else { - Global.dealloc(right_node.node.cast(), Layout::new::>()); + right_node.inner.deallocate(); } Handle::new_edge(self.node, self.idx) @@ -1466,14 +1529,14 @@ unsafe fn move_edges( impl NodeRef { /// Removes any static information asserting that this node is a `Leaf` node. pub fn forget_type(self) -> NodeRef { - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + NodeRef { inner: self.inner, _marker: PhantomData } } } impl NodeRef { /// Removes any static information asserting that this node is an `Internal` node. pub fn forget_type(self) -> NodeRef { - NodeRef { height: self.height, node: self.node, _marker: PhantomData } + NodeRef { inner: self.inner, _marker: PhantomData } } } @@ -1545,7 +1608,7 @@ impl<'a, K, V> Handle, K, V, marker::LeafOrInternal>, ma let mut right_node = right.reborrow_mut(); assert!(right_node.len() == 0); - assert!(left_node.height == right_node.height); + assert!(left_node.height() == right_node.height()); if right_new_len > 0 { let left_kv = left_node.reborrow_mut().into_kv_pointers_mut(); diff --git a/src/etc/gdb_providers.py b/src/etc/gdb_providers.py index fa21305424de7..b5aad41ac4d60 100644 --- a/src/etc/gdb_providers.py +++ b/src/etc/gdb_providers.py @@ -207,25 +207,31 @@ def children(self): yield "borrow", self.borrow -# Yields children (in a provider's sense of the word) for a tree headed by a node. -# In particular, yields each key/value pair in the node and in any child nodes. -def children_of_node(node_ptr, height): +# Extracts leaf portion and edges array (if any) from a BoxedNode. +def unbox_node(unique, height): def cast_to_internal(node): internal_type_name = node.type.target().name.replace("LeafNode", "InternalNode", 1) internal_type = lookup_type(internal_type_name) return node.cast(internal_type.pointer()) + node_ptr = unwrap_unique_or_non_null(unique) leaf = node_ptr.dereference() + edges = cast_to_internal(node_ptr)["edges"] if height > 0 else None + return leaf, edges + + +# Yields children (in a provider's sense of the word) for a tree headed by a BoxedNode. +# In particular, yields each key/value pair in the node and in any child nodes. +def children_of_node(leaf, edges, height): keys = leaf["keys"] vals = leaf["vals"] - edges = cast_to_internal(node_ptr)["edges"] if height > 0 else None length = leaf["len"] for i in xrange(0, length + 1): if height > 0: boxed_child_node = edges[i]["value"]["value"] - child_node = unwrap_unique_or_non_null(boxed_child_node["ptr"]) - for child in children_of_node(child_node, height - 1): + child_leaf, child_edges = unbox_node(boxed_child_node["ptr"], height - 1) + for child in children_of_node(child_leaf, child_edges, height - 1): yield child if i < length: # Avoid "Cannot perform pointer math on incomplete type" on zero-sized arrays. @@ -240,12 +246,30 @@ def children_of_map(map): root = map["root"] if root.type.name.startswith("core::option::Option<"): root = root.cast(gdb.lookup_type(root.type.name[21:-1])) - node_ptr = root["node"] - if node_ptr.type.name.startswith("alloc::collections::btree::node::BoxedNode<"): - node_ptr = node_ptr["ptr"] - node_ptr = unwrap_unique_or_non_null(node_ptr) - height = root["height"] - for child in children_of_node(node_ptr, height): + if root.type.has_key("inner"): + root = root["inner"] + content = root[root.type.fields()[0]] + fields = content.type.fields() + discriminant = int(content[fields[0]]) + 1 + variant = fields[discriminant] + root = content[variant] + node_ptr = unwrap_unique_or_non_null(root["node"]) + node = node_ptr.dereference() + if variant.name == "Leaf": + height = 0 + leaf = node + edges = None + else: + height = root["height"][ZERO_FIELD] # unwrap NonZeroUsize + leaf = node["data"] + edges = node["edges"] + else: + height = root["height"] + unique = root["node"] + if unique.type.name.startswith("alloc::collections::btree::node::BoxedNode<"): + unique = unique["ptr"] + leaf, edges = unbox_node(unique, height) + for child in children_of_node(leaf, edges, height): yield child