From 4de8cefbdf032077154576a646f340883927c5b3 Mon Sep 17 00:00:00 2001 From: Boxy Date: Fri, 10 Jan 2025 23:19:17 +0000 Subject: [PATCH] De-abstract tagged pointer abstraction --- compiler/rustc_data_structures/src/marker.rs | 4 +- .../rustc_data_structures/src/tagged_ptr.rs | 357 +++++++++--------- .../src/tagged_ptr/copy.rs | 330 ---------------- .../src/tagged_ptr/copy/tests.rs | 50 --- .../src/tagged_ptr/drop.rs | 178 --------- .../src/tagged_ptr/drop/tests.rs | 72 ---- .../src/tagged_ptr/impl_tag.rs | 144 ------- .../src/tagged_ptr/impl_tag/tests.rs | 34 -- .../src/tagged_ptr/tests.rs | 105 ++++++ compiler/rustc_middle/src/ty/list.rs | 2 +- 10 files changed, 291 insertions(+), 985 deletions(-) delete mode 100644 compiler/rustc_data_structures/src/tagged_ptr/copy.rs delete mode 100644 compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs delete mode 100644 compiler/rustc_data_structures/src/tagged_ptr/drop.rs delete mode 100644 compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs delete mode 100644 compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs delete mode 100644 compiler/rustc_data_structures/src/tagged_ptr/impl_tag/tests.rs create mode 100644 compiler/rustc_data_structures/src/tagged_ptr/tests.rs diff --git a/compiler/rustc_data_structures/src/marker.rs b/compiler/rustc_data_structures/src/marker.rs index 2b629024bfe45..6ae97222f77fd 100644 --- a/compiler/rustc_data_structures/src/marker.rs +++ b/compiler/rustc_data_structures/src/marker.rs @@ -72,7 +72,7 @@ impl_dyn_send!( [Vec where T: DynSend, A: std::alloc::Allocator + DynSend] [Box where T: ?Sized + DynSend, A: std::alloc::Allocator + DynSend] [crate::sync::RwLock where T: DynSend] - [crate::tagged_ptr::CopyTaggedPtr where P: Send + crate::tagged_ptr::Pointer, T: Send + crate::tagged_ptr::Tag, const CP: bool] + [crate::tagged_ptr::TaggedRef<'a, P, T> where 'a, P: Sync, T: Send + crate::tagged_ptr::Tag] [rustc_arena::TypedArena where T: DynSend] [indexmap::IndexSet where V: DynSend, S: DynSend] [indexmap::IndexMap where K: DynSend, V: DynSend, S: DynSend] @@ -148,7 +148,7 @@ impl_dyn_sync!( [crate::sync::RwLock where T: DynSend + DynSync] [crate::sync::WorkerLocal where T: DynSend] [crate::intern::Interned<'a, T> where 'a, T: DynSync] - [crate::tagged_ptr::CopyTaggedPtr where P: Sync + crate::tagged_ptr::Pointer, T: Sync + crate::tagged_ptr::Tag, const CP: bool] + [crate::tagged_ptr::TaggedRef<'a, P, T> where 'a, P: Sync, T: Sync + crate::tagged_ptr::Tag] [parking_lot::lock_api::Mutex where R: DynSync, T: ?Sized + DynSend] [parking_lot::lock_api::RwLock where R: DynSync, T: ?Sized + DynSend + DynSync] [indexmap::IndexSet where V: DynSync, S: DynSync] diff --git a/compiler/rustc_data_structures/src/tagged_ptr.rs b/compiler/rustc_data_structures/src/tagged_ptr.rs index 2914eece6796b..94db421f77e89 100644 --- a/compiler/rustc_data_structures/src/tagged_ptr.rs +++ b/compiler/rustc_data_structures/src/tagged_ptr.rs @@ -1,116 +1,26 @@ -//! This module implements tagged pointers. +//! This module implements tagged pointers. In order to utilize the pointer +//! packing, you must have a tag type implementing the [`Tag`] trait. //! -//! In order to utilize the pointer packing, you must have two types: a pointer, -//! and a tag. -//! -//! The pointer must implement the [`Pointer`] trait, with the primary -//! requirement being convertible to and from a raw pointer. Note that the -//! pointer must be dereferenceable, so raw pointers generally cannot implement -//! the [`Pointer`] trait. This implies that the pointer must also be non-null. -//! -//! Many common pointer types already implement the [`Pointer`] trait. -//! -//! The tag must implement the [`Tag`] trait. -//! -//! We assert that the tag and the [`Pointer`] types are compatible at compile +//! We assert that the tag and the reference type is compatible at compile //! time. +use std::fmt; +use std::hash::{Hash, Hasher}; +use std::marker::PhantomData; +use std::num::NonZero; use std::ops::Deref; use std::ptr::NonNull; -use std::rc::Rc; -use std::sync::Arc; use crate::aligned::Aligned; +use crate::stable_hasher::{HashStable, StableHasher}; -mod copy; -mod drop; -mod impl_tag; - -pub use copy::CopyTaggedPtr; -pub use drop::TaggedPtr; - -/// This describes the pointer type encapsulated by [`TaggedPtr`] and -/// [`CopyTaggedPtr`]. -/// -/// # Safety -/// -/// The pointer returned from [`into_ptr`] must be a [valid], pointer to -/// [`::Target`]. -/// -/// Note that if `Self` implements [`DerefMut`] the pointer returned from -/// [`into_ptr`] must be valid for writes (and thus calling [`NonNull::as_mut`] -/// on it must be safe). -/// -/// The [`BITS`] constant must be correct. [`BITS`] least-significant bits, -/// must be zero on all pointers returned from [`into_ptr`]. -/// -/// For example, if the alignment of [`Self::Target`] is 2, then `BITS` should be 1. -/// -/// [`BITS`]: Pointer::BITS -/// [`into_ptr`]: Pointer::into_ptr -/// [valid]: std::ptr#safety -/// [`::Target`]: Deref::Target -/// [`Self::Target`]: Deref::Target -/// [`DerefMut`]: std::ops::DerefMut -pub unsafe trait Pointer: Deref { - /// Number of unused (always zero) **least-significant bits** in this - /// pointer, usually related to the pointees alignment. - /// - /// For example if [`BITS`] = `2`, then given `ptr = Self::into_ptr(..)`, - /// `ptr.addr() & 0b11 == 0` must be true. - /// - /// Most likely the value you want to use here is the following, unless - /// your [`Self::Target`] type is unsized (e.g., `ty::List` in rustc) - /// or your pointer is over/under aligned, in which case you'll need to - /// manually figure out what the right type to pass to [`bits_for`] is, or - /// what the value to set here. - /// - /// ```rust - /// # use std::ops::Deref; - /// # use rustc_data_structures::tagged_ptr::bits_for; - /// # struct T; - /// # impl Deref for T { type Target = u8; fn deref(&self) -> &u8 { &0 } } - /// # impl T { - /// const BITS: u32 = bits_for::<::Target>(); - /// # } - /// ``` - /// - /// [`BITS`]: Pointer::BITS - /// [`Self::Target`]: Deref::Target - const BITS: u32; - - /// Turns this pointer into a raw, non-null pointer. - /// - /// The inverse of this function is [`from_ptr`]. - /// - /// This function guarantees that the least-significant [`Self::BITS`] bits - /// are zero. - /// - /// [`from_ptr`]: Pointer::from_ptr - /// [`Self::BITS`]: Pointer::BITS - fn into_ptr(self) -> NonNull; - - /// Re-creates the original pointer, from a raw pointer returned by [`into_ptr`]. - /// - /// # Safety - /// - /// The passed `ptr` must be returned from [`into_ptr`]. - /// - /// This acts as [`ptr::read::()`] semantically, it should not be called more than - /// once on non-[`Copy`] `Pointer`s. - /// - /// [`into_ptr`]: Pointer::into_ptr - /// [`ptr::read::()`]: std::ptr::read - unsafe fn from_ptr(ptr: NonNull) -> Self; -} - -/// This describes tags that the [`TaggedPtr`] struct can hold. +/// This describes tags that the [`TaggedRef`] struct can hold. /// /// # Safety /// -/// The [`BITS`] constant must be correct. -/// -/// No more than [`BITS`] least-significant bits may be set in the returned usize. +/// - The [`BITS`] constant must be correct. +/// - No more than [`BITS`] least-significant bits may be set in the returned usize. +/// - [`Eq`] and [`Hash`] must be implementable with the returned `usize` from `into_usize`. /// /// [`BITS`]: Tag::BITS pub unsafe trait Tag: Copy { @@ -166,118 +76,217 @@ pub const fn bits_for_tags(mut tags: &[usize]) -> u32 { bits } -unsafe impl Pointer for Box { - const BITS: u32 = bits_for::(); +/// A covariant [`Copy`] tagged borrow. This is essentially `{ pointer: &'a P, tag: T }` packed +/// in a single reference. +pub struct TaggedRef<'a, Pointee: Aligned + ?Sized, T: Tag> { + /// This is semantically a pair of `pointer: &'a P` and `tag: T` fields, + /// however we pack them in a single pointer, to save space. + /// + /// We pack the tag into the **most**-significant bits of the pointer to + /// ease retrieval of the value. A left shift is a multiplication and + /// those are embeddable in instruction encoding, for example: + /// + /// ```asm + /// // () + /// example::shift_read3: + /// mov eax, dword ptr [8*rdi] + /// ret + /// + /// example::mask_read3: + /// and rdi, -8 + /// mov eax, dword ptr [rdi] + /// ret + /// ``` + /// + /// This is ASM outputted by rustc for reads of values behind tagged + /// pointers for different approaches of tagging: + /// - `shift_read3` uses `<< 3` (the tag is in the most-significant bits) + /// - `mask_read3` uses `& !0b111` (the tag is in the least-significant bits) + /// + /// The shift approach thus produces less instructions and is likely faster + /// (see ). + /// + /// Encoding diagram: + /// ```text + /// [ packed.addr ] + /// [ tag ] [ pointer.addr >> T::BITS ] <-- usize::BITS - T::BITS bits + /// ^ + /// | + /// T::BITS bits + /// ``` + /// + /// The tag can be retrieved by `packed.addr() >> T::BITS` and the pointer + /// can be retrieved by `packed.map_addr(|addr| addr << T::BITS)`. + packed: NonNull, + tag_pointer_ghost: PhantomData<(&'a Pointee, T)>, +} +impl<'a, P, T> TaggedRef<'a, P, T> +where + P: Aligned + ?Sized, + T: Tag, +{ + /// Tags `pointer` with `tag`. + /// + /// [`TaggedRef`]: crate::tagged_ptr::TaggedRef #[inline] - fn into_ptr(self) -> NonNull { - // Safety: pointers from `Box::into_raw` are valid & non-null - unsafe { NonNull::new_unchecked(Box::into_raw(self)) } + pub fn new(pointer: &'a P, tag: T) -> Self { + Self { packed: Self::pack(NonNull::from(pointer), tag), tag_pointer_ghost: PhantomData } } + /// Retrieves the pointer. #[inline] - unsafe fn from_ptr(ptr: NonNull) -> Self { - // Safety: `ptr` comes from `into_ptr` which calls `Box::into_raw` - unsafe { Box::from_raw(ptr.as_ptr()) } + pub fn pointer(self) -> &'a P { + // SAFETY: pointer_raw returns the original pointer + unsafe { self.pointer_raw().as_ref() } } -} - -unsafe impl Pointer for Rc { - const BITS: u32 = bits_for::(); + /// Retrieves the tag. #[inline] - fn into_ptr(self) -> NonNull { - // Safety: pointers from `Rc::into_raw` are valid & non-null - unsafe { NonNull::new_unchecked(Rc::into_raw(self).cast_mut()) } + pub fn tag(&self) -> T { + // Unpack the tag, according to the `self.packed` encoding scheme + let tag = self.packed.addr().get() >> Self::TAG_BIT_SHIFT; + + // Safety: + // The shift retrieves the original value from `T::into_usize`, + // satisfying `T::from_usize`'s preconditions. + unsafe { T::from_usize(tag) } } + /// Sets the tag to a new value. #[inline] - unsafe fn from_ptr(ptr: NonNull) -> Self { - // Safety: `ptr` comes from `into_ptr` which calls `Rc::into_raw` - unsafe { Rc::from_raw(ptr.as_ptr()) } + pub fn set_tag(&mut self, tag: T) { + self.packed = Self::pack(self.pointer_raw(), tag); } -} -unsafe impl Pointer for Arc { - const BITS: u32 = bits_for::(); + const TAG_BIT_SHIFT: u32 = usize::BITS - T::BITS; + const ASSERTION: () = { assert!(T::BITS <= bits_for::

()) }; + /// Pack pointer `ptr` with a `tag`, according to `self.packed` encoding scheme. #[inline] - fn into_ptr(self) -> NonNull { - // Safety: pointers from `Arc::into_raw` are valid & non-null - unsafe { NonNull::new_unchecked(Arc::into_raw(self).cast_mut()) } + fn pack(ptr: NonNull

, tag: T) -> NonNull

{ + // Trigger assert! + let () = Self::ASSERTION; + + let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT; + + ptr.map_addr(|addr| { + // Safety: + // - The pointer is `NonNull` => it's address is `NonZero` + // - `P::BITS` least significant bits are always zero (`Pointer` contract) + // - `T::BITS <= P::BITS` (from `Self::ASSERTION`) + // + // Thus `addr >> T::BITS` is guaranteed to be non-zero. + // + // `{non_zero} | packed_tag` can't make the value zero. + + let packed = (addr.get() >> T::BITS) | packed_tag; + unsafe { NonZero::new_unchecked(packed) } + }) } + /// Retrieves the original raw pointer from `self.packed`. #[inline] - unsafe fn from_ptr(ptr: NonNull) -> Self { - // Safety: `ptr` comes from `into_ptr` which calls `Arc::into_raw` - unsafe { Arc::from_raw(ptr.as_ptr()) } + pub(super) fn pointer_raw(&self) -> NonNull

{ + self.packed.map_addr(|addr| unsafe { NonZero::new_unchecked(addr.get() << T::BITS) }) } } -unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a T { - const BITS: u32 = bits_for::(); +impl Copy for TaggedRef<'_, P, T> +where + P: Aligned + ?Sized, + T: Tag, +{ +} +impl Clone for TaggedRef<'_, P, T> +where + P: Aligned + ?Sized, + T: Tag, +{ #[inline] - fn into_ptr(self) -> NonNull { - NonNull::from(self) + fn clone(&self) -> Self { + *self } +} + +impl Deref for TaggedRef<'_, P, T> +where + P: Aligned + ?Sized, + T: Tag, +{ + type Target = P; #[inline] - unsafe fn from_ptr(ptr: NonNull) -> Self { - // Safety: - // `ptr` comes from `into_ptr` which gets the pointer from a reference - unsafe { ptr.as_ref() } + fn deref(&self) -> &Self::Target { + self.pointer() } } -unsafe impl<'a, T: 'a + ?Sized + Aligned> Pointer for &'a mut T { - const BITS: u32 = bits_for::(); +impl fmt::Debug for TaggedRef<'_, P, T> +where + P: Aligned + fmt::Debug + ?Sized, + T: Tag + fmt::Debug, +{ + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("TaggedRef") + .field("pointer", &self.pointer()) + .field("tag", &self.tag()) + .finish() + } +} +impl PartialEq for TaggedRef<'_, P, T> +where + P: Aligned + ?Sized, + T: Tag, +{ #[inline] - fn into_ptr(self) -> NonNull { - NonNull::from(self) + #[allow(ambiguous_wide_pointer_comparisons)] + fn eq(&self, other: &Self) -> bool { + self.packed == other.packed } +} +impl Eq for TaggedRef<'_, P, T> {} + +impl Hash for TaggedRef<'_, P, T> { #[inline] - unsafe fn from_ptr(mut ptr: NonNull) -> Self { - // Safety: - // `ptr` comes from `into_ptr` which gets the pointer from a reference - unsafe { ptr.as_mut() } + fn hash(&self, state: &mut H) { + self.packed.hash(state); } } -/// A tag type used in [`CopyTaggedPtr`] and [`TaggedPtr`] tests. -#[derive(Copy, Clone, Debug, PartialEq, Eq)] -#[cfg(test)] -enum Tag2 { - B00 = 0b00, - B01 = 0b01, - B10 = 0b10, - B11 = 0b11, +impl<'a, P, T, HCX> HashStable for TaggedRef<'a, P, T> +where + P: HashStable + Aligned + ?Sized, + T: Tag + HashStable, +{ + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { + self.pointer().hash_stable(hcx, hasher); + self.tag().hash_stable(hcx, hasher); + } } -#[cfg(test)] -unsafe impl Tag for Tag2 { - const BITS: u32 = 2; - - fn into_usize(self) -> usize { - self as _ - } +// Safety: +// `TaggedRef` is semantically just `{ ptr: P, tag: T }`, as such +// it's ok to implement `Sync` as long as `P: Sync, T: Sync` +unsafe impl Sync for TaggedRef<'_, P, T> +where + P: Sync + Aligned + ?Sized, + T: Sync + Tag, +{ +} - unsafe fn from_usize(tag: usize) -> Self { - match tag { - 0b00 => Tag2::B00, - 0b01 => Tag2::B01, - 0b10 => Tag2::B10, - 0b11 => Tag2::B11, - _ => unreachable!(), - } - } +// Safety: +// `TaggedRef` is semantically just `{ ptr: P, tag: T }`, as such +// it's ok to implement `Send` as long as `P: Send, T: Send` +unsafe impl Send for TaggedRef<'_, P, T> +where + P: Sync + Aligned + ?Sized, + T: Send + Tag, +{ } #[cfg(test)] -impl crate::stable_hasher::HashStable for Tag2 { - fn hash_stable(&self, hcx: &mut HCX, hasher: &mut crate::stable_hasher::StableHasher) { - (*self as u8).hash_stable(hcx, hasher); - } -} +mod tests; diff --git a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs b/compiler/rustc_data_structures/src/tagged_ptr/copy.rs deleted file mode 100644 index 25e107b0f413c..0000000000000 --- a/compiler/rustc_data_structures/src/tagged_ptr/copy.rs +++ /dev/null @@ -1,330 +0,0 @@ -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::marker::PhantomData; -use std::mem::ManuallyDrop; -use std::num::NonZero; -use std::ops::{Deref, DerefMut}; -use std::ptr::NonNull; - -use super::{Pointer, Tag}; -use crate::stable_hasher::{HashStable, StableHasher}; - -/// A [`Copy`] tagged pointer. -/// -/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer. -/// -/// You should use this instead of the [`TaggedPtr`] type in all cases where -/// `P` implements [`Copy`]. -/// -/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without -/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`]; -/// if you want that, wrap the [`CopyTaggedPtr`]. -/// -/// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr -pub struct CopyTaggedPtr -where - P: Pointer, - T: Tag, -{ - /// This is semantically a pair of `pointer: P` and `tag: T` fields, - /// however we pack them in a single pointer, to save space. - /// - /// We pack the tag into the **most**-significant bits of the pointer to - /// ease retrieval of the value. A left shift is a multiplication and - /// those are embeddable in instruction encoding, for example: - /// - /// ```asm - /// // () - /// example::shift_read3: - /// mov eax, dword ptr [8*rdi] - /// ret - /// - /// example::mask_read3: - /// and rdi, -8 - /// mov eax, dword ptr [rdi] - /// ret - /// ``` - /// - /// This is ASM outputted by rustc for reads of values behind tagged - /// pointers for different approaches of tagging: - /// - `shift_read3` uses `<< 3` (the tag is in the most-significant bits) - /// - `mask_read3` uses `& !0b111` (the tag is in the least-significant bits) - /// - /// The shift approach thus produces less instructions and is likely faster - /// (see ). - /// - /// Encoding diagram: - /// ```text - /// [ packed.addr ] - /// [ tag ] [ pointer.addr >> T::BITS ] <-- usize::BITS - T::BITS bits - /// ^ - /// | - /// T::BITS bits - /// ``` - /// - /// The tag can be retrieved by `packed.addr() >> T::BITS` and the pointer - /// can be retrieved by `packed.map_addr(|addr| addr << T::BITS)`. - packed: NonNull, - tag_ghost: PhantomData, -} - -// Note that even though `CopyTaggedPtr` is only really expected to work with -// `P: Copy`, can't add `P: Copy` bound, because `CopyTaggedPtr` is used in the -// `TaggedPtr`'s implementation. -impl CopyTaggedPtr -where - P: Pointer, - T: Tag, -{ - /// Tags `pointer` with `tag`. - /// - /// Note that this leaks `pointer`: it won't be dropped when - /// `CopyTaggedPtr` is dropped. If you have a pointer with a significant - /// drop, use [`TaggedPtr`] instead. - /// - /// [`TaggedPtr`]: crate::tagged_ptr::TaggedPtr - #[inline] - pub fn new(pointer: P, tag: T) -> Self { - Self { packed: Self::pack(P::into_ptr(pointer), tag), tag_ghost: PhantomData } - } - - /// Retrieves the pointer. - #[inline] - pub fn pointer(self) -> P - where - P: Copy, - { - // SAFETY: pointer_raw returns the original pointer - // - // Note that this isn't going to double-drop or anything because we have - // P: Copy - unsafe { P::from_ptr(self.pointer_raw()) } - } - - /// Retrieves the tag. - #[inline] - pub fn tag(&self) -> T { - // Unpack the tag, according to the `self.packed` encoding scheme - let tag = self.packed.addr().get() >> Self::TAG_BIT_SHIFT; - - // Safety: - // The shift retrieves the original value from `T::into_usize`, - // satisfying `T::from_usize`'s preconditions. - unsafe { T::from_usize(tag) } - } - - /// Sets the tag to a new value. - #[inline] - pub fn set_tag(&mut self, tag: T) { - self.packed = Self::pack(self.pointer_raw(), tag); - } - - const TAG_BIT_SHIFT: u32 = usize::BITS - T::BITS; - const ASSERTION: () = { assert!(T::BITS <= P::BITS) }; - - /// Pack pointer `ptr` that comes from [`P::into_ptr`] with a `tag`, - /// according to `self.packed` encoding scheme. - /// - /// [`P::into_ptr`]: Pointer::into_ptr - #[inline] - fn pack(ptr: NonNull, tag: T) -> NonNull { - // Trigger assert! - let () = Self::ASSERTION; - - let packed_tag = tag.into_usize() << Self::TAG_BIT_SHIFT; - - ptr.map_addr(|addr| { - // Safety: - // - The pointer is `NonNull` => it's address is `NonZero` - // - `P::BITS` least significant bits are always zero (`Pointer` contract) - // - `T::BITS <= P::BITS` (from `Self::ASSERTION`) - // - // Thus `addr >> T::BITS` is guaranteed to be non-zero. - // - // `{non_zero} | packed_tag` can't make the value zero. - - let packed = (addr.get() >> T::BITS) | packed_tag; - unsafe { NonZero::new_unchecked(packed) } - }) - } - - /// Retrieves the original raw pointer from `self.packed`. - #[inline] - pub(super) fn pointer_raw(&self) -> NonNull { - self.packed.map_addr(|addr| unsafe { NonZero::new_unchecked(addr.get() << T::BITS) }) - } - - /// This provides a reference to the `P` pointer itself, rather than the - /// `Deref::Target`. It is used for cases where we want to call methods - /// that may be implement differently for the Pointer than the Pointee - /// (e.g., `Rc::clone` vs cloning the inner value). - pub(super) fn with_pointer_ref(&self, f: impl FnOnce(&P) -> R) -> R { - // Safety: - // - `self.raw.pointer_raw()` is originally returned from `P::into_ptr` - // and as such is valid for `P::from_ptr`. - // - This also allows us to not care whatever `f` panics or not. - // - Even though we create a copy of the pointer, we store it inside - // `ManuallyDrop` and only access it by-ref, so we don't double-drop. - // - // Semantically this is just `f(&self.pointer)` (where `self.pointer` - // is non-packed original pointer). - // - // Note that even though `CopyTaggedPtr` is only really expected to - // work with `P: Copy`, we have to assume `P: ?Copy`, because - // `CopyTaggedPtr` is used in the `TaggedPtr`'s implementation. - let ptr = unsafe { ManuallyDrop::new(P::from_ptr(self.pointer_raw())) }; - f(&ptr) - } -} - -impl Copy for CopyTaggedPtr -where - P: Pointer + Copy, - T: Tag, -{ -} - -impl Clone for CopyTaggedPtr -where - P: Pointer + Copy, - T: Tag, -{ - #[inline] - fn clone(&self) -> Self { - *self - } -} - -impl Deref for CopyTaggedPtr -where - P: Pointer, - T: Tag, -{ - type Target = P::Target; - - #[inline] - fn deref(&self) -> &Self::Target { - // Safety: - // `pointer_raw` returns the original pointer from `P::into_ptr` which, - // by the `Pointer`'s contract, must be valid. - unsafe { self.pointer_raw().as_ref() } - } -} - -impl DerefMut for CopyTaggedPtr -where - P: Pointer + DerefMut, - T: Tag, -{ - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - // Safety: - // `pointer_raw` returns the original pointer from `P::into_ptr` which, - // by the `Pointer`'s contract, must be valid for writes if - // `P: DerefMut`. - unsafe { self.pointer_raw().as_mut() } - } -} - -impl fmt::Debug for CopyTaggedPtr -where - P: Pointer + fmt::Debug, - T: Tag + fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.with_pointer_ref(|ptr| { - f.debug_struct("CopyTaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish() - }) - } -} - -impl PartialEq for CopyTaggedPtr -where - P: Pointer, - T: Tag, -{ - #[inline] - #[allow(ambiguous_wide_pointer_comparisons)] - fn eq(&self, other: &Self) -> bool { - self.packed == other.packed - } -} - -impl Eq for CopyTaggedPtr -where - P: Pointer, - T: Tag, -{ -} - -impl Hash for CopyTaggedPtr -where - P: Pointer, - T: Tag, -{ - #[inline] - fn hash(&self, state: &mut H) { - self.packed.hash(state); - } -} - -impl HashStable for CopyTaggedPtr -where - P: Pointer + HashStable, - T: Tag + HashStable, -{ - fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { - self.with_pointer_ref(|ptr| ptr.hash_stable(hcx, hasher)); - self.tag().hash_stable(hcx, hasher); - } -} - -// Safety: -// `CopyTaggedPtr` is semantically just `{ ptr: P, tag: T }`, as such -// it's ok to implement `Sync` as long as `P: Sync, T: Sync` -unsafe impl Sync for CopyTaggedPtr -where - P: Sync + Pointer, - T: Sync + Tag, -{ -} - -// Safety: -// `CopyTaggedPtr` is semantically just `{ ptr: P, tag: T }`, as such -// it's ok to implement `Send` as long as `P: Send, T: Send` -unsafe impl Send for CopyTaggedPtr -where - P: Send + Pointer, - T: Send + Tag, -{ -} - -/// Test that `new` does not compile if there is not enough alignment for the -/// tag in the pointer. -/// -/// ```compile_fail,E0080 -/// use rustc_data_structures::tagged_ptr::{CopyTaggedPtr, Tag}; -/// -/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] -/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 }; -/// -/// unsafe impl Tag for Tag2 { -/// const BITS: u32 = 2; -/// -/// fn into_usize(self) -> usize { todo!() } -/// unsafe fn from_usize(tag: usize) -> Self { todo!() } -/// } -/// -/// let value = 12u16; -/// let reference = &value; -/// let tag = Tag2::B01; -/// -/// let _ptr = CopyTaggedPtr::<_, _, true>::new(reference, tag); -/// ``` -// For some reason miri does not get the compile error -// probably it `check`s instead of `build`ing? -#[cfg(not(miri))] -const _: () = (); - -#[cfg(test)] -mod tests; diff --git a/compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs b/compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs deleted file mode 100644 index 160af8a65d9c2..0000000000000 --- a/compiler/rustc_data_structures/src/tagged_ptr/copy/tests.rs +++ /dev/null @@ -1,50 +0,0 @@ -use std::ptr; - -use crate::hashes::Hash128; -use crate::stable_hasher::{HashStable, StableHasher}; -use crate::tagged_ptr::{CopyTaggedPtr, Pointer, Tag, Tag2}; - -#[test] -fn smoke() { - let value = 12u32; - let reference = &value; - let tag = Tag2::B01; - - let ptr = tag_ptr(reference, tag); - - assert_eq!(ptr.tag(), tag); - assert_eq!(*ptr, 12); - assert!(ptr::eq(ptr.pointer(), reference)); - - let copy = ptr; - - let mut ptr = ptr; - ptr.set_tag(Tag2::B00); - assert_eq!(ptr.tag(), Tag2::B00); - - assert_eq!(copy.tag(), tag); - assert_eq!(*copy, 12); - assert!(ptr::eq(copy.pointer(), reference)); -} - -#[test] -fn stable_hash_hashes_as_tuple() { - let hash_packed = { - let mut hasher = StableHasher::new(); - tag_ptr(&12, Tag2::B11).hash_stable(&mut (), &mut hasher); - hasher.finish::() - }; - - let hash_tupled = { - let mut hasher = StableHasher::new(); - (&12, Tag2::B11).hash_stable(&mut (), &mut hasher); - hasher.finish::() - }; - - assert_eq!(hash_packed, hash_tupled); -} - -/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter. -fn tag_ptr(ptr: P, tag: T) -> CopyTaggedPtr { - CopyTaggedPtr::new(ptr, tag) -} diff --git a/compiler/rustc_data_structures/src/tagged_ptr/drop.rs b/compiler/rustc_data_structures/src/tagged_ptr/drop.rs deleted file mode 100644 index 319a8cdd3990a..0000000000000 --- a/compiler/rustc_data_structures/src/tagged_ptr/drop.rs +++ /dev/null @@ -1,178 +0,0 @@ -use std::fmt; -use std::hash::{Hash, Hasher}; -use std::ops::{Deref, DerefMut}; - -use super::{CopyTaggedPtr, Pointer, Tag}; -use crate::stable_hasher::{HashStable, StableHasher}; - -/// A tagged pointer that supports pointers that implement [`Drop`]. -/// -/// This is essentially `{ pointer: P, tag: T }` packed in a single pointer. -/// -/// You should use [`CopyTaggedPtr`] instead of the this type in all cases -/// where `P` implements [`Copy`]. -/// -/// If `COMPARE_PACKED` is true, then the pointers will be compared and hashed without -/// unpacking. Otherwise we don't implement [`PartialEq`], [`Eq`] and [`Hash`]; -/// if you want that, wrap the [`TaggedPtr`]. -pub struct TaggedPtr -where - P: Pointer, - T: Tag, -{ - raw: CopyTaggedPtr, -} - -impl TaggedPtr -where - P: Pointer, - T: Tag, -{ - /// Tags `pointer` with `tag`. - #[inline] - pub fn new(pointer: P, tag: T) -> Self { - TaggedPtr { raw: CopyTaggedPtr::new(pointer, tag) } - } - - /// Retrieves the tag. - #[inline] - pub fn tag(&self) -> T { - self.raw.tag() - } - - /// Sets the tag to a new value. - #[inline] - pub fn set_tag(&mut self, tag: T) { - self.raw.set_tag(tag) - } -} - -impl Clone for TaggedPtr -where - P: Pointer + Clone, - T: Tag, -{ - fn clone(&self) -> Self { - let ptr = self.raw.with_pointer_ref(P::clone); - - Self::new(ptr, self.tag()) - } -} - -impl Deref for TaggedPtr -where - P: Pointer, - T: Tag, -{ - type Target = P::Target; - - #[inline] - fn deref(&self) -> &Self::Target { - self.raw.deref() - } -} - -impl DerefMut for TaggedPtr -where - P: Pointer + DerefMut, - T: Tag, -{ - #[inline] - fn deref_mut(&mut self) -> &mut Self::Target { - self.raw.deref_mut() - } -} - -impl Drop for TaggedPtr -where - P: Pointer, - T: Tag, -{ - fn drop(&mut self) { - // No need to drop the tag, as it's Copy - unsafe { - drop(P::from_ptr(self.raw.pointer_raw())); - } - } -} - -impl fmt::Debug for TaggedPtr -where - P: Pointer + fmt::Debug, - T: Tag + fmt::Debug, -{ - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - self.raw.with_pointer_ref(|ptr| { - f.debug_struct("TaggedPtr").field("pointer", ptr).field("tag", &self.tag()).finish() - }) - } -} - -impl PartialEq for TaggedPtr -where - P: Pointer, - T: Tag, -{ - #[inline] - fn eq(&self, other: &Self) -> bool { - self.raw.eq(&other.raw) - } -} - -impl Eq for TaggedPtr -where - P: Pointer, - T: Tag, -{ -} - -impl Hash for TaggedPtr -where - P: Pointer, - T: Tag, -{ - #[inline] - fn hash(&self, state: &mut H) { - self.raw.hash(state); - } -} - -impl HashStable for TaggedPtr -where - P: Pointer + HashStable, - T: Tag + HashStable, -{ - fn hash_stable(&self, hcx: &mut HCX, hasher: &mut StableHasher) { - self.raw.hash_stable(hcx, hasher); - } -} - -/// Test that `new` does not compile if there is not enough alignment for the -/// tag in the pointer. -/// -/// ```compile_fail,E0080 -/// use rustc_data_structures::tagged_ptr::{TaggedPtr, Tag}; -/// -/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] -/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 }; -/// -/// unsafe impl Tag for Tag2 { -/// const BITS: u32 = 2; -/// -/// fn into_usize(self) -> usize { todo!() } -/// unsafe fn from_usize(tag: usize) -> Self { todo!() } -/// } -/// -/// let value = 12u16; -/// let reference = &value; -/// let tag = Tag2::B01; -/// -/// let _ptr = TaggedPtr::<_, _, true>::new(reference, tag); -/// ``` -// For some reason miri does not get the compile error -// probably it `check`s instead of `build`ing? -#[cfg(not(miri))] -const _: () = (); - -#[cfg(test)] -mod tests; diff --git a/compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs b/compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs deleted file mode 100644 index 4d342c72cc52a..0000000000000 --- a/compiler/rustc_data_structures/src/tagged_ptr/drop/tests.rs +++ /dev/null @@ -1,72 +0,0 @@ -use std::ptr; -use std::sync::Arc; - -use crate::tagged_ptr::{Pointer, Tag, Tag2, TaggedPtr}; - -#[test] -fn smoke() { - let value = 12u32; - let reference = &value; - let tag = Tag2::B01; - - let ptr = tag_ptr(reference, tag); - - assert_eq!(ptr.tag(), tag); - assert_eq!(*ptr, 12); - - let clone = ptr.clone(); - assert_eq!(clone.tag(), tag); - assert_eq!(*clone, 12); - - let mut ptr = ptr; - ptr.set_tag(Tag2::B00); - assert_eq!(ptr.tag(), Tag2::B00); - - assert_eq!(clone.tag(), tag); - assert_eq!(*clone, 12); - assert!(ptr::eq(&*ptr, &*clone)) -} - -#[test] -fn boxed() { - let value = 12u32; - let boxed = Box::new(value); - let tag = Tag2::B01; - - let ptr = tag_ptr(boxed, tag); - - assert_eq!(ptr.tag(), tag); - assert_eq!(*ptr, 12); - - let clone = ptr.clone(); - assert_eq!(clone.tag(), tag); - assert_eq!(*clone, 12); - - let mut ptr = ptr; - ptr.set_tag(Tag2::B00); - assert_eq!(ptr.tag(), Tag2::B00); - - assert_eq!(clone.tag(), tag); - assert_eq!(*clone, 12); - assert!(!ptr::eq(&*ptr, &*clone)) -} - -#[test] -fn arclones() { - let value = 12u32; - let arc = Arc::new(value); - let tag = Tag2::B01; - - let ptr = tag_ptr(arc, tag); - - assert_eq!(ptr.tag(), tag); - assert_eq!(*ptr, 12); - - let clone = ptr.clone(); - assert!(ptr::eq(&*ptr, &*clone)) -} - -/// Helper to create tagged pointers without specifying `COMPARE_PACKED` if it does not matter. -fn tag_ptr(ptr: P, tag: T) -> TaggedPtr { - TaggedPtr::new(ptr, tag) -} diff --git a/compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs b/compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs deleted file mode 100644 index f17a0bf26d747..0000000000000 --- a/compiler/rustc_data_structures/src/tagged_ptr/impl_tag.rs +++ /dev/null @@ -1,144 +0,0 @@ -/// Implements [`Tag`] for a given type. -/// -/// You can use `impl_tag` on structs and enums. -/// You need to specify the type and all its possible values, -/// which can only be paths with optional fields. -/// -/// [`Tag`]: crate::tagged_ptr::Tag -/// -/// # Examples -/// -/// Basic usage: -/// -/// ``` -/// #![feature(macro_metavar_expr)] -/// use rustc_data_structures::{impl_tag, tagged_ptr::Tag}; -/// -/// #[derive(Copy, Clone, PartialEq, Debug)] -/// enum SomeTag { -/// A, -/// B, -/// X { v: bool }, -/// Y(bool, bool), -/// } -/// -/// impl_tag! { -/// // The type for which the `Tag` will be implemented -/// impl Tag for SomeTag; -/// // You need to specify all possible tag values: -/// SomeTag::A, // 0 -/// SomeTag::B, // 1 -/// // For variants with fields, you need to specify the fields: -/// SomeTag::X { v: true }, // 2 -/// SomeTag::X { v: false }, // 3 -/// // For tuple variants use named syntax: -/// SomeTag::Y { 0: true, 1: true }, // 4 -/// SomeTag::Y { 0: false, 1: true }, // 5 -/// SomeTag::Y { 0: true, 1: false }, // 6 -/// SomeTag::Y { 0: false, 1: false }, // 7 -/// } -/// -/// // Tag values are assigned in order: -/// assert_eq!(SomeTag::A.into_usize(), 0); -/// assert_eq!(SomeTag::X { v: false }.into_usize(), 3); -/// assert_eq!(SomeTag::Y(false, true).into_usize(), 5); -/// -/// assert_eq!(unsafe { SomeTag::from_usize(1) }, SomeTag::B); -/// assert_eq!(unsafe { SomeTag::from_usize(2) }, SomeTag::X { v: true }); -/// assert_eq!(unsafe { SomeTag::from_usize(7) }, SomeTag::Y(false, false)); -/// ``` -/// -/// Structs are supported: -/// -/// ``` -/// #![feature(macro_metavar_expr)] -/// # use rustc_data_structures::impl_tag; -/// #[derive(Copy, Clone)] -/// struct Flags { a: bool, b: bool } -/// -/// impl_tag! { -/// impl Tag for Flags; -/// Flags { a: true, b: true }, -/// Flags { a: false, b: true }, -/// Flags { a: true, b: false }, -/// Flags { a: false, b: false }, -/// } -/// ``` -/// -/// Not specifying all values results in a compile error: -/// -/// ```compile_fail,E0004 -/// #![feature(macro_metavar_expr)] -/// # use rustc_data_structures::impl_tag; -/// #[derive(Copy, Clone)] -/// enum E { -/// A, -/// B, -/// } -/// -/// impl_tag! { -/// impl Tag for E; -/// E::A, -/// } -/// ``` -#[macro_export] -macro_rules! impl_tag { - ( - impl Tag for $Self:ty; - $( - $($path:ident)::* $( { $( $fields:tt )* })?, - )* - ) => { - // Safety: - // `bits_for_tags` is called on the same `${index()}`-es as - // `into_usize` returns, thus `BITS` constant is correct. - unsafe impl $crate::tagged_ptr::Tag for $Self { - const BITS: u32 = $crate::tagged_ptr::bits_for_tags(&[ - $( - ${index()}, - $( ${ignore($path)} )* - )* - ]); - - #[inline] - fn into_usize(self) -> usize { - // This forbids use of repeating patterns (`Enum::V`&`Enum::V`, etc) - // (or at least it should, see ) - #[forbid(unreachable_patterns)] - match self { - // `match` is doing heavy lifting here, by requiring exhaustiveness - $( - $($path)::* $( { $( $fields )* } )? => ${index()}, - )* - } - } - - #[inline] - unsafe fn from_usize(tag: usize) -> Self { - match tag { - $( - ${index()} => $($path)::* $( { $( $fields )* } )?, - )* - - // Safety: - // `into_usize` only returns `${index()}` of the same - // repetition as we are filtering above, thus if this is - // reached, the safety contract of this function was - // already breached. - _ => unsafe { - debug_assert!( - false, - "invalid tag: {tag}\ - (this is a bug in the caller of `from_usize`)" - ); - std::hint::unreachable_unchecked() - }, - } - } - - } - }; -} - -#[cfg(test)] -mod tests; diff --git a/compiler/rustc_data_structures/src/tagged_ptr/impl_tag/tests.rs b/compiler/rustc_data_structures/src/tagged_ptr/impl_tag/tests.rs deleted file mode 100644 index 62c926153e1e9..0000000000000 --- a/compiler/rustc_data_structures/src/tagged_ptr/impl_tag/tests.rs +++ /dev/null @@ -1,34 +0,0 @@ -#[test] -fn bits_constant() { - use crate::tagged_ptr::Tag; - - #[derive(Copy, Clone)] - struct Unit; - impl_tag! { impl Tag for Unit; Unit, } - assert_eq!(Unit::BITS, 0); - - #[derive(Copy, Clone)] - enum Enum3 { - A, - B, - C, - } - impl_tag! { impl Tag for Enum3; Enum3::A, Enum3::B, Enum3::C, } - assert_eq!(Enum3::BITS, 2); - - #[derive(Copy, Clone)] - struct Eight(bool, bool, bool); - impl_tag! { - impl Tag for Eight; - Eight { 0: true, 1: true, 2: true }, - Eight { 0: true, 1: true, 2: false }, - Eight { 0: true, 1: false, 2: true }, - Eight { 0: true, 1: false, 2: false }, - Eight { 0: false, 1: true, 2: true }, - Eight { 0: false, 1: true, 2: false }, - Eight { 0: false, 1: false, 2: true }, - Eight { 0: false, 1: false, 2: false }, - } - - assert_eq!(Eight::BITS, 3); -} diff --git a/compiler/rustc_data_structures/src/tagged_ptr/tests.rs b/compiler/rustc_data_structures/src/tagged_ptr/tests.rs new file mode 100644 index 0000000000000..b1bdee18d6d43 --- /dev/null +++ b/compiler/rustc_data_structures/src/tagged_ptr/tests.rs @@ -0,0 +1,105 @@ +use std::ptr; + +use super::*; +use crate::hashes::Hash128; +use crate::stable_hasher::{HashStable, StableHasher}; + +/// A tag type used in [`TaggedRef`] tests. +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +pub enum Tag2 { + B00 = 0b00, + B01 = 0b01, + B10 = 0b10, + B11 = 0b11, +} + +unsafe impl Tag for Tag2 { + const BITS: u32 = 2; + + fn into_usize(self) -> usize { + self as _ + } + + unsafe fn from_usize(tag: usize) -> Self { + match tag { + 0b00 => Tag2::B00, + 0b01 => Tag2::B01, + 0b10 => Tag2::B10, + 0b11 => Tag2::B11, + _ => unreachable!(), + } + } +} + +impl crate::stable_hasher::HashStable for Tag2 { + fn hash_stable(&self, hcx: &mut HCX, hasher: &mut crate::stable_hasher::StableHasher) { + (*self as u8).hash_stable(hcx, hasher); + } +} + +#[test] +fn smoke() { + let value = 12u32; + let reference = &value; + let tag = Tag2::B01; + + let ptr = TaggedRef::new(reference, tag); + + assert_eq!(ptr.tag(), tag); + assert_eq!(*ptr, 12); + assert!(ptr::eq(ptr.pointer(), reference)); + + let copy = ptr; + + let mut ptr = ptr; + ptr.set_tag(Tag2::B00); + assert_eq!(ptr.tag(), Tag2::B00); + + assert_eq!(copy.tag(), tag); + assert_eq!(*copy, 12); + assert!(ptr::eq(copy.pointer(), reference)); +} + +#[test] +fn stable_hash_hashes_as_tuple() { + let hash_packed = { + let mut hasher = StableHasher::new(); + TaggedRef::new(&12, Tag2::B11).hash_stable(&mut (), &mut hasher); + hasher.finish::() + }; + + let hash_tupled = { + let mut hasher = StableHasher::new(); + (&12, Tag2::B11).hash_stable(&mut (), &mut hasher); + hasher.finish::() + }; + + assert_eq!(hash_packed, hash_tupled); +} + +/// Test that `new` does not compile if there is not enough alignment for the +/// tag in the pointer. +/// +/// ```compile_fail,E0080 +/// use rustc_data_structures::tagged_ptr::{TaggedRef, Tag}; +/// +/// #[derive(Copy, Clone, Debug, PartialEq, Eq)] +/// enum Tag2 { B00 = 0b00, B01 = 0b01, B10 = 0b10, B11 = 0b11 }; +/// +/// unsafe impl Tag for Tag2 { +/// const BITS: u32 = 2; +/// +/// fn into_usize(self) -> usize { todo!() } +/// unsafe fn from_usize(tag: usize) -> Self { todo!() } +/// } +/// +/// let value = 12u16; +/// let reference = &value; +/// let tag = Tag2::B01; +/// +/// let _ptr = TaggedRef::<_, _, true>::new(reference, tag); +/// ``` +// For some reason miri does not get the compile error +// probably it `check`s instead of `build`ing? +#[cfg(not(miri))] +const _: () = (); diff --git a/compiler/rustc_middle/src/ty/list.rs b/compiler/rustc_middle/src/ty/list.rs index 30a5586f59ca4..6718493f6b32a 100644 --- a/compiler/rustc_middle/src/ty/list.rs +++ b/compiler/rustc_middle/src/ty/list.rs @@ -21,7 +21,7 @@ use crate::arena::Arena; /// pointer. /// - Because of this, you cannot get a `List` that is a sub-list of another /// `List`. You can get a sub-slice `&[T]`, however. -/// - `List` can be used with `CopyTaggedPtr`, which is useful within +/// - `List` can be used with `TaggedRef`, which is useful within /// structs whose size must be minimized. /// - Because of the uniqueness assumption, we can use the address of a /// `List` for faster equality comparisons and hashing.