Rollup merge of #99011 - oli-obk:UnsoundCell, r=eddyb

`UnsafeCell` blocks niches inside its nested type from being available outside

fixes #87341

This implements the plan by `@eddyb` in #87341 (comment)

Somewhat related PR (not strictly necessary, but that cleanup made this PR simpler): #94527

compiler/rustc_attr/src/builtin.rs

@@ -856,7 +856,6 @@ pub enum ReprAttr {
     ReprSimd,
     ReprTransparent,
     ReprAlign(u32),
-    ReprNoNiche,
 }
 
 #[derive(Eq, PartialEq, Debug, Copy, Clone)]
@@ -904,7 +903,6 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                     sym::packed => Some(ReprPacked(1)),
                     sym::simd => Some(ReprSimd),
                     sym::transparent => Some(ReprTransparent),
-                    sym::no_niche => Some(ReprNoNiche),
                     sym::align => {
                         let mut err = struct_span_err!(
                             diagnostic,
@@ -943,7 +941,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         Ok(literal) => acc.push(ReprPacked(literal)),
                         Err(message) => literal_error = Some(message),
                     };
-                } else if matches!(name, sym::C | sym::simd | sym::transparent | sym::no_niche)
+                } else if matches!(name, sym::C | sym::simd | sym::transparent)
                     || int_type_of_word(name).is_some()
                 {
                     recognised = true;
@@ -1001,7 +999,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                     } else {
                         if matches!(
                             meta_item.name_or_empty(),
-                            sym::C | sym::simd | sym::transparent | sym::no_niche
+                            sym::C | sym::simd | sym::transparent
                         ) || int_type_of_word(meta_item.name_or_empty()).is_some()
                         {
                             recognised = true;
@@ -1039,7 +1037,7 @@ pub fn parse_repr_attr(sess: &Session, attr: &Attribute) -> Vec<ReprAttr> {
                         .emit();
                     } else if matches!(
                         meta_item.name_or_empty(),
-                        sym::C | sym::simd | sym::transparent | sym::no_niche
+                        sym::C | sym::simd | sym::transparent
                     ) || int_type_of_word(meta_item.name_or_empty()).is_some()
                     {
                         recognised = true;

compiler/rustc_const_eval/src/interpret/intern.rs

@@ -217,7 +217,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: CompileTimeMachine<'mir, 'tcx, const_eval::Memory
         }
 
         if let Some(def) = mplace.layout.ty.ty_adt_def() {
-            if Some(def.did()) == self.ecx.tcx.lang_items().unsafe_cell_type() {
+            if def.is_unsafe_cell() {
                 // We are crossing over an `UnsafeCell`, we can mutate again. This means that
                 // References we encounter inside here are interned as pointing to mutable
                 // allocations.

compiler/rustc_const_eval/src/interpret/validity.rs

@@ -821,7 +821,7 @@ impl<'rt, 'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> ValueVisitor<'mir, 'tcx, M>
         // Special check preventing `UnsafeCell` in the inner part of constants
         if let Some(def) = op.layout.ty.ty_adt_def() {
             if matches!(self.ctfe_mode, Some(CtfeValidationMode::Const { inner: true, .. }))
-                && Some(def.did()) == self.ecx.tcx.lang_items().unsafe_cell_type()
+                && def.is_unsafe_cell()
             {
                 throw_validation_failure!(self.path, { "`UnsafeCell` in a `const`" });
             }

compiler/rustc_const_eval/src/transform/check_consts/qualifs.rs

@@ -96,13 +96,13 @@ impl Qualif for HasMutInterior {
     }
 
     fn in_adt_inherently<'tcx>(
-        cx: &ConstCx<'_, 'tcx>,
+        _cx: &ConstCx<'_, 'tcx>,
         adt: AdtDef<'tcx>,
         _: SubstsRef<'tcx>,
     ) -> bool {
         // Exactly one type, `UnsafeCell`, has the `HasMutInterior` qualif inherently.
         // It arises structurally for all other types.
-        Some(adt.did()) == cx.tcx.lang_items().unsafe_cell_type()
+        adt.is_unsafe_cell()
     }
 }
 

compiler/rustc_feature/src/active.rs

@@ -156,9 +156,6 @@ declare_features! (
     (active, intrinsics, "1.0.0", None, None),
     /// Allows using `#[lang = ".."]` attribute for linking items to special compiler logic.
     (active, lang_items, "1.0.0", None, None),
-    /// Allows `#[repr(no_niche)]` (an implementation detail of `rustc`,
-    /// it is not on path for eventual stabilization).
-    (active, no_niche, "1.42.0", None, None),
     /// Allows using `#[omit_gdb_pretty_printer_section]`.
     (active, omit_gdb_pretty_printer_section, "1.5.0", None, None),
     /// Allows using `#[prelude_import]` on glob `use` items.

compiler/rustc_lint/src/types.rs

@@ -703,9 +703,8 @@ fn ty_is_known_nonnull<'tcx>(cx: &LateContext<'tcx>, ty: Ty<'tcx>, mode: CItemKi
                 return true;
             }
 
-            // Types with a `#[repr(no_niche)]` attribute have their niche hidden.
-            // The attribute is used by the UnsafeCell for example (the only use so far).
-            if def.repr().hide_niche() {
+            // `UnsafeCell` has its niche hidden.
+            if def.is_unsafe_cell() {
                 return false;
             }
 

compiler/rustc_middle/src/ty/adt.rs

@@ -52,6 +52,8 @@ bitflags! {
         /// Indicates whether the variant list of this ADT is `#[non_exhaustive]`.
         /// (i.e., this flag is never set unless this ADT is an enum).
         const IS_VARIANT_LIST_NON_EXHAUSTIVE = 1 << 8;
+        /// Indicates whether the type is `UnsafeCell`.
+        const IS_UNSAFE_CELL              = 1 << 9;
     }
 }
 
@@ -247,6 +249,9 @@ impl AdtDefData {
         if Some(did) == tcx.lang_items().manually_drop() {
             flags |= AdtFlags::IS_MANUALLY_DROP;
         }
+        if Some(did) == tcx.lang_items().unsafe_cell_type() {
+            flags |= AdtFlags::IS_UNSAFE_CELL;
+        }
 
         AdtDefData { did, variants, flags, repr }
     }
@@ -333,6 +338,12 @@ impl<'tcx> AdtDef<'tcx> {
         self.flags().contains(AdtFlags::IS_BOX)
     }
 
+    /// Returns `true` if this is UnsafeCell<T>.
+    #[inline]
+    pub fn is_unsafe_cell(self) -> bool {
+        self.flags().contains(AdtFlags::IS_UNSAFE_CELL)
+    }
+
     /// Returns `true` if this is `ManuallyDrop<T>`.
     #[inline]
     pub fn is_manually_drop(self) -> bool {

compiler/rustc_middle/src/ty/layout.rs

@@ -542,14 +542,12 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
             debug!("univariant offset: {:?} field: {:#?}", offset, field);
             offsets[i as usize] = offset;
 
-            if !repr.hide_niche() {
-                if let Some(mut niche) = field.largest_niche {
-                    let available = niche.available(dl);
-                    if available > largest_niche_available {
-                        largest_niche_available = available;
-                        niche.offset += offset;
-                        largest_niche = Some(niche);
-                    }
+            if let Some(mut niche) = field.largest_niche {
+                let available = niche.available(dl);
+                if available > largest_niche_available {
+                    largest_niche_available = available;
+                    niche.offset += offset;
+                    largest_niche = Some(niche);
                 }
             }
 
@@ -1078,6 +1076,29 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
 
                     let mut st = self.univariant_uninterned(ty, &variants[v], &def.repr(), kind)?;
                     st.variants = Variants::Single { index: v };
+
+                    if def.is_unsafe_cell() {
+                        let hide_niches = |scalar: &mut _| match scalar {
+                            Scalar::Initialized { value, valid_range } => {
+                                *valid_range = WrappingRange::full(value.size(dl))
+                            }
+                            // Already doesn't have any niches
+                            Scalar::Union { .. } => {}
+                        };
+                        match &mut st.abi {
+                            Abi::Uninhabited => {}
+                            Abi::Scalar(scalar) => hide_niches(scalar),
+                            Abi::ScalarPair(a, b) => {
+                                hide_niches(a);
+                                hide_niches(b);
+                            }
+                            Abi::Vector { element, count: _ } => hide_niches(element),
+                            Abi::Aggregate { sized: _ } => {}
+                        }
+                        st.largest_niche = None;
+                        return Ok(tcx.intern_layout(st));
+                    }
+
                     let (start, end) = self.tcx.layout_scalar_valid_range(def.did());
                     match st.abi {
                         Abi::Scalar(ref mut scalar) | Abi::ScalarPair(ref mut scalar, _) => {
@@ -1106,11 +1127,7 @@ impl<'tcx> LayoutCx<'tcx, TyCtxt<'tcx>> {
                             }
 
                             // Update `largest_niche` if we have introduced a larger niche.
-                            let niche = if def.repr().hide_niche() {
-                                None
-                            } else {
-                                Niche::from_scalar(dl, Size::ZERO, *scalar)
-                            };
+                            let niche = Niche::from_scalar(dl, Size::ZERO, *scalar);
                             if let Some(niche) = niche {
                                 match st.largest_niche {
                                     Some(largest_niche) => {

compiler/rustc_middle/src/ty/mod.rs

@@ -1720,11 +1720,9 @@ bitflags! {
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
         const IS_LINEAR          = 1 << 3;
-        // If true, don't expose any niche to type's context.
-        const HIDE_NICHE         = 1 << 4;
         // If true, the type's layout can be randomized using
         // the seed stored in `ReprOptions.layout_seed`
-        const RANDOMIZE_LAYOUT   = 1 << 5;
+        const RANDOMIZE_LAYOUT   = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
         const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits
                                  | ReprFlags::IS_SIMD.bits
@@ -1781,7 +1779,6 @@ impl ReprOptions {
                         ReprFlags::empty()
                     }
                     attr::ReprTransparent => ReprFlags::IS_TRANSPARENT,
-                    attr::ReprNoNiche => ReprFlags::HIDE_NICHE,
                     attr::ReprSimd => ReprFlags::IS_SIMD,
                     attr::ReprInt(i) => {
                         size = Some(i);
@@ -1834,11 +1831,6 @@ impl ReprOptions {
         self.flags.contains(ReprFlags::IS_LINEAR)
     }
 
-    #[inline]
-    pub fn hide_niche(&self) -> bool {
-        self.flags.contains(ReprFlags::HIDE_NICHE)
-    }
-
     /// Returns the discriminant type, given these `repr` options.
     /// This must only be called on enums!
     pub fn discr_type(&self) -> attr::IntType {

compiler/rustc_passes/src/check_attr.rs

@@ -1808,21 +1808,6 @@ impl CheckAttrVisitor<'_> {
                         _ => ("a", "struct, enum, or union"),
                     }
                 }
-                sym::no_niche => {
-                    if !self.tcx.features().enabled(sym::no_niche) {
-                        feature_err(
-                            &self.tcx.sess.parse_sess,
-                            sym::no_niche,
-                            hint.span(),
-                            "the attribute `repr(no_niche)` is currently unstable",
-                        )
-                        .emit();
-                    }
-                    match target {
-                        Target::Struct | Target::Enum => continue,
-                        _ => ("a", "struct or enum"),
-                    }
-                }
                 sym::i8
                 | sym::u8
                 | sym::i16
@@ -1870,10 +1855,8 @@ impl CheckAttrVisitor<'_> {
         // This is not ideal, but tracking precisely which ones are at fault is a huge hassle.
         let hint_spans = hints.iter().map(|hint| hint.span());
 
-        // Error on repr(transparent, <anything else apart from no_niche>).
-        let non_no_niche = |hint: &&NestedMetaItem| hint.name_or_empty() != sym::no_niche;
-        let non_no_niche_count = hints.iter().filter(non_no_niche).count();
-        if is_transparent && non_no_niche_count > 1 {
+        // Error on repr(transparent, <anything else>).
+        if is_transparent && hints.len() > 1 {
             let hint_spans: Vec<_> = hint_spans.clone().collect();
             struct_span_err!(
                 self.tcx.sess,

compiler/rustc_span/src/symbol.rs

@@ -980,7 +980,6 @@ symbols! {
         no_link,
         no_main,
         no_mangle,
-        no_niche,
         no_sanitize,
         no_stack_check,
         no_start,
@@ -1153,7 +1152,6 @@ symbols! {
         repr128,
         repr_align,
         repr_align_enum,
-        repr_no_niche,
         repr_packed,
         repr_simd,
         repr_transparent,

library/core/src/cell.rs

@@ -1856,7 +1856,6 @@ impl<T: ?Sized + fmt::Display> fmt::Display for RefMut<'_, T> {
 #[lang = "unsafe_cell"]
 #[stable(feature = "rust1", since = "1.0.0")]
 #[repr(transparent)]
-#[repr(no_niche)] // rust-lang/rust#68303.
 pub struct UnsafeCell<T: ?Sized> {
     value: T,
 }

library/core/src/lib.rs

@@ -191,7 +191,6 @@
 #![feature(never_type)]
 #![feature(no_core)]
 #![feature(no_coverage)] // rust-lang/rust#84605
-#![feature(no_niche)] // rust-lang/rust#68303
 #![feature(platform_intrinsics)]
 #![feature(prelude_import)]
 #![feature(repr_simd)]

src/test/ui/layout/unsafe-cell-hides-niche.rs

@@ -3,30 +3,80 @@
 // test checks that an `Option<UnsafeCell<NonZeroU32>>` has the same
 // size in memory as an `Option<UnsafeCell<u32>>` (namely, 8 bytes).
 
-// run-pass
+// check-pass
+// compile-flags: --crate-type=lib
+// only-x86
 
-#![feature(no_niche)]
+#![feature(repr_simd)]
 
-use std::cell::UnsafeCell;
+use std::cell::{UnsafeCell, RefCell, Cell};
 use std::mem::size_of;
 use std::num::NonZeroU32 as N32;
+use std::sync::{Mutex, RwLock};
 
 struct Wrapper<T>(T);
 
 #[repr(transparent)]
 struct Transparent<T>(T);
 
-#[repr(no_niche)]
-struct NoNiche<T>(T);
+struct NoNiche<T>(UnsafeCell<T>);
 
-fn main() {
-    assert_eq!(size_of::<Option<Wrapper<u32>>>(),     8);
-    assert_eq!(size_of::<Option<Wrapper<N32>>>(),     4);
-    assert_eq!(size_of::<Option<Transparent<u32>>>(), 8);
-    assert_eq!(size_of::<Option<Transparent<N32>>>(), 4);
-    assert_eq!(size_of::<Option<NoNiche<u32>>>(),     8);
-    assert_eq!(size_of::<Option<NoNiche<N32>>>(),     8);
+struct Size<const S: usize>;
 
-    assert_eq!(size_of::<Option<UnsafeCell<u32>>>(),  8);
-    assert_eq!(size_of::<Option<UnsafeCell<N32>>>(),  8);
+macro_rules! check_sizes {
+    (check_one_specific_size: $ty:ty, $size:expr) => {
+        const _: Size::<{$size}> = Size::<{size_of::<$ty>()}>;
+    };
+    // Any tests run on `UnsafeCell` must be the same for `Cell`
+    (UnsafeCell<$ty:ty>: $size:expr => $optioned_size:expr) => {
+        check_sizes!(Cell<$ty>: $size => $optioned_size);
+        check_sizes!(@actual_check: UnsafeCell<$ty>: $size => $optioned_size);
+    };
+    ($ty:ty: $size:expr => $optioned_size:expr) => {
+        check_sizes!(@actual_check: $ty: $size => $optioned_size);
+    };
+    // This branch does the actual checking logic, the `@actual_check` prefix is here to distinguish
+    // it from other branches and not accidentally match any.
+    (@actual_check: $ty:ty: $size:expr => $optioned_size:expr) => {
+        check_sizes!(check_one_specific_size: $ty, $size);
+        check_sizes!(check_one_specific_size: Option<$ty>, $optioned_size);
+        check_sizes!(check_no_niche_opt: $size != $optioned_size, $ty);
+    };
+    // only check that there is no niche (size goes up when wrapped in an option),
+    // don't check actual sizes
+    ($ty:ty) => {
+        check_sizes!(check_no_niche_opt: true, $ty);
+    };
+    (check_no_niche_opt: $no_niche_opt:expr, $ty:ty) => {
+        const _: () = if $no_niche_opt { assert!(size_of::<$ty>() < size_of::<Option<$ty>>()); };
+    };
 }
+
+const PTR_SIZE: usize = std::mem::size_of::<*const ()>();
+
+check_sizes!(Wrapper<u32>:     4 => 8);
+check_sizes!(Wrapper<N32>:     4 => 4); // (✓ niche opt)
+check_sizes!(Transparent<u32>: 4 => 8);
+check_sizes!(Transparent<N32>: 4 => 4); // (✓ niche opt)
+check_sizes!(NoNiche<u32>:     4 => 8);
+check_sizes!(NoNiche<N32>:     4 => 8);
+
+check_sizes!(UnsafeCell<u32>:  4 => 8);
+check_sizes!(UnsafeCell<N32>:  4 => 8);
+
+check_sizes!(UnsafeCell<&()>: PTR_SIZE => PTR_SIZE * 2);
+check_sizes!(   RefCell<&()>: PTR_SIZE * 2 => PTR_SIZE * 3);
+
+check_sizes!(RwLock<&()>);
+check_sizes!(Mutex<&()>);
+
+check_sizes!(UnsafeCell<&[i32]>: PTR_SIZE * 2 => PTR_SIZE * 3);
+check_sizes!(UnsafeCell<(&(), &())>: PTR_SIZE * 2 => PTR_SIZE * 3);
+
+trait Trait {}
+check_sizes!(UnsafeCell<&dyn Trait>: PTR_SIZE * 2 => PTR_SIZE * 3);
+
+#[repr(simd)]
+pub struct Vec4<T>([T; 4]);
+
+check_sizes!(UnsafeCell<Vec4<N32>>: 16 => 32);