Added randomized field padding to -Z randomize-layout

compiler/rustc_abi/src/layout.rs

@@ -8,7 +8,7 @@ use std::{
 };
 
 #[cfg(feature = "randomize")]
-use rand::{seq::SliceRandom, SeedableRng};
+use rand::{seq::SliceRandom, Rng, SeedableRng};
 #[cfg(feature = "randomize")]
 use rand_xoshiro::Xoshiro128StarStar;
 
@@ -61,18 +61,30 @@ pub trait LayoutCalculator {
         }
     }
 
-    fn univariant<'a, V: Idx, F: Deref<Target = &'a LayoutS<V>> + Debug>(
+    fn univariant<'a, V, F, N>(
         &self,
         dl: &TargetDataLayout,
         fields: &[F],
         repr: &ReprOptions,
         kind: StructKind,
-    ) -> Option<LayoutS<V>> {
+        option_niche_guaranteed: N,
+    ) -> Option<LayoutS<V>>
+    where
+        V: Idx,
+        F: Deref<Target = &'a LayoutS<V>> + Debug,
+        N: Fn(&Self) -> bool + Copy,
+    {
         let pack = repr.pack;
         let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
         let mut inverse_memory_index: Vec<u32> = (0..fields.len() as u32).collect();
-        let optimize = !repr.inhibit_struct_field_reordering_opt();
-        if optimize {
+
+        // `ReprOptions.layout_seed` is a deterministic seed that we can use to
+        // randomize field ordering with
+        #[cfg(feature = "randomize")]
+        let mut rng = Xoshiro128StarStar::seed_from_u64(repr.field_shuffle_seed);
+
+        let can_optimize = !repr.inhibit_struct_field_reordering_opt();
+        if can_optimize {
             let end =
                 if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
             let optimizing = &mut inverse_memory_index[..end];
@@ -94,16 +106,11 @@ pub trait LayoutCalculator {
             // the field ordering to try and catch some code making assumptions about layouts
             // we don't guarantee
             if repr.can_randomize_type_layout() && cfg!(feature = "randomize") {
+                // Shuffle the ordering of the fields
                 #[cfg(feature = "randomize")]
-                {
-                    // `ReprOptions.layout_seed` is a deterministic seed that we can use to
-                    // randomize field ordering with
-                    let mut rng = Xoshiro128StarStar::seed_from_u64(repr.field_shuffle_seed);
+                optimizing.shuffle(&mut rng);
 
-                    // Shuffle the ordering of the fields
-                    optimizing.shuffle(&mut rng);
-                }
-                // Otherwise we just leave things alone and actually optimize the type's fields
+            // Otherwise we just leave things alone and actually optimize the type's fields
             } else {
                 match kind {
                     StructKind::AlwaysSized | StructKind::MaybeUnsized => {
@@ -173,6 +180,32 @@ pub trait LayoutCalculator {
             offset = offset.align_to(field_align.abi);
             align = align.max(field_align);
 
+            // If `-Z randomize-layout` is enabled, we pad each field by a multiple of its alignment
+            // If layout randomization is disabled, we don't pad it by anything and if it is
+            // we multiply the field's alignment by anything from zero to the user provided
+            // maximum multiple (defaults to three)
+            //
+            // When `-Z randomize-layout` is enabled that doesn't necessarily mean we can
+            // go ham on every type that at first glance looks valid for layout optimization.
+            // `Option` specifically has layout guarantees when it has specific `T` substitutions,
+            // such as `Option<NonNull<_>>` or `Option<NonZeroUsize>` both being exactly one `usize`
+            // large. As such, we have to ensure that the type doesn't guarantee niche optimization
+            // with the current payload
+            #[cfg(feature = "randomize")]
+            if repr.can_randomize_type_layout() && !option_niche_guaranteed(self) {
+                let align_bytes = field_align.abi.bytes();
+                let random_padding = align_bytes
+                    .checked_mul(rng.gen_range(0..=repr.random_padding_max_factor as u64))
+                    .unwrap_or(align_bytes);
+
+                // Attempt to add our extra padding, defaulting to the type's alignment
+                if let Some(randomized_offset) =
+                    offset.checked_add(Size::from_bytes(random_padding), dl)
+                {
+                    offset = randomized_offset;
+                }
+            }
+
             debug!("univariant offset: {:?} field: {:#?}", offset, field);
             offsets[i as usize] = offset;
 
@@ -199,7 +232,7 @@ pub trait LayoutCalculator {
         // Field 5 would be the first element, so memory_index is i:
         // Note: if we didn't optimize, it's already right.
         let memory_index =
-            if optimize { invert_mapping(&inverse_memory_index) } else { inverse_memory_index };
+            if can_optimize { invert_mapping(&inverse_memory_index) } else { inverse_memory_index };
         let size = min_size.align_to(align.abi);
         let mut abi = Abi::Aggregate { sized };
         // Unpack newtype ABIs and find scalar pairs.
@@ -216,7 +249,7 @@ pub trait LayoutCalculator {
                         match field.abi {
                             // For plain scalars, or vectors of them, we can't unpack
                             // newtypes for `#[repr(C)]`, as that affects C ABIs.
-                            Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
+                            Abi::Scalar(_) | Abi::Vector { .. } if can_optimize => {
                                 abi = field.abi;
                             }
                             // But scalar pairs are Rust-specific and get
@@ -290,7 +323,7 @@ pub trait LayoutCalculator {
         }
     }
 
-    fn layout_of_struct_or_enum<'a, V: Idx, F: Deref<Target = &'a LayoutS<V>> + Debug>(
+    fn layout_of_struct_or_enum<'a, V, F, N>(
         &self,
         repr: &ReprOptions,
         variants: &IndexVec<V, Vec<F>>,
@@ -301,7 +334,13 @@ pub trait LayoutCalculator {
         discriminants: impl Iterator<Item = (V, i128)>,
         niche_optimize_enum: bool,
         always_sized: bool,
-    ) -> Option<LayoutS<V>> {
+        option_niche_guaranteed: N,
+    ) -> Option<LayoutS<V>>
+    where
+        V: Idx,
+        F: Deref<Target = &'a LayoutS<V>> + Debug,
+        N: Fn(&Self) -> bool + Copy,
+    {
         let dl = self.current_data_layout();
         let dl = dl.borrow();
 
@@ -354,7 +393,7 @@ pub trait LayoutCalculator {
                 if !always_sized { StructKind::MaybeUnsized } else { StructKind::AlwaysSized }
             };
 
-            let mut st = self.univariant(dl, &variants[v], repr, kind)?;
+            let mut st = self.univariant(dl, &variants[v], repr, kind, option_niche_guaranteed)?;
             st.variants = Variants::Single { index: v };
 
             if is_unsafe_cell {
@@ -457,7 +496,13 @@ pub trait LayoutCalculator {
             let mut variant_layouts = variants
                 .iter_enumerated()
                 .map(|(j, v)| {
-                    let mut st = self.univariant(dl, v, repr, StructKind::AlwaysSized)?;
+                    let mut st = self.univariant(
+                        dl,
+                        v,
+                        repr,
+                        StructKind::AlwaysSized,
+                        option_niche_guaranteed,
+                    )?;
                     st.variants = Variants::Single { index: j };
 
                     align = align.max(st.align);
@@ -650,6 +695,7 @@ pub trait LayoutCalculator {
                     field_layouts,
                     repr,
                     StructKind::Prefixed(min_ity.size(), prefix_align),
+                    option_niche_guaranteed,
                 )?;
                 st.variants = Variants::Single { index: i };
                 // Find the first field we can't move later

compiler/rustc_abi/src/lib.rs

@@ -83,6 +83,7 @@ pub struct ReprOptions {
     /// Everything's a tradeoff, a `u64` seed should be sufficient for our
     /// purposes (primarily `-Z randomize-layout`)
     pub field_shuffle_seed: u64,
+    pub random_padding_max_factor: u8,
 }
 
 impl ReprOptions {
@@ -139,8 +140,10 @@ impl ReprOptions {
     /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`
     /// was enabled for its declaration crate
     pub fn can_randomize_type_layout(&self) -> bool {
-        !self.inhibit_struct_field_reordering_opt()
-            && self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT)
+        self.flags.contains(ReprFlags::RANDOMIZE_LAYOUT)
+            && !self.flags.contains(ReprFlags::IS_TRANSPARENT)
+            && !self.inhibit_struct_field_reordering_opt()
+            && !self.inhibit_enum_layout_opt()
     }
 
     /// Returns `true` if this `#[repr()]` should inhibit union ABI optimisations.

compiler/rustc_middle/src/ty/mod.rs

@@ -2030,9 +2030,17 @@ impl<'tcx> TyCtxt<'tcx> {
         // path hash with the user defined seed, this will allowing determinism while
         // still allowing users to further randomize layout generation for e.g. fuzzing
         if let Some(user_seed) = self.sess.opts.unstable_opts.layout_seed {
-            field_shuffle_seed ^= user_seed;
+            // Order-sensitive hash combination
+            field_shuffle_seed ^= user_seed
+                .wrapping_add(0x9e3779b9)
+                .wrapping_add(field_shuffle_seed << 6)
+                .wrapping_add(field_shuffle_seed >> 2);
         }
 
+        // Sets the maximum layout padding multiple, defaults to three
+        let random_padding_max_factor =
+            self.sess.opts.unstable_opts.layout_random_padding_max_factor.unwrap_or(3);
+
         for attr in self.get_attrs(did, sym::repr) {
             for r in attr::parse_repr_attr(&self.sess, attr) {
                 flags.insert(match r {
@@ -2088,7 +2096,14 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::IS_LINEAR);
         }
 
-        ReprOptions { int: size, align: max_align, pack: min_pack, flags, field_shuffle_seed }
+        ReprOptions {
+            int: size,
+            align: max_align,
+            pack: min_pack,
+            flags,
+            field_shuffle_seed,
+            random_padding_max_factor,
+        }
     }
 
     /// Look up the name of a definition across crates. This does not look at HIR.

compiler/rustc_session/src/config.rs

@@ -2826,6 +2826,7 @@ pub(crate) mod dep_tracking {
         lint::Level,
         WasiExecModel,
         u32,
+        u8,
         RelocModel,
         CodeModel,
         TlsModel,

compiler/rustc_session/src/options.rs

@@ -1364,8 +1364,9 @@ options! {
         "insert function instrument code for mcount-based tracing (default: no)"),
     keep_hygiene_data: bool = (false, parse_bool, [UNTRACKED],
         "keep hygiene data after analysis (default: no)"),
-    layout_seed: Option<u64> = (None, parse_opt_number, [TRACKED],
-        "seed layout randomization"),
+    layout_random_padding_max_factor: Option<u8> = (None, parse_opt_number, [TRACKED],
+        "set the maximum field alignment padding multiple when using 'randomize-layout' (default: 3)"),
+    layout_seed: Option<u64> = (None, parse_opt_number, [TRACKED], "seed layout randomization"),
     link_native_libraries: bool = (true, parse_bool, [UNTRACKED],
         "link native libraries in the linker invocation (default: yes)"),
     link_only: bool = (false, parse_bool, [TRACKED],

compiler/rustc_ty_utils/src/layout.rs

@@ -9,8 +9,8 @@ use rustc_middle::ty::{
     self, subst::SubstsRef, EarlyBinder, ReprOptions, Ty, TyCtxt, TypeVisitable,
 };
 use rustc_session::{DataTypeKind, FieldInfo, SizeKind, VariantInfo};
-use rustc_span::symbol::Symbol;
 use rustc_span::DUMMY_SP;
+use rustc_span::{sym, symbol::Symbol};
 use rustc_target::abi::*;
 
 use std::fmt::Debug;
@@ -88,7 +88,8 @@ fn univariant_uninterned<'tcx>(
         return Err(LayoutError::Unknown(ty));
     }
 
-    cx.univariant(dl, fields, repr, kind).ok_or(LayoutError::SizeOverflow(ty))
+    cx.univariant(dl, fields, repr, kind, |tcx| option_niche_guaranteed(tcx.tcx, ty))
+        .ok_or(LayoutError::SizeOverflow(ty))
 }
 
 fn layout_of_uncached<'tcx>(
@@ -437,6 +438,7 @@ fn layout_of_uncached<'tcx>(
                                 None => false,
                             }
                     },
+                    |tcx| option_niche_guaranteed(tcx.tcx, ty),
                 )
                 .ok_or(LayoutError::SizeOverflow(ty))?,
             )
@@ -459,6 +461,90 @@ fn layout_of_uncached<'tcx>(
     })
 }
 
+/// Returns `true` if nullable pointer optimizations apply to the given type,
+/// e.g. `Option<NonNull<_>>` as well as `Option<T>`s where `T` is a transparent
+/// newtype around another niche-able type.
+///
+/// Currently supports function pointers (`fn(...) -> ..`), `Box<_>`, references, `NonNull<_>`,
+/// `NonZero*`, `bool` and `#[repr(transparent)]` newtypes around them
+fn option_niche_guaranteed<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>) -> bool {
+    // This filters out types that match the following criteria (enums that look like `Option`):
+    // - The type is an adt
+    // - The type is an enum with two variants
+    // - One enum variant is empty, one carries a single data member
+    if let ty::Adt(ty_def, substs) = ty.kind()
+        && let [var_one, var_two] =  &*ty_def.variants().raw
+        && let ([], [field]) | ([field], []) = (&*var_one.fields, &*var_two.fields)
+    {
+        let mut required_niches = 1;
+        let mut field_ty = field.ty(tcx, substs);
+
+        // We apply these criteria recursively until we've peeled our way back to center of the type
+        // before checking if that type is non-null
+        // FIXME(Kixiron): This also accepts enums that just *look* like `Option`, any arbitrary
+        // `enum Maybe<T> { Just(T), Nothing }` also fits these criteria even though they don't have guaranteed
+        // niche optimization. I'm fairly sure that the only workaround for this is to make `Option` a lang item
+        // or otherwise well-known to the compiler, which is a little unfortunate
+        while let ty::Adt(ty_def, substs) = field_ty.kind()
+            && let [var_one, var_two] =  &*ty_def.variants().raw
+            && let ([], [field]) | ([field], []) = (&*var_one.fields, &*var_two.fields)
+        {
+            required_niches += 1;
+            field_ty = field.ty(tcx, substs);
+        }
+
+        // Now that we know the type is an option-shaped enum, we can check its contents to see if
+        // they're nullable
+        return type_is_nicheable(tcx, field_ty, required_niches);
+    }
+
+    false
+}
+
+/// Is the given type known to be non-null?
+fn type_is_nicheable<'tcx>(tcx: TyCtxt<'tcx>, ty: Ty<'tcx>, required_niches: usize) -> bool {
+    match ty.kind() {
+        // `bool` has 254 niches available (2⁸ - 2)
+        ty::Bool => true, // required_niches <= 254,
+
+        // Pointers only have one niche available
+        ty::FnPtr(_) | ty::Ref(..) => true, //  required_niches <= 1,
+
+        // If the type is `Box` or has `#[rustc_nonnull_optimization_guaranteed]`
+        ty::Adt(def, _)
+            if def.is_box()
+                || tcx.has_attr(def.did(), sym::rustc_nonnull_optimization_guaranteed) =>
+        {
+            true // required_niches <= 1
+        }
+
+        // `UnsafeCell` and unions have their niches hidden
+        ty::Adt(def, _) if def.is_unsafe_cell() || def.is_union() => false,
+
+        ty::Adt(def, substs) if def.repr().transparent() => def
+            .variants()
+            .iter()
+            .filter_map(|variant| transparent_newtype_field(tcx, variant))
+            .any(|field| type_is_nicheable(tcx, field.ty(tcx, substs), required_niches)),
+
+        _ => false,
+    }
+}
+
+/// `repr(transparent)` structs can have a single non-ZST field, this function returns that
+/// field.
+fn transparent_newtype_field<'tcx, 'a>(
+    tcx: TyCtxt<'tcx>,
+    variant: &'a ty::VariantDef,
+) -> Option<&'a ty::FieldDef> {
+    let param_env = tcx.param_env(variant.def_id);
+    variant.fields.iter().find(|field| {
+        let field_ty = tcx.type_of(field.did);
+        let is_zst = tcx.layout_of(param_env.and(field_ty)).map_or(false, |layout| layout.is_zst());
+        !is_zst
+    })
+}
+
 /// Overlap eligibility and variant assignment for each GeneratorSavedLocal.
 #[derive(Clone, Debug, PartialEq)]
 enum SavedLocalEligibility {