enable -Zrandomize-layout in debug CI builds

Cargo.lock

@@ -3569,6 +3569,7 @@ dependencies = [
  "rustc_hir_pretty",
  "rustc_hir_typeck",
  "rustc_incremental",
+ "rustc_index",
  "rustc_infer",
  "rustc_interface",
  "rustc_lint",

compiler/rustc/Cargo.toml

@@ -30,5 +30,6 @@ features = ['unprefixed_malloc_on_supported_platforms']
 jemalloc = ['dep:jemalloc-sys']
 llvm = ['rustc_driver_impl/llvm']
 max_level_info = ['rustc_driver_impl/max_level_info']
+rustc_randomized_layouts = ['rustc_driver_impl/rustc_randomized_layouts']
 rustc_use_parallel_compiler = ['rustc_driver_impl/rustc_use_parallel_compiler']
 # tidy-alphabetical-end

compiler/rustc_abi/src/layout.rs

@@ -968,8 +968,8 @@ fn univariant<
     let mut align = if pack.is_some() { dl.i8_align } else { dl.aggregate_align };
     let mut max_repr_align = repr.align;
     let mut inverse_memory_index: IndexVec<u32, FieldIdx> = fields.indices().collect();
-    let optimize = !repr.inhibit_struct_field_reordering();
-    if optimize && fields.len() > 1 {
+    let optimize_field_order = !repr.inhibit_struct_field_reordering();
+    if optimize_field_order && fields.len() > 1 {
         let end = if let StructKind::MaybeUnsized = kind { fields.len() - 1 } else { fields.len() };
         let optimizing = &mut inverse_memory_index.raw[..end];
         let fields_excluding_tail = &fields.raw[..end];
@@ -1176,7 +1176,7 @@ fn univariant<
     // If field 5 has offset 0, offsets[0] is 5, and memory_index[5] should be 0.
     // Field 5 would be the first element, so memory_index is i:
     // Note: if we didn't optimize, it's already right.
-    let memory_index = if optimize {
+    let memory_index = if optimize_field_order {
         inverse_memory_index.invert_bijective_mapping()
     } else {
         debug_assert!(inverse_memory_index.iter().copied().eq(fields.indices()));
@@ -1189,6 +1189,9 @@ fn univariant<
     }
     let mut layout_of_single_non_zst_field = None;
     let mut abi = Abi::Aggregate { sized };
+
+    let optimize_abi = !repr.inhibit_newtype_abi_optimization();
+
     // Try to make this a Scalar/ScalarPair.
     if sized && size.bytes() > 0 {
         // We skip *all* ZST here and later check if we are good in terms of alignment.
@@ -1205,7 +1208,7 @@ fn univariant<
                     match field.abi {
                         // For plain scalars, or vectors of them, we can't unpack
                         // newtypes for `#[repr(C)]`, as that affects C ABIs.
-                        Abi::Scalar(_) | Abi::Vector { .. } if optimize => {
+                        Abi::Scalar(_) | Abi::Vector { .. } if optimize_abi => {
                             abi = field.abi;
                         }
                         // But scalar pairs are Rust-specific and get

compiler/rustc_abi/src/lib.rs

@@ -43,14 +43,17 @@ bitflags! {
         const IS_SIMD            = 1 << 1;
         const IS_TRANSPARENT     = 1 << 2;
         // Internal only for now. If true, don't reorder fields.
+        // On its own it does not prevent ABI optimizations.
         const IS_LINEAR          = 1 << 3;
-        // If true, the type's layout can be randomized using
-        // the seed stored in `ReprOptions.field_shuffle_seed`
+        // If true, the type's crate has opted into layout randomization.
+        // Other flags can still inhibit reordering and thus randomization.
+        // The seed stored in `ReprOptions.field_shuffle_seed`.
         const RANDOMIZE_LAYOUT   = 1 << 4;
         // Any of these flags being set prevent field reordering optimisation.
-        const IS_UNOPTIMISABLE   = ReprFlags::IS_C.bits()
+        const FIELD_ORDER_UNOPTIMIZABLE   = ReprFlags::IS_C.bits()
                                  | ReprFlags::IS_SIMD.bits()
                                  | ReprFlags::IS_LINEAR.bits();
+        const ABI_UNOPTIMIZABLE = ReprFlags::IS_C.bits() | ReprFlags::IS_SIMD.bits();
     }
 }
 
@@ -139,10 +142,14 @@ impl ReprOptions {
         self.c() || self.int.is_some()
     }
 
+    pub fn inhibit_newtype_abi_optimization(&self) -> bool {
+        self.flags.intersects(ReprFlags::ABI_UNOPTIMIZABLE)
+    }
+
     /// Returns `true` if this `#[repr()]` guarantees a fixed field order,
     /// e.g. `repr(C)` or `repr(<int>)`.
     pub fn inhibit_struct_field_reordering(&self) -> bool {
-        self.flags.intersects(ReprFlags::IS_UNOPTIMISABLE) || self.int.is_some()
+        self.flags.intersects(ReprFlags::FIELD_ORDER_UNOPTIMIZABLE) || self.int.is_some()
     }
 
     /// Returns `true` if this type is valid for reordering and `-Z randomize-layout`

compiler/rustc_driver_impl/Cargo.toml

@@ -23,6 +23,7 @@ rustc_hir_analysis = { path = "../rustc_hir_analysis" }
 rustc_hir_pretty = { path = "../rustc_hir_pretty" }
 rustc_hir_typeck = { path = "../rustc_hir_typeck" }
 rustc_incremental = { path = "../rustc_incremental" }
+rustc_index = { path = "../rustc_index" }
 rustc_infer = { path = "../rustc_infer" }
 rustc_interface = { path = "../rustc_interface" }
 rustc_lint = { path = "../rustc_lint" }
@@ -72,6 +73,10 @@ ctrlc = "3.4.4"
 # tidy-alphabetical-start
 llvm = ['rustc_interface/llvm']
 max_level_info = ['rustc_log/max_level_info']
+rustc_randomized_layouts = [
+    'rustc_index/rustc_randomized_layouts',
+    'rustc_middle/rustc_randomized_layouts'
+]
 rustc_use_parallel_compiler = [
     'rustc_data_structures/rustc_use_parallel_compiler',
     'rustc_interface/rustc_use_parallel_compiler',

compiler/rustc_index/Cargo.toml

@@ -20,4 +20,5 @@ nightly = [
     "dep:rustc_macros",
     "rustc_index_macros/nightly",
 ]
+rustc_randomized_layouts = []
 # tidy-alphabetical-end

compiler/rustc_index/src/lib.rs

@@ -33,8 +33,19 @@ pub use vec::IndexVec;
 ///
 /// </div>
 #[macro_export]
+#[cfg(not(feature = "rustc_randomized_layouts"))]
 macro_rules! static_assert_size {
     ($ty:ty, $size:expr) => {
         const _: [(); $size] = [(); ::std::mem::size_of::<$ty>()];
     };
 }
+
+#[macro_export]
+#[cfg(feature = "rustc_randomized_layouts")]
+macro_rules! static_assert_size {
+    ($ty:ty, $size:expr) => {
+        // no effect other than using the statements.
+        // struct sizes are not deterministic under randomized layouts
+        const _: (usize, usize) = ($size, ::std::mem::size_of::<$ty>());
+    };
+}

compiler/rustc_middle/Cargo.toml

@@ -40,5 +40,6 @@ tracing = "0.1"
 
 [features]
 # tidy-alphabetical-start
+rustc_randomized_layouts = []
 rustc_use_parallel_compiler = ["dep:rustc-rayon-core"]
 # tidy-alphabetical-end

compiler/rustc_middle/src/query/plumbing.rs

@@ -337,6 +337,7 @@ macro_rules! define_callbacks {
                 // Ensure that values grow no larger than 64 bytes by accident.
                 // Increase this limit if necessary, but do try to keep the size low if possible
                 #[cfg(target_pointer_width = "64")]
+                #[cfg(not(feature = "rustc_randomized_layouts"))]
                 const _: () = {
                     if mem::size_of::<Value<'static>>() > 64 {
                         panic!("{}", concat!(

compiler/rustc_middle/src/ty/mod.rs

@@ -35,6 +35,7 @@ use rustc_data_structures::tagged_ptr::CopyTaggedPtr;
 use rustc_errors::{Diag, ErrorGuaranteed, StashKey};
 use rustc_hir::def::{CtorKind, CtorOf, DefKind, DocLinkResMap, LifetimeRes, Res};
 use rustc_hir::def_id::{CrateNum, DefId, DefIdMap, LocalDefId, LocalDefIdMap};
+use rustc_hir::LangItem;
 use rustc_index::IndexVec;
 use rustc_macros::{
     extension, Decodable, Encodable, HashStable, TyDecodable, TyEncodable, TypeFoldable,
@@ -1570,8 +1571,15 @@ impl<'tcx> TyCtxt<'tcx> {
             flags.insert(ReprFlags::RANDOMIZE_LAYOUT);
         }
 
+        // box is special, on the one hand the compiler assumes an ordered layout, with the pointer
+        // always at offset zero. On the other hand we want scalar abi optimizations.
+        let is_box = self.is_lang_item(did.to_def_id(), LangItem::OwnedBox);
+
         // This is here instead of layout because the choice must make it into metadata.
-        if !self.consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did))) {
+        if is_box
+            || !self
+                .consider_optimizing(|| format!("Reorder fields of {:?}", self.def_path_str(did)))
+        {
             flags.insert(ReprFlags::IS_LINEAR);
         }
 

config.example.toml

@@ -512,6 +512,9 @@
 # are disabled statically" because `max_level_info` is enabled, set this value to `true`.
 #debug-logging = rust.debug-assertions (boolean)
 
+# Whether or not to build rustc, tools and the libraries with randomized type layout
+#randomize-layout = false
+
 # Whether or not overflow checks are enabled for the compiler and standard
 # library.
 #

library/alloc/Cargo.toml

@@ -52,4 +52,5 @@ check-cfg = [
     'cfg(no_global_oom_handling)',
     'cfg(no_rc)',
     'cfg(no_sync)',
+    'cfg(randomized_layouts)',
 ]

library/alloc/src/collections/btree/node/tests.rs

@@ -90,7 +90,7 @@ fn test_partial_eq() {
 
 #[test]
 #[cfg(target_arch = "x86_64")]
-#[cfg_attr(miri, ignore)] // We'd like to run Miri with layout randomization
+#[cfg_attr(any(miri, randomized_layouts), ignore)] // We'd like to run Miri with layout randomization
 fn test_sizes() {
     assert_eq!(core::mem::size_of::<LeafNode<(), ()>>(), 16);
     assert_eq!(core::mem::size_of::<LeafNode<i64, i64>>(), 16 + CAPACITY * 2 * 8);

src/bootstrap/src/core/build_steps/test.rs

@@ -1810,6 +1810,9 @@ NOTE: if you're sure you want to do this, please open an issue as to why. In the
         if builder.config.rust_optimize_tests {
             cmd.arg("--optimize-tests");
         }
+        if builder.config.rust_randomize_layout {
+            cmd.arg("--rust-randomized-layout");
+        }
         if builder.config.cmd.only_modified() {
             cmd.arg("--only-modified");
         }

src/bootstrap/src/core/builder.rs

@@ -1614,6 +1614,15 @@ impl<'a> Builder<'a> {
             rustflags.arg("-Csymbol-mangling-version=legacy");
         }
 
+        // FIXME: the following components don't build with `-Zrandomize-layout` yet:
+        // - wasm-component-ld, due to the `wast`crate
+        // - rust-analyzer, due to the rowan crate
+        // so we exclude entire categories of steps here due to lack of fine-grained control over
+        // rustflags.
+        if self.config.rust_randomize_layout && mode != Mode::ToolStd && mode != Mode::ToolRustc {
+            rustflags.arg("-Zrandomize-layout");
+        }
+
         // Enable compile-time checking of `cfg` names, values and Cargo `features`.
         //
         // Note: `std`, `alloc` and `core` imports some dependencies by #[path] (like
@@ -2189,6 +2198,9 @@ impl<'a> Builder<'a> {
                 rustflags.arg("-Zvalidate-mir");
                 rustflags.arg(&format!("-Zmir-opt-level={mir_opt_level}"));
             }
+            if self.config.rust_randomize_layout {
+                rustflags.arg("--cfg=randomized_layouts");
+            }
             // Always enable inlining MIR when building the standard library.
             // Without this flag, MIR inlining is disabled when incremental compilation is enabled.
             // That causes some mir-opt tests which inline functions from the standard library to

src/bootstrap/src/core/config/config.rs

@@ -280,6 +280,7 @@ pub struct Config {
     pub rust_codegen_backends: Vec<String>,
     pub rust_verify_llvm_ir: bool,
     pub rust_thin_lto_import_instr_limit: Option<u32>,
+    pub rust_randomize_layout: bool,
     pub rust_remap_debuginfo: bool,
     pub rust_new_symbol_mangling: Option<bool>,
     pub rust_profile_use: Option<String>,
@@ -1088,6 +1089,7 @@ define_config! {
         codegen_units: Option<u32> = "codegen-units",
         codegen_units_std: Option<u32> = "codegen-units-std",
         debug_assertions: Option<bool> = "debug-assertions",
+        randomize_layout: Option<bool> = "randomize-layout",
         debug_assertions_std: Option<bool> = "debug-assertions-std",
         overflow_checks: Option<bool> = "overflow-checks",
         overflow_checks_std: Option<bool> = "overflow-checks-std",
@@ -1179,6 +1181,7 @@ impl Config {
             backtrace: true,
             rust_optimize: RustOptimize::Bool(true),
             rust_optimize_tests: true,
+            rust_randomize_layout: false,
             submodules: None,
             docs: true,
             docs_minification: true,
@@ -1629,6 +1632,7 @@ impl Config {
                 backtrace,
                 incremental,
                 parallel_compiler,
+                randomize_layout,
                 default_linker,
                 channel,
                 description,
@@ -1718,6 +1722,7 @@ impl Config {
             set(&mut config.lld_mode, lld_mode);
             set(&mut config.llvm_bitcode_linker_enabled, llvm_bitcode_linker);
 
+            config.rust_randomize_layout = randomize_layout.unwrap_or_default();
             config.llvm_tools_enabled = llvm_tools.unwrap_or(true);
             config.rustc_parallel =
                 parallel_compiler.unwrap_or(config.channel == "dev" || config.channel == "nightly");
@@ -2889,6 +2894,7 @@ fn check_incompatible_options_for_ci_rustc(
     let Rust {
         // Following options are the CI rustc incompatible ones.
         optimize,
+        randomize_layout,
         debug_logging,
         debuginfo_level_rustc,
         llvm_tools,
@@ -2953,6 +2959,7 @@ fn check_incompatible_options_for_ci_rustc(
     // otherwise, we just print a warning with `warn` macro.
 
     err!(current_rust_config.optimize, optimize);
+    err!(current_rust_config.randomize_layout, randomize_layout);
     err!(current_rust_config.debug_logging, debug_logging);
     err!(current_rust_config.debuginfo_level_rustc, debuginfo_level_rustc);
     err!(current_rust_config.rpath, rpath);

src/bootstrap/src/lib.rs

@@ -678,6 +678,9 @@ impl Build {
         if self.config.rustc_parallel {
             features.push("rustc_use_parallel_compiler");
         }
+        if self.config.rust_randomize_layout {
+            features.push("rustc_randomized_layouts");
+        }
 
         // If debug logging is on, then we want the default for tracing:
         // https://github.com/tokio-rs/tracing/blob/3dd5c03d907afdf2c39444a29931833335171554/tracing/src/level_filters.rs#L26

src/ci/docker/host-x86_64/x86_64-gnu-llvm-17/Dockerfile

@@ -50,6 +50,7 @@ ENV RUST_CONFIGURE_ARGS \
       --build=x86_64-unknown-linux-gnu \
       --llvm-root=/usr/lib/llvm-17 \
       --enable-llvm-link-shared \
+      --set rust.randomize-layout=true \
       --set rust.thin-lto-import-instr-limit=10
 
 COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/

src/tools/compiletest/src/command-list.rs

@@ -136,6 +136,7 @@ const KNOWN_DIRECTIVE_NAMES: &[&str] = &[
     "min-llvm-version",
     "min-system-llvm-version",
     "needs-asm-support",
+    "needs-deterministic-layouts",
     "needs-dlltool",
     "needs-dynamic-linking",
     "needs-force-clang-based-tests",

src/tools/compiletest/src/common.rs

@@ -274,6 +274,9 @@ pub struct Config {
     /// Flags to pass to the compiler when building for the target
     pub target_rustcflags: Vec<String>,
 
+    /// Whether the compiler and stdlib has been built with randomized struct layouts
+    pub rust_randomized_layout: bool,
+
     /// Whether tests should be optimized by default. Individual test-suites and test files may
     /// override this setting.
     pub optimize_tests: bool,

src/tools/compiletest/src/header/needs.rs

@@ -134,6 +134,11 @@ pub(super) fn handle_needs(
             condition: config.target_cfg().relocation_model == "pic",
             ignore_reason: "ignored on targets without PIC relocation model",
         },
+        Need {
+            name: "needs-deterministic-layouts",
+            condition: !config.rust_randomized_layout,
+            ignore_reason: "ignored when randomizing layouts",
+        },
         Need {
             name: "needs-wasmtime",
             condition: config.runner.as_ref().is_some_and(|r| r.contains("wasmtime")),

src/tools/compiletest/src/lib.rs

@@ -99,6 +99,11 @@ pub fn parse_config(args: Vec<String>) -> Config {
         )
         .optmulti("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS")
         .optmulti("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS")
+        .optflag(
+            "",
+            "rust-randomized-layout",
+            "set this when rustc/stdlib were compiled with randomized layouts",
+        )
         .optflag("", "optimize-tests", "run tests with optimizations enabled")
         .optflag("", "verbose", "run tests verbosely, showing all output")
         .optflag(
@@ -286,6 +291,7 @@ pub fn parse_config(args: Vec<String>) -> Config {
         host_rustcflags: matches.opt_strs("host-rustcflags"),
         target_rustcflags: matches.opt_strs("target-rustcflags"),
         optimize_tests: matches.opt_present("optimize-tests"),
+        rust_randomized_layout: matches.opt_present("rust-randomized-layout"),
         target,
         host: opt_str2(matches.opt_str("host")),
         cdb,

tests/codegen/issues/issue-86106.rs

@@ -1,5 +1,6 @@
 //@ only-64bit llvm appears to use stores instead of memset on 32bit
 //@ compile-flags: -C opt-level=3 -Z merge-functions=disabled
+//@ needs-deterministic-layouts
 
 // The below two functions ensure that both `String::new()` and `"".to_string()`
 // produce the identical code.

tests/codegen/mem-replace-big-type.rs

@@ -5,6 +5,7 @@
 
 //@ compile-flags: -C no-prepopulate-passes -Zinline-mir=no
 //@ ignore-debug: precondition checks in ptr::read make them a bad candidate for MIR inlining
+//@ needs-deterministic-layouts
 
 #![crate_type = "lib"]
 

tests/codegen/slice-iter-nonnull.rs

@@ -1,4 +1,5 @@
 //@ compile-flags: -O
+//@ needs-deterministic-layouts
 #![crate_type = "lib"]
 #![feature(exact_size_is_empty)]
 

tests/codegen/vecdeque-drain.rs

@@ -1,6 +1,7 @@
 // Check that draining at the front or back doesn't copy memory.
 
 //@ compile-flags: -O
+//@ needs-deterministic-layouts
 //@ ignore-debug: FIXME: checks for call detect scoped noalias metadata
 
 #![crate_type = "lib"]

tests/mir-opt/pre-codegen/issue_117368_print_invalid_constant.rs

@@ -1,3 +1,4 @@
+//@needs-deterministic-layouts
 // Verify that we do not ICE when printing an invalid constant.
 // EMIT_MIR_FOR_EACH_BIT_WIDTH
 // EMIT_MIR_FOR_EACH_PANIC_STRATEGY