Rollup merge of #118647 - onur-ozkan:bootstrap-shims-dump, r=clubby789

dump bootstrap shims

When making changes to the bootstrap that shouldn't change its behavior, this feature will help developers perform comparisons to check whether the bootstrap behavior has changed or not. As an example, when removing Python from the bootstrap by migrating to Rust, this feature will be super useful for ensuring that the behavior remains unaffected. It will assist me in performing comparisons to verify that the bootstrap behavior and its outputs remains consistent throughout the migration process.

This can also be used for different purposes. For example, allowing CI to dump the shims and upload them so that developers can download them and compare with their local dump to see if CI affects the bootstrap unexpectedly. Or, make CI perform comparisons on specific bootstrap tests to check for behavior changes between the master and PR branches.

Author: matthiaskrgr

src/bootstrap/src/bin/main.rs

@@ -10,8 +10,9 @@ use std::io::Write;
 #[cfg(all(any(unix, windows), not(target_os = "solaris")))]
 use std::process;
 use std::{
-    env, fs,
-    io::{self, IsTerminal},
+    env,
+    fs::{self, OpenOptions},
+    io::{self, BufRead, BufReader, IsTerminal},
 };
 
 use bootstrap::{
@@ -79,6 +80,9 @@ fn main() {
     }
 
     let pre_commit = config.src.join(".git").join("hooks").join("pre-commit");
+    let dump_bootstrap_shims = config.dump_bootstrap_shims;
+    let out_dir = config.out.clone();
+
     Build::new(config).build();
 
     if suggest_setup {
@@ -107,6 +111,29 @@ fn main() {
     if suggest_setup || changelog_suggestion.is_some() {
         println!("NOTE: this message was printed twice to make it more likely to be seen");
     }
+
+    if dump_bootstrap_shims {
+        let dump_dir = out_dir.join("bootstrap-shims-dump");
+        assert!(dump_dir.exists());
+
+        for entry in walkdir::WalkDir::new(&dump_dir) {
+            let entry = t!(entry);
+
+            if !entry.file_type().is_file() {
+                continue;
+            }
+
+            let file = t!(fs::File::open(&entry.path()));
+
+            // To ensure deterministic results we must sort the dump lines.
+            // This is necessary because the order of rustc invocations different
+            // almost all the time.
+            let mut lines: Vec<String> = t!(BufReader::new(&file).lines().collect());
+            lines.sort_by_key(|t| t.to_lowercase());
+            let mut file = t!(OpenOptions::new().write(true).truncate(true).open(&entry.path()));
+            t!(file.write_all(lines.join("\n").as_bytes()));
+        }
+    }
 }
 
 fn check_version(config: &Config) -> Option<String> {

src/bootstrap/src/bin/rustc.rs

@@ -32,9 +32,7 @@ fn main() {
     let args = env::args_os().skip(1).collect::<Vec<_>>();
     let arg = |name| args.windows(2).find(|args| args[0] == name).and_then(|args| args[1].to_str());
 
-    // We don't use the stage in this shim, but let's parse it to make sure that we're invoked
-    // by bootstrap, or that we provide a helpful error message if not.
-    bin_helpers::parse_rustc_stage();
+    let stage = bin_helpers::parse_rustc_stage();
     let verbose = bin_helpers::parse_rustc_verbose();
 
     // Detect whether or not we're a build script depending on whether --target
@@ -214,6 +212,8 @@ fn main() {
         }
     }
 
+    bin_helpers::maybe_dump(format!("stage{stage}-rustc"), &cmd);
+
     let start = Instant::now();
     let (child, status) = {
         let errmsg = format!("\nFailed to run:\n{cmd:?}\n-------------");

src/bootstrap/src/bin/rustdoc.rs

@@ -62,6 +62,8 @@ fn main() {
     cmd.arg("-Zunstable-options");
     cmd.arg("--check-cfg=cfg(bootstrap)");
 
+    bin_helpers::maybe_dump(format!("stage{stage}-rustdoc"), &cmd);
+
     if verbose > 1 {
         eprintln!(
             "rustdoc command: {:?}={:?} {:?}",

src/bootstrap/src/core/build_steps/clean.rs

@@ -146,6 +146,7 @@ fn clean_default(build: &Build) {
     rm_rf(&build.out.join("tmp"));
     rm_rf(&build.out.join("dist"));
     rm_rf(&build.out.join("bootstrap").join(".last-warned-change-id"));
+    rm_rf(&build.out.join("bootstrap-shims-dump"));
     rm_rf(&build.out.join("rustfmt.stamp"));
 
     for host in &build.hosts {

src/bootstrap/src/core/builder.rs

@@ -18,12 +18,12 @@ use crate::core::build_steps::tool::{self, SourceType};
 use crate::core::build_steps::{check, clean, compile, dist, doc, install, run, setup, test};
 use crate::core::config::flags::{Color, Subcommand};
 use crate::core::config::{DryRun, SplitDebuginfo, TargetSelection};
+use crate::prepare_behaviour_dump_dir;
 use crate::utils::cache::{Cache, Interned, INTERNER};
 use crate::utils::helpers::{self, add_dylib_path, add_link_lib_path, exe, linker_args};
 use crate::utils::helpers::{libdir, linker_flags, output, t, LldThreads};
-use crate::Crate;
 use crate::EXTRA_CHECK_CFGS;
-use crate::{Build, CLang, DocTests, GitRepo, Mode};
+use crate::{Build, CLang, Crate, DocTests, GitRepo, Mode};
 
 pub use crate::Compiler;
 
@@ -1788,6 +1788,16 @@ impl<'a> Builder<'a> {
 
         // Enable usage of unstable features
         cargo.env("RUSTC_BOOTSTRAP", "1");
+
+        if self.config.dump_bootstrap_shims {
+            prepare_behaviour_dump_dir(&self.build);
+
+            cargo
+                .env("DUMP_BOOTSTRAP_SHIMS", self.build.out.join("bootstrap-shims-dump"))
+                .env("BUILD_OUT", &self.build.out)
+                .env("CARGO_HOME", t!(home::cargo_home()));
+        };
+
         self.add_rust_test_threads(&mut cargo);
 
         // Almost all of the crates that we compile as part of the bootstrap may

src/bootstrap/src/core/config/config.rs

@@ -193,6 +193,7 @@ pub struct Config {
     pub cmd: Subcommand,
     pub incremental: bool,
     pub dry_run: DryRun,
+    pub dump_bootstrap_shims: bool,
     /// Arguments appearing after `--` to be forwarded to tools,
     /// e.g. `--fix-broken` or test arguments.
     pub free_args: Vec<String>,
@@ -1210,6 +1211,7 @@ impl Config {
         config.cmd = flags.cmd;
         config.incremental = flags.incremental;
         config.dry_run = if flags.dry_run { DryRun::UserSelected } else { DryRun::Disabled };
+        config.dump_bootstrap_shims = flags.dump_bootstrap_shims;
         config.keep_stage = flags.keep_stage;
         config.keep_stage_std = flags.keep_stage_std;
         config.color = flags.color;

src/bootstrap/src/core/config/flags.rs

@@ -85,6 +85,9 @@ pub struct Flags {
     #[arg(global(true), long)]
     /// dry run; don't build anything
     pub dry_run: bool,
+    /// Indicates whether to dump the work done from bootstrap shims
+    #[arg(global(true), long)]
+    pub dump_bootstrap_shims: bool,
     #[arg(global(true), value_hint = clap::ValueHint::Other, long, value_name = "N")]
     /// stage to build (indicates compiler to use/test, e.g., stage 0 uses the
     /// bootstrap compiler, stage 1 the stage 0 rustc artifacts, etc.)

src/bootstrap/src/lib.rs

@@ -1871,3 +1871,22 @@ pub fn generate_smart_stamp_hash(dir: &Path, additional_input: &str) -> String {
 
     hex::encode(hasher.finalize().as_slice())
 }
+
+/// Ensures that the behavior dump directory is properly initialized.
+pub fn prepare_behaviour_dump_dir(build: &Build) {
+    static INITIALIZED: OnceLock<bool> = OnceLock::new();
+
+    let dump_path = build.out.join("bootstrap-shims-dump");
+
+    let initialized = INITIALIZED.get().unwrap_or_else(|| &false);
+    if !initialized {
+        // clear old dumps
+        if dump_path.exists() {
+            t!(fs::remove_dir_all(&dump_path));
+        }
+
+        t!(fs::create_dir_all(&dump_path));
+
+        t!(INITIALIZED.set(true));
+    }
+}

src/bootstrap/src/utils/bin_helpers.rs

@@ -2,14 +2,18 @@
 //! dependency on the bootstrap library. This reduces the binary size and
 //! improves compilation time by reducing the linking time.
 
+use std::env;
+use std::fs::OpenOptions;
+use std::io::Write;
+use std::process::Command;
+use std::str::FromStr;
+
 /// Parses the value of the "RUSTC_VERBOSE" environment variable and returns it as a `usize`.
 /// If it was not defined, returns 0 by default.
 ///
 /// Panics if "RUSTC_VERBOSE" is defined with the value that is not an unsigned integer.
 pub(crate) fn parse_rustc_verbose() -> usize {
-    use std::str::FromStr;
-
-    match std::env::var("RUSTC_VERBOSE") {
+    match env::var("RUSTC_VERBOSE") {
         Ok(s) => usize::from_str(&s).expect("RUSTC_VERBOSE should be an integer"),
         Err(_) => 0,
     }
@@ -19,10 +23,29 @@ pub(crate) fn parse_rustc_verbose() -> usize {
 ///
 /// If "RUSTC_STAGE" was not set, the program will be terminated with 101.
 pub(crate) fn parse_rustc_stage() -> String {
-    std::env::var("RUSTC_STAGE").unwrap_or_else(|_| {
+    env::var("RUSTC_STAGE").unwrap_or_else(|_| {
         // Don't panic here; it's reasonable to try and run these shims directly. Give a helpful error instead.
         eprintln!("rustc shim: FATAL: RUSTC_STAGE was not set");
         eprintln!("rustc shim: NOTE: use `x.py build -vvv` to see all environment variables set by bootstrap");
         std::process::exit(101);
     })
 }
+
+/// Writes the command invocation to a file if `DUMP_BOOTSTRAP_SHIMS` is set during bootstrap.
+///
+/// Before writing it, replaces user-specific values to create generic dumps for cross-environment
+/// comparisons.
+pub(crate) fn maybe_dump(dump_name: String, cmd: &Command) {
+    if let Ok(dump_dir) = env::var("DUMP_BOOTSTRAP_SHIMS") {
+        let dump_file = format!("{dump_dir}/{dump_name}");
+
+        let mut file =
+            OpenOptions::new().create(true).write(true).append(true).open(&dump_file).unwrap();
+
+        let cmd_dump = format!("{:?}\n", cmd);
+        let cmd_dump = cmd_dump.replace(&env::var("BUILD_OUT").unwrap(), "${BUILD_OUT}");
+        let cmd_dump = cmd_dump.replace(&env::var("CARGO_HOME").unwrap(), "${CARGO_HOME}");
+
+        file.write_all(cmd_dump.as_bytes()).expect("Unable to write file");
+    }
+}