From 58306c6a0027f3d36dbbd10b23d30ed453514c9a Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 08:38:50 +0300 Subject: [PATCH 01/23] implement `BuildStamp` that is stricter impl for build stamps Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/build_stamp.rs | 73 ++++++++++++++++++++++++++ src/bootstrap/src/utils/mod.rs | 7 ++- 2 files changed, 78 insertions(+), 2 deletions(-) create mode 100644 src/bootstrap/src/utils/build_stamp.rs diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs new file mode 100644 index 0000000000000..fc695b2b477ae --- /dev/null +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -0,0 +1,73 @@ +use std::{ + fs, io, + path::{Path, PathBuf}, +}; + +#[derive(Clone)] +pub struct BuildStamp { + path: PathBuf, + stamp: String, +} + +impl From for PathBuf { + fn from(value: BuildStamp) -> Self { + value.path + } +} + +impl AsRef for BuildStamp { + fn as_ref(&self) -> &Path { + &self.path + } +} + +impl BuildStamp { + pub fn new(dir: &Path) -> Self { + Self { path: dir.join(".stamp"), stamp: String::new() } + } + + pub fn with_stamp(mut self, stamp: String) -> Self { + self.stamp = stamp; + self + } + + pub fn with_prefix(mut self, prefix: &str) -> Self { + assert!( + !prefix.starts_with('.') && !prefix.ends_with('.'), + "prefix can not start or end with '.'" + ); + + let stamp_filename = self.path.components().last().unwrap().as_os_str().to_str().unwrap(); + let stamp_filename = stamp_filename.strip_prefix('.').unwrap_or(stamp_filename); + self.path.set_file_name(format!(".{prefix}-{stamp_filename}")); + + self + } + + pub fn remove(self) -> io::Result<()> { + match fs::remove_file(&self.path) { + Ok(()) => Ok(()), + Err(e) => { + if e.kind() == io::ErrorKind::NotFound { + Ok(()) + } else { + Err(e) + } + } + } + } + + pub fn write(&self) -> io::Result<()> { + fs::write(&self.path, &self.stamp) + } + + pub fn is_up_to_date(&self) -> bool { + match fs::read(&self.path) { + Ok(h) => self.stamp.as_bytes() == h.as_slice(), + Err(e) if e.kind() == io::ErrorKind::NotFound => false, + Err(e) => { + panic!("failed to read stamp file `{}`: {}", self.path.display(), e); + } + } + } +} diff --git a/src/bootstrap/src/utils/mod.rs b/src/bootstrap/src/utils/mod.rs index b5f5e2ba6dc8b..ea56932b40437 100644 --- a/src/bootstrap/src/utils/mod.rs +++ b/src/bootstrap/src/utils/mod.rs @@ -2,6 +2,7 @@ //! support for a wide range of tasks and operations such as caching, tarballs, release //! channels, job management, etc. +pub(crate) mod build_stamp; pub(crate) mod cache; pub(crate) mod cc_detect; pub(crate) mod change_tracker; @@ -9,10 +10,12 @@ pub(crate) mod channel; pub(crate) mod exec; pub(crate) mod helpers; pub(crate) mod job; -#[cfg(feature = "build-metrics")] -pub(crate) mod metrics; pub(crate) mod render_tests; pub(crate) mod shared_helpers; pub(crate) mod tarball; + +#[cfg(feature = "build-metrics")] +pub(crate) mod metrics; + #[cfg(test)] mod tests; From e3de3c767e5438016dcd909b51dc6262f9a1fd88 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 08:51:16 +0300 Subject: [PATCH 02/23] use `BuildStamp` instead of std paths and strings Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/check.rs | 37 ++++++++++--------- src/bootstrap/src/core/build_steps/clean.rs | 3 +- src/bootstrap/src/core/build_steps/clippy.rs | 7 ++-- src/bootstrap/src/core/build_steps/compile.rs | 30 ++++++++------- src/bootstrap/src/core/build_steps/dist.rs | 10 ++++- src/bootstrap/src/core/build_steps/format.rs | 5 ++- src/bootstrap/src/core/build_steps/test.rs | 7 ++-- src/bootstrap/src/core/download.rs | 11 +++--- src/bootstrap/src/lib.rs | 15 ++++---- src/bootstrap/src/utils/build_stamp.rs | 6 +-- src/bootstrap/src/utils/helpers.rs | 10 ++--- src/bootstrap/src/utils/helpers/tests.rs | 14 ++++--- 12 files changed, 86 insertions(+), 69 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index 46fad688ebaa0..4681b4b83eb68 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -1,7 +1,5 @@ //! Implementation of compiling the compiler and standard library, in "check"-based modes. -use std::path::PathBuf; - use crate::core::build_steps::compile::{ add_to_sysroot, run_cargo, rustc_cargo, rustc_cargo_env, std_cargo, std_crates_for_run_make, }; @@ -10,6 +8,7 @@ use crate::core::builder::{ self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, crate_description, }; use crate::core::config::TargetSelection; +use crate::utils::build_stamp::BuildStamp; use crate::{Compiler, Mode, Subcommand}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] @@ -393,8 +392,9 @@ impl Step for RustAnalyzer { /// Cargo's output path in a given stage, compiled by a particular /// compiler for the specified target. - fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::ToolRustc, target).join(".rust-analyzer-check.stamp") + fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target)) + .with_prefix("rust-analyzer-check") } } } @@ -469,9 +469,8 @@ fn run_tool_check_step( cargo.arg("--all-targets"); } - let stamp = builder - .cargo_out(compiler, Mode::ToolRustc, target) - .join(format!(".{}-check.stamp", step_type_name.to_lowercase())); + let stamp = BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target)) + .with_prefix(&format!("{}-check", step_type_name.to_lowercase())); let _guard = builder.msg_check(format!("{display_name} artifacts"), target); run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); @@ -502,8 +501,8 @@ tool_check_step!(Compiletest { path: "src/tools/compiletest", default: false }); /// Cargo's output path for the standard library in a given stage, compiled /// by a particular compiler for the specified target. -fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check.stamp") +fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Std, target)).with_prefix("libstd-check") } /// Cargo's output path for the standard library in a given stage, compiled @@ -512,14 +511,19 @@ fn libstd_test_stamp( builder: &Builder<'_>, compiler: Compiler, target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd-check-test.stamp") +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Std, target)) + .with_prefix("libstd-check-test") } /// Cargo's output path for librustc in a given stage, compiled by a particular /// compiler for the specified target. -fn librustc_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc-check.stamp") +fn librustc_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Rustc, target)).with_prefix("librustc-check") } /// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular @@ -529,8 +533,7 @@ fn codegen_backend_stamp( compiler: Compiler, target: TargetSelection, backend: &str, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}-check.stamp")) +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Codegen, target)) + .with_prefix(&format!("librustc_codegen_{backend}-check")) } diff --git a/src/bootstrap/src/core/build_steps/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs index 61cc9eeed5555..13627d8e93114 100644 --- a/src/bootstrap/src/core/build_steps/clean.rs +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -10,6 +10,7 @@ use std::io::{self, ErrorKind}; use std::path::Path; use crate::core::builder::{Builder, RunConfig, ShouldRun, Step, crate_description}; +use crate::utils::build_stamp::BuildStamp; use crate::utils::helpers::t; use crate::{Build, Compiler, Kind, Mode, Subcommand}; @@ -146,7 +147,7 @@ fn clean_default(build: &Build) { rm_rf(&build.out.join("dist")); rm_rf(&build.out.join("bootstrap").join(".last-warned-change-id")); rm_rf(&build.out.join("bootstrap-shims-dump")); - rm_rf(&build.out.join("rustfmt.stamp")); + rm_rf(BuildStamp::new(&build.out).with_prefix("rustfmt").as_ref()); let mut hosts: Vec<_> = build.hosts.iter().map(|t| build.out.join(t)).collect(); // After cross-compilation, artifacts of the host architecture (which may differ from build.host) diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs index 518db156fea97..be65496035f02 100644 --- a/src/bootstrap/src/core/build_steps/clippy.rs +++ b/src/bootstrap/src/core/build_steps/clippy.rs @@ -7,6 +7,7 @@ use crate::builder::{Builder, ShouldRun}; use crate::core::build_steps::compile::std_crates_for_run_make; use crate::core::builder; use crate::core::builder::{Alias, Kind, RunConfig, Step, crate_description}; +use crate::utils::build_stamp::BuildStamp; use crate::{Mode, Subcommand, TargetSelection}; /// Disable the most spammy clippy lints @@ -307,9 +308,9 @@ macro_rules! lint_any { &target, ); - let stamp = builder - .cargo_out(compiler, Mode::ToolRustc, target) - .join(format!(".{}-check.stamp", stringify!($name).to_lowercase())); + let stringified_name = stringify!($name).to_lowercase(); + let stamp = BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target)) + .with_prefix(&format!("{}-check", stringified_name)); run_cargo( builder, diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index ce8e7e6eb2b8d..8d196c321d096 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -24,6 +24,7 @@ use crate::core::builder::{ Builder, Cargo, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath, crate_description, }; use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; +use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::command; use crate::utils::helpers::{ exe, get_clang_cl_resource_dir, is_debug_info, is_dylib, symlink_dir, t, up_to_date, @@ -984,7 +985,7 @@ impl Step for Rustc { true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. ); - let target_root_dir = stamp.parent().unwrap(); + let target_root_dir = stamp.as_ref().parent().unwrap(); // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain // unexpected debuginfo from dependencies, for example from the C++ standard library used in // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with @@ -1447,7 +1448,7 @@ impl Step for CodegenBackend { .arg(builder.src.join(format!("compiler/rustc_codegen_{backend}/Cargo.toml"))); rustc_cargo_env(builder, &mut cargo, target, compiler.stage); - let tmp_stamp = out_dir.join(".tmp.stamp"); + let tmp_stamp = BuildStamp::new(&out_dir).with_prefix("tmp"); let _guard = builder.msg_build(compiler, format_args!("codegen backend {backend}"), target); let files = run_cargo(builder, cargo, vec![], &tmp_stamp, vec![], false, false); @@ -1525,8 +1526,12 @@ fn copy_codegen_backends_to_sysroot( /// Cargo's output path for the standard library in a given stage, compiled /// by a particular compiler for the specified target. -pub fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf { - builder.cargo_out(compiler, Mode::Std, target).join(".libstd.stamp") +pub fn libstd_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Std, target)).with_prefix("libstd") } /// Cargo's output path for librustc in a given stage, compiled by a particular @@ -1535,8 +1540,8 @@ pub fn librustc_stamp( builder: &Builder<'_>, compiler: Compiler, target: TargetSelection, -) -> PathBuf { - builder.cargo_out(compiler, Mode::Rustc, target).join(".librustc.stamp") +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Rustc, target)).with_prefix("librustc") } /// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular @@ -1546,10 +1551,9 @@ fn codegen_backend_stamp( compiler: Compiler, target: TargetSelection, backend: &str, -) -> PathBuf { - builder - .cargo_out(compiler, Mode::Codegen, target) - .join(format!(".librustc_codegen_{backend}.stamp")) +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Codegen, target)) + .with_prefix(&format!("librustc_codegen_{backend}")) } pub fn compiler_file( @@ -2014,7 +2018,7 @@ pub fn add_to_sysroot( builder: &Builder<'_>, sysroot_dst: &Path, sysroot_host_dst: &Path, - stamp: &Path, + stamp: &BuildStamp, ) { let self_contained_dst = &sysroot_dst.join("self-contained"); t!(fs::create_dir_all(sysroot_dst)); @@ -2034,13 +2038,13 @@ pub fn run_cargo( builder: &Builder<'_>, cargo: Cargo, tail_args: Vec, - stamp: &Path, + stamp: &BuildStamp, additional_target_deps: Vec<(PathBuf, DependencyType)>, is_check: bool, rlib_only_metadata: bool, ) -> Vec { // `target_root_dir` looks like $dir/$target/release - let target_root_dir = stamp.parent().unwrap(); + let target_root_dir = stamp.as_ref().parent().unwrap(); // `target_deps_dir` looks like $dir/$target/release/deps let target_deps_dir = target_root_dir.join("deps"); // `host_root_dir` looks like $dir/release diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 6f7f844280008..c6fb1424ac694 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -23,6 +23,7 @@ use crate::core::build_steps::vendor::default_paths_to_vendor; use crate::core::build_steps::{compile, llvm}; use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; +use crate::utils::build_stamp::BuildStamp; use crate::utils::channel::{self, Info}; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ @@ -584,7 +585,7 @@ fn skip_host_target_lib(builder: &Builder<'_>, compiler: Compiler) -> bool { /// Check that all objects in rlibs for UEFI targets are COFF. This /// ensures that the C compiler isn't producing ELF objects, which would /// not link correctly with the COFF objects. -fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &Path) { +fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp: &BuildStamp) { if !target.ends_with("-uefi") { return; } @@ -615,7 +616,12 @@ fn verify_uefi_rlib_format(builder: &Builder<'_>, target: TargetSelection, stamp } /// Copy stamped files into an image's `target/lib` directory. -fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path, stamp: &Path) { +fn copy_target_libs( + builder: &Builder<'_>, + target: TargetSelection, + image: &Path, + stamp: &BuildStamp, +) { let dst = image.join("lib/rustlib").join(target).join("lib"); let self_contained_dst = dst.join("self-contained"); t!(fs::create_dir_all(&dst)); diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs index 590da1fb514cf..cfabfcf7f22d3 100644 --- a/src/bootstrap/src/core/build_steps/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -11,6 +11,7 @@ use build_helper::git::get_git_modified_files; use ignore::WalkBuilder; use crate::core::builder::Builder; +use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::command; use crate::utils::helpers::{self, program_out_of_date, t}; @@ -55,8 +56,8 @@ fn rustfmt( } } -fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, PathBuf)> { - let stamp_file = build.out.join("rustfmt.stamp"); +fn get_rustfmt_version(build: &Builder<'_>) -> Option<(String, BuildStamp)> { + let stamp_file = BuildStamp::new(&build.out).with_prefix("rustfmt"); let mut cmd = command(build.initial_rustfmt()?); cmd.arg("--version"); diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 914260e38d1d9..9fe59be7dd69a 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -21,6 +21,7 @@ use crate::core::builder::{ }; use crate::core::config::TargetSelection; use crate::core::config::flags::{Subcommand, get_completion}; +use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ self, LldThreads, add_link_lib_path, add_rustdoc_cargo_linker_args, dylib_path, dylib_path_var, @@ -2258,10 +2259,8 @@ impl BookTest { &[], ); - let stamp = builder - .cargo_out(compiler, mode, target) - .join(PathBuf::from(dep).file_name().unwrap()) - .with_extension("stamp"); + let stamp = BuildStamp::new(&builder.cargo_out(compiler, mode, target)) + .with_prefix(PathBuf::from(dep).file_name().and_then(|v| v.to_str()).unwrap()); let output_paths = run_cargo(builder, cargo, vec![], &stamp, vec![], false, false); let directories = output_paths diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index b5f7ed5313122..c18f7c2aae49e 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -10,6 +10,7 @@ use build_helper::ci::CiEnv; use xz2::bufread::XzDecoder; use crate::core::config::BUILDER_CONFIG_FILENAME; +use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{check_run, exe, hex_encode, move_file, program_out_of_date}; use crate::{Config, t}; @@ -46,7 +47,7 @@ impl Config { self.verbose > 0 } - pub(crate) fn create(&self, path: &Path, s: &str) { + pub(crate) fn create>(&self, path: P, s: &str) { if self.dry_run() { return; } @@ -426,7 +427,7 @@ impl Config { let version = &self.stage0_metadata.compiler.version; let host = self.build; - let clippy_stamp = self.initial_sysroot.join(".clippy-stamp"); + let clippy_stamp = BuildStamp::new(&self.initial_sysroot).with_prefix("clippy"); let cargo_clippy = self.initial_sysroot.join("bin").join(exe("cargo-clippy", host)); if cargo_clippy.exists() && !program_out_of_date(&clippy_stamp, date) { return cargo_clippy; @@ -460,7 +461,7 @@ impl Config { let host = self.build; let bin_root = self.out.join(host).join("rustfmt"); let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); - let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); + let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt"); if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { return Some(rustfmt_path); } @@ -567,7 +568,7 @@ impl Config { ) { let host = self.build.triple; let bin_root = self.out.join(host).join(sysroot); - let rustc_stamp = bin_root.join(".rustc-stamp"); + let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc"); if !bin_root.join("bin").join(exe("rustc", self.build)).exists() || program_out_of_date(&rustc_stamp, stamp_key) @@ -728,7 +729,7 @@ download-rustc = false } let llvm_root = self.ci_llvm_root(); - let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm"); let llvm_sha = detect_llvm_sha(self, self.rust_info.is_managed_git_subrepository()); let key = format!("{}{}", llvm_sha, self.llvm_assertions); if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 4cc812829f9b6..c5794aec7380f 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -30,6 +30,7 @@ use build_helper::ci::gha; use build_helper::exit; use sha2::digest::Digest; use termcolor::{ColorChoice, StandardStream, WriteColor}; +use utils::build_stamp::BuildStamp; use utils::channel::GitInfo; use utils::helpers::hex_encode; @@ -602,13 +603,13 @@ impl Build { /// /// After this executes, it will also ensure that `dir` exists. fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { - let stamp = dir.join(".stamp"); + let stamp = BuildStamp::new(dir); let mut cleared = false; - if mtime(&stamp) < mtime(input) { + if mtime(stamp.as_ref()) < mtime(input) { self.verbose(|| println!("Dirty - {}", dir.display())); let _ = fs::remove_dir_all(dir); cleared = true; - } else if stamp.exists() { + } else if stamp.as_ref().exists() { return cleared; } t!(fs::create_dir_all(dir)); @@ -1622,21 +1623,21 @@ Executed at: {executed_at}"#, ret } - fn read_stamp_file(&self, stamp: &Path) -> Vec<(PathBuf, DependencyType)> { + fn read_stamp_file(&self, stamp: &BuildStamp) -> Vec<(PathBuf, DependencyType)> { if self.config.dry_run() { return Vec::new(); } - if !stamp.exists() { + if !stamp.as_ref().exists() { eprintln!( "ERROR: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", - stamp.display() + stamp.as_ref().display() ); crate::exit!(1); } let mut paths = Vec::new(); - let contents = t!(fs::read(stamp), &stamp); + let contents = t!(fs::read(stamp.as_ref()), stamp.as_ref()); // This is the method we use for extracting paths from the stamp file passed to us. See // run_cargo for more information (in compile.rs). for part in contents.split(|b| *b == 0) { diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index fc695b2b477ae..84eaab93ebcbd 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -1,7 +1,5 @@ -use std::{ - fs, io, - path::{Path, PathBuf}, -}; +use std::path::{Path, PathBuf}; +use std::{fs, io}; #[derive(Clone)] pub struct BuildStamp { diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 50fea43aee07c..42041bb594416 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -13,9 +13,11 @@ use std::{env, fs, io, str}; use build_helper::util::fail; use object::read::archive::ArchiveFile; +use super::build_stamp::BuildStamp; use crate::LldMode; use crate::core::builder::Builder; use crate::core::config::{Config, TargetSelection}; +use crate::utils::exec::{BootstrapCommand, command}; pub use crate::utils::shared_helpers::{dylib_path, dylib_path_var}; #[cfg(test)] @@ -45,10 +47,8 @@ macro_rules! t { } }; } -pub use t; - -use crate::utils::exec::{BootstrapCommand, command}; +pub use t; pub fn exe(name: &str, target: TargetSelection) -> String { crate::utils::shared_helpers::exe(name, &target.triple) } @@ -149,8 +149,8 @@ impl Drop for TimeIt { } /// Used for download caching -pub(crate) fn program_out_of_date(stamp: &Path, key: &str) -> bool { - if !stamp.exists() { +pub(crate) fn program_out_of_date(stamp: &BuildStamp, key: &str) -> bool { + if !stamp.as_ref().exists() { return true; } t!(fs::read_to_string(stamp)) != key diff --git a/src/bootstrap/src/utils/helpers/tests.rs b/src/bootstrap/src/utils/helpers/tests.rs index 613286cfaa89a..611d9f299b7e9 100644 --- a/src/bootstrap/src/utils/helpers/tests.rs +++ b/src/bootstrap/src/utils/helpers/tests.rs @@ -2,6 +2,7 @@ use std::fs::{self, File, remove_file}; use std::io::Write; use std::path::PathBuf; +use crate::utils::build_stamp::BuildStamp; use crate::utils::helpers::{ check_cfg_arg, extract_beta_rev, hex_encode, make, program_out_of_date, set_file_times, submodule_path_of, symlink_dir, @@ -61,16 +62,17 @@ fn test_check_cfg_arg() { fn test_program_out_of_date() { let config = Config::parse(Flags::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()])); - let tempfile = config.tempdir().join(".tmp-stamp-file"); - File::create(&tempfile).unwrap().write_all(b"dummy value").unwrap(); - assert!(tempfile.exists()); + let temp_stampfile = BuildStamp::new(&config.tempdir()).with_prefix("tmp"); + + File::create(temp_stampfile.as_ref()).unwrap().write_all(b"dummy value").unwrap(); + assert!(temp_stampfile.as_ref().exists()); // up-to-date - assert!(!program_out_of_date(&tempfile, "dummy value")); + assert!(!program_out_of_date(&temp_stampfile, "dummy value")); // out-of-date - assert!(program_out_of_date(&tempfile, "")); + assert!(program_out_of_date(&temp_stampfile, "")); - remove_file(tempfile).unwrap(); + remove_file(temp_stampfile).unwrap(); } #[test] From c68c721b506b4dd2688eaad8dba40a0a4150a874 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 10:33:30 +0300 Subject: [PATCH 03/23] migrate `HashStamp` to `BuildStamp` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/gcc.rs | 14 ++++---- src/bootstrap/src/core/build_steps/llvm.rs | 35 ++++++++++---------- src/bootstrap/src/utils/build_stamp.rs | 8 ++--- src/bootstrap/src/utils/helpers.rs | 38 ---------------------- 4 files changed, 29 insertions(+), 66 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index b950bec11fd01..e67ad7b166e69 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -14,12 +14,13 @@ use std::sync::OnceLock; use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; +use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::command; -use crate::utils::helpers::{self, HashStamp, t}; +use crate::utils::helpers::{self, t}; use crate::{Kind, generate_smart_stamp_hash}; pub struct Meta { - stamp: HashStamp, + stamp: BuildStamp, out_dir: PathBuf, install_dir: PathBuf, root: PathBuf, @@ -54,18 +55,17 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc ) }); - let stamp = out_dir.join("gcc-finished-building"); - let stamp = HashStamp::new(stamp, Some(smart_stamp_hash)); + let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").with_stamp(smart_stamp_hash); - if stamp.is_done() { - if stamp.hash.is_none() { + if stamp.is_up_to_date() { + if stamp.stamp.is_empty() { builder.info( "Could not determine the GCC submodule commit hash. \ Assuming that an GCC rebuild is not necessary.", ); builder.info(&format!( "To force GCC to rebuild, remove the file `{}`", - stamp.path.display() + stamp.as_ref().display() )); } return GccBuildStatus::AlreadyBuilt; diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index be5b405705162..c4cbcd6bfed65 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -20,9 +20,10 @@ use build_helper::git::get_closest_merge_commit; use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::core::config::{Config, TargetSelection}; +use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::command; use crate::utils::helpers::{ - self, HashStamp, exe, get_clang_cl_resource_dir, t, unhashed_basename, up_to_date, + self, exe, get_clang_cl_resource_dir, t, unhashed_basename, up_to_date, }; use crate::{CLang, GitRepo, Kind, generate_smart_stamp_hash}; @@ -36,7 +37,7 @@ pub struct LlvmResult { } pub struct Meta { - stamp: HashStamp, + stamp: BuildStamp, res: LlvmResult, out_dir: PathBuf, root: String, @@ -135,18 +136,17 @@ pub fn prebuilt_llvm_config( ) }); - let stamp = out_dir.join("llvm-finished-building"); - let stamp = HashStamp::new(stamp, Some(smart_stamp_hash)); + let stamp = BuildStamp::new(&out_dir).with_prefix("llvm").with_stamp(smart_stamp_hash); - if stamp.is_done() { - if stamp.hash.is_none() { + if stamp.is_up_to_date() { + if stamp.stamp.is_empty() { builder.info( "Could not determine the LLVM submodule commit hash. \ Assuming that an LLVM rebuild is not necessary.", ); builder.info(&format!( "To force LLVM to rebuild, remove the file `{}`", - stamp.path.display() + stamp.as_ref().display() )); } return LlvmBuildStatus::AlreadyBuilt(res); @@ -922,18 +922,17 @@ impl Step for Enzyme { }); let out_dir = builder.enzyme_out(target); - let stamp = out_dir.join("enzyme-finished-building"); - let stamp = HashStamp::new(stamp, Some(smart_stamp_hash)); + let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").with_prefix(smart_stamp_hash); - if stamp.is_done() { - if stamp.hash.is_none() { + if stamp.is_up_to_date() { + if stamp.stamp.is_empty() { builder.info( "Could not determine the Enzyme submodule commit hash. \ Assuming that an Enzyme rebuild is not necessary.", ); builder.info(&format!( "To force Enzyme to rebuild, remove the file `{}`", - stamp.path.display() + stamp.as_ref().display() )); } return out_dir; @@ -1131,16 +1130,18 @@ impl Step for Sanitizers { return runtimes; } - let stamp = out_dir.join("sanitizers-finished-building"); - let stamp = HashStamp::new(stamp, builder.in_tree_llvm_info.sha()); + let stamp = BuildStamp::new(&out_dir) + .with_prefix("sanitizers") + .with_stamp(builder.in_tree_llvm_info.sha().map(String::from).unwrap_or_default()); - if stamp.is_done() { - if stamp.hash.is_none() { + if stamp.is_up_to_date() { + if stamp.stamp.is_empty() { builder.info(&format!( "Rebuild sanitizers by removing the file `{}`", - stamp.path.display() + stamp.as_ref().display() )); } + return runtimes; } diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 84eaab93ebcbd..67a9e9d12045e 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -4,7 +4,7 @@ use std::{fs, io}; #[derive(Clone)] pub struct BuildStamp { path: PathBuf, - stamp: String, + pub(crate) stamp: String, } impl From for PathBuf { @@ -24,8 +24,8 @@ impl BuildStamp { Self { path: dir.join(".stamp"), stamp: String::new() } } - pub fn with_stamp(mut self, stamp: String) -> Self { - self.stamp = stamp; + pub fn with_stamp(mut self, stamp: S) -> Self { + self.stamp = stamp.to_string(); self } @@ -42,7 +42,7 @@ impl BuildStamp { self } - pub fn remove(self) -> io::Result<()> { + pub fn remove(&self) -> io::Result<()> { match fs::remove_file(&self.path) { Ok(()) => Ok(()), Err(e) => { diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 42041bb594416..55797af4c61cd 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -598,41 +598,3 @@ pub fn set_file_times>(path: P, times: fs::FileTimes) -> io::Resu }; f.set_times(times) } - -pub struct HashStamp { - pub path: PathBuf, - pub hash: Option>, -} - -impl HashStamp { - pub fn new(path: PathBuf, hash: Option<&str>) -> Self { - HashStamp { path, hash: hash.map(|s| s.as_bytes().to_owned()) } - } - - pub fn is_done(&self) -> bool { - match fs::read(&self.path) { - Ok(h) => self.hash.as_deref().unwrap_or(b"") == h.as_slice(), - Err(e) if e.kind() == io::ErrorKind::NotFound => false, - Err(e) => { - panic!("failed to read stamp file `{}`: {}", self.path.display(), e); - } - } - } - - pub fn remove(&self) -> io::Result<()> { - match fs::remove_file(&self.path) { - Ok(()) => Ok(()), - Err(e) => { - if e.kind() == io::ErrorKind::NotFound { - Ok(()) - } else { - Err(e) - } - } - } - } - - pub fn write(&self) -> io::Result<()> { - fs::write(&self.path, self.hash.as_deref().unwrap_or(b"")) - } -} From 9e929754b2bba1bee0431feac879fec71b525382 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 11:03:43 +0300 Subject: [PATCH 04/23] migrate helper stamp functions Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/check.rs | 118 ++++-------------- src/bootstrap/src/core/build_steps/clippy.rs | 8 +- src/bootstrap/src/core/build_steps/compile.rs | 52 ++------ src/bootstrap/src/core/build_steps/dist.rs | 6 +- src/bootstrap/src/utils/build_stamp.rs | 36 ++++++ 5 files changed, 79 insertions(+), 141 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/check.rs b/src/bootstrap/src/core/build_steps/check.rs index 4681b4b83eb68..d8d862caf6a9a 100644 --- a/src/bootstrap/src/core/build_steps/check.rs +++ b/src/bootstrap/src/core/build_steps/check.rs @@ -8,8 +8,8 @@ use crate::core::builder::{ self, Alias, Builder, Kind, RunConfig, ShouldRun, Step, crate_description, }; use crate::core::config::TargetSelection; -use crate::utils::build_stamp::BuildStamp; -use crate::{Compiler, Mode, Subcommand}; +use crate::utils::build_stamp::{self, BuildStamp}; +use crate::{Mode, Subcommand}; #[derive(Debug, Clone, PartialEq, Eq, Hash)] pub struct Std { @@ -82,22 +82,16 @@ impl Step for Std { format_args!("library artifacts{}", crate_description(&self.crates)), target, ); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &libstd_stamp(builder, compiler, target), - vec![], - true, - false, - ); + + let stamp = build_stamp::libstd_stamp(builder, compiler, target).with_prefix("check"); + run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); // We skip populating the sysroot in non-zero stage because that'll lead // to rlib/rmeta conflicts if std gets built during this session. if compiler.stage == 0 { let libdir = builder.sysroot_target_libdir(compiler, target); let hostdir = builder.sysroot_target_libdir(compiler, compiler.host); - add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + add_to_sysroot(builder, &libdir, &hostdir, &stamp); } drop(_guard); @@ -138,16 +132,9 @@ impl Step for Std { cargo.arg("-p").arg(krate); } + let stamp = build_stamp::libstd_stamp(builder, compiler, target).with_prefix("check-test"); let _guard = builder.msg_check("library test/bench/example targets", target); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &libstd_test_stamp(builder, compiler, target), - vec![], - true, - false, - ); + run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } } @@ -248,19 +235,14 @@ impl Step for Rustc { format_args!("compiler artifacts{}", crate_description(&self.crates)), target, ); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &librustc_stamp(builder, compiler, target), - vec![], - true, - false, - ); + + let stamp = build_stamp::librustc_stamp(builder, compiler, target).with_prefix("check"); + + run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); let libdir = builder.sysroot_target_libdir(compiler, target); let hostdir = builder.sysroot_target_libdir(compiler, compiler.host); - add_to_sysroot(builder, &libdir, &hostdir, &librustc_stamp(builder, compiler, target)); + add_to_sysroot(builder, &libdir, &hostdir, &stamp); } } @@ -314,15 +296,10 @@ impl Step for CodegenBackend { let _guard = builder.msg_check(backend, target); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &codegen_backend_stamp(builder, compiler, target, backend), - vec![], - true, - false, - ); + let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, backend) + .with_prefix("check"); + + run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } } @@ -379,23 +356,13 @@ impl Step for RustAnalyzer { cargo.arg("--benches"); } - let _guard = builder.msg_check("rust-analyzer artifacts", target); - run_cargo( - builder, - cargo, - builder.config.free_args.clone(), - &stamp(builder, compiler, target), - vec![], - true, - false, - ); + // Cargo's output path in a given stage, compiled by a particular + // compiler for the specified target. + let stamp = BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target)) + .with_prefix("rust-analyzer-check"); - /// Cargo's output path in a given stage, compiled by a particular - /// compiler for the specified target. - fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::ToolRustc, target)) - .with_prefix("rust-analyzer-check") - } + let _guard = builder.msg_check("rust-analyzer artifacts", target); + run_cargo(builder, cargo, builder.config.free_args.clone(), &stamp, vec![], true, false); } } @@ -498,42 +465,3 @@ tool_check_step!(RunMakeSupport { path: "src/tools/run-make-support", default: f // Compiletest is implicitly "checked" when it gets built in order to run tests, // so this is mainly for people working on compiletest to run locally. tool_check_step!(Compiletest { path: "src/tools/compiletest", default: false }); - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::Std, target)).with_prefix("libstd-check") -} - -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -fn libstd_test_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::Std, target)) - .with_prefix("libstd-check-test") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -fn librustc_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::Rustc, target)).with_prefix("librustc-check") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: &str, -) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::Codegen, target)) - .with_prefix(&format!("librustc_codegen_{backend}-check")) -} diff --git a/src/bootstrap/src/core/build_steps/clippy.rs b/src/bootstrap/src/core/build_steps/clippy.rs index be65496035f02..fe7e4a77f71a9 100644 --- a/src/bootstrap/src/core/build_steps/clippy.rs +++ b/src/bootstrap/src/core/build_steps/clippy.rs @@ -1,13 +1,13 @@ //! Implementation of running clippy on the compiler, standard library and various tools. -use super::compile::{librustc_stamp, libstd_stamp, run_cargo, rustc_cargo, std_cargo}; +use super::compile::{run_cargo, rustc_cargo, std_cargo}; use super::tool::{SourceType, prepare_tool_cargo}; use super::{check, compile}; use crate::builder::{Builder, ShouldRun}; use crate::core::build_steps::compile::std_crates_for_run_make; use crate::core::builder; use crate::core::builder::{Alias, Kind, RunConfig, Step, crate_description}; -use crate::utils::build_stamp::BuildStamp; +use crate::utils::build_stamp::{self, BuildStamp}; use crate::{Mode, Subcommand, TargetSelection}; /// Disable the most spammy clippy lints @@ -168,7 +168,7 @@ impl Step for Std { builder, cargo, lint_args(builder, &self.config, IGNORED_RULES_FOR_STD_AND_RUSTC), - &libstd_stamp(builder, compiler, target), + &build_stamp::libstd_stamp(builder, compiler, target), vec![], true, false, @@ -244,7 +244,7 @@ impl Step for Rustc { builder, cargo, lint_args(builder, &self.config, IGNORED_RULES_FOR_STD_AND_RUSTC), - &librustc_stamp(builder, compiler, target), + &build_stamp::librustc_stamp(builder, compiler, target), vec![], true, false, diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 8d196c321d096..9be8b9cded24b 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -24,6 +24,7 @@ use crate::core::builder::{ Builder, Cargo, Kind, PathSet, RunConfig, ShouldRun, Step, TaskPath, crate_description, }; use crate::core::config::{DebuginfoLevel, LlvmLibunwind, RustcLto, TargetSelection}; +use crate::utils::build_stamp; use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::command; use crate::utils::helpers::{ @@ -255,7 +256,7 @@ impl Step for Std { builder, cargo, vec![], - &libstd_stamp(builder, compiler, target), + &build_stamp::libstd_stamp(builder, compiler, target), target_deps, self.is_for_mir_opt_tests, // is_check false, @@ -645,7 +646,12 @@ impl Step for StdLink { (libdir, hostdir) }; - add_to_sysroot(builder, &libdir, &hostdir, &libstd_stamp(builder, compiler, target)); + add_to_sysroot( + builder, + &libdir, + &hostdir, + &build_stamp::libstd_stamp(builder, compiler, target), + ); // Special case for stage0, to make `rustup toolchain link` and `x dist --stage 0` // work for stage0-sysroot. We only do this if the stage0 compiler comes from beta, @@ -974,7 +980,7 @@ impl Step for Rustc { compiler.host, target, ); - let stamp = librustc_stamp(builder, compiler, target); + let stamp = build_stamp::librustc_stamp(builder, compiler, target); run_cargo( builder, cargo, @@ -1330,7 +1336,7 @@ impl Step for RustcLink { builder, &builder.sysroot_target_libdir(previous_stage_compiler, target), &builder.sysroot_target_libdir(previous_stage_compiler, compiler.host), - &librustc_stamp(builder, compiler, target), + &build_stamp::librustc_stamp(builder, compiler, target), ); } } @@ -1470,7 +1476,7 @@ impl Step for CodegenBackend { f.display() ); } - let stamp = codegen_backend_stamp(builder, compiler, target, &backend); + let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, &backend); let codegen_backend = codegen_backend.to_str().unwrap(); t!(fs::write(stamp, codegen_backend)); } @@ -1509,7 +1515,7 @@ fn copy_codegen_backends_to_sysroot( continue; // Already built as part of rustc } - let stamp = codegen_backend_stamp(builder, compiler, target, backend); + let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, backend); let dylib = t!(fs::read_to_string(&stamp)); let file = Path::new(&dylib); let filename = file.file_name().unwrap().to_str().unwrap(); @@ -1524,38 +1530,6 @@ fn copy_codegen_backends_to_sysroot( } } -/// Cargo's output path for the standard library in a given stage, compiled -/// by a particular compiler for the specified target. -pub fn libstd_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::Std, target)).with_prefix("libstd") -} - -/// Cargo's output path for librustc in a given stage, compiled by a particular -/// compiler for the specified target. -pub fn librustc_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, -) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::Rustc, target)).with_prefix("librustc") -} - -/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular -/// compiler for the specified target and backend. -fn codegen_backend_stamp( - builder: &Builder<'_>, - compiler: Compiler, - target: TargetSelection, - backend: &str, -) -> BuildStamp { - BuildStamp::new(&builder.cargo_out(compiler, Mode::Codegen, target)) - .with_prefix(&format!("librustc_codegen_{backend}")) -} - pub fn compiler_file( builder: &Builder<'_>, compiler: &Path, @@ -1912,7 +1886,7 @@ impl Step for Assemble { builder.info(&msg); // Link in all dylibs to the libdir - let stamp = librustc_stamp(builder, build_compiler, target_compiler.host); + let stamp = build_stamp::librustc_stamp(builder, build_compiler, target_compiler.host); let proc_macros = builder .read_stamp_file(&stamp) .into_iter() diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index c6fb1424ac694..8ca087f9941fa 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -23,7 +23,7 @@ use crate::core::build_steps::vendor::default_paths_to_vendor; use crate::core::build_steps::{compile, llvm}; use crate::core::builder::{Builder, Kind, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; -use crate::utils::build_stamp::BuildStamp; +use crate::utils::build_stamp::{self, BuildStamp}; use crate::utils::channel::{self, Info}; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ @@ -674,7 +674,7 @@ impl Step for Std { tarball.include_target_in_component_name(true); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::libstd_stamp(builder, compiler_to_use, target); + let stamp = build_stamp::libstd_stamp(builder, compiler_to_use, target); verify_uefi_rlib_format(builder, target, &stamp); copy_target_libs(builder, target, tarball.image_dir(), &stamp); @@ -724,7 +724,7 @@ impl Step for RustcDev { let tarball = Tarball::new(builder, "rustc-dev", &target.triple); let compiler_to_use = builder.compiler_for(compiler.stage, compiler.host, target); - let stamp = compile::librustc_stamp(builder, compiler_to_use, target); + let stamp = build_stamp::librustc_stamp(builder, compiler_to_use, target); copy_target_libs(builder, target, tarball.image_dir(), &stamp); let src_files = &["Cargo.lock"]; diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 67a9e9d12045e..0137795790201 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -1,6 +1,10 @@ use std::path::{Path, PathBuf}; use std::{fs, io}; +use crate::core::builder::Builder; +use crate::core::config::TargetSelection; +use crate::{Compiler, Mode}; + #[derive(Clone)] pub struct BuildStamp { path: PathBuf, @@ -69,3 +73,35 @@ impl BuildStamp { } } } + +/// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular +/// compiler for the specified target and backend. +pub fn codegen_backend_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, + backend: &str, +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Codegen, target)) + .with_prefix(&format!("librustc_codegen_{backend}")) +} + +/// Cargo's output path for the standard library in a given stage, compiled +/// by a particular compiler for the specified target. +pub fn libstd_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Std, target)).with_prefix("libstd") +} + +/// Cargo's output path for librustc in a given stage, compiled by a particular +/// compiler for the specified target. +pub fn librustc_stamp( + builder: &Builder<'_>, + compiler: Compiler, + target: TargetSelection, +) -> BuildStamp { + BuildStamp::new(&builder.cargo_out(compiler, Mode::Rustc, target)).with_prefix("librustc") +} From 236d5804bf210f4067fa09dff0c9e8b98e5e592b Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 11:06:52 +0300 Subject: [PATCH 05/23] migrate `Builder::clear_if_dirty` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/test.rs | 6 +++--- src/bootstrap/src/core/builder/cargo.rs | 7 +++--- src/bootstrap/src/lib.rs | 25 ++-------------------- src/bootstrap/src/utils/build_stamp.rs | 21 +++++++++++++++++- 4 files changed, 29 insertions(+), 30 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index 9fe59be7dd69a..acfca7f952357 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -21,7 +21,7 @@ use crate::core::builder::{ }; use crate::core::config::TargetSelection; use crate::core::config::flags::{Subcommand, get_completion}; -use crate::utils::build_stamp::BuildStamp; +use crate::utils::build_stamp::{self, BuildStamp}; use crate::utils::exec::{BootstrapCommand, command}; use crate::utils::helpers::{ self, LldThreads, add_link_lib_path, add_rustdoc_cargo_linker_args, dylib_path, dylib_path_var, @@ -545,7 +545,7 @@ impl Step for Miri { // The mtime of `miri_sysroot` changes when the sysroot gets rebuilt (also see // ). // We can hence use that directly as a signal to clear the ui test dir. - builder.clear_if_dirty(&ui_test_dep_dir, &miri_sysroot); + build_stamp::clear_if_dirty(builder, &ui_test_dep_dir, &miri_sysroot); } // Run `cargo test`. @@ -982,7 +982,7 @@ impl Step for RustdocGUI { let mut cmd = builder.tool_cmd(Tool::RustdocGUITest); let out_dir = builder.test_out(self.target).join("rustdoc-gui"); - builder.clear_if_dirty(&out_dir, &builder.rustdoc(self.compiler)); + build_stamp::clear_if_dirty(builder, &out_dir, &builder.rustdoc(self.compiler)); if let Some(src) = builder.config.src.to_str() { cmd.arg("--rust-src").arg(src); diff --git a/src/bootstrap/src/core/builder/cargo.rs b/src/bootstrap/src/core/builder/cargo.rs index c121543462cf2..2692a129ef3b9 100644 --- a/src/bootstrap/src/core/builder/cargo.rs +++ b/src/bootstrap/src/core/builder/cargo.rs @@ -7,6 +7,7 @@ use crate::core::build_steps::tool::SourceType; use crate::core::build_steps::{compile, test}; use crate::core::config::SplitDebuginfo; use crate::core::config::flags::Color; +use crate::utils::build_stamp; use crate::utils::helpers::{ self, LldThreads, add_link_lib_path, check_cfg_arg, linker_args, linker_flags, }; @@ -454,7 +455,7 @@ impl Builder<'_> { // Codegen backends are not yet tracked by -Zbinary-dep-depinfo, // so we need to explicitly clear out if they've been updated. for backend in self.codegen_backends(compiler) { - self.clear_if_dirty(&out_dir, &backend); + build_stamp::clear_if_dirty(self, &out_dir, &backend); } if cmd_kind == Kind::Doc { @@ -471,7 +472,7 @@ impl Builder<'_> { _ => panic!("doc mode {mode:?} not expected"), }; let rustdoc = self.rustdoc(compiler); - self.clear_if_dirty(&my_out, &rustdoc); + build_stamp::clear_if_dirty(self, &my_out, &rustdoc); } let profile_var = |name: &str| { @@ -763,7 +764,7 @@ impl Builder<'_> { // Only clear out the directory if we're compiling std; otherwise, we // should let Cargo take care of things for us (via depdep info) if !self.config.dry_run() && mode == Mode::Std && cmd_kind == Kind::Build { - self.clear_if_dirty(&out_dir, &self.rustc(compiler)); + build_stamp::clear_if_dirty(self, &out_dir, &self.rustc(compiler)); } let rustdoc_path = match cmd_kind { diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index c5794aec7380f..fd32904ea7b27 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -19,12 +19,11 @@ use std::cell::{Cell, RefCell}; use std::collections::{BTreeSet, HashMap, HashSet}; use std::fmt::Display; -use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::process::Command; use std::sync::OnceLock; use std::time::SystemTime; -use std::{env, io, str}; +use std::{env, fs, io, str}; use build_helper::ci::gha; use build_helper::exit; @@ -38,9 +37,7 @@ use crate::core::builder; use crate::core::builder::{Builder, Kind}; use crate::core::config::{DryRun, LldMode, LlvmLibunwind, Target, TargetSelection, flags}; use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, command}; -use crate::utils::helpers::{ - self, dir_is_empty, exe, libdir, mtime, output, set_file_times, symlink_dir, -}; +use crate::utils::helpers::{self, dir_is_empty, exe, libdir, output, set_file_times, symlink_dir}; mod core; mod utils; @@ -599,24 +596,6 @@ impl Build { self.metrics.persist(self); } - /// Clear out `dir` if `input` is newer. - /// - /// After this executes, it will also ensure that `dir` exists. - fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool { - let stamp = BuildStamp::new(dir); - let mut cleared = false; - if mtime(stamp.as_ref()) < mtime(input) { - self.verbose(|| println!("Dirty - {}", dir.display())); - let _ = fs::remove_dir_all(dir); - cleared = true; - } else if stamp.as_ref().exists() { - return cleared; - } - t!(fs::create_dir_all(dir)); - t!(File::create(stamp)); - cleared - } - fn rust_info(&self) -> &GitInfo { &self.config.rust_info } diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 0137795790201..167ef44594bae 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -3,7 +3,8 @@ use std::{fs, io}; use crate::core::builder::Builder; use crate::core::config::TargetSelection; -use crate::{Compiler, Mode}; +use crate::utils::helpers::mtime; +use crate::{Compiler, Mode, t}; #[derive(Clone)] pub struct BuildStamp { @@ -74,6 +75,24 @@ impl BuildStamp { } } +/// Clear out `dir` if `input` is newer. +/// +/// After this executes, it will also ensure that `dir` exists. +pub fn clear_if_dirty(builder: &Builder<'_>, dir: &Path, input: &Path) -> bool { + let stamp = BuildStamp::new(dir); + let mut cleared = false; + if mtime(stamp.as_ref()) < mtime(input) { + builder.verbose(|| println!("Dirty - {}", dir.display())); + let _ = fs::remove_dir_all(dir); + cleared = true; + } else if stamp.as_ref().exists() { + return cleared; + } + t!(fs::create_dir_all(dir)); + t!(fs::File::create(stamp)); + cleared +} + /// Cargo's output path for librustc_codegen_llvm in a given stage, compiled by a particular /// compiler for the specified target and backend. pub fn codegen_backend_stamp( From 9e1c9fd65417b04c3a0b63a4e0f8ffba50293867 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 11:22:10 +0300 Subject: [PATCH 06/23] document `build_stamp` implementation Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/build_stamp.rs | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 167ef44594bae..5450558c86e49 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -1,3 +1,7 @@ +//! Module for managing build stamp files. +//! +//! Contains the core implementation of how bootstrap utilizes stamp files on build processes. + use std::path::{Path, PathBuf}; use std::{fs, io}; @@ -6,18 +10,14 @@ use crate::core::config::TargetSelection; use crate::utils::helpers::mtime; use crate::{Compiler, Mode, t}; +/// Manages a stamp file to track build state. The file is created in the given +/// directory and can have custom content and name. #[derive(Clone)] pub struct BuildStamp { path: PathBuf, pub(crate) stamp: String, } -impl From for PathBuf { - fn from(value: BuildStamp) -> Self { - value.path - } -} - impl AsRef for BuildStamp { fn as_ref(&self) -> &Path { &self.path @@ -25,15 +25,22 @@ impl AsRef for BuildStamp { } impl BuildStamp { + /// Creates a new `BuildStamp` for a given directory. + /// + /// By default, stamp will be an empty file named `.stamp` within the specified directory. pub fn new(dir: &Path) -> Self { Self { path: dir.join(".stamp"), stamp: String::new() } } + /// Sets stamp content to the specified value. pub fn with_stamp(mut self, stamp: S) -> Self { self.stamp = stamp.to_string(); self } + /// Adds a prefix to stamp's name. + /// + /// Prefix cannot start or end with a dot (`.`). pub fn with_prefix(mut self, prefix: &str) -> Self { assert!( !prefix.starts_with('.') && !prefix.ends_with('.'), @@ -47,6 +54,7 @@ impl BuildStamp { self } + /// Removes the stamp file if it exists. pub fn remove(&self) -> io::Result<()> { match fs::remove_file(&self.path) { Ok(()) => Ok(()), @@ -60,10 +68,14 @@ impl BuildStamp { } } + /// Creates the stamp file. pub fn write(&self) -> io::Result<()> { fs::write(&self.path, &self.stamp) } + /// Checks if the stamp file is up-to-date. + /// + /// It is considered up-to-date if file content matches with the stamp string. pub fn is_up_to_date(&self) -> bool { match fs::read(&self.path) { Ok(h) => self.stamp.as_bytes() == h.as_slice(), From cacb4fe93a5c75eb4f69774a4ebf6d1549c95201 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 11:35:13 +0300 Subject: [PATCH 07/23] add test coverage for `build_stamp` implementation Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/build_stamp.rs | 3 ++ src/bootstrap/src/utils/build_stamp/tests.rs | 44 ++++++++++++++++++++ 2 files changed, 47 insertions(+) create mode 100644 src/bootstrap/src/utils/build_stamp/tests.rs diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 5450558c86e49..d76b395709afb 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -10,6 +10,9 @@ use crate::core::config::TargetSelection; use crate::utils::helpers::mtime; use crate::{Compiler, Mode, t}; +#[cfg(test)] +mod tests; + /// Manages a stamp file to track build state. The file is created in the given /// directory and can have custom content and name. #[derive(Clone)] diff --git a/src/bootstrap/src/utils/build_stamp/tests.rs b/src/bootstrap/src/utils/build_stamp/tests.rs new file mode 100644 index 0000000000000..4e8a43078444e --- /dev/null +++ b/src/bootstrap/src/utils/build_stamp/tests.rs @@ -0,0 +1,44 @@ +use std::path::PathBuf; + +use crate::{BuildStamp, Config, Flags}; + +fn temp_dir() -> PathBuf { + let config = + Config::parse(Flags::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()])); + config.tempdir() +} + +#[test] +#[should_panic(expected = "prefix can not start or end with '.'")] +fn test_with_invalid_prefix() { + let dir = temp_dir(); + BuildStamp::new(&dir).with_prefix(".invalid"); +} + +#[test] +#[should_panic(expected = "prefix can not start or end with '.'")] +fn test_with_invalid_prefix2() { + let dir = temp_dir(); + BuildStamp::new(&dir).with_prefix("invalid."); +} + +#[test] +fn test_is_up_to_date() { + let dir = temp_dir(); + + let mut build_stamp = BuildStamp::new(&dir).with_stamp("v1.0.0"); + build_stamp.write().unwrap(); + + assert!( + build_stamp.is_up_to_date(), + "Expected stamp file to be up-to-date, but contents do not match the expected value." + ); + + build_stamp.stamp = "dummy value".to_owned(); + assert!( + !build_stamp.is_up_to_date(), + "Stamp should no longer be up-to-date as we changed its content right above." + ); + + build_stamp.remove().unwrap(); +} From 615131b4d40b348cf56da02f25b67483c369f5b5 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 11:41:45 +0300 Subject: [PATCH 08/23] migrate `generate_smart_stamp_hash` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/gcc.rs | 4 +- src/bootstrap/src/core/build_steps/llvm.rs | 4 +- src/bootstrap/src/lib.rs | 50 +-------------------- src/bootstrap/src/utils/build_stamp.rs | 52 +++++++++++++++++++++- 4 files changed, 55 insertions(+), 55 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index e67ad7b166e69..f55a17aab0468 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -12,12 +12,12 @@ use std::fs; use std::path::PathBuf; use std::sync::OnceLock; +use crate::Kind; use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::core::config::TargetSelection; -use crate::utils::build_stamp::BuildStamp; +use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash}; use crate::utils::exec::command; use crate::utils::helpers::{self, t}; -use crate::{Kind, generate_smart_stamp_hash}; pub struct Meta { stamp: BuildStamp, diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index c4cbcd6bfed65..b31a2aeb4c1aa 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -20,12 +20,12 @@ use build_helper::git::get_closest_merge_commit; use crate::core::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::core::config::{Config, TargetSelection}; -use crate::utils::build_stamp::BuildStamp; +use crate::utils::build_stamp::{BuildStamp, generate_smart_stamp_hash}; use crate::utils::exec::command; use crate::utils::helpers::{ self, exe, get_clang_cl_resource_dir, t, unhashed_basename, up_to_date, }; -use crate::{CLang, GitRepo, Kind, generate_smart_stamp_hash}; +use crate::{CLang, GitRepo, Kind}; #[derive(Clone)] pub struct LlvmResult { diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index fd32904ea7b27..590b5da252406 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -27,14 +27,12 @@ use std::{env, fs, io, str}; use build_helper::ci::gha; use build_helper::exit; -use sha2::digest::Digest; use termcolor::{ColorChoice, StandardStream, WriteColor}; use utils::build_stamp::BuildStamp; use utils::channel::GitInfo; -use utils::helpers::hex_encode; use crate::core::builder; -use crate::core::builder::{Builder, Kind}; +use crate::core::builder::Kind; use crate::core::config::{DryRun, LldMode, LlvmLibunwind, Target, TargetSelection, flags}; use crate::utils::exec::{BehaviorOnFailure, BootstrapCommand, CommandOutput, OutputMode, command}; use crate::utils::helpers::{self, dir_is_empty, exe, libdir, output, set_file_times, symlink_dir}; @@ -1902,52 +1900,6 @@ fn envify(s: &str) -> String { .collect() } -/// Computes a hash representing the state of a repository/submodule and additional input. -/// -/// It uses `git diff` for the actual changes, and `git status` for including the untracked -/// files in the specified directory. The additional input is also incorporated into the -/// computation of the hash. -/// -/// # Parameters -/// -/// - `dir`: A reference to the directory path of the target repository/submodule. -/// - `additional_input`: An additional input to be included in the hash. -/// -/// # Panics -/// -/// In case of errors during `git` command execution (e.g., in tarball sources), default values -/// are used to prevent panics. -pub fn generate_smart_stamp_hash( - builder: &Builder<'_>, - dir: &Path, - additional_input: &str, -) -> String { - let diff = helpers::git(Some(dir)) - .allow_failure() - .arg("diff") - .run_capture_stdout(builder) - .stdout_if_ok() - .unwrap_or_default(); - - let status = helpers::git(Some(dir)) - .allow_failure() - .arg("status") - .arg("--porcelain") - .arg("-z") - .arg("--untracked-files=normal") - .run_capture_stdout(builder) - .stdout_if_ok() - .unwrap_or_default(); - - let mut hasher = sha2::Sha256::new(); - - hasher.update(diff); - hasher.update(status); - hasher.update(additional_input); - - hex_encode(hasher.finalize().as_slice()) -} - /// Ensures that the behavior dump directory is properly initialized. pub fn prepare_behaviour_dump_dir(build: &Build) { static INITIALIZED: OnceLock = OnceLock::new(); diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index d76b395709afb..7c3935e1db2a9 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -5,10 +5,12 @@ use std::path::{Path, PathBuf}; use std::{fs, io}; +use sha2::digest::Digest; + use crate::core::builder::Builder; use crate::core::config::TargetSelection; -use crate::utils::helpers::mtime; -use crate::{Compiler, Mode, t}; +use crate::utils::helpers::{hex_encode, mtime}; +use crate::{Compiler, Mode, helpers, t}; #[cfg(test)] mod tests; @@ -139,3 +141,49 @@ pub fn librustc_stamp( ) -> BuildStamp { BuildStamp::new(&builder.cargo_out(compiler, Mode::Rustc, target)).with_prefix("librustc") } + +/// Computes a hash representing the state of a repository/submodule and additional input. +/// +/// It uses `git diff` for the actual changes, and `git status` for including the untracked +/// files in the specified directory. The additional input is also incorporated into the +/// computation of the hash. +/// +/// # Parameters +/// +/// - `dir`: A reference to the directory path of the target repository/submodule. +/// - `additional_input`: An additional input to be included in the hash. +/// +/// # Panics +/// +/// In case of errors during `git` command execution (e.g., in tarball sources), default values +/// are used to prevent panics. +pub fn generate_smart_stamp_hash( + builder: &Builder<'_>, + dir: &Path, + additional_input: &str, +) -> String { + let diff = helpers::git(Some(dir)) + .allow_failure() + .arg("diff") + .run_capture_stdout(builder) + .stdout_if_ok() + .unwrap_or_default(); + + let status = helpers::git(Some(dir)) + .allow_failure() + .arg("status") + .arg("--porcelain") + .arg("-z") + .arg("--untracked-files=normal") + .run_capture_stdout(builder) + .stdout_if_ok() + .unwrap_or_default(); + + let mut hasher = sha2::Sha256::new(); + + hasher.update(diff); + hasher.update(status); + hasher.update(additional_input); + + hex_encode(hasher.finalize().as_slice()) +} From 9e86d76ad9613a49427f7765fa1cea4f753eb686 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 11:45:59 +0300 Subject: [PATCH 09/23] fix compiler errors Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/exec.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/bootstrap/src/utils/exec.rs b/src/bootstrap/src/utils/exec.rs index 530d760a584c8..60216395c2f6e 100644 --- a/src/bootstrap/src/utils/exec.rs +++ b/src/bootstrap/src/utils/exec.rs @@ -120,7 +120,7 @@ impl BootstrapCommand { Self { failure_behavior: BehaviorOnFailure::DelayFail, ..self } } - #[must_use] + #[allow(dead_code)] pub fn fail_fast(self) -> Self { Self { failure_behavior: BehaviorOnFailure::Exit, ..self } } @@ -275,7 +275,7 @@ impl CommandOutput { !self.is_success() } - #[must_use] + #[allow(dead_code)] pub fn status(&self) -> Option { match self.status { CommandStatus::Finished(status) => Some(status), From 1fa66573cdd267bbc6eafb64fbe8235d806ad61d Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 11:57:37 +0300 Subject: [PATCH 10/23] fix an invalid prefix usage on enzyme Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/llvm.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index b31a2aeb4c1aa..7bf24675d061f 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -922,7 +922,7 @@ impl Step for Enzyme { }); let out_dir = builder.enzyme_out(target); - let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").with_prefix(smart_stamp_hash); + let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").with_stamp(smart_stamp_hash); if stamp.is_up_to_date() { if stamp.stamp.is_empty() { From 9878d63acbd95c63b4cc2ec4034e4eddbb9cd6bb Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 12:01:51 +0300 Subject: [PATCH 11/23] add coverage for `BuildStamp::with_prefix` Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/build_stamp/tests.rs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/bootstrap/src/utils/build_stamp/tests.rs b/src/bootstrap/src/utils/build_stamp/tests.rs index 4e8a43078444e..0637897507a1b 100644 --- a/src/bootstrap/src/utils/build_stamp/tests.rs +++ b/src/bootstrap/src/utils/build_stamp/tests.rs @@ -42,3 +42,19 @@ fn test_is_up_to_date() { build_stamp.remove().unwrap(); } + +#[test] +fn test_with_prefix() { + let dir = temp_dir(); + + let stamp = BuildStamp::new(&dir).with_stamp("v1.0.0"); + assert_eq!(stamp.path.file_name().unwrap(), ".stamp"); + + let stamp = stamp.with_prefix("test"); + let expected_filename = ".test-stamp"; + assert_eq!(stamp.path.file_name().unwrap(), expected_filename); + + let stamp = stamp.with_prefix("extra-prefix"); + let expected_filename = ".extra-prefix-test-stamp"; + assert_eq!(stamp.path.file_name().unwrap(), expected_filename); +} From a72068adeed913068350327d0a5f6fdbb04e0b4a Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 12:34:03 +0300 Subject: [PATCH 12/23] migrate `program_out_of_date` to `BuildStamp::is_up_to_date` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/format.rs | 4 +-- src/bootstrap/src/core/download.rs | 29 ++++++++++---------- src/bootstrap/src/utils/helpers.rs | 9 ------ src/bootstrap/src/utils/helpers/tests.rs | 24 ++-------------- 4 files changed, 20 insertions(+), 46 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs index cfabfcf7f22d3..253a33183bfd1 100644 --- a/src/bootstrap/src/core/build_steps/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -13,7 +13,7 @@ use ignore::WalkBuilder; use crate::core::builder::Builder; use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::command; -use crate::utils::helpers::{self, program_out_of_date, t}; +use crate::utils::helpers::{self, t}; #[must_use] enum RustfmtStatus { @@ -74,7 +74,7 @@ fn verify_rustfmt_version(build: &Builder<'_>) -> bool { let Some((version, stamp_file)) = get_rustfmt_version(build) else { return false; }; - !program_out_of_date(&stamp_file, &version) + stamp_file.with_stamp(version).is_up_to_date() } /// Updates the last rustfmt version used. diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index c18f7c2aae49e..78bb5747ffd90 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -12,7 +12,7 @@ use xz2::bufread::XzDecoder; use crate::core::config::BUILDER_CONFIG_FILENAME; use crate::utils::build_stamp::BuildStamp; use crate::utils::exec::{BootstrapCommand, command}; -use crate::utils::helpers::{check_run, exe, hex_encode, move_file, program_out_of_date}; +use crate::utils::helpers::{check_run, exe, hex_encode, move_file}; use crate::{Config, t}; static SHOULD_FIX_BINS_AND_DYLIBS: OnceLock = OnceLock::new(); @@ -427,9 +427,10 @@ impl Config { let version = &self.stage0_metadata.compiler.version; let host = self.build; - let clippy_stamp = BuildStamp::new(&self.initial_sysroot).with_prefix("clippy"); + let clippy_stamp = + BuildStamp::new(&self.initial_sysroot).with_prefix("clippy").with_stamp(date); let cargo_clippy = self.initial_sysroot.join("bin").join(exe("cargo-clippy", host)); - if cargo_clippy.exists() && !program_out_of_date(&clippy_stamp, date) { + if cargo_clippy.exists() && clippy_stamp.is_up_to_date() { return cargo_clippy; } @@ -440,7 +441,7 @@ impl Config { self.fix_bin_or_dylib(&cargo_clippy.with_file_name(exe("clippy-driver", host))); } - self.create(&clippy_stamp, date); + t!(clippy_stamp.write()); cargo_clippy } @@ -461,8 +462,8 @@ impl Config { let host = self.build; let bin_root = self.out.join(host).join("rustfmt"); let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); - let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt"); - if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { + let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").with_stamp(channel); + if rustfmt_path.exists() && rustfmt_stamp.is_up_to_date() { return Some(rustfmt_path); } @@ -493,7 +494,7 @@ impl Config { } } - self.create(&rustfmt_stamp, &channel); + t!(rustfmt_stamp.write()); Some(rustfmt_path) } @@ -568,10 +569,10 @@ impl Config { ) { let host = self.build.triple; let bin_root = self.out.join(host).join(sysroot); - let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc"); + let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").with_stamp(stamp_key); if !bin_root.join("bin").join(exe("rustc", self.build)).exists() - || program_out_of_date(&rustc_stamp, stamp_key) + || !rustc_stamp.is_up_to_date() { if bin_root.exists() { t!(fs::remove_dir_all(&bin_root)); @@ -602,7 +603,7 @@ impl Config { } } - t!(fs::write(rustc_stamp, stamp_key)); + t!(rustc_stamp.write()); } } @@ -729,10 +730,10 @@ download-rustc = false } let llvm_root = self.ci_llvm_root(); - let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm"); let llvm_sha = detect_llvm_sha(self, self.rust_info.is_managed_git_subrepository()); - let key = format!("{}{}", llvm_sha, self.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { + let stamp_key = format!("{}{}", llvm_sha, self.llvm_assertions); + let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm").with_stamp(stamp_key); + if !llvm_stamp.is_up_to_date() && !self.dry_run() { self.download_ci_llvm(&llvm_sha); if self.should_fix_bins_and_dylibs() { @@ -765,7 +766,7 @@ download-rustc = false } } - t!(fs::write(llvm_stamp, key)); + t!(llvm_stamp.write()); } if let Some(config_path) = &self.config { diff --git a/src/bootstrap/src/utils/helpers.rs b/src/bootstrap/src/utils/helpers.rs index 55797af4c61cd..516c314024b28 100644 --- a/src/bootstrap/src/utils/helpers.rs +++ b/src/bootstrap/src/utils/helpers.rs @@ -13,7 +13,6 @@ use std::{env, fs, io, str}; use build_helper::util::fail; use object::read::archive::ArchiveFile; -use super::build_stamp::BuildStamp; use crate::LldMode; use crate::core::builder::Builder; use crate::core::config::{Config, TargetSelection}; @@ -148,14 +147,6 @@ impl Drop for TimeIt { } } -/// Used for download caching -pub(crate) fn program_out_of_date(stamp: &BuildStamp, key: &str) -> bool { - if !stamp.as_ref().exists() { - return true; - } - t!(fs::read_to_string(stamp)) != key -} - /// Symlinks two directories, using junctions on Windows and normal symlinks on /// Unix. pub fn symlink_dir(config: &Config, original: &Path, link: &Path) -> io::Result<()> { diff --git a/src/bootstrap/src/utils/helpers/tests.rs b/src/bootstrap/src/utils/helpers/tests.rs index 611d9f299b7e9..9030ca2820a8b 100644 --- a/src/bootstrap/src/utils/helpers/tests.rs +++ b/src/bootstrap/src/utils/helpers/tests.rs @@ -1,11 +1,10 @@ -use std::fs::{self, File, remove_file}; +use std::fs::{self, File}; use std::io::Write; use std::path::PathBuf; -use crate::utils::build_stamp::BuildStamp; use crate::utils::helpers::{ - check_cfg_arg, extract_beta_rev, hex_encode, make, program_out_of_date, set_file_times, - submodule_path_of, symlink_dir, + check_cfg_arg, extract_beta_rev, hex_encode, make, set_file_times, submodule_path_of, + symlink_dir, }; use crate::{Config, Flags}; @@ -58,23 +57,6 @@ fn test_check_cfg_arg() { ); } -#[test] -fn test_program_out_of_date() { - let config = - Config::parse(Flags::parse(&["check".to_owned(), "--config=/does/not/exist".to_owned()])); - let temp_stampfile = BuildStamp::new(&config.tempdir()).with_prefix("tmp"); - - File::create(temp_stampfile.as_ref()).unwrap().write_all(b"dummy value").unwrap(); - assert!(temp_stampfile.as_ref().exists()); - - // up-to-date - assert!(!program_out_of_date(&temp_stampfile, "dummy value")); - // out-of-date - assert!(program_out_of_date(&temp_stampfile, "")); - - remove_file(temp_stampfile).unwrap(); -} - #[test] fn test_symlink_dir() { let config = From 375165259c45f005a86c083e4faccbed9a76549e Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Thu, 9 Jan 2025 12:39:59 +0300 Subject: [PATCH 13/23] auto label `A-bootstrap-stamp` Signed-off-by: onur-ozkan --- triagebot.toml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/triagebot.toml b/triagebot.toml index 67412d9e60b05..5ee52bbf435c9 100644 --- a/triagebot.toml +++ b/triagebot.toml @@ -391,6 +391,11 @@ trigger_files = [ "x.ps1" ] +[autolabel."A-bootstrap-stamp"] +trigger_files = [ + "src/bootstrap/src/utils/build_stamp.rs", +] + [autolabel."T-infra"] trigger_files = [ "src/ci", From bdb7518203c58417d018788e51c75b6ede849836 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 11 Jan 2025 09:25:54 +0300 Subject: [PATCH 14/23] apply minor improvements on build_stamp Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/build_stamp.rs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 7c3935e1db2a9..79b9951f5fbae 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -34,6 +34,9 @@ impl BuildStamp { /// /// By default, stamp will be an empty file named `.stamp` within the specified directory. pub fn new(dir: &Path) -> Self { + // Avoid using `is_dir()` as the directory may not exist yet. + // It is more appropriate to assert that the path is not a file. + assert!(!dir.is_file(), "can't be a file path"); Self { path: dir.join(".stamp"), stamp: String::new() } } @@ -52,7 +55,7 @@ impl BuildStamp { "prefix can not start or end with '.'" ); - let stamp_filename = self.path.components().last().unwrap().as_os_str().to_str().unwrap(); + let stamp_filename = self.path.file_name().unwrap().to_str().unwrap(); let stamp_filename = stamp_filename.strip_prefix('.').unwrap_or(stamp_filename); self.path.set_file_name(format!(".{prefix}-{stamp_filename}")); From ffd4c5b51ce61e0bf1e1f24e143a390a5fbded13 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 11 Jan 2025 09:29:47 +0300 Subject: [PATCH 15/23] migrate lld build stamp Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/llvm.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 7bf24675d061f..1159ba712306c 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -8,12 +8,11 @@ //! LLVM and compiler-rt are essentially just wired up to everything else to //! ensure that they're always in place if needed. -use std::env; use std::env::consts::EXE_EXTENSION; use std::ffi::{OsStr, OsString}; -use std::fs::{self, File}; use std::path::{Path, PathBuf}; use std::sync::OnceLock; +use std::{env, fs}; use build_helper::ci::CiEnv; use build_helper::git::get_closest_merge_commit; @@ -1015,8 +1014,9 @@ impl Step for Lld { } let out_dir = builder.lld_out(target); - let done_stamp = out_dir.join("lld-finished-building"); - if done_stamp.exists() { + + let done_stamp = BuildStamp::new(&out_dir).with_prefix("lld"); + if done_stamp.as_ref().exists() { return out_dir; } @@ -1091,7 +1091,7 @@ impl Step for Lld { cfg.build(); - t!(File::create(&done_stamp)); + t!(done_stamp.write()); out_dir } } From 216969b8c2c454ccf3daf7de0df49846ffd501ef Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 11 Jan 2025 16:00:43 +0300 Subject: [PATCH 16/23] run git only inside the current directory Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/build_stamp.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 79b9951f5fbae..ae35b4c8d61c4 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -168,6 +168,7 @@ pub fn generate_smart_stamp_hash( let diff = helpers::git(Some(dir)) .allow_failure() .arg("diff") + .arg(".") .run_capture_stdout(builder) .stdout_if_ok() .unwrap_or_default(); @@ -175,6 +176,7 @@ pub fn generate_smart_stamp_hash( let status = helpers::git(Some(dir)) .allow_failure() .arg("status") + .arg(".") .arg("--porcelain") .arg("-z") .arg("--untracked-files=normal") From 99322a5158e274decf1552b3abc041b3abbacbd2 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sat, 11 Jan 2025 16:01:33 +0300 Subject: [PATCH 17/23] extend sanitizers stamp calculation Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/llvm.rs | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 1159ba712306c..51df2eba6e9ce 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -1121,18 +1121,24 @@ impl Step for Sanitizers { let out_dir = builder.native_dir(self.target).join("sanitizers"); let runtimes = supported_sanitizers(&out_dir, self.target, &builder.config.channel); - if runtimes.is_empty() { + + if builder.config.dry_run() || runtimes.is_empty() { return runtimes; } let LlvmResult { llvm_config, .. } = builder.ensure(Llvm { target: builder.config.build }); - if builder.config.dry_run() { - return runtimes; - } - let stamp = BuildStamp::new(&out_dir) - .with_prefix("sanitizers") - .with_stamp(builder.in_tree_llvm_info.sha().map(String::from).unwrap_or_default()); + static STAMP_HASH_MEMO: OnceLock = OnceLock::new(); + let smart_stamp_hash = STAMP_HASH_MEMO.get_or_init(|| { + generate_smart_stamp_hash( + builder, + &builder.config.src.join("src/llvm-project/compiler-rt"), + builder.in_tree_llvm_info.sha().unwrap_or_default(), + ) + }); + + let stamp = + BuildStamp::new(&out_dir).with_prefix("sanitizers").with_stamp(smart_stamp_hash); if stamp.is_up_to_date() { if stamp.stamp.is_empty() { From dcc001adbbfd9378d08593ab2f4b3ce780308ad8 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 12 Jan 2025 08:20:42 +0300 Subject: [PATCH 18/23] refactor `with_stamp` as `add_stamp` for incrementality Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/format.rs | 2 +- src/bootstrap/src/core/build_steps/gcc.rs | 2 +- src/bootstrap/src/core/build_steps/llvm.rs | 7 +++---- src/bootstrap/src/core/download.rs | 8 ++++---- src/bootstrap/src/utils/build_stamp.rs | 4 ++-- src/bootstrap/src/utils/build_stamp/tests.rs | 4 ++-- 6 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs index 253a33183bfd1..5556e5e631bce 100644 --- a/src/bootstrap/src/core/build_steps/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -74,7 +74,7 @@ fn verify_rustfmt_version(build: &Builder<'_>) -> bool { let Some((version, stamp_file)) = get_rustfmt_version(build) else { return false; }; - stamp_file.with_stamp(version).is_up_to_date() + stamp_file.add_stamp(version).is_up_to_date() } /// Updates the last rustfmt version used. diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index f55a17aab0468..d090644c5b7fa 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -55,7 +55,7 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc ) }); - let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").with_stamp(smart_stamp_hash); + let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { if stamp.stamp.is_empty() { diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 51df2eba6e9ce..a70dc85aab4ac 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -135,7 +135,7 @@ pub fn prebuilt_llvm_config( ) }); - let stamp = BuildStamp::new(&out_dir).with_prefix("llvm").with_stamp(smart_stamp_hash); + let stamp = BuildStamp::new(&out_dir).with_prefix("llvm").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { if stamp.stamp.is_empty() { @@ -921,7 +921,7 @@ impl Step for Enzyme { }); let out_dir = builder.enzyme_out(target); - let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").with_stamp(smart_stamp_hash); + let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { if stamp.stamp.is_empty() { @@ -1137,8 +1137,7 @@ impl Step for Sanitizers { ) }); - let stamp = - BuildStamp::new(&out_dir).with_prefix("sanitizers").with_stamp(smart_stamp_hash); + let stamp = BuildStamp::new(&out_dir).with_prefix("sanitizers").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { if stamp.stamp.is_empty() { diff --git a/src/bootstrap/src/core/download.rs b/src/bootstrap/src/core/download.rs index 78bb5747ffd90..c477bdb829a91 100644 --- a/src/bootstrap/src/core/download.rs +++ b/src/bootstrap/src/core/download.rs @@ -428,7 +428,7 @@ impl Config { let host = self.build; let clippy_stamp = - BuildStamp::new(&self.initial_sysroot).with_prefix("clippy").with_stamp(date); + BuildStamp::new(&self.initial_sysroot).with_prefix("clippy").add_stamp(date); let cargo_clippy = self.initial_sysroot.join("bin").join(exe("cargo-clippy", host)); if cargo_clippy.exists() && clippy_stamp.is_up_to_date() { return cargo_clippy; @@ -462,7 +462,7 @@ impl Config { let host = self.build; let bin_root = self.out.join(host).join("rustfmt"); let rustfmt_path = bin_root.join("bin").join(exe("rustfmt", host)); - let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").with_stamp(channel); + let rustfmt_stamp = BuildStamp::new(&bin_root).with_prefix("rustfmt").add_stamp(channel); if rustfmt_path.exists() && rustfmt_stamp.is_up_to_date() { return Some(rustfmt_path); } @@ -569,7 +569,7 @@ impl Config { ) { let host = self.build.triple; let bin_root = self.out.join(host).join(sysroot); - let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").with_stamp(stamp_key); + let rustc_stamp = BuildStamp::new(&bin_root).with_prefix("rustc").add_stamp(stamp_key); if !bin_root.join("bin").join(exe("rustc", self.build)).exists() || !rustc_stamp.is_up_to_date() @@ -732,7 +732,7 @@ download-rustc = false let llvm_root = self.ci_llvm_root(); let llvm_sha = detect_llvm_sha(self, self.rust_info.is_managed_git_subrepository()); let stamp_key = format!("{}{}", llvm_sha, self.llvm_assertions); - let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm").with_stamp(stamp_key); + let llvm_stamp = BuildStamp::new(&llvm_root).with_prefix("llvm").add_stamp(stamp_key); if !llvm_stamp.is_up_to_date() && !self.dry_run() { self.download_ci_llvm(&llvm_sha); diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index ae35b4c8d61c4..831385c762ad0 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -41,8 +41,8 @@ impl BuildStamp { } /// Sets stamp content to the specified value. - pub fn with_stamp(mut self, stamp: S) -> Self { - self.stamp = stamp.to_string(); + pub fn add_stamp(mut self, stamp: S) -> Self { + self.stamp.push_str(&stamp.to_string()); self } diff --git a/src/bootstrap/src/utils/build_stamp/tests.rs b/src/bootstrap/src/utils/build_stamp/tests.rs index 0637897507a1b..2d7d1f7124655 100644 --- a/src/bootstrap/src/utils/build_stamp/tests.rs +++ b/src/bootstrap/src/utils/build_stamp/tests.rs @@ -26,7 +26,7 @@ fn test_with_invalid_prefix2() { fn test_is_up_to_date() { let dir = temp_dir(); - let mut build_stamp = BuildStamp::new(&dir).with_stamp("v1.0.0"); + let mut build_stamp = BuildStamp::new(&dir).add_stamp("v1.0.0"); build_stamp.write().unwrap(); assert!( @@ -47,7 +47,7 @@ fn test_is_up_to_date() { fn test_with_prefix() { let dir = temp_dir(); - let stamp = BuildStamp::new(&dir).with_stamp("v1.0.0"); + let stamp = BuildStamp::new(&dir).add_stamp("v1.0.0"); assert_eq!(stamp.path.file_name().unwrap(), ".stamp"); let stamp = stamp.with_prefix("test"); From 9611d8ea1359bc687cb5cca5bf3875dbe4790904 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 12 Jan 2025 08:35:12 +0300 Subject: [PATCH 19/23] avoid magical `AsRef` implementation Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/clean.rs | 2 +- src/bootstrap/src/core/build_steps/compile.rs | 10 +++---- src/bootstrap/src/core/build_steps/format.rs | 2 +- src/bootstrap/src/core/build_steps/gcc.rs | 4 +-- src/bootstrap/src/core/build_steps/llvm.rs | 14 +++++----- src/bootstrap/src/lib.rs | 6 ++-- src/bootstrap/src/utils/build_stamp.rs | 28 ++++++++++++------- 7 files changed, 37 insertions(+), 29 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/clean.rs b/src/bootstrap/src/core/build_steps/clean.rs index 13627d8e93114..21e9fea936387 100644 --- a/src/bootstrap/src/core/build_steps/clean.rs +++ b/src/bootstrap/src/core/build_steps/clean.rs @@ -147,7 +147,7 @@ fn clean_default(build: &Build) { rm_rf(&build.out.join("dist")); rm_rf(&build.out.join("bootstrap").join(".last-warned-change-id")); rm_rf(&build.out.join("bootstrap-shims-dump")); - rm_rf(BuildStamp::new(&build.out).with_prefix("rustfmt").as_ref()); + rm_rf(BuildStamp::new(&build.out).with_prefix("rustfmt").path()); let mut hosts: Vec<_> = build.hosts.iter().map(|t| build.out.join(t)).collect(); // After cross-compilation, artifacts of the host architecture (which may differ from build.host) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 9be8b9cded24b..4c4df922b3799 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -991,7 +991,7 @@ impl Step for Rustc { true, // Only ship rustc_driver.so and .rmeta files, not all intermediate .rlib files. ); - let target_root_dir = stamp.as_ref().parent().unwrap(); + let target_root_dir = stamp.path().parent().unwrap(); // When building `librustc_driver.so` (like `libLLVM.so`) on linux, it can contain // unexpected debuginfo from dependencies, for example from the C++ standard library used in // our LLVM wrapper. Unless we're explicitly requesting `librustc_driver` to be built with @@ -1478,7 +1478,7 @@ impl Step for CodegenBackend { } let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, &backend); let codegen_backend = codegen_backend.to_str().unwrap(); - t!(fs::write(stamp, codegen_backend)); + t!(stamp.add_stamp(codegen_backend).write()); } } @@ -1516,7 +1516,7 @@ fn copy_codegen_backends_to_sysroot( } let stamp = build_stamp::codegen_backend_stamp(builder, compiler, target, backend); - let dylib = t!(fs::read_to_string(&stamp)); + let dylib = t!(fs::read_to_string(stamp.path())); let file = Path::new(&dylib); let filename = file.file_name().unwrap().to_str().unwrap(); // change `librustc_codegen_cranelift-xxxxxx.so` to @@ -2018,7 +2018,7 @@ pub fn run_cargo( rlib_only_metadata: bool, ) -> Vec { // `target_root_dir` looks like $dir/$target/release - let target_root_dir = stamp.as_ref().parent().unwrap(); + let target_root_dir = stamp.path().parent().unwrap(); // `target_deps_dir` looks like $dir/$target/release/deps let target_deps_dir = target_root_dir.join("deps"); // `host_root_dir` looks like $dir/release @@ -2171,7 +2171,7 @@ pub fn run_cargo( new_contents.extend(dep.to_str().unwrap().as_bytes()); new_contents.extend(b"\0"); } - t!(fs::write(stamp, &new_contents)); + t!(fs::write(stamp.path(), &new_contents)); deps.into_iter().map(|(d, _)| d).collect() } diff --git a/src/bootstrap/src/core/build_steps/format.rs b/src/bootstrap/src/core/build_steps/format.rs index 5556e5e631bce..40c908c3fa996 100644 --- a/src/bootstrap/src/core/build_steps/format.rs +++ b/src/bootstrap/src/core/build_steps/format.rs @@ -82,7 +82,7 @@ fn update_rustfmt_version(build: &Builder<'_>) { let Some((version, stamp_file)) = get_rustfmt_version(build) else { return; }; - t!(std::fs::write(stamp_file, version)) + t!(std::fs::write(stamp_file.path(), version)) } /// Returns the Rust files modified between the `merge-base` of HEAD and diff --git a/src/bootstrap/src/core/build_steps/gcc.rs b/src/bootstrap/src/core/build_steps/gcc.rs index d090644c5b7fa..563b715fa6400 100644 --- a/src/bootstrap/src/core/build_steps/gcc.rs +++ b/src/bootstrap/src/core/build_steps/gcc.rs @@ -58,14 +58,14 @@ pub fn prebuilt_gcc_config(builder: &Builder<'_>, target: TargetSelection) -> Gc let stamp = BuildStamp::new(&out_dir).with_prefix("gcc").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { - if stamp.stamp.is_empty() { + if stamp.stamp().is_empty() { builder.info( "Could not determine the GCC submodule commit hash. \ Assuming that an GCC rebuild is not necessary.", ); builder.info(&format!( "To force GCC to rebuild, remove the file `{}`", - stamp.as_ref().display() + stamp.path().display() )); } return GccBuildStatus::AlreadyBuilt; diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index a70dc85aab4ac..92efd5412978a 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -138,14 +138,14 @@ pub fn prebuilt_llvm_config( let stamp = BuildStamp::new(&out_dir).with_prefix("llvm").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { - if stamp.stamp.is_empty() { + if stamp.stamp().is_empty() { builder.info( "Could not determine the LLVM submodule commit hash. \ Assuming that an LLVM rebuild is not necessary.", ); builder.info(&format!( "To force LLVM to rebuild, remove the file `{}`", - stamp.as_ref().display() + stamp.path().display() )); } return LlvmBuildStatus::AlreadyBuilt(res); @@ -924,14 +924,14 @@ impl Step for Enzyme { let stamp = BuildStamp::new(&out_dir).with_prefix("enzyme").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { - if stamp.stamp.is_empty() { + if stamp.stamp().is_empty() { builder.info( "Could not determine the Enzyme submodule commit hash. \ Assuming that an Enzyme rebuild is not necessary.", ); builder.info(&format!( "To force Enzyme to rebuild, remove the file `{}`", - stamp.as_ref().display() + stamp.path().display() )); } return out_dir; @@ -1016,7 +1016,7 @@ impl Step for Lld { let out_dir = builder.lld_out(target); let done_stamp = BuildStamp::new(&out_dir).with_prefix("lld"); - if done_stamp.as_ref().exists() { + if done_stamp.path().exists() { return out_dir; } @@ -1140,10 +1140,10 @@ impl Step for Sanitizers { let stamp = BuildStamp::new(&out_dir).with_prefix("sanitizers").add_stamp(smart_stamp_hash); if stamp.is_up_to_date() { - if stamp.stamp.is_empty() { + if stamp.stamp().is_empty() { builder.info(&format!( "Rebuild sanitizers by removing the file `{}`", - stamp.as_ref().display() + stamp.path().display() )); } diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 590b5da252406..ebe2c332258d7 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -1605,16 +1605,16 @@ Executed at: {executed_at}"#, return Vec::new(); } - if !stamp.as_ref().exists() { + if !stamp.path().exists() { eprintln!( "ERROR: Unable to find the stamp file {}, did you try to keep a nonexistent build stage?", - stamp.as_ref().display() + stamp.path().display() ); crate::exit!(1); } let mut paths = Vec::new(); - let contents = t!(fs::read(stamp.as_ref()), stamp.as_ref()); + let contents = t!(fs::read(stamp.path()), stamp.path()); // This is the method we use for extracting paths from the stamp file passed to us. See // run_cargo for more information (in compile.rs). for part in contents.split(|b| *b == 0) { diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 831385c762ad0..7b56aa66adf64 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -20,13 +20,7 @@ mod tests; #[derive(Clone)] pub struct BuildStamp { path: PathBuf, - pub(crate) stamp: String, -} - -impl AsRef for BuildStamp { - fn as_ref(&self) -> &Path { - &self.path - } + stamp: String, } impl BuildStamp { @@ -40,6 +34,20 @@ impl BuildStamp { Self { path: dir.join(".stamp"), stamp: String::new() } } + /// Returns path of the stamp file. + pub fn path(&self) -> &Path { + &self.path + } + + /// Returns the value of the stamp. + /// + /// Note that this is empty by default and is populated using `BuildStamp::add_stamp`. + /// It is not read from an actual file, but rather it holds the value that will be used + /// when `BuildStamp::write` is called. + pub fn stamp(&self) -> &str { + &self.stamp + } + /// Sets stamp content to the specified value. pub fn add_stamp(mut self, stamp: S) -> Self { self.stamp.push_str(&stamp.to_string()); @@ -101,15 +109,15 @@ impl BuildStamp { pub fn clear_if_dirty(builder: &Builder<'_>, dir: &Path, input: &Path) -> bool { let stamp = BuildStamp::new(dir); let mut cleared = false; - if mtime(stamp.as_ref()) < mtime(input) { + if mtime(stamp.path()) < mtime(input) { builder.verbose(|| println!("Dirty - {}", dir.display())); let _ = fs::remove_dir_all(dir); cleared = true; - } else if stamp.as_ref().exists() { + } else if stamp.path().exists() { return cleared; } t!(fs::create_dir_all(dir)); - t!(fs::File::create(stamp)); + t!(fs::File::create(stamp.path())); cleared } From fae26e7ffae16ef3f550c83ad2f45792621bb6b7 Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 12 Jan 2025 08:46:58 +0300 Subject: [PATCH 20/23] add change entry for renamings Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/change_tracker.rs | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/bootstrap/src/utils/change_tracker.rs b/src/bootstrap/src/utils/change_tracker.rs index 75c9c677ff591..85d17c8fa5077 100644 --- a/src/bootstrap/src/utils/change_tracker.rs +++ b/src/bootstrap/src/utils/change_tracker.rs @@ -330,4 +330,9 @@ pub const CONFIG_CHANGE_HISTORY: &[ChangeInfo] = &[ severity: ChangeSeverity::Warning, summary: "It is now possible to configure `optimized-compiler-builtins` for per target.", }, + ChangeInfo { + change_id: 135281, + severity: ChangeSeverity::Warning, + summary: "Some stamp names in the build artifacts may have changed slightly (e.g., from `llvm-finished-building` to `.llvm-stamp`).", + }, ]; From b54d65230f2c4cb3ca14fc2ff3fd1153531327ec Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 12 Jan 2025 08:47:57 +0300 Subject: [PATCH 21/23] rustc-dev-guide: update outdated LLVM stamp filename Signed-off-by: onur-ozkan --- src/doc/rustc-dev-guide/src/backend/debugging.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/doc/rustc-dev-guide/src/backend/debugging.md b/src/doc/rustc-dev-guide/src/backend/debugging.md index 275a1777a9964..805291017c274 100644 --- a/src/doc/rustc-dev-guide/src/backend/debugging.md +++ b/src/doc/rustc-dev-guide/src/backend/debugging.md @@ -56,7 +56,7 @@ These tools include: By default, the Rust build system does not check for changes to the LLVM source code or its build configuration settings. So, if you need to rebuild the LLVM that is linked -into `rustc`, first delete the file `llvm-finished-building`, which should be located +into `rustc`, first delete the file `.llvm-stamp`, which should be located in `build//llvm/`. The default rustc compilation pipeline has multiple codegen units, which is From b03fba7ab4bb519103e16ccc4995949c9ae8fdfc Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 12 Jan 2025 08:53:22 +0300 Subject: [PATCH 22/23] rename `done_stamp` to `lld_stamp` Signed-off-by: onur-ozkan --- src/bootstrap/src/core/build_steps/llvm.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/llvm.rs b/src/bootstrap/src/core/build_steps/llvm.rs index 92efd5412978a..cf55fff4078ac 100644 --- a/src/bootstrap/src/core/build_steps/llvm.rs +++ b/src/bootstrap/src/core/build_steps/llvm.rs @@ -1015,8 +1015,8 @@ impl Step for Lld { let out_dir = builder.lld_out(target); - let done_stamp = BuildStamp::new(&out_dir).with_prefix("lld"); - if done_stamp.path().exists() { + let lld_stamp = BuildStamp::new(&out_dir).with_prefix("lld"); + if lld_stamp.path().exists() { return out_dir; } @@ -1091,7 +1091,7 @@ impl Step for Lld { cfg.build(); - t!(done_stamp.write()); + t!(lld_stamp.write()); out_dir } } From 2a4bcf597b1b7e3a852e826153bab8082969681f Mon Sep 17 00:00:00 2001 From: onur-ozkan Date: Sun, 12 Jan 2025 10:15:36 +0300 Subject: [PATCH 23/23] update doc-comment of `BuildStamp::add_stamp` Signed-off-by: onur-ozkan --- src/bootstrap/src/utils/build_stamp.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/bootstrap/src/utils/build_stamp.rs b/src/bootstrap/src/utils/build_stamp.rs index 7b56aa66adf64..f43d860893f6b 100644 --- a/src/bootstrap/src/utils/build_stamp.rs +++ b/src/bootstrap/src/utils/build_stamp.rs @@ -48,7 +48,9 @@ impl BuildStamp { &self.stamp } - /// Sets stamp content to the specified value. + /// Adds specified stamp content to the current value. + /// + /// This method can be used incrementally e.g., `add_stamp("x").add_stamp("y").add_stamp("z")`. pub fn add_stamp(mut self, stamp: S) -> Self { self.stamp.push_str(&stamp.to_string()); self