From 0ec5d374fe901488c9bd73cd2162bd62daca8261 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Sat, 16 Mar 2024 12:24:49 +0100 Subject: [PATCH 1/2] bootstrap: Don't name things copy that are not copies The bootstrap copy methods don't actually copy, they just hard link. Simply lying about it being "copying" can be very confusing! (ask me how I know!). --- src/bootstrap/src/core/build_steps/compile.rs | 41 ++++++++-------- src/bootstrap/src/core/build_steps/dist.rs | 29 +++++------ src/bootstrap/src/core/build_steps/doc.rs | 9 ++-- src/bootstrap/src/core/build_steps/test.rs | 2 +- src/bootstrap/src/core/build_steps/tool.rs | 8 ++-- src/bootstrap/src/lib.rs | 48 ++++++++++++------- src/bootstrap/src/utils/tarball.rs | 4 +- 7 files changed, 79 insertions(+), 62 deletions(-) diff --git a/src/bootstrap/src/core/build_steps/compile.rs b/src/bootstrap/src/core/build_steps/compile.rs index 94ea2a01a4057..e927b491c71ea 100644 --- a/src/bootstrap/src/core/build_steps/compile.rs +++ b/src/bootstrap/src/core/build_steps/compile.rs @@ -231,7 +231,7 @@ impl Step for Std { let target_sysroot_bin = builder.sysroot_libdir(compiler, target).parent().unwrap().join("bin"); t!(fs::create_dir_all(&target_sysroot_bin)); - builder.cp_r(&src_sysroot_bin, &target_sysroot_bin); + builder.cp_link_r(&src_sysroot_bin, &target_sysroot_bin); } } @@ -307,7 +307,7 @@ fn copy_and_stamp( dependency_type: DependencyType, ) { let target = libdir.join(name); - builder.copy(&sourcedir.join(name), &target); + builder.copy_link(&sourcedir.join(name), &target); target_deps.push((target, dependency_type)); } @@ -316,7 +316,7 @@ fn copy_llvm_libunwind(builder: &Builder<'_>, target: TargetSelection, libdir: & let libunwind_path = builder.ensure(llvm::Libunwind { target }); let libunwind_source = libunwind_path.join("libunwind.a"); let libunwind_target = libdir.join("libunwind.a"); - builder.copy(&libunwind_source, &libunwind_target); + builder.copy_link(&libunwind_source, &libunwind_target); libunwind_target } @@ -385,7 +385,7 @@ fn copy_self_contained_objects( for &obj in &["crtbegin.o", "crtbeginS.o", "crtend.o", "crtendS.o"] { let src = crt_path.join(obj); let target = libdir_self_contained.join(obj); - builder.copy(&src, &target); + builder.copy_link(&src, &target); target_deps.push((target, DependencyType::TargetSelfContained)); } @@ -418,7 +418,7 @@ fn copy_self_contained_objects( for obj in ["crt2.o", "dllcrt2.o"].iter() { let src = compiler_file(builder, &builder.cc(target), target, CLang::C, obj); let target = libdir_self_contained.join(obj); - builder.copy(&src, &target); + builder.copy_link(&src, &target); target_deps.push((target, DependencyType::TargetSelfContained)); } } @@ -637,7 +637,7 @@ impl Step for StdLink { let stage0_bin_dir = builder.out.join(host).join("stage0/bin"); let sysroot_bin_dir = sysroot.join("bin"); t!(fs::create_dir_all(&sysroot_bin_dir)); - builder.cp_r(&stage0_bin_dir, &sysroot_bin_dir); + builder.cp_link_r(&stage0_bin_dir, &sysroot_bin_dir); // Copy all *.so files from stage0/lib to stage0-sysroot/lib let stage0_lib_dir = builder.out.join(host).join("stage0/lib"); @@ -646,7 +646,8 @@ impl Step for StdLink { let file = t!(file); let path = file.path(); if path.is_file() && is_dylib(&file.file_name().into_string().unwrap()) { - builder.copy(&path, &sysroot.join("lib").join(path.file_name().unwrap())); + builder + .copy_link(&path, &sysroot.join("lib").join(path.file_name().unwrap())); } } } @@ -661,7 +662,7 @@ impl Step for StdLink { .join(host) .join("codegen-backends"); if stage0_codegen_backends.exists() { - builder.cp_r(&stage0_codegen_backends, &sysroot_codegen_backends); + builder.cp_link_r(&stage0_codegen_backends, &sysroot_codegen_backends); } } } @@ -684,7 +685,7 @@ fn copy_sanitizers( for runtime in &runtimes { let dst = libdir.join(&runtime.name); - builder.copy(&runtime.path, &dst); + builder.copy_link(&runtime.path, &dst); // The `aarch64-apple-ios-macabi` and `x86_64-apple-ios-macabi` are also supported for // sanitizers, but they share a sanitizer runtime with `${arch}-apple-darwin`, so we do @@ -790,7 +791,7 @@ impl Step for StartupObjects { } let target = sysroot_dir.join((*file).to_string() + ".o"); - builder.copy(dst_file, &target); + builder.copy_link(dst_file, &target); target_deps.push((target, DependencyType::Target)); } @@ -812,7 +813,7 @@ fn cp_rustc_component_to_ci_sysroot( if src.is_dir() { t!(fs::create_dir_all(dst)); } else { - builder.copy(&src, &dst); + builder.copy_link(&src, &dst); } } } @@ -1443,7 +1444,7 @@ fn copy_codegen_backends_to_sysroot( let dot = filename.find('.').unwrap(); format!("{}-{}{}", &filename[..dash], builder.rust_release(), &filename[dot..]) }; - builder.copy(file, &dst.join(target_filename)); + builder.copy_link(file, &dst.join(target_filename)); } } @@ -1599,7 +1600,7 @@ impl Step for Sysroot { OsStr::new(std::env::consts::DLL_EXTENSION), ]; let ci_rustc_dir = builder.config.ci_rustc_dir(); - builder.cp_filtered(&ci_rustc_dir, &sysroot, &|path| { + builder.cp_link_filtered(&ci_rustc_dir, &sysroot, &|path| { if path.extension().map_or(true, |ext| !filtered_extensions.contains(&ext)) { return true; } @@ -1791,7 +1792,7 @@ impl Step for Assemble { let filename = f.file_name().into_string().unwrap(); if (is_dylib(&filename) || is_debug_info(&filename)) && !proc_macros.contains(&filename) { - builder.copy(&f.path(), &rustc_libdir.join(&filename)); + builder.copy_link(&f.path(), &rustc_libdir.join(&filename)); } } @@ -1805,7 +1806,7 @@ impl Step for Assemble { if let Some(lld_install) = lld_install { let src_exe = exe("lld", target_compiler.host); let dst_exe = exe("rust-lld", target_compiler.host); - builder.copy(&lld_install.join("bin").join(src_exe), &libdir_bin.join(dst_exe)); + builder.copy_link(&lld_install.join("bin").join(src_exe), &libdir_bin.join(dst_exe)); let self_contained_lld_dir = libdir_bin.join("gcc-ld"); t!(fs::create_dir_all(&self_contained_lld_dir)); let lld_wrapper_exe = builder.ensure(crate::core::build_steps::tool::LldWrapper { @@ -1813,7 +1814,7 @@ impl Step for Assemble { target: target_compiler.host, }); for name in crate::LLD_FILE_NAMES { - builder.copy( + builder.copy_link( &lld_wrapper_exe, &self_contained_lld_dir.join(exe(name, target_compiler.host)), ); @@ -1838,7 +1839,7 @@ impl Step for Assemble { // When using `download-ci-llvm`, some of the tools // may not exist, so skip trying to copy them. if src_path.exists() { - builder.copy(&src_path, &libdir_bin.join(&tool_exe)); + builder.copy_link(&src_path, &libdir_bin.join(&tool_exe)); } } } @@ -1851,7 +1852,7 @@ impl Step for Assemble { extra_features: vec![], }); let tool_exe = exe("llvm-bitcode-linker", target_compiler.host); - builder.copy(&src_path, &libdir_bin.join(&tool_exe)); + builder.copy_link(&src_path, &libdir_bin.join(&tool_exe)); } // Ensure that `libLLVM.so` ends up in the newly build compiler directory, @@ -1865,7 +1866,7 @@ impl Step for Assemble { let bindir = sysroot.join("bin"); t!(fs::create_dir_all(bindir)); let compiler = builder.rustc(target_compiler); - builder.copy(&rustc, &compiler); + builder.copy_link(&rustc, &compiler); target_compiler } @@ -1891,7 +1892,7 @@ pub fn add_to_sysroot( DependencyType::Target => sysroot_dst, DependencyType::TargetSelfContained => self_contained_dst, }; - builder.copy(&path, &dst.join(path.file_name().unwrap())); + builder.copy_link(&path, &dst.join(path.file_name().unwrap())); } } diff --git a/src/bootstrap/src/core/build_steps/dist.rs b/src/bootstrap/src/core/build_steps/dist.rs index 3efdfc324b86c..012d64e534439 100644 --- a/src/bootstrap/src/core/build_steps/dist.rs +++ b/src/bootstrap/src/core/build_steps/dist.rs @@ -272,7 +272,7 @@ fn make_win_dist( let dist_bin_dir = rust_root.join("bin/"); fs::create_dir_all(&dist_bin_dir).expect("creating dist_bin_dir failed"); for src in rustc_dlls { - builder.copy_to_folder(&src, &dist_bin_dir); + builder.copy_link_to_folder(&src, &dist_bin_dir); } //Copy platform tools to platform-specific bin directory @@ -284,7 +284,7 @@ fn make_win_dist( .join("self-contained"); fs::create_dir_all(&target_bin_dir).expect("creating target_bin_dir failed"); for src in target_tools { - builder.copy_to_folder(&src, &target_bin_dir); + builder.copy_link_to_folder(&src, &target_bin_dir); } // Warn windows-gnu users that the bundled GCC cannot compile C files @@ -304,7 +304,7 @@ fn make_win_dist( .join("self-contained"); fs::create_dir_all(&target_lib_dir).expect("creating target_lib_dir failed"); for src in target_libs { - builder.copy_to_folder(&src, &target_lib_dir); + builder.copy_link_to_folder(&src, &target_lib_dir); } } @@ -400,7 +400,7 @@ impl Step for Rustc { // Copy rustc binary t!(fs::create_dir_all(image.join("bin"))); - builder.cp_r(&src.join("bin"), &image.join("bin")); + builder.cp_link_r(&src.join("bin"), &image.join("bin")); // If enabled, copy rustdoc binary if builder @@ -458,13 +458,13 @@ impl Step for Rustc { if builder.config.lld_enabled { let src_dir = builder.sysroot_libdir(compiler, host).parent().unwrap().join("bin"); let rust_lld = exe("rust-lld", compiler.host); - builder.copy(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); + builder.copy_link(&src_dir.join(&rust_lld), &dst_dir.join(&rust_lld)); let self_contained_lld_src_dir = src_dir.join("gcc-ld"); let self_contained_lld_dst_dir = dst_dir.join("gcc-ld"); t!(fs::create_dir(&self_contained_lld_dst_dir)); for name in crate::LLD_FILE_NAMES { let exe_name = exe(name, compiler.host); - builder.copy( + builder.copy_link( &self_contained_lld_src_dir.join(&exe_name), &self_contained_lld_dst_dir.join(&exe_name), ); @@ -609,9 +609,9 @@ fn copy_target_libs(builder: &Builder<'_>, target: TargetSelection, image: &Path t!(fs::create_dir_all(&self_contained_dst)); for (path, dependency_type) in builder.read_stamp_file(stamp) { if dependency_type == DependencyType::TargetSelfContained { - builder.copy(&path, &self_contained_dst.join(path.file_name().unwrap())); + builder.copy_link(&path, &self_contained_dst.join(path.file_name().unwrap())); } else if dependency_type == DependencyType::Target || builder.config.build == target { - builder.copy(&path, &dst.join(path.file_name().unwrap())); + builder.copy_link(&path, &dst.join(path.file_name().unwrap())); } } } @@ -865,7 +865,8 @@ fn copy_src_dirs( for item in src_dirs { let dst = &dst_dir.join(item); t!(fs::create_dir_all(dst)); - builder.cp_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); + builder + .cp_link_filtered(&base.join(item), dst, &|path| filter_fn(exclude_dirs, item, path)); } } @@ -923,7 +924,7 @@ impl Step for Src { &dst_src, ); for file in src_files.iter() { - builder.copy(&builder.src.join(file), &dst_src.join(file)); + builder.copy_link(&builder.src.join(file), &dst_src.join(file)); } tarball.generate() @@ -979,7 +980,7 @@ impl Step for PlainSourceTarball { // Copy the files normally for item in &src_files { - builder.copy(&builder.src.join(item), &plain_dst_src.join(item)); + builder.copy_link(&builder.src.join(item), &plain_dst_src.join(item)); } // Create the version file @@ -1608,7 +1609,7 @@ impl Step for Extended { let prepare = |name: &str| { builder.create_dir(&pkg.join(name)); - builder.cp_r( + builder.cp_link_r( &work.join(format!("{}-{}", pkgname(builder, name), target.triple)), &pkg.join(name), ); @@ -1672,7 +1673,7 @@ impl Step for Extended { } else { name.to_string() }; - builder.cp_r( + builder.cp_link_r( &work.join(format!("{}-{}", pkgname(builder, name), target.triple)).join(dir), &exe.join(name), ); @@ -2040,7 +2041,7 @@ fn install_llvm_file( if install_symlink { // For download-ci-llvm, also install the symlink, to match what LLVM does. Using a // symlink is fine here, as this is not a rustup component. - builder.copy(&source, &full_dest); + builder.copy_link(&source, &full_dest); } else { // Otherwise, replace the symlink with an equivalent linker script. This is used when // projects like miri link against librustc_driver.so. We don't use a symlink, as diff --git a/src/bootstrap/src/core/build_steps/doc.rs b/src/bootstrap/src/core/build_steps/doc.rs index 1d4d9d4c2e1be..51b5cdc056577 100644 --- a/src/bootstrap/src/core/build_steps/doc.rs +++ b/src/bootstrap/src/core/build_steps/doc.rs @@ -520,7 +520,10 @@ impl Step for SharedAssets { t!(fs::write(&version_info, info)); } - builder.copy(&builder.src.join("src").join("doc").join("rust.css"), &out.join("rust.css")); + builder.copy_link( + &builder.src.join("src").join("doc").join("rust.css"), + &out.join("rust.css"), + ); SharedAssetsPaths { version_info } } @@ -718,7 +721,7 @@ fn doc_std( let _guard = builder.msg_doc(compiler, description, target); builder.run(&mut cargo.into()); - builder.cp_r(&out_dir, out); + builder.cp_link_r(&out_dir, out); } #[derive(Debug, Clone, Hash, PartialEq, Eq)] @@ -1151,7 +1154,7 @@ impl Step for RustcBook { let out_base = builder.md_doc_out(self.target).join("rustc"); t!(fs::create_dir_all(&out_base)); let out_listing = out_base.join("src/lints"); - builder.cp_r(&builder.src.join("src/doc/rustc"), &out_base); + builder.cp_link_r(&builder.src.join("src/doc/rustc"), &out_base); builder.info(&format!("Generating lint docs ({})", self.target)); let rustc = builder.rustc(self.compiler); diff --git a/src/bootstrap/src/core/build_steps/test.rs b/src/bootstrap/src/core/build_steps/test.rs index e9e2a881d111d..690e20fb6b978 100644 --- a/src/bootstrap/src/core/build_steps/test.rs +++ b/src/bootstrap/src/core/build_steps/test.rs @@ -1367,7 +1367,7 @@ impl Step for RunMakeSupport { let cargo_out = builder.cargo_out(self.compiler, Mode::ToolStd, self.target).join(&lib_name); - builder.copy(&cargo_out, &lib); + builder.copy_link(&cargo_out, &lib); lib } } diff --git a/src/bootstrap/src/core/build_steps/tool.rs b/src/bootstrap/src/core/build_steps/tool.rs index 53dc1cff0aeba..3c20011210373 100644 --- a/src/bootstrap/src/core/build_steps/tool.rs +++ b/src/bootstrap/src/core/build_steps/tool.rs @@ -127,7 +127,7 @@ impl Step for ToolBuild { } let cargo_out = builder.cargo_out(compiler, self.mode, target).join(exe(tool, target)); let bin = builder.tools_dir(compiler).join(exe(tool, target)); - builder.copy(&cargo_out, &bin); + builder.copy_link(&cargo_out, &bin); bin } } @@ -507,7 +507,7 @@ impl Step for Rustdoc { t!(fs::create_dir_all(&bindir)); let bin_rustdoc = bindir.join(exe("rustdoc", target_compiler.host)); let _ = fs::remove_file(&bin_rustdoc); - builder.copy(&tool_rustdoc, &bin_rustdoc); + builder.copy_link(&tool_rustdoc, &bin_rustdoc); bin_rustdoc } else { tool_rustdoc @@ -686,7 +686,7 @@ impl Step for RustAnalyzerProcMacroSrv { // so that r-a can use it. let libexec_path = builder.sysroot(self.compiler).join("libexec"); t!(fs::create_dir_all(&libexec_path)); - builder.copy(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); + builder.copy_link(&path, &libexec_path.join("rust-analyzer-proc-macro-srv")); Some(path) } @@ -765,7 +765,7 @@ macro_rules! tool_extended { $(for add_bin in $add_bins_to_sysroot { let bin_source = tools_out.join(exe(add_bin, $sel.target)); let bin_destination = bindir.join(exe(add_bin, $sel.compiler.host)); - $builder.copy(&bin_source, &bin_destination); + $builder.copy_link(&bin_source, &bin_destination); })? let tool = bindir.join(exe($tool_name, $sel.compiler.host)); diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index 85211aabb74c4..ea25015ba2719 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -1646,16 +1646,19 @@ impl Build { paths } - /// Copies a file from `src` to `dst` - pub fn copy(&self, src: &Path, dst: &Path) { - self.copy_internal(src, dst, false); + /// Links a file from `src` to `dst`. + /// Attempts to use hard links if possible, falling back to copying. + /// You can neither rely on this being a copy nor it being a link, + /// so do not write to dst. + pub fn copy_link(&self, src: &Path, dst: &Path) { + self.copy_link_internal(src, dst, false); } - fn copy_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { + fn copy_link_internal(&self, src: &Path, dst: &Path, dereference_symlinks: bool) { if self.config.dry_run() { return; } - self.verbose_than(1, || println!("Copy {src:?} to {dst:?}")); + self.verbose_than(1, || println!("Copy/Link {src:?} to {dst:?}")); if src == dst { return; } @@ -1686,9 +1689,10 @@ impl Build { } } - /// Copies the `src` directory recursively to `dst`. Both are assumed to exist + /// Links the `src` directory recursively to `dst`. Both are assumed to exist /// when this function is called. - pub fn cp_r(&self, src: &Path, dst: &Path) { + /// Will attempt to use hard links if possible and fall back to copying. + pub fn cp_link_r(&self, src: &Path, dst: &Path) { if self.config.dry_run() { return; } @@ -1698,24 +1702,32 @@ impl Build { let dst = dst.join(name); if t!(f.file_type()).is_dir() { t!(fs::create_dir_all(&dst)); - self.cp_r(&path, &dst); + self.cp_link_r(&path, &dst); } else { let _ = fs::remove_file(&dst); - self.copy(&path, &dst); + self.copy_link(&path, &dst); } } } /// Copies the `src` directory recursively to `dst`. Both are assumed to exist - /// when this function is called. Unwanted files or directories can be skipped + /// when this function is called. + /// Will attempt to use hard links if possible and fall back to copying. + /// Unwanted files or directories can be skipped /// by returning `false` from the filter function. - pub fn cp_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { + pub fn cp_link_filtered(&self, src: &Path, dst: &Path, filter: &dyn Fn(&Path) -> bool) { // Immediately recurse with an empty relative path - self.recurse_(src, dst, Path::new(""), filter) + self.cp_link_filtered_recurse(src, dst, Path::new(""), filter) } // Inner function does the actual work - fn recurse_(&self, src: &Path, dst: &Path, relative: &Path, filter: &dyn Fn(&Path) -> bool) { + fn cp_link_filtered_recurse( + &self, + src: &Path, + dst: &Path, + relative: &Path, + filter: &dyn Fn(&Path) -> bool, + ) { for f in self.read_dir(src) { let path = f.path(); let name = path.file_name().unwrap(); @@ -1726,19 +1738,19 @@ impl Build { if t!(f.file_type()).is_dir() { let _ = fs::remove_dir_all(&dst); self.create_dir(&dst); - self.recurse_(&path, &dst, &relative, filter); + self.cp_link_filtered_recurse(&path, &dst, &relative, filter); } else { let _ = fs::remove_file(&dst); - self.copy(&path, &dst); + self.copy_link(&path, &dst); } } } } - fn copy_to_folder(&self, src: &Path, dest_folder: &Path) { + fn copy_link_to_folder(&self, src: &Path, dest_folder: &Path) { let file_name = src.file_name().unwrap(); let dest = dest_folder.join(file_name); - self.copy(src, &dest); + self.copy_link(src, &dest); } fn install(&self, src: &Path, dstdir: &Path, perms: u32) { @@ -1751,7 +1763,7 @@ impl Build { if !src.exists() { panic!("ERROR: File \"{}\" not found!", src.display()); } - self.copy_internal(src, &dst, true); + self.copy_link_internal(src, &dst, true); chmod(&dst, perms); } diff --git a/src/bootstrap/src/utils/tarball.rs b/src/bootstrap/src/utils/tarball.rs index 03f56cba29d8d..4f99079a57f36 100644 --- a/src/bootstrap/src/utils/tarball.rs +++ b/src/bootstrap/src/utils/tarball.rs @@ -197,7 +197,7 @@ impl<'a> Tarball<'a> { ) { let destdir = self.image_dir.join(destdir.as_ref()); t!(std::fs::create_dir_all(&destdir)); - self.builder.copy(src.as_ref(), &destdir.join(new_name)); + self.builder.copy_link(src.as_ref(), &destdir.join(new_name)); } pub(crate) fn add_legal_and_readme_to(&self, destdir: impl AsRef) { @@ -210,7 +210,7 @@ impl<'a> Tarball<'a> { let dest = self.image_dir.join(dest.as_ref()); t!(std::fs::create_dir_all(&dest)); - self.builder.cp_r(src.as_ref(), &dest); + self.builder.cp_link_r(src.as_ref(), &dest); } pub(crate) fn add_bulk_dir(&mut self, src: impl AsRef, dest: impl AsRef) { From 52f84fa7c2a1c096bda71475e4e3007110a60887 Mon Sep 17 00:00:00 2001 From: Nilstrieb <48135649+Nilstrieb@users.noreply.github.com> Date: Sat, 16 Mar 2024 12:57:36 +0100 Subject: [PATCH 2/2] Remove double remove_file --- src/bootstrap/src/lib.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/src/bootstrap/src/lib.rs b/src/bootstrap/src/lib.rs index ea25015ba2719..f0fe994d155d0 100644 --- a/src/bootstrap/src/lib.rs +++ b/src/bootstrap/src/lib.rs @@ -1704,7 +1704,6 @@ impl Build { t!(fs::create_dir_all(&dst)); self.cp_link_r(&path, &dst); } else { - let _ = fs::remove_file(&dst); self.copy_link(&path, &dst); } }