From fb471de5a91a3426872c654681a8e6330ae84aa4 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sun, 30 Oct 2022 17:29:51 -0500 Subject: [PATCH] Make all download functions need only Config, not Builder This also adds a new `mod download` instead of scattering the download code across `config.rs` and `native.rs`. --- src/bootstrap/builder.rs | 261 +------------------ src/bootstrap/channel.rs | 3 + src/bootstrap/compile.rs | 4 +- src/bootstrap/config.rs | 314 +++++++---------------- src/bootstrap/dist.rs | 4 +- src/bootstrap/doc.rs | 6 +- src/bootstrap/download.rs | 519 ++++++++++++++++++++++++++++++++++++++ src/bootstrap/lib.rs | 101 +++----- src/bootstrap/native.rs | 84 +----- src/bootstrap/sanity.rs | 2 +- src/bootstrap/tarball.rs | 2 +- 11 files changed, 663 insertions(+), 637 deletions(-) create mode 100644 src/bootstrap/download.rs diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 31158870f394e..8d999302a6d7b 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -2,14 +2,13 @@ use std::any::{type_name, Any}; use std::cell::{Cell, RefCell}; use std::collections::BTreeSet; use std::env; -use std::ffi::{OsStr, OsString}; +use std::ffi::OsStr; use std::fmt::{Debug, Write}; -use std::fs::{self, File}; +use std::fs::{self}; use std::hash::Hash; -use std::io::{BufRead, BufReader, ErrorKind}; use std::ops::Deref; use std::path::{Component, Path, PathBuf}; -use std::process::{Command, Stdio}; +use std::process::Command; use std::time::{Duration, Instant}; use crate::cache::{Cache, Interned, INTERNER}; @@ -24,14 +23,12 @@ use crate::test; use crate::tool::{self, SourceType}; use crate::util::{self, add_dylib_path, add_link_lib_path, exe, libdir, output, t}; use crate::EXTRA_CHECK_CFGS; -use crate::{check, Config}; -use crate::{compile, Crate}; +use crate::{check, compile, Crate}; use crate::{Build, CLang, DocTests, GitRepo, Mode}; pub use crate::Compiler; // FIXME: replace with std::lazy after it gets stabilized and reaches beta -use once_cell::sync::{Lazy, OnceCell}; -use xz2::bufread::XzDecoder; +use once_cell::sync::Lazy; pub struct Builder<'a> { pub build: &'a Build, @@ -853,241 +850,6 @@ impl<'a> Builder<'a> { StepDescription::run(v, self, paths); } - /// Modifies the interpreter section of 'fname' to fix the dynamic linker, - /// or the RPATH section, to fix the dynamic library search path - /// - /// This is only required on NixOS and uses the PatchELF utility to - /// change the interpreter/RPATH of ELF executables. - /// - /// Please see https://nixos.org/patchelf.html for more information - pub(crate) fn fix_bin_or_dylib(&self, fname: &Path) { - // FIXME: cache NixOS detection? - match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { - Err(_) => return, - Ok(output) if !output.status.success() => return, - Ok(output) => { - let mut s = output.stdout; - if s.last() == Some(&b'\n') { - s.pop(); - } - if s != b"Linux" { - return; - } - } - } - - // If the user has asked binaries to be patched for Nix, then - // don't check for NixOS or `/lib`, just continue to the patching. - // NOTE: this intentionally comes after the Linux check: - // - patchelf only works with ELF files, so no need to run it on Mac or Windows - // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. - if !self.config.patch_binaries_for_nix { - // Use `/etc/os-release` instead of `/etc/NIXOS`. - // The latter one does not exist on NixOS when using tmpfs as root. - const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; - let os_release = match File::open("/etc/os-release") { - Err(e) if e.kind() == ErrorKind::NotFound => return, - Err(e) => panic!("failed to access /etc/os-release: {}", e), - Ok(f) => f, - }; - if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { - return; - } - if Path::new("/lib").exists() { - return; - } - } - - // At this point we're pretty sure the user is running NixOS or using Nix - println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); - - // Only build `.nix-deps` once. - static NIX_DEPS_DIR: OnceCell = OnceCell::new(); - let mut nix_build_succeeded = true; - let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { - // Run `nix-build` to "build" each dependency (which will likely reuse - // the existing `/nix/store` copy, or at most download a pre-built copy). - // - // Importantly, we create a gc-root called `.nix-deps` in the `build/` - // directory, but still reference the actual `/nix/store` path in the rpath - // as it makes it significantly more robust against changes to the location of - // the `.nix-deps` location. - // - // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). - // zlib: Needed as a system dependency of `libLLVM-*.so`. - // patchelf: Needed for patching ELF binaries (see doc comment above). - let nix_deps_dir = self.out.join(".nix-deps"); - const NIX_EXPR: &str = " - with (import {}); - symlinkJoin { - name = \"rust-stage0-dependencies\"; - paths = [ - zlib - patchelf - stdenv.cc.bintools - ]; - } - "; - nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[ - Path::new("-E"), - Path::new(NIX_EXPR), - Path::new("-o"), - &nix_deps_dir, - ])); - nix_deps_dir - }); - if !nix_build_succeeded { - return; - } - - let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); - let rpath_entries = { - // ORIGIN is a relative default, all binary and dynamic libraries we ship - // appear to have this (even when `../lib` is redundant). - // NOTE: there are only two paths here, delimited by a `:` - let mut entries = OsString::from("$ORIGIN/../lib:"); - entries.push(t!(fs::canonicalize(nix_deps_dir))); - entries.push("/lib"); - entries - }; - patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); - if !fname.extension().map_or(false, |ext| ext == "so") { - // Finally, set the correct .interp for binaries - let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); - // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... - let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); - patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); - } - - self.try_run(patchelf.arg(fname)); - } - - pub(crate) fn download_component(&self, url: &str, dest_path: &Path, help_on_error: &str) { - self.verbose(&format!("download {url}")); - // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. - let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - // While bootstrap itself only supports http and https downloads, downstream forks might - // need to download components from other protocols. The match allows them adding more - // protocols without worrying about merge conflicts if we change the HTTP implementation. - match url.split_once("://").map(|(proto, _)| proto) { - Some("http") | Some("https") => { - self.download_http_with_retries(&tempfile, url, help_on_error) - } - Some(other) => panic!("unsupported protocol {other} in {url}"), - None => panic!("no protocol in {url}"), - } - t!(std::fs::rename(&tempfile, dest_path)); - } - - fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { - println!("downloading {}", url); - // Try curl. If that fails and we are on windows, fallback to PowerShell. - let mut curl = Command::new("curl"); - curl.args(&[ - "-#", - "-y", - "30", - "-Y", - "10", // timeout if speed is < 10 bytes/sec for > 30 seconds - "--connect-timeout", - "30", // timeout if cannot connect within 30 seconds - "--retry", - "3", - "-Sf", - "-o", - ]); - curl.arg(tempfile); - curl.arg(url); - if !self.check_run(&mut curl) { - if self.build.build.contains("windows-msvc") { - println!("Fallback to PowerShell"); - for _ in 0..3 { - if self.try_run(Command::new("PowerShell.exe").args(&[ - "/nologo", - "-Command", - "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", - &format!( - "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", - url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), - ), - ])) { - return; - } - println!("\nspurious failure, trying again"); - } - } - if !help_on_error.is_empty() { - eprintln!("{}", help_on_error); - } - crate::detail_exit(1); - } - } - - pub(crate) fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { - println!("extracting {} to {}", tarball.display(), dst.display()); - if !dst.exists() { - t!(fs::create_dir_all(dst)); - } - - // `tarball` ends with `.tar.xz`; strip that suffix - // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` - let uncompressed_filename = - Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); - let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); - - // decompress the file - let data = t!(File::open(tarball)); - let decompressor = XzDecoder::new(BufReader::new(data)); - - let mut tar = tar::Archive::new(decompressor); - for member in t!(tar.entries()) { - let mut member = t!(member); - let original_path = t!(member.path()).into_owned(); - // skip the top-level directory - if original_path == directory_prefix { - continue; - } - let mut short_path = t!(original_path.strip_prefix(directory_prefix)); - if !short_path.starts_with(pattern) { - continue; - } - short_path = t!(short_path.strip_prefix(pattern)); - let dst_path = dst.join(short_path); - self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); - if !t!(member.unpack_in(dst)) { - panic!("path traversal attack ??"); - } - let src_path = dst.join(original_path); - if src_path.is_dir() && dst_path.exists() { - continue; - } - t!(fs::rename(src_path, dst_path)); - } - t!(fs::remove_dir_all(dst.join(directory_prefix))); - } - - /// Returns whether the SHA256 checksum of `path` matches `expected`. - pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { - use sha2::Digest; - - self.verbose(&format!("verifying {}", path.display())); - let mut hasher = sha2::Sha256::new(); - // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. - // Consider using streaming IO instead? - let contents = if self.config.dry_run() { vec![] } else { t!(fs::read(path)) }; - hasher.update(&contents); - let found = hex::encode(hasher.finalize().as_slice()); - let verified = found == expected; - if !verified && !self.config.dry_run() { - println!( - "invalid checksum: \n\ - found: {found}\n\ - expected: {expected}", - ); - } - return verified; - } - /// Obtain a compiler at a given stage and for a given host. Explicitly does /// not take `Compiler` since all `Compiler` instances are meant to be /// obtained through this function, since it ensures that they are valid @@ -1301,19 +1063,6 @@ impl<'a> Builder<'a> { None } - /// Convenience wrapper to allow `builder.llvm_link_shared()` instead of `builder.config.llvm_link_shared(&builder)`. - pub(crate) fn llvm_link_shared(&self) -> bool { - Config::llvm_link_shared(self) - } - - pub(crate) fn download_rustc(&self) -> bool { - Config::download_rustc(self) - } - - pub(crate) fn initial_rustfmt(&self) -> Option { - Config::initial_rustfmt(self) - } - /// Prepares an invocation of `cargo` to be run. /// /// This will create a `Command` that represents a pending execution of diff --git a/src/bootstrap/channel.rs b/src/bootstrap/channel.rs index 258352a21a4ad..eae81b9fc69c8 100644 --- a/src/bootstrap/channel.rs +++ b/src/bootstrap/channel.rs @@ -13,8 +13,10 @@ use crate::util::output; use crate::util::t; use crate::Build; +#[derive(Clone, Default)] pub enum GitInfo { /// This is not a git repository. + #[default] Absent, /// This is a git repository. /// If the info should be used (`ignore_git` is false), this will be @@ -25,6 +27,7 @@ pub enum GitInfo { RecordedForTarball(Info), } +#[derive(Clone)] pub struct Info { pub commit_date: String, pub sha: String, diff --git a/src/bootstrap/compile.rs b/src/bootstrap/compile.rs index 67947a263dc87..54906a4918bc1 100644 --- a/src/bootstrap/compile.rs +++ b/src/bootstrap/compile.rs @@ -763,10 +763,10 @@ pub fn rustc_cargo_env(builder: &Builder<'_>, cargo: &mut Cargo, target: TargetS cargo.env("CFG_LIBDIR_RELATIVE", libdir_relative); - if let Some(ref ver_date) = builder.rust_info.commit_date() { + if let Some(ref ver_date) = builder.rust_info().commit_date() { cargo.env("CFG_VER_DATE", ver_date); } - if let Some(ref ver_hash) = builder.rust_info.sha() { + if let Some(ref ver_hash) = builder.rust_info().sha() { cargo.env("CFG_VER_HASH", ver_hash); } if !builder.unstable_features() { diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index e843bd411c172..af004aa509854 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -7,19 +7,18 @@ use std::cell::{Cell, RefCell}; use std::cmp; use std::collections::{HashMap, HashSet}; use std::env; -use std::ffi::OsStr; use std::fmt; use std::fs; use std::path::{Path, PathBuf}; use std::process::Command; use std::str::FromStr; -use crate::builder::{Builder, TaskPath}; +use crate::builder::TaskPath; use crate::cache::{Interned, INTERNER}; -use crate::channel::GitInfo; +use crate::channel::{self, GitInfo}; pub use crate::flags::Subcommand; use crate::flags::{Color, Flags}; -use crate::util::{exe, output, program_out_of_date, t}; +use crate::util::{exe, output, t}; use once_cell::sync::OnceCell; use serde::{Deserialize, Deserializer}; @@ -224,6 +223,7 @@ pub struct Config { #[cfg(test)] pub initial_rustfmt: RefCell, pub out: PathBuf, + pub rust_info: channel::GitInfo, } #[derive(Default, Deserialize)] @@ -1204,7 +1204,7 @@ impl Config { config.rust_codegen_units_std = rust.codegen_units_std.map(threads_from_config); config.rust_profile_use = flags.rust_profile_use.or(rust.profile_use); config.rust_profile_generate = flags.rust_profile_generate.or(rust.profile_generate); - config.download_rustc_commit = download_ci_rustc_commit(&config, rust.download_rustc); + config.download_rustc_commit = config.download_ci_rustc_commit(rust.download_rustc); config.rust_lto = rust .lto @@ -1326,6 +1326,7 @@ impl Config { let default = config.channel == "dev"; config.ignore_git = ignore_git.unwrap_or(default); + config.rust_info = GitInfo::new(config.ignore_git, &config.src); let download_rustc = config.download_rustc_commit.is_some(); // See https://github.com/rust-lang/compiler-team/issues/326 @@ -1401,8 +1402,8 @@ impl Config { /// Bootstrap embeds a version number into the name of shared libraries it uploads in CI. /// Return the version it would have used for the given commit. - pub(crate) fn artifact_version_part(&self, builder: &Builder<'_>, commit: &str) -> String { - let (channel, version) = if builder.rust_info.is_managed_git_subrepository() { + pub(crate) fn artifact_version_part(&self, commit: &str) -> String { + let (channel, version) = if self.rust_info.is_managed_git_subrepository() { let mut channel = self.git(); channel.arg("show").arg(format!("{}:src/ci/channel", commit)); let channel = output(&mut channel); @@ -1411,14 +1412,14 @@ impl Config { let version = output(&mut version); (channel.trim().to_owned(), version.trim().to_owned()) } else { - let channel = fs::read_to_string(builder.src.join("src/ci/channel")); - let version = fs::read_to_string(builder.src.join("src/version")); + let channel = fs::read_to_string(self.src.join("src/ci/channel")); + let version = fs::read_to_string(self.src.join("src/version")); match (channel, version) { (Ok(channel), Ok(version)) => { (channel.trim().to_owned(), version.trim().to_owned()) } (channel, version) => { - let src = builder.src.display(); + let src = self.src.display(); eprintln!("error: failed to determine artifact channel and/or version"); eprintln!( "help: consider using a git checkout or ensure these files are readable" @@ -1477,17 +1478,17 @@ impl Config { /// /// If `false`, llvm should be linked statically. /// This is computed on demand since LLVM might have to first be downloaded from CI. - pub(crate) fn llvm_link_shared(builder: &Builder<'_>) -> bool { - let mut opt = builder.config.llvm_link_shared.get(); - if opt.is_none() && builder.config.dry_run() { + pub(crate) fn llvm_link_shared(&self) -> bool { + let mut opt = self.llvm_link_shared.get(); + if opt.is_none() && self.dry_run() { // just assume static for now - dynamic linking isn't supported on all platforms return false; } let llvm_link_shared = *opt.get_or_insert_with(|| { - if builder.config.llvm_from_ci { - crate::native::maybe_download_ci_llvm(builder); - let ci_llvm = builder.config.ci_llvm_root(); + if self.llvm_from_ci { + self.maybe_download_ci_llvm(); + let ci_llvm = self.ci_llvm_root(); let link_type = t!( std::fs::read_to_string(ci_llvm.join("link-type.txt")), format!("CI llvm missing: {}", ci_llvm.display()) @@ -1499,36 +1500,36 @@ impl Config { false } }); - builder.config.llvm_link_shared.set(opt); + self.llvm_link_shared.set(opt); llvm_link_shared } /// Return whether we will use a downloaded, pre-compiled version of rustc, or just build from source. - pub(crate) fn download_rustc(builder: &Builder<'_>) -> bool { + pub(crate) fn download_rustc(&self) -> bool { static DOWNLOAD_RUSTC: OnceCell = OnceCell::new(); - if builder.config.dry_run() && DOWNLOAD_RUSTC.get().is_none() { + if self.dry_run() && DOWNLOAD_RUSTC.get().is_none() { // avoid trying to actually download the commit return false; } - *DOWNLOAD_RUSTC.get_or_init(|| match &builder.config.download_rustc_commit { + *DOWNLOAD_RUSTC.get_or_init(|| match &self.download_rustc_commit { None => false, Some(commit) => { - download_ci_rustc(builder, commit); + self.download_ci_rustc(commit); true } }) } - pub(crate) fn initial_rustfmt(builder: &Builder<'_>) -> Option { - match &mut *builder.config.initial_rustfmt.borrow_mut() { + pub(crate) fn initial_rustfmt(&self) -> Option { + match &mut *self.initial_rustfmt.borrow_mut() { RustfmtState::SystemToolchain(p) | RustfmtState::Downloaded(p) => Some(p.clone()), RustfmtState::Unavailable => None, r @ RustfmtState::LazyEvaluated => { - if builder.config.dry_run() { + if self.dry_run() { return Some(PathBuf::new()); } - let path = maybe_download_rustfmt(builder); + let path = self.maybe_download_rustfmt(); *r = if let Some(p) = &path { RustfmtState::Downloaded(p.clone()) } else { @@ -1539,8 +1540,10 @@ impl Config { } } - pub fn verbose(&self) -> bool { - self.verbose > 0 + pub fn verbose(&self, msg: &str) { + if self.verbose > 0 { + println!("{}", msg); + } } pub fn sanitizers_enabled(&self, target: TargetSelection) -> bool { @@ -1578,218 +1581,77 @@ impl Config { pub fn submodules(&self, rust_info: &GitInfo) -> bool { self.submodules.unwrap_or(rust_info.is_managed_git_subrepository()) } -} - -fn set(field: &mut T, val: Option) { - if let Some(v) = val { - *field = v; - } -} -fn threads_from_config(v: u32) -> u32 { - match v { - 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, - n => n, - } -} + /// Returns the commit to download, or `None` if we shouldn't download CI artifacts. + fn download_ci_rustc_commit(&self, download_rustc: Option) -> Option { + // If `download-rustc` is not set, default to rebuilding. + let if_unchanged = match download_rustc { + None | Some(StringOrBool::Bool(false)) => return None, + Some(StringOrBool::Bool(true)) => false, + Some(StringOrBool::String(s)) if s == "if-unchanged" => true, + Some(StringOrBool::String(other)) => { + panic!("unrecognized option for download-rustc: {}", other) + } + }; -/// Returns the commit to download, or `None` if we shouldn't download CI artifacts. -fn download_ci_rustc_commit( - config: &Config, - download_rustc: Option, -) -> Option { - // If `download-rustc` is not set, default to rebuilding. - let if_unchanged = match download_rustc { - None | Some(StringOrBool::Bool(false)) => return None, - Some(StringOrBool::Bool(true)) => false, - Some(StringOrBool::String(s)) if s == "if-unchanged" => true, - Some(StringOrBool::String(other)) => { - panic!("unrecognized option for download-rustc: {}", other) + // Handle running from a directory other than the top level + let top_level = output(self.git().args(&["rev-parse", "--show-toplevel"])); + let top_level = top_level.trim_end(); + let compiler = format!("{top_level}/compiler/"); + let library = format!("{top_level}/library/"); + + // Look for a version to compare to based on the current commit. + // Only commits merged by bors will have CI artifacts. + let merge_base = output( + self.git() + .arg("rev-list") + .arg(format!("--author={}", self.stage0_metadata.config.git_merge_commit_email)) + .args(&["-n1", "--first-parent", "HEAD"]), + ); + let commit = merge_base.trim_end(); + if commit.is_empty() { + println!("error: could not find commit hash for downloading rustc"); + println!("help: maybe your repository history is too shallow?"); + println!("help: consider disabling `download-rustc`"); + println!("help: or fetch enough history to include one upstream commit"); + crate::detail_exit(1); } - }; - - // Handle running from a directory other than the top level - let top_level = output(config.git().args(&["rev-parse", "--show-toplevel"])); - let top_level = top_level.trim_end(); - let compiler = format!("{top_level}/compiler/"); - let library = format!("{top_level}/library/"); - // Look for a version to compare to based on the current commit. - // Only commits merged by bors will have CI artifacts. - let merge_base = output( - config + // Warn if there were changes to the compiler or standard library since the ancestor commit. + let has_changes = !t!(self .git() - .arg("rev-list") - .arg(format!("--author={}", config.stage0_metadata.config.git_merge_commit_email)) - .args(&["-n1", "--first-parent", "HEAD"]), - ); - let commit = merge_base.trim_end(); - if commit.is_empty() { - println!("error: could not find commit hash for downloading rustc"); - println!("help: maybe your repository history is too shallow?"); - println!("help: consider disabling `download-rustc`"); - println!("help: or fetch enough history to include one upstream commit"); - crate::detail_exit(1); - } - - // Warn if there were changes to the compiler or standard library since the ancestor commit. - let has_changes = !t!(config - .git() - .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) - .status()) - .success(); - if has_changes { - if if_unchanged { - if config.verbose > 0 { - println!( - "warning: saw changes to compiler/ or library/ since {commit}; \ - ignoring `download-rustc`" - ); + .args(&["diff-index", "--quiet", &commit, "--", &compiler, &library]) + .status()) + .success(); + if has_changes { + if if_unchanged { + if self.verbose > 0 { + println!( + "warning: saw changes to compiler/ or library/ since {commit}; \ + ignoring `download-rustc`" + ); + } + return None; } - return None; + println!( + "warning: `download-rustc` is enabled, but there are changes to \ + compiler/ or library/" + ); } - println!( - "warning: `download-rustc` is enabled, but there are changes to \ - compiler/ or library/" - ); - } - - Some(commit.to_string()) -} - -fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { - let RustfmtMetadata { date, version } = builder.config.stage0_metadata.rustfmt.as_ref()?; - let channel = format!("{version}-{date}"); - let host = builder.config.build; - let rustfmt_path = builder.config.initial_rustc.with_file_name(exe("rustfmt", host)); - let bin_root = builder.config.out.join(host.triple).join("stage0"); - let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); - if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { - return Some(rustfmt_path); + Some(commit.to_string()) } - - let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); - download_component(builder, DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0"); - - builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); - builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); - - builder.create(&rustfmt_stamp, &channel); - Some(rustfmt_path) } -fn download_ci_rustc(builder: &Builder<'_>, commit: &str) { - builder.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); - let version = builder.config.artifact_version_part(builder, commit); - let host = builder.config.build.triple; - let bin_root = builder.out.join(host).join("ci-rustc"); - let rustc_stamp = bin_root.join(".rustc-stamp"); - - if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) { - if bin_root.exists() { - t!(fs::remove_dir_all(&bin_root)); - } - let filename = format!("rust-std-{version}-{host}.tar.xz"); - let pattern = format!("rust-std-{host}"); - download_ci_component(builder, filename, &pattern, commit); - let filename = format!("rustc-{version}-{host}.tar.xz"); - download_ci_component(builder, filename, "rustc", commit); - // download-rustc doesn't need its own cargo, it can just use beta's. - let filename = format!("rustc-dev-{version}-{host}.tar.xz"); - download_ci_component(builder, filename, "rustc-dev", commit); - - builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); - builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); - let lib_dir = bin_root.join("lib"); - for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { - let lib = t!(lib); - if lib.path().extension() == Some(OsStr::new("so")) { - builder.fix_bin_or_dylib(&lib.path()); - } - } - t!(fs::write(rustc_stamp, commit)); +fn set(field: &mut T, val: Option) { + if let Some(v) = val { + *field = v; } } -pub(crate) enum DownloadSource { - CI, - Dist, -} - -/// Download a single component of a CI-built toolchain (not necessarily a published nightly). -// NOTE: intentionally takes an owned string to avoid downloading multiple times by accident -fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, commit: &str) { - download_component(builder, DownloadSource::CI, filename, prefix, commit, "ci-rustc") -} - -fn download_component( - builder: &Builder<'_>, - mode: DownloadSource, - filename: String, - prefix: &str, - key: &str, - destination: &str, -) { - let cache_dst = builder.out.join("cache"); - let cache_dir = cache_dst.join(key); - if !cache_dir.exists() { - t!(fs::create_dir_all(&cache_dir)); - } - - let bin_root = builder.out.join(builder.config.build.triple).join(destination); - let tarball = cache_dir.join(&filename); - let (base_url, url, should_verify) = match mode { - DownloadSource::CI => ( - builder.config.stage0_metadata.config.artifacts_server.clone(), - format!("{key}/{filename}"), - false, - ), - DownloadSource::Dist => { - let dist_server = env::var("RUSTUP_DIST_SERVER") - .unwrap_or(builder.config.stage0_metadata.config.dist_server.to_string()); - // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json - (dist_server, format!("dist/{key}/{filename}"), true) - } - }; - - // For the beta compiler, put special effort into ensuring the checksums are valid. - // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update - // this on each and every nightly ... - let checksum = if should_verify { - let error = format!( - "src/stage0.json doesn't contain a checksum for {url}. \ - Pre-built artifacts might not be available for this \ - target at this time, see https://doc.rust-lang.org/nightly\ - /rustc/platform-support.html for more information." - ); - let sha256 = builder.config.stage0_metadata.checksums_sha256.get(&url).expect(&error); - if tarball.exists() { - if builder.verify(&tarball, sha256) { - builder.unpack(&tarball, &bin_root, prefix); - return; - } else { - builder.verbose(&format!( - "ignoring cached file {} due to failed verification", - tarball.display() - )); - builder.remove(&tarball); - } - } - Some(sha256) - } else if tarball.exists() { - builder.unpack(&tarball, &bin_root, prefix); - return; - } else { - None - }; - - builder.download_component(&format!("{base_url}/{url}"), &tarball, ""); - if let Some(sha256) = checksum { - if !builder.verify(&tarball, sha256) { - panic!("failed to verify {}", tarball.display()); - } +fn threads_from_config(v: u32) -> u32 { + match v { + 0 => std::thread::available_parallelism().map_or(1, std::num::NonZeroUsize::get) as u32, + n => n, } - - builder.unpack(&tarball, &bin_root, prefix); } diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 9fbe476534eb7..aacd2c7eab981 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -924,13 +924,13 @@ impl Step for PlainSourceTarball { // Create the version file builder.create(&plain_dst_src.join("version"), &builder.rust_version()); - if let Some(info) = builder.rust_info.info() { + if let Some(info) = builder.rust_info().info() { channel::write_commit_hash_file(&plain_dst_src, &info.sha); channel::write_commit_info_file(&plain_dst_src, info); } // If we're building from git sources, we need to vendor a complete distribution. - if builder.rust_info.is_managed_git_subrepository() { + if builder.rust_info().is_managed_git_subrepository() { // Ensure we have the submodules checked out. builder.update_submodule(Path::new("src/tools/rust-analyzer")); diff --git a/src/bootstrap/doc.rs b/src/bootstrap/doc.rs index c7d21bf3cdb3f..3180a12c85be7 100644 --- a/src/bootstrap/doc.rs +++ b/src/bootstrap/doc.rs @@ -405,8 +405,8 @@ impl Step for SharedAssets { if !builder.config.dry_run() && !up_to_date(&version_input, &version_info) { let info = t!(fs::read_to_string(&version_input)) .replace("VERSION", &builder.rust_release()) - .replace("SHORT_HASH", builder.rust_info.sha_short().unwrap_or("")) - .replace("STAMP", builder.rust_info.sha().unwrap_or("")); + .replace("SHORT_HASH", builder.rust_info().sha_short().unwrap_or("")) + .replace("STAMP", builder.rust_info().sha().unwrap_or("")); t!(fs::write(&version_info, &info)); } @@ -965,7 +965,7 @@ impl Step for RustcBook { cmd.arg("--rustc"); cmd.arg(&rustc); cmd.arg("--rustc-target").arg(&self.target.rustc_target_arg()); - if builder.config.verbose() { + if builder.is_verbose() { cmd.arg("--verbose"); } if self.validate { diff --git a/src/bootstrap/download.rs b/src/bootstrap/download.rs new file mode 100644 index 0000000000000..d0f389df97344 --- /dev/null +++ b/src/bootstrap/download.rs @@ -0,0 +1,519 @@ +use std::{ + env, + ffi::{OsStr, OsString}, + fs::{self, File}, + io::{BufRead, BufReader, ErrorKind}, + path::{Path, PathBuf}, + process::{Command, Stdio}, +}; + +use once_cell::sync::OnceCell; +use xz2::bufread::XzDecoder; + +use crate::{ + config::RustfmtMetadata, + native::detect_llvm_sha, + t, + util::{check_run, exe, program_out_of_date, try_run}, + Config, +}; + +/// Generic helpers that are useful anywhere in bootstrap. +impl Config { + pub fn is_verbose(&self) -> bool { + self.verbose > 0 + } + + pub(crate) fn create(&self, path: &Path, s: &str) { + if self.dry_run() { + return; + } + t!(fs::write(path, s)); + } + + pub(crate) fn remove(&self, f: &Path) { + if self.dry_run() { + return; + } + fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); + } + + /// Create a temporary directory in `out` and return its path. + /// + /// NOTE: this temporary directory is shared between all steps; + /// if you need an empty directory, create a new subdirectory inside it. + pub(crate) fn tempdir(&self) -> PathBuf { + let tmp = self.out.join("tmp"); + t!(fs::create_dir_all(&tmp)); + tmp + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Exits if the command failed to execute at all, otherwise returns its + /// `status.success()`. + pub(crate) fn try_run(&self, cmd: &mut Command) -> bool { + if self.dry_run() { + return true; + } + self.verbose(&format!("running: {:?}", cmd)); + try_run(cmd, self.is_verbose()) + } + + /// Runs a command, printing out nice contextual information if it fails. + /// Returns false if do not execute at all, otherwise returns its + /// `status.success()`. + pub(crate) fn check_run(&self, cmd: &mut Command) -> bool { + if self.dry_run() { + return true; + } + self.verbose(&format!("running: {:?}", cmd)); + check_run(cmd, self.is_verbose()) + } + + /// Modifies the interpreter section of 'fname' to fix the dynamic linker, + /// or the RPATH section, to fix the dynamic library search path + /// + /// This is only required on NixOS and uses the PatchELF utility to + /// change the interpreter/RPATH of ELF executables. + /// + /// Please see https://nixos.org/patchelf.html for more information + fn fix_bin_or_dylib(&self, fname: &Path) { + // FIXME: cache NixOS detection? + match Command::new("uname").arg("-s").stderr(Stdio::inherit()).output() { + Err(_) => return, + Ok(output) if !output.status.success() => return, + Ok(output) => { + let mut s = output.stdout; + if s.last() == Some(&b'\n') { + s.pop(); + } + if s != b"Linux" { + return; + } + } + } + + // If the user has asked binaries to be patched for Nix, then + // don't check for NixOS or `/lib`, just continue to the patching. + // NOTE: this intentionally comes after the Linux check: + // - patchelf only works with ELF files, so no need to run it on Mac or Windows + // - On other Unix systems, there is no stable syscall interface, so Nix doesn't manage the global libc. + if !self.patch_binaries_for_nix { + // Use `/etc/os-release` instead of `/etc/NIXOS`. + // The latter one does not exist on NixOS when using tmpfs as root. + const NIX_IDS: &[&str] = &["ID=nixos", "ID='nixos'", "ID=\"nixos\""]; + let os_release = match File::open("/etc/os-release") { + Err(e) if e.kind() == ErrorKind::NotFound => return, + Err(e) => panic!("failed to access /etc/os-release: {}", e), + Ok(f) => f, + }; + if !BufReader::new(os_release).lines().any(|l| NIX_IDS.contains(&t!(l).trim())) { + return; + } + if Path::new("/lib").exists() { + return; + } + } + + // At this point we're pretty sure the user is running NixOS or using Nix + println!("info: you seem to be using Nix. Attempting to patch {}", fname.display()); + + // Only build `.nix-deps` once. + static NIX_DEPS_DIR: OnceCell = OnceCell::new(); + let mut nix_build_succeeded = true; + let nix_deps_dir = NIX_DEPS_DIR.get_or_init(|| { + // Run `nix-build` to "build" each dependency (which will likely reuse + // the existing `/nix/store` copy, or at most download a pre-built copy). + // + // Importantly, we create a gc-root called `.nix-deps` in the `build/` + // directory, but still reference the actual `/nix/store` path in the rpath + // as it makes it significantly more robust against changes to the location of + // the `.nix-deps` location. + // + // bintools: Needed for the path of `ld-linux.so` (via `nix-support/dynamic-linker`). + // zlib: Needed as a system dependency of `libLLVM-*.so`. + // patchelf: Needed for patching ELF binaries (see doc comment above). + let nix_deps_dir = self.out.join(".nix-deps"); + const NIX_EXPR: &str = " + with (import {}); + symlinkJoin { + name = \"rust-stage0-dependencies\"; + paths = [ + zlib + patchelf + stdenv.cc.bintools + ]; + } + "; + nix_build_succeeded = self.try_run(Command::new("nix-build").args(&[ + Path::new("-E"), + Path::new(NIX_EXPR), + Path::new("-o"), + &nix_deps_dir, + ])); + nix_deps_dir + }); + if !nix_build_succeeded { + return; + } + + let mut patchelf = Command::new(nix_deps_dir.join("bin/patchelf")); + let rpath_entries = { + // ORIGIN is a relative default, all binary and dynamic libraries we ship + // appear to have this (even when `../lib` is redundant). + // NOTE: there are only two paths here, delimited by a `:` + let mut entries = OsString::from("$ORIGIN/../lib:"); + entries.push(t!(fs::canonicalize(nix_deps_dir))); + entries.push("/lib"); + entries + }; + patchelf.args(&[OsString::from("--set-rpath"), rpath_entries]); + if !fname.extension().map_or(false, |ext| ext == "so") { + // Finally, set the correct .interp for binaries + let dynamic_linker_path = nix_deps_dir.join("nix-support/dynamic-linker"); + // FIXME: can we support utf8 here? `args` doesn't accept Vec, only OsString ... + let dynamic_linker = t!(String::from_utf8(t!(fs::read(dynamic_linker_path)))); + patchelf.args(&["--set-interpreter", dynamic_linker.trim_end()]); + } + + self.try_run(patchelf.arg(fname)); + } + + fn download_file(&self, url: &str, dest_path: &Path, help_on_error: &str) { + self.verbose(&format!("download {url}")); + // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. + let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); + // While bootstrap itself only supports http and https downloads, downstream forks might + // need to download components from other protocols. The match allows them adding more + // protocols without worrying about merge conflicts if we change the HTTP implementation. + match url.split_once("://").map(|(proto, _)| proto) { + Some("http") | Some("https") => { + self.download_http_with_retries(&tempfile, url, help_on_error) + } + Some(other) => panic!("unsupported protocol {other} in {url}"), + None => panic!("no protocol in {url}"), + } + t!(std::fs::rename(&tempfile, dest_path)); + } + + fn download_http_with_retries(&self, tempfile: &Path, url: &str, help_on_error: &str) { + println!("downloading {}", url); + // Try curl. If that fails and we are on windows, fallback to PowerShell. + let mut curl = Command::new("curl"); + curl.args(&[ + "-#", + "-y", + "30", + "-Y", + "10", // timeout if speed is < 10 bytes/sec for > 30 seconds + "--connect-timeout", + "30", // timeout if cannot connect within 30 seconds + "--retry", + "3", + "-Sf", + "-o", + ]); + curl.arg(tempfile); + curl.arg(url); + if !self.check_run(&mut curl) { + if self.build.contains("windows-msvc") { + println!("Fallback to PowerShell"); + for _ in 0..3 { + if self.try_run(Command::new("PowerShell.exe").args(&[ + "/nologo", + "-Command", + "[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12;", + &format!( + "(New-Object System.Net.WebClient).DownloadFile('{}', '{}')", + url, tempfile.to_str().expect("invalid UTF-8 not supported with powershell downloads"), + ), + ])) { + return; + } + println!("\nspurious failure, trying again"); + } + } + if !help_on_error.is_empty() { + eprintln!("{}", help_on_error); + } + crate::detail_exit(1); + } + } + + fn unpack(&self, tarball: &Path, dst: &Path, pattern: &str) { + println!("extracting {} to {}", tarball.display(), dst.display()); + if !dst.exists() { + t!(fs::create_dir_all(dst)); + } + + // `tarball` ends with `.tar.xz`; strip that suffix + // example: `rust-dev-nightly-x86_64-unknown-linux-gnu` + let uncompressed_filename = + Path::new(tarball.file_name().expect("missing tarball filename")).file_stem().unwrap(); + let directory_prefix = Path::new(Path::new(uncompressed_filename).file_stem().unwrap()); + + // decompress the file + let data = t!(File::open(tarball)); + let decompressor = XzDecoder::new(BufReader::new(data)); + + let mut tar = tar::Archive::new(decompressor); + for member in t!(tar.entries()) { + let mut member = t!(member); + let original_path = t!(member.path()).into_owned(); + // skip the top-level directory + if original_path == directory_prefix { + continue; + } + let mut short_path = t!(original_path.strip_prefix(directory_prefix)); + if !short_path.starts_with(pattern) { + continue; + } + short_path = t!(short_path.strip_prefix(pattern)); + let dst_path = dst.join(short_path); + self.verbose(&format!("extracting {} to {}", original_path.display(), dst.display())); + if !t!(member.unpack_in(dst)) { + panic!("path traversal attack ??"); + } + let src_path = dst.join(original_path); + if src_path.is_dir() && dst_path.exists() { + continue; + } + t!(fs::rename(src_path, dst_path)); + } + t!(fs::remove_dir_all(dst.join(directory_prefix))); + } + + /// Returns whether the SHA256 checksum of `path` matches `expected`. + fn verify(&self, path: &Path, expected: &str) -> bool { + use sha2::Digest; + + self.verbose(&format!("verifying {}", path.display())); + let mut hasher = sha2::Sha256::new(); + // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. + // Consider using streaming IO instead? + let contents = if self.dry_run() { vec![] } else { t!(fs::read(path)) }; + hasher.update(&contents); + let found = hex::encode(hasher.finalize().as_slice()); + let verified = found == expected; + if !verified && !self.dry_run() { + println!( + "invalid checksum: \n\ + found: {found}\n\ + expected: {expected}", + ); + } + return verified; + } +} + +enum DownloadSource { + CI, + Dist, +} + +/// Functions that are only ever called once, but named for clarify and to avoid thousand-line functions. +impl Config { + pub(crate) fn maybe_download_rustfmt(&self) -> Option { + let RustfmtMetadata { date, version } = self.stage0_metadata.rustfmt.as_ref()?; + let channel = format!("{version}-{date}"); + + let host = self.build; + let rustfmt_path = self.initial_rustc.with_file_name(exe("rustfmt", host)); + let bin_root = self.out.join(host.triple).join("stage0"); + let rustfmt_stamp = bin_root.join(".rustfmt-stamp"); + if rustfmt_path.exists() && !program_out_of_date(&rustfmt_stamp, &channel) { + return Some(rustfmt_path); + } + + let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); + self.download_component(DownloadSource::Dist, filename, "rustfmt-preview", &date, "stage0"); + + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); + self.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); + + self.create(&rustfmt_stamp, &channel); + Some(rustfmt_path) + } + + pub(crate) fn download_ci_rustc(&self, commit: &str) { + self.verbose(&format!("using downloaded stage2 artifacts from CI (commit {commit})")); + let version = self.artifact_version_part(commit); + let host = self.build.triple; + let bin_root = self.out.join(host).join("ci-rustc"); + let rustc_stamp = bin_root.join(".rustc-stamp"); + + if !bin_root.join("bin").join("rustc").exists() || program_out_of_date(&rustc_stamp, commit) + { + if bin_root.exists() { + t!(fs::remove_dir_all(&bin_root)); + } + let filename = format!("rust-std-{version}-{host}.tar.xz"); + let pattern = format!("rust-std-{host}"); + self.download_ci_component(filename, &pattern, commit); + let filename = format!("rustc-{version}-{host}.tar.xz"); + self.download_ci_component(filename, "rustc", commit); + // download-rustc doesn't need its own cargo, it can just use beta's. + let filename = format!("rustc-dev-{version}-{host}.tar.xz"); + self.download_ci_component(filename, "rustc-dev", commit); + let filename = format!("rust-src-{version}.tar.xz"); + self.download_ci_component(filename, "rust-src", commit); + + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustc")); + self.fix_bin_or_dylib(&bin_root.join("bin").join("rustdoc")); + let lib_dir = bin_root.join("lib"); + for lib in t!(fs::read_dir(&lib_dir), lib_dir.display().to_string()) { + let lib = t!(lib); + if lib.path().extension() == Some(OsStr::new("so")) { + self.fix_bin_or_dylib(&lib.path()); + } + } + t!(fs::write(rustc_stamp, commit)); + } + } + + /// Download a single component of a CI-built toolchain (not necessarily a published nightly). + // NOTE: intentionally takes an owned string to avoid downloading multiple times by accident + fn download_ci_component(&self, filename: String, prefix: &str, commit: &str) { + Self::download_component(self, DownloadSource::CI, filename, prefix, commit, "ci-rustc") + } + + fn download_component( + &self, + mode: DownloadSource, + filename: String, + prefix: &str, + key: &str, + destination: &str, + ) { + let cache_dst = self.out.join("cache"); + let cache_dir = cache_dst.join(key); + if !cache_dir.exists() { + t!(fs::create_dir_all(&cache_dir)); + } + + let bin_root = self.out.join(self.build.triple).join(destination); + let tarball = cache_dir.join(&filename); + let (base_url, url, should_verify) = match mode { + DownloadSource::CI => ( + self.stage0_metadata.config.artifacts_server.clone(), + format!("{key}/{filename}"), + false, + ), + DownloadSource::Dist => { + let dist_server = env::var("RUSTUP_DIST_SERVER") + .unwrap_or(self.stage0_metadata.config.dist_server.to_string()); + // NOTE: make `dist` part of the URL because that's how it's stored in src/stage0.json + (dist_server, format!("dist/{key}/{filename}"), true) + } + }; + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if should_verify { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + let sha256 = self.stage0_metadata.checksums_sha256.get(&url).expect(&error); + if tarball.exists() { + if self.verify(&tarball, sha256) { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + self.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + self.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + self.unpack(&tarball, &bin_root, prefix); + return; + } else { + None + }; + + self.download_file(&format!("{base_url}/{url}"), &tarball, ""); + if let Some(sha256) = checksum { + if !self.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } + } + + self.unpack(&tarball, &bin_root, prefix); + } + + pub(crate) fn maybe_download_ci_llvm(&self) { + if !self.llvm_from_ci { + return; + } + let llvm_root = self.ci_llvm_root(); + let llvm_stamp = llvm_root.join(".llvm-stamp"); + let llvm_sha = detect_llvm_sha(&self, self.rust_info.is_managed_git_subrepository()); + let key = format!("{}{}", llvm_sha, self.llvm_assertions); + if program_out_of_date(&llvm_stamp, &key) && !self.dry_run() { + self.download_ci_llvm(&llvm_sha); + for entry in t!(fs::read_dir(llvm_root.join("bin"))) { + self.fix_bin_or_dylib(&t!(entry).path()); + } + + // Update the timestamp of llvm-config to force rustc_llvm to be + // rebuilt. This is a hacky workaround for a deficiency in Cargo where + // the rerun-if-changed directive doesn't handle changes very well. + // https://github.com/rust-lang/cargo/issues/10791 + // Cargo only compares the timestamp of the file relative to the last + // time `rustc_llvm` build script ran. However, the timestamps of the + // files in the tarball are in the past, so it doesn't trigger a + // rebuild. + let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); + let llvm_config = llvm_root.join("bin").join(exe("llvm-config", self.build)); + t!(filetime::set_file_times(&llvm_config, now, now)); + + let llvm_lib = llvm_root.join("lib"); + for entry in t!(fs::read_dir(&llvm_lib)) { + let lib = t!(entry).path(); + if lib.extension().map_or(false, |ext| ext == "so") { + self.fix_bin_or_dylib(&lib); + } + } + t!(fs::write(llvm_stamp, key)); + } + } + + fn download_ci_llvm(&self, llvm_sha: &str) { + let llvm_assertions = self.llvm_assertions; + + let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); + let cache_dst = self.out.join("cache"); + let rustc_cache = cache_dst.join(cache_prefix); + if !rustc_cache.exists() { + t!(fs::create_dir_all(&rustc_cache)); + } + let base = if llvm_assertions { + &self.stage0_metadata.config.artifacts_with_llvm_assertions_server + } else { + &self.stage0_metadata.config.artifacts_server + }; + let version = self.artifact_version_part(llvm_sha); + let filename = format!("rust-dev-{}-{}.tar.xz", version, self.build.triple); + let tarball = rustc_cache.join(&filename); + if !tarball.exists() { + let help_on_error = "error: failed to download llvm from ci + + help: old builds get deleted after a certain time + help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: + + [llvm] + download-ci-llvm = false + "; + self.download_file(&format!("{base}/{llvm_sha}/{filename}"), &tarball, help_on_error); + } + let llvm_root = self.ci_llvm_root(); + self.unpack(&tarball, &llvm_root, "rust-dev"); + } +} diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index bbb5a18ba07ba..f4fa556b97450 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -112,15 +112,14 @@ use std::path::{Path, PathBuf}; use std::process::Command; use std::str; +use channel::GitInfo; use config::{DryRun, Target}; use filetime::FileTime; use once_cell::sync::OnceCell; use crate::builder::Kind; use crate::config::{LlvmLibunwind, TargetSelection}; -use crate::util::{ - check_run, exe, libdir, mtime, output, run, run_suppressed, try_run, try_run_suppressed, CiEnv, -}; +use crate::util::{exe, libdir, mtime, output, run, run_suppressed, try_run_suppressed, CiEnv}; mod bolt; mod builder; @@ -133,6 +132,7 @@ mod compile; mod config; mod dist; mod doc; +mod download; mod flags; mod format; mod install; @@ -281,7 +281,6 @@ pub struct Build { src: PathBuf, out: PathBuf, bootstrap_out: PathBuf, - rust_info: channel::GitInfo, cargo_info: channel::GitInfo, rust_analyzer_info: channel::GitInfo, clippy_info: channel::GitInfo, @@ -396,6 +395,28 @@ pub enum CLang { Cxx, } +macro_rules! forward { + ( $( $fn:ident( $($param:ident: $ty:ty),* ) $( -> $ret:ty)? ),+ $(,)? ) => { + impl Build { + $( fn $fn(&self, $($param: $ty),* ) $( -> $ret)? { + self.config.$fn( $($param),* ) + } )+ + } + } +} + +forward! { + verbose(msg: &str), + is_verbose() -> bool, + create(path: &Path, s: &str), + remove(f: &Path), + tempdir() -> PathBuf, + try_run(cmd: &mut Command) -> bool, + llvm_link_shared() -> bool, + download_rustc() -> bool, + initial_rustfmt() -> Option, +} + impl Build { /// Creates a new set of build configuration from the `flags` on the command /// line and the filesystem `config`. @@ -499,7 +520,6 @@ impl Build { out, bootstrap_out, - rust_info, cargo_info, rust_analyzer_info, clippy_info, @@ -570,7 +590,7 @@ impl Build { t!(std::fs::read_dir(dir)).next().is_none() } - if !self.config.submodules(&self.rust_info) { + if !self.config.submodules(&self.rust_info()) { return; } @@ -636,7 +656,7 @@ impl Build { /// This avoids contributors checking in a submodule change by accident. pub fn maybe_update_submodules(&self) { // Avoid running git when there isn't a git checkout. - if !self.config.submodules(&self.rust_info) { + if !self.config.submodules(&self.rust_info()) { return; } let output = output( @@ -735,6 +755,10 @@ impl Build { cleared } + fn rust_info(&self) -> &GitInfo { + &self.config.rust_info + } + /// Gets the space-separated set of activated features for the standard /// library. fn std_features(&self, target: TargetSelection) -> String { @@ -963,17 +987,6 @@ impl Build { run_suppressed(cmd) } - /// Runs a command, printing out nice contextual information if it fails. - /// Exits if the command failed to execute at all, otherwise returns its - /// `status.success()`. - fn try_run(&self, cmd: &mut Command) -> bool { - if self.config.dry_run() { - return true; - } - self.verbose(&format!("running: {:?}", cmd)); - try_run(cmd, self.is_verbose()) - } - /// Runs a command, printing out nice contextual information if it fails. /// Exits if the command failed to execute at all, otherwise returns its /// `status.success()`. @@ -985,28 +998,6 @@ impl Build { try_run_suppressed(cmd) } - /// Runs a command, printing out nice contextual information if it fails. - /// Returns false if do not execute at all, otherwise returns its - /// `status.success()`. - fn check_run(&self, cmd: &mut Command) -> bool { - if self.config.dry_run() { - return true; - } - self.verbose(&format!("running: {:?}", cmd)); - check_run(cmd, self.is_verbose()) - } - - pub fn is_verbose(&self) -> bool { - self.verbosity > 0 - } - - /// Prints a message if this build is configured in verbose mode. - fn verbose(&self, msg: &str) { - if self.is_verbose() { - println!("{}", msg); - } - } - pub fn is_verbose_than(&self, level: usize) -> bool { self.verbosity > level } @@ -1269,7 +1260,7 @@ impl Build { match &self.config.channel[..] { "stable" => num.to_string(), "beta" => { - if self.rust_info.is_managed_git_subrepository() && !self.config.ignore_git { + if self.rust_info().is_managed_git_subrepository() && !self.config.ignore_git { format!("{}-beta.{}", num, self.beta_prerelease_version()) } else { format!("{}-beta", num) @@ -1329,7 +1320,7 @@ impl Build { /// Note that this is a descriptive string which includes the commit date, /// sha, version, etc. fn rust_version(&self) -> String { - let mut version = self.rust_info.version(self, &self.version); + let mut version = self.rust_info().version(self, &self.version); if let Some(ref s) = self.config.description { version.push_str(" ("); version.push_str(s); @@ -1340,7 +1331,7 @@ impl Build { /// Returns the full commit hash. fn rust_sha(&self) -> Option<&str> { - self.rust_info.sha() + self.rust_info().sha() } /// Returns the `a.b.c` version that the given package is at. @@ -1426,16 +1417,6 @@ impl Build { paths } - /// Create a temporary directory in `out` and return its path. - /// - /// NOTE: this temporary directory is shared between all steps; - /// if you need an empty directory, create a new subdirectory inside it. - fn tempdir(&self) -> PathBuf { - let tmp = self.out.join("tmp"); - t!(fs::create_dir_all(&tmp)); - tmp - } - /// Copies a file from `src` to `dst` pub fn copy(&self, src: &Path, dst: &Path) { self.copy_internal(src, dst, false); @@ -1545,13 +1526,6 @@ impl Build { chmod(&dst, perms); } - fn create(&self, path: &Path, s: &str) { - if self.config.dry_run() { - return; - } - t!(fs::write(path, s)); - } - fn read(&self, path: &Path) -> String { if self.config.dry_run() { return String::new(); @@ -1590,13 +1564,6 @@ impl Build { if !self.config.dry_run() { symlink_file(src.as_ref(), link.as_ref()) } else { Ok(()) } } - fn remove(&self, f: &Path) { - if self.config.dry_run() { - return; - } - fs::remove_file(f).unwrap_or_else(|_| panic!("failed to remove {:?}", f)); - } - /// Returns if config.ninja is enabled, and checks for ninja existence, /// exiting with a nicer error message if not. fn ninja(&self) -> bool { diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 2407291ceea30..f6c453ebe107b 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -19,9 +19,9 @@ use std::process::Command; use crate::bolt::{instrument_with_bolt_inplace, optimize_library_with_bolt_inplace}; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::channel; -use crate::config::TargetSelection; +use crate::config::{Config, TargetSelection}; use crate::util::get_clang_cl_resource_dir; -use crate::util::{self, exe, output, program_out_of_date, t, up_to_date}; +use crate::util::{self, exe, output, t, up_to_date}; use crate::{CLang, GitRepo}; pub struct Meta { @@ -65,7 +65,7 @@ pub fn prebuilt_llvm_config( builder: &Builder<'_>, target: TargetSelection, ) -> Result { - maybe_download_ci_llvm(builder); + builder.config.maybe_download_ci_llvm(); // If we're using a custom LLVM bail out here, but we can only use a // custom LLVM for the build triple. @@ -117,7 +117,7 @@ pub fn prebuilt_llvm_config( } /// This retrieves the LLVM sha we *want* to use, according to git history. -pub(crate) fn detect_llvm_sha(config: &crate::config::Config, is_git: bool) -> String { +pub(crate) fn detect_llvm_sha(config: &Config, is_git: bool) -> String { let llvm_sha = if is_git { let mut rev_list = config.git(); rev_list.args(&[ @@ -155,7 +155,7 @@ pub(crate) fn detect_llvm_sha(config: &crate::config::Config, is_git: bool) -> S /// This checks both the build triple platform to confirm we're usable at all, /// and then verifies if the current HEAD matches the detected LLVM SHA head, /// in which case LLVM is indicated as not available. -pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool) -> bool { +pub(crate) fn is_ci_llvm_available(config: &Config, asserts: bool) -> bool { // This is currently all tier 1 targets and tier 2 targets with host tools // (since others may not have CI artifacts) // https://doc.rust-lang.org/rustc/platform-support.html#tier-1 @@ -217,80 +217,6 @@ pub(crate) fn is_ci_llvm_available(config: &crate::config::Config, asserts: bool true } -pub(crate) fn maybe_download_ci_llvm(builder: &Builder<'_>) { - let config = &builder.config; - if !config.llvm_from_ci { - return; - } - let llvm_root = config.ci_llvm_root(); - let llvm_stamp = llvm_root.join(".llvm-stamp"); - let llvm_sha = detect_llvm_sha(&config, builder.rust_info.is_managed_git_subrepository()); - let key = format!("{}{}", llvm_sha, config.llvm_assertions); - if program_out_of_date(&llvm_stamp, &key) && !config.dry_run() { - download_ci_llvm(builder, &llvm_sha); - for entry in t!(fs::read_dir(llvm_root.join("bin"))) { - builder.fix_bin_or_dylib(&t!(entry).path()); - } - - // Update the timestamp of llvm-config to force rustc_llvm to be - // rebuilt. This is a hacky workaround for a deficiency in Cargo where - // the rerun-if-changed directive doesn't handle changes very well. - // https://github.com/rust-lang/cargo/issues/10791 - // Cargo only compares the timestamp of the file relative to the last - // time `rustc_llvm` build script ran. However, the timestamps of the - // files in the tarball are in the past, so it doesn't trigger a - // rebuild. - let now = filetime::FileTime::from_system_time(std::time::SystemTime::now()); - let llvm_config = llvm_root.join("bin").join(exe("llvm-config", builder.config.build)); - t!(filetime::set_file_times(&llvm_config, now, now)); - - let llvm_lib = llvm_root.join("lib"); - for entry in t!(fs::read_dir(&llvm_lib)) { - let lib = t!(entry).path(); - if lib.extension().map_or(false, |ext| ext == "so") { - builder.fix_bin_or_dylib(&lib); - } - } - t!(fs::write(llvm_stamp, key)); - } -} - -fn download_ci_llvm(builder: &Builder<'_>, llvm_sha: &str) { - let llvm_assertions = builder.config.llvm_assertions; - - let cache_prefix = format!("llvm-{}-{}", llvm_sha, llvm_assertions); - let cache_dst = builder.out.join("cache"); - let rustc_cache = cache_dst.join(cache_prefix); - if !rustc_cache.exists() { - t!(fs::create_dir_all(&rustc_cache)); - } - let base = if llvm_assertions { - &builder.config.stage0_metadata.config.artifacts_with_llvm_assertions_server - } else { - &builder.config.stage0_metadata.config.artifacts_server - }; - let version = builder.config.artifact_version_part(builder, llvm_sha); - let filename = format!("rust-dev-{}-{}.tar.xz", version, builder.build.build.triple); - let tarball = rustc_cache.join(&filename); - if !tarball.exists() { - let help_on_error = "error: failed to download llvm from ci - -help: old builds get deleted after a certain time -help: if trying to compile an old commit of rustc, disable `download-ci-llvm` in config.toml: - -[llvm] -download-ci-llvm = false -"; - builder.download_component( - &format!("{base}/{llvm_sha}/{filename}"), - &tarball, - help_on_error, - ); - } - let llvm_root = builder.config.ci_llvm_root(); - builder.unpack(&tarball, &llvm_root, "rust-dev"); -} - #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Llvm { pub target: TargetSelection, diff --git a/src/bootstrap/sanity.rs b/src/bootstrap/sanity.rs index 35c66cfd95f2f..631d42acb93fc 100644 --- a/src/bootstrap/sanity.rs +++ b/src/bootstrap/sanity.rs @@ -74,7 +74,7 @@ pub fn check(build: &mut Build) { let mut cmd_finder = Finder::new(); // If we've got a git directory we're gonna need git to update // submodules and learn about various other aspects. - if build.rust_info.is_managed_git_subrepository() { + if build.rust_info().is_managed_git_subrepository() { cmd_finder.must_have("git"); } diff --git a/src/bootstrap/tarball.rs b/src/bootstrap/tarball.rs index 82b063583c9f9..fc850a22b2f6f 100644 --- a/src/bootstrap/tarball.rs +++ b/src/bootstrap/tarball.rs @@ -298,7 +298,7 @@ impl<'a> Tarball<'a> { fn run(self, build_cli: impl FnOnce(&Tarball<'a>, &mut Command)) -> GeneratedTarball { t!(std::fs::create_dir_all(&self.overlay_dir)); self.builder.create(&self.overlay_dir.join("version"), &self.overlay.version(self.builder)); - if let Some(info) = self.builder.rust_info.info() { + if let Some(info) = self.builder.rust_info().info() { channel::write_commit_hash_file(&self.overlay_dir, &info.sha); channel::write_commit_info_file(&self.overlay_dir, info); }