From f4210fc12f6090288c85dc0c9d9cb341c9acf72d Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Fri, 4 Jun 2021 22:17:01 -0400 Subject: [PATCH 1/4] Revert "Revert "Move llvm submodule updates to rustbuild"" This reverts commit ad308264a38531bc8d2179324bac3652a1cda640. --- src/bootstrap/bootstrap.py | 16 ++------ src/bootstrap/lib.rs | 10 +++++ src/bootstrap/native.rs | 84 +++++++++++++++++++++++++++++++++++++- src/build_helper/lib.rs | 1 + 4 files changed, 98 insertions(+), 13 deletions(-) diff --git a/src/bootstrap/bootstrap.py b/src/bootstrap/bootstrap.py index 149a899cef7a0..bd5b3797ea825 100644 --- a/src/bootstrap/bootstrap.py +++ b/src/bootstrap/bootstrap.py @@ -991,28 +991,20 @@ def update_submodules(self): ).decode(default_encoding).splitlines()] filtered_submodules = [] submodules_names = [] - llvm_checked_out = os.path.exists(os.path.join(self.rust_root, "src/llvm-project/.git")) - external_llvm_provided = self.get_toml('llvm-config') or self.downloading_llvm() - llvm_needed = not self.get_toml('codegen-backends', 'rust') \ - or "llvm" in self.get_toml('codegen-backends', 'rust') for module in submodules: + # This is handled by native::Llvm in rustbuild, not here if module.endswith("llvm-project"): - # Don't sync the llvm-project submodule if an external LLVM was - # provided, if we are downloading LLVM or if the LLVM backend is - # not being built. Also, if the submodule has been initialized - # already, sync it anyways so that it doesn't mess up contributor - # pull requests. - if external_llvm_provided or not llvm_needed: - if self.get_toml('lld') != 'true' and not llvm_checked_out: - continue + continue check = self.check_submodule(module, slow_submodules) filtered_submodules.append((module, check)) submodules_names.append(module) recorded = subprocess.Popen(["git", "ls-tree", "HEAD"] + submodules_names, cwd=self.rust_root, stdout=subprocess.PIPE) recorded = recorded.communicate()[0].decode(default_encoding).strip().splitlines() + # { filename: hash } recorded_submodules = {} for data in recorded: + # [mode, kind, hash, filename] data = data.split() recorded_submodules[data[3]] = data[2] for module in filtered_submodules: diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 2960dd3df6bf4..a351290a4206f 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -472,12 +472,22 @@ impl Build { slice::from_ref(&self.build.triple) } + /// If the LLVM submodule has been initialized already, sync it unconditionally. This avoids + /// contributors checking in a submodule change by accident. + pub fn maybe_update_llvm_submodule(&self) { + if self.in_tree_llvm_info.is_git() { + native::update_llvm_submodule(self); + } + } + /// Executes the entire build, as configured by the flags and configuration. pub fn build(&mut self) { unsafe { job::setup(self); } + self.maybe_update_llvm_submodule(); + if let Subcommand::Format { check, paths } = &self.config.cmd { return format::format(self, *check, &paths); } diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 4f33a13c2536a..20155c0fcde05 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -21,7 +21,7 @@ use build_helper::{output, t}; use crate::builder::{Builder, RunConfig, ShouldRun, Step}; use crate::config::TargetSelection; use crate::util::{self, exe}; -use crate::GitRepo; +use crate::{Build, GitRepo}; use build_helper::up_to_date; pub struct Meta { @@ -91,6 +91,85 @@ pub fn prebuilt_llvm_config( Err(Meta { stamp, build_llvm_config, out_dir, root: root.into() }) } +// modified from `check_submodule` and `update_submodule` in bootstrap.py +pub(crate) fn update_llvm_submodule(build: &Build) { + let llvm_project = &Path::new("src").join("llvm-project"); + + fn dir_is_empty(dir: &Path) -> bool { + t!(std::fs::read_dir(dir)).next().is_none() + } + + // NOTE: The check for the empty directory is here because when running x.py + // the first time, the llvm submodule won't be checked out. Check it out + // now so we can build it. + if !build.in_tree_llvm_info.is_git() && !dir_is_empty(&build.config.src.join(llvm_project)) { + return; + } + + // check_submodule + let checked_out = if build.config.fast_submodules { + Some(output( + Command::new("git") + .args(&["rev-parse", "HEAD"]) + .current_dir(build.config.src.join(llvm_project)), + )) + } else { + None + }; + + // update_submodules + let recorded = output( + Command::new("git") + .args(&["ls-tree", "HEAD"]) + .arg(llvm_project) + .current_dir(&build.config.src), + ); + let hash = + recorded.split(' ').nth(2).unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + + // update_submodule + if let Some(llvm_hash) = checked_out { + if hash == llvm_hash { + // already checked out + return; + } + } + + println!("Updating submodule {}", llvm_project.display()); + build.run( + Command::new("git") + .args(&["submodule", "-q", "sync"]) + .arg(llvm_project) + .current_dir(&build.config.src), + ); + + // Try passing `--progress` to start, then run git again without if that fails. + let update = |progress: bool| { + let mut git = Command::new("git"); + git.args(&["submodule", "update", "--init", "--recursive"]); + if progress { + git.arg("--progress"); + } + git.arg(llvm_project).current_dir(&build.config.src); + git + }; + // NOTE: doesn't use `try_run` because this shouldn't print an error if it fails. + if !update(true).status().map_or(false, |status| status.success()) { + build.run(&mut update(false)); + } + + build.run( + Command::new("git") + .args(&["reset", "-q", "--hard"]) + .current_dir(build.config.src.join(llvm_project)), + ); + build.run( + Command::new("git") + .args(&["clean", "-qdfx"]) + .current_dir(build.config.src.join(llvm_project)), + ); +} + #[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)] pub struct Llvm { pub target: TargetSelection, @@ -128,6 +207,9 @@ impl Step for Llvm { Err(m) => m, }; + if !builder.config.dry_run { + update_llvm_submodule(builder); + } if builder.config.llvm_link_shared && (target.contains("windows") || target.contains("apple-darwin")) { diff --git a/src/build_helper/lib.rs b/src/build_helper/lib.rs index 80f804174ed08..b1ec072f3f8aa 100644 --- a/src/build_helper/lib.rs +++ b/src/build_helper/lib.rs @@ -130,6 +130,7 @@ pub fn make(host: &str) -> PathBuf { } } +#[track_caller] pub fn output(cmd: &mut Command) -> String { let output = match cmd.stderr(Stdio::inherit()).output() { Ok(status) => status, From 28d0d0c38b791bf60a83b34e7bb3eec165a2734c Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Fri, 4 Jun 2021 22:22:30 -0400 Subject: [PATCH 2/4] Fix commit check --- src/bootstrap/native.rs | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 20155c0fcde05..3a5ef3e308bf0 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -107,12 +107,14 @@ pub(crate) fn update_llvm_submodule(build: &Build) { } // check_submodule + let buf; let checked_out = if build.config.fast_submodules { - Some(output( + buf = output( Command::new("git") .args(&["rev-parse", "HEAD"]) .current_dir(build.config.src.join(llvm_project)), - )) + ); + Some(buf.trim_end()) } else { None }; @@ -124,8 +126,10 @@ pub(crate) fn update_llvm_submodule(build: &Build) { .arg(llvm_project) .current_dir(&build.config.src), ); - let hash = - recorded.split(' ').nth(2).unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); + let hash = recorded + .split_whitespace() + .nth(2) + .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); // update_submodule if let Some(llvm_hash) = checked_out { From 2fbe2ca916e4a815a15c5100bc5ad1452d33a314 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 5 Jun 2021 12:47:10 -0400 Subject: [PATCH 3/4] Simplify commit check --- src/bootstrap/native.rs | 37 +++++++++++++++---------------------- 1 file changed, 15 insertions(+), 22 deletions(-) diff --git a/src/bootstrap/native.rs b/src/bootstrap/native.rs index 3a5ef3e308bf0..7632889689f36 100644 --- a/src/bootstrap/native.rs +++ b/src/bootstrap/native.rs @@ -107,33 +107,26 @@ pub(crate) fn update_llvm_submodule(build: &Build) { } // check_submodule - let buf; - let checked_out = if build.config.fast_submodules { - buf = output( + if build.config.fast_submodules { + let checked_out_hash = output( Command::new("git") .args(&["rev-parse", "HEAD"]) .current_dir(build.config.src.join(llvm_project)), ); - Some(buf.trim_end()) - } else { - None - }; + // update_submodules + let recorded = output( + Command::new("git") + .args(&["ls-tree", "HEAD"]) + .arg(llvm_project) + .current_dir(&build.config.src), + ); + let actual_hash = recorded + .split_whitespace() + .nth(2) + .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); - // update_submodules - let recorded = output( - Command::new("git") - .args(&["ls-tree", "HEAD"]) - .arg(llvm_project) - .current_dir(&build.config.src), - ); - let hash = recorded - .split_whitespace() - .nth(2) - .unwrap_or_else(|| panic!("unexpected output `{}`", recorded)); - - // update_submodule - if let Some(llvm_hash) = checked_out { - if hash == llvm_hash { + // update_submodule + if actual_hash == checked_out_hash.trim_end() { // already checked out return; } From 73a40ac1a0bf9e7adfe69a886727d64bc69fec05 Mon Sep 17 00:00:00 2001 From: Joshua Nelson Date: Sat, 5 Jun 2021 12:49:51 -0400 Subject: [PATCH 4/4] Inline maybe_update_submodule It was a trivial function only used once. --- src/bootstrap/lib.rs | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index a351290a4206f..a5b1664ed69a1 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -472,21 +472,17 @@ impl Build { slice::from_ref(&self.build.triple) } - /// If the LLVM submodule has been initialized already, sync it unconditionally. This avoids - /// contributors checking in a submodule change by accident. - pub fn maybe_update_llvm_submodule(&self) { - if self.in_tree_llvm_info.is_git() { - native::update_llvm_submodule(self); - } - } - /// Executes the entire build, as configured by the flags and configuration. pub fn build(&mut self) { unsafe { job::setup(self); } - self.maybe_update_llvm_submodule(); + // If the LLVM submodule has been initialized already, sync it unconditionally. This avoids + // contributors checking in a submodule change by accident. + if self.in_tree_llvm_info.is_git() { + native::update_llvm_submodule(self); + } if let Subcommand::Format { check, paths } = &self.config.cmd { return format::format(self, *check, &paths);