Auto merge of #122709 - onur-ozkan:use-precompiled-rustc-by-default, …

…r=<try>

use precompiled rustc for non-dist builders

Makes non-dist builders to use precompiled CI rustc by default if they are available for the target triple.

As we are going to make `rust.download-rustc=if-unchanged` default option with #119899, we need to make sure `if-unchanged` logic never breaks and works as expected.

As an addition, this will significantly improve the build times on CI when there's no change on the compiler.

blocker for #119899

try-job: x86_64-gnu-nopt

src/bootstrap/src/core/builder.rs

@@ -418,6 +418,15 @@ impl StepDescription {
  .map(|desc| (desc.should_run)(ShouldRun::new(builder, desc.kind)))
  .collect::<Vec<_>>();
 
+ if builder.download_rustc() && (builder.kind == Kind::Dist || builder.kind == Kind::Install)
+ {
+ eprintln!(
+ "ERROR: '{}' subcommand is incompatible with `rust.download-rustc`.",
+ builder.kind.as_str()
+ );
+ crate::exit!(1);
+ }
+
  // sanity checks on rules
  for (desc, should_run) in v.iter().zip(&should_runs) {
  assert!(

src/bootstrap/src/core/builder/tests.rs

@@ -1,5 +1,7 @@
 use std::thread;
 
+use build_helper::git::get_closest_merge_commit;
+
 use super::*;
 use crate::core::build_steps::doc::DocumentationFormat;
 use crate::core::config::Config;
@@ -212,6 +214,52 @@ fn alias_and_path_for_library() {
  assert_eq!(first(cache.all::<doc::Std>()), &[doc_std!(A => A, stage = 0)]);
 }
 
+#[test]
+fn ci_rustc_if_unchanged_logic() {
+ let config = Config::parse_inner(
+ Flags::parse(&[
+ "build".to_owned(),
+ "--dry-run".to_owned(),
+ "--set=rust.download-rustc='if-unchanged'".to_owned(),
+ ]),
+ |&_| Ok(Default::default()),
+ );
+
+ if config.rust_info.is_from_tarball() {
+ return;
+ }
+
+ let build = Build::new(config.clone());
+ let builder = Builder::new(&build);
+
+ if config.out.exists() {
+ fs::remove_dir_all(&config.out).unwrap();
+ }
+
+ builder.run_step_descriptions(&Builder::get_step_descriptions(config.cmd.kind()), &[]);
+
+ let compiler_path = build.src.join("compiler");
+ let library_path = build.src.join("compiler");
+
+ let commit = get_closest_merge_commit(
+ Some(&builder.config.src),
+ &builder.config.git_config(),
+ &[compiler_path.clone(), library_path.clone()],
+ )
+ .unwrap();
+
+ let has_changes = !helpers::git(Some(&builder.src))
+ .args(["diff-index", "--quiet", &commit])
+ .arg("--")
+ .args([compiler_path, library_path])
+ .as_command_mut()
+ .status()
+ .unwrap()
+ .success();
+
+ assert!(has_changes == config.download_rustc_commit.is_none());
+}
+
 mod defaults {
  use pretty_assertions::assert_eq;
 

src/bootstrap/src/core/config/config.rs

@@ -13,6 +13,7 @@ use std::str::FromStr;
 use std::sync::OnceLock;
 use std::{cmp, env, fs};
 
+use build_helper::ci::CiEnv;
 use build_helper::exit;
 use build_helper::git::{get_closest_merge_commit, output_result, GitConfig};
 use serde::{Deserialize, Deserializer};
@@ -22,6 +23,7 @@ use crate::core::build_steps::compile::CODEGEN_BACKEND_PREFIX;
 use crate::core::build_steps::llvm;
 pub use crate::core::config::flags::Subcommand;
 use crate::core::config::flags::{Color, Flags, Warnings};
+use crate::core::download::is_download_ci_available;
 use crate::utils::cache::{Interned, INTERNER};
 use crate::utils::channel::{self, GitInfo};
 use crate::utils::helpers::{self, exe, output, t};
@@ -1616,9 +1618,11 @@ impl Config {
  config.mandir = mandir.map(PathBuf::from);
  }
 
+ config.llvm_assertions =
+ toml.llvm.as_ref().map_or(false, |llvm| llvm.assertions.unwrap_or(false));
+
  // Store off these values as options because if they're not provided
  // we'll infer default values for them later
- let mut llvm_assertions = None;
  let mut llvm_tests = None;
  let mut llvm_enzyme = None;
  let mut llvm_plugins = None;
@@ -1699,7 +1703,8 @@ impl Config {
  is_user_configured_rust_channel = channel.is_some();
  set(&mut config.channel, channel.clone());
 
- config.download_rustc_commit = config.download_ci_rustc_commit(download_rustc);
+ config.download_rustc_commit =
+ config.download_ci_rustc_commit(download_rustc, config.llvm_assertions);
 
  debug = debug_toml;
  debug_assertions = debug_assertions_toml;
@@ -1824,7 +1829,7 @@ impl Config {
  optimize: optimize_toml,
  thin_lto,
  release_debuginfo,
- assertions,
+ assertions: _,
  tests,
  enzyme,
  plugins,
@@ -1858,7 +1863,6 @@ impl Config {
  Some(StringOrBool::Bool(false)) | None => {}
  }
  set(&mut config.ninja_in_file, ninja);
- llvm_assertions = assertions;
  llvm_tests = tests;
  llvm_enzyme = enzyme;
  llvm_plugins = plugins;
@@ -1887,8 +1891,8 @@ impl Config {
  config.llvm_enable_warnings = enable_warnings.unwrap_or(false);
  config.llvm_build_config = build_config.clone().unwrap_or(Default::default());
 
- let asserts = llvm_assertions.unwrap_or(false);
- config.llvm_from_ci = config.parse_download_ci_llvm(download_ci_llvm, asserts);
+ config.llvm_from_ci =
+  config.parse_download_ci_llvm(download_ci_llvm, config.llvm_assertions);
 
  if config.llvm_from_ci {
  let warn = |option: &str| {
@@ -2056,7 +2060,6 @@ impl Config {
  // Now that we've reached the end of our configuration, infer the
  // default values for all options that we haven't otherwise stored yet.
 
- config.llvm_assertions = llvm_assertions.unwrap_or(false);
  config.llvm_tests = llvm_tests.unwrap_or(false);
  config.llvm_enzyme = llvm_enzyme.unwrap_or(false);
  config.llvm_plugins = llvm_plugins.unwrap_or(false);
@@ -2392,8 +2395,9 @@ impl Config {
  ci_config_toml,
  );
 
- let disable_ci_rustc_if_incompatible =
- env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
+ // Primarily used by CI runners to avoid handling download-rustc incompatible
+ // options one by one on shell scripts.
+ let disable_ci_rustc_if_incompatible = env::var_os("DISABLE_CI_RUSTC_IF_INCOMPATIBLE")
  .is_some_and(|s| s == "1" || s == "true");
 
  if disable_ci_rustc_if_incompatible && res.is_err() {
@@ -2684,7 +2688,15 @@ impl Config {
  }
 
  /// Returns the commit to download, or `None` if we shouldn't download CI artifacts.
- fn download_ci_rustc_commit(&self, download_rustc: Option<StringOrBool>) -> Option<String> {
+ fn download_ci_rustc_commit(
+ &self,
+ download_rustc: Option<StringOrBool>,
+ llvm_assertions: bool,
+ ) -> Option<String> {
+ if !is_download_ci_available(&self.build.triple, llvm_assertions) {
+ return None;
+ }
+
  // If `download-rustc` is not set, default to rebuilding.
  let if_unchanged = match download_rustc {
  None | Some(StringOrBool::Bool(false)) => return None,
@@ -2697,7 +2709,12 @@ impl Config {
 
  // Look for a version to compare to based on the current commit.
  // Only commits merged by bors will have CI artifacts.
- let commit = get_closest_merge_commit(Some(&self.src), &self.git_config(), &[]).unwrap();
+ let commit = get_closest_merge_commit(
+ Some(&self.src),
+ &self.git_config(),
+ &[self.src.join("compiler"), self.src.join("library")],
+ )
+ .unwrap();
  if commit.is_empty() {
  println!("ERROR: could not find commit hash for downloading rustc");
  println!("HELP: maybe your repository history is too shallow?");
@@ -2706,6 +2723,19 @@ impl Config {
  crate::exit!(1);
  }
 
+ if CiEnv::is_ci() && {
+ let head_sha =
+ output(helpers::git(Some(&self.src)).arg("rev-parse").arg("HEAD").as_command_mut());
+ let head_sha = head_sha.trim();
+ commit == head_sha
+ } {
+ eprintln!("CI rustc commit matches with HEAD and we are in CI.");
+ eprintln!(
+ "`rustc.download-ci` functionality will be skipped as artifacts are not available."
+ );
+ return None;
+ }
+
  // Warn if there were changes to the compiler or standard library since the ancestor commit.
  let has_changes = !t!(helpers::git(Some(&self.src))
  .args(["diff-index", "--quiet", &commit])

src/bootstrap/src/core/config/tests.rs

@@ -11,7 +11,7 @@ use super::{ChangeIdWrapper, Config};
 use crate::core::build_steps::clippy::get_clippy_rules_in_order;
 use crate::core::config::{LldMode, Target, TargetSelection, TomlConfig};
 
-fn parse(config: &str) -> Config {
+pub(crate) fn parse(config: &str) -> Config {
  Config::parse_inner(
  Flags::parse(&["check".to_string(), "--config=/does/not/exist".to_string()]),
  |&_| toml::from_str(&config),

src/bootstrap/src/core/download.rs

@@ -832,3 +832,43 @@ fn path_is_dylib(path: &Path) -> bool {
  // The .so is not necessarily the extension, it might be libLLVM.so.18.1
  path.to_str().map_or(false, |path| path.contains(".so"))
 }
+
+/// Checks whether the CI rustc is available for the given target triple.
+pub(crate) fn is_download_ci_available(target_triple: &str, llvm_assertions: bool) -> bool {
+ // All tier 1 targets and tier 2 targets with host tools.
+ const SUPPORTED_PLATFORMS: &[&str] = &[
+ "aarch64-apple-darwin",
+ "aarch64-pc-windows-msvc",
+ "aarch64-unknown-linux-gnu",
+ "aarch64-unknown-linux-musl",
+ "arm-unknown-linux-gnueabi",
+ "arm-unknown-linux-gnueabihf",
+ "armv7-unknown-linux-gnueabihf",
+ "i686-pc-windows-gnu",
+ "i686-pc-windows-msvc",
+ "i686-unknown-linux-gnu",
+ "loongarch64-unknown-linux-gnu",
+ "powerpc-unknown-linux-gnu",
+ "powerpc64-unknown-linux-gnu",
+ "powerpc64le-unknown-linux-gnu",
+ "riscv64gc-unknown-linux-gnu",
+ "s390x-unknown-linux-gnu",
+ "x86_64-apple-darwin",
+ "x86_64-pc-windows-gnu",
+ "x86_64-pc-windows-msvc",
+ "x86_64-unknown-freebsd",
+ "x86_64-unknown-illumos",
+ "x86_64-unknown-linux-gnu",
+ "x86_64-unknown-linux-musl",
+ "x86_64-unknown-netbsd",
+ ];
+
+ const SUPPORTED_PLATFORMS_WITH_ASSERTIONS: &[&str] =
+ &["x86_64-unknown-linux-gnu", "x86_64-pc-windows-msvc"];
+
+ if llvm_assertions {
+ SUPPORTED_PLATFORMS_WITH_ASSERTIONS.contains(&target_triple)
+ } else {
+ SUPPORTED_PLATFORMS.contains(&target_triple)
+ }
+}

src/ci/docker/host-x86_64/mingw-check/Dockerfile

@@ -46,7 +46,20 @@ ENV RUN_CHECK_WITH_PARALLEL_QUERIES 1
 # Check library crates on all tier 1 targets.
 # We disable optimized compiler built-ins because that requires a C toolchain for the target.
 # We also skip the x86_64-unknown-linux-gnu target as it is well-tested by other jobs.
-ENV SCRIPT python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
+ENV SCRIPT \
+ # `core::builder::tests::ci_rustc_if_unchanged_logic` bootstrap test covers the `rust.download-rustc=if-unchanged` logic.
+ # Here we are adding a dummy commit on compiler and running that test to ensure when there is a change on the compiler,
+ # we never download ci rustc with `rust.download-rustc=if-unchanged` option.
+ echo \"\" >> ../compiler/rustc/src/main.rs && \
+ git config --global user.email \"dummy@dummy.com\" && \
+ git config --global user.name \"dummy\" && \
+ git add ../compiler/rustc/src/main.rs && \
+ git commit -m \"test commit for rust.download-rustc=if-unchanged logic\" && \
+ DISABLE_CI_RUSTC_IF_INCOMPATIBLE=0 python3 ../x.py test bootstrap -- core::builder::tests::ci_rustc_if_unchanged_logic && \
+ # Revert the dummy commit
+ git reset --hard HEAD~1 && \
+
+ python3 ../x.py check --stage 0 --set build.optimized-compiler-builtins=false core alloc std --target=aarch64-unknown-linux-gnu,i686-pc-windows-msvc,i686-unknown-linux-gnu,x86_64-apple-darwin,x86_64-pc-windows-gnu,x86_64-pc-windows-msvc && \
  /scripts/check-default-config-profiles.sh && \
  python3 ../x.py check --target=i686-pc-windows-gnu --host=i686-pc-windows-gnu && \
  python3 ../x.py clippy bootstrap -Dwarnings && \

src/ci/docker/host-x86_64/x86_64-fuchsia/Dockerfile

@@ -58,6 +58,9 @@ RUN mkdir -p $RUST_INSTALL_DIR/etc
 # Fuchsia only supports LLVM.
 ENV CODEGEN_BACKENDS llvm
 
+# download-rustc is not allowed for `x install`
+ENV NO_DOWNLOAD_CI_RUSTC 1
+
 ENV RUST_CONFIGURE_ARGS \
  --prefix=$RUST_INSTALL_DIR \
  --sysconfdir=etc \
@@ -70,6 +73,7 @@ ENV RUST_CONFIGURE_ARGS \
  --set target.x86_64-unknown-fuchsia.ar=/usr/local/bin/llvm-ar \
  --set target.x86_64-unknown-fuchsia.ranlib=/usr/local/bin/llvm-ranlib \
  --set target.x86_64-unknown-fuchsia.linker=/usr/local/bin/ld.lld
+
 ENV SCRIPT \
  python3 ../x.py install --target $TARGETS compiler/rustc library/std clippy && \
  bash ../src/ci/docker/host-x86_64/x86_64-fuchsia/build-fuchsia.sh

src/ci/docker/host-x86_64/x86_64-gnu-nopt/Dockerfile

@@ -29,5 +29,4 @@ RUN echo "optimize = false" >> /config/nopt-std-config.toml
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu \
  --disable-optimize-tests \
  --set rust.test-compare-mode
-ENV SCRIPT python3 ../x.py test --stage 0 --config /config/nopt-std-config.toml library/std \
- && python3 ../x.py --stage 2 test
+ENV SCRIPT python3 ../x.py test --stage 2 tests/ui/allocator/no_std-alloc-error-handler-custom.rs -- --force-rerun

src/ci/docker/host-x86_64/x86_64-gnu-tools/Dockerfile

@@ -84,6 +84,7 @@ ENV RUST_CONFIGURE_ARGS \
  --enable-new-symbol-mangling
 
 ENV HOST_TARGET x86_64-unknown-linux-gnu
+ENV FORCE_CI_RUSTC 1
 
 COPY host-x86_64/dist-x86_64-linux/shared.sh /scripts/
 COPY host-x86_64/dist-x86_64-linux/build-gccjit.sh /scripts/

src/ci/docker/run.sh

@@ -343,6 +343,7 @@ docker \
  --env PR_CI_JOB \
  --env OBJDIR_ON_HOST="$objdir" \
  --env CODEGEN_BACKENDS \
+ --env DISABLE_CI_RUSTC_IF_INCOMPATIBLE="$DISABLE_CI_RUSTC_IF_INCOMPATIBLE" \
  --init \
  --rm \
  rust-ci \

src/ci/github-actions/jobs.yml

@@ -85,6 +85,9 @@ envs:
 # it in each job definition.
 pr:
  - image: mingw-check
+ env:
+ # We are adding (temporarily) a dummy commit on the compiler
+ READ_ONLY_SRC: "0"
  <<: *job-linux-4c
  - image: mingw-check-tidy
  continue_on_error: true
@@ -207,6 +210,8 @@ auto:
  <<: *job-linux-8c
 
  - image: mingw-check
+ env:
+ READ_ONLY_SRC: 0
  <<: *job-linux-4c
 
  - image: test-various

src/ci/run.sh

@@ -52,6 +52,13 @@ if [ "$CI" != "" ]; then
  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set change-id=99999999"
 fi
 
+# If runner uses an incompatible option and `FORCE_CI_RUSTC` is not defined,
+# switch to in-tree rustc.
+if [ "$FORCE_CI_RUSTC" == "" ]; then
+ echo "debug: `DISABLE_CI_RUSTC_IF_INCOMPATIBLE` configured."
+ DISABLE_CI_RUSTC_IF_INCOMPATIBLE=1
+fi
+
 if ! isCI || isCiBranch auto || isCiBranch beta || isCiBranch try || isCiBranch try-perf || \
  isCiBranch automation/bors/try; then
  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set build.print-step-timings --enable-verbose-tests"
@@ -169,10 +176,16 @@ else
  if [ "$NO_DOWNLOAD_CI_LLVM" = "" ]; then
  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.download-ci-llvm=if-unchanged"
  else
+ # CI rustc requires CI LLVM to be enabled (see https://github.com/rust-lang/rust/issues/123586).
+ NO_DOWNLOAD_CI_RUSTC=1
  # When building for CI we want to use the static C++ Standard library
  # included with LLVM, since a dynamic libstdcpp may not be available.
  RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set llvm.static-libstdcpp"
  fi
+
+ if [ "$NO_DOWNLOAD_CI_RUSTC" = "" ]; then
+ RUST_CONFIGURE_ARGS="$RUST_CONFIGURE_ARGS --set rust.download-rustc=if-unchanged"
+ fi
 fi
 
 if [ "$ENABLE_GCC_CODEGEN" = "1" ]; then