Auto merge of rust-lang#77133 - tmandry:bootstrap-host, r=Mark-Simula…

…crum

bootstrap: Always build for host, even when target is given

This changes the behavior from *not* building for host whenever an
explicit target is specified. I find this much less confusing.

You can still disable host steps by passing an explicit empty list for
host.

Fixes rust-lang#76990.

r? `@Mark-Simulacrum`

src/bootstrap/CHANGELOG.md

@@ -6,6 +6,15 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Non-breaking changes since the last major version]
 
+None.
+
+## [Version 2] - 2020-09-25
+
+- `host` now defaults to the value of `build` in all cases
+ + Previously `host` defaulted to an empty list when `target` was overridden, and to `build` otherwise
+
+### Non-breaking changes
+
 - Add `x.py setup` [#76631](https://github.com/rust-lang/rust/pull/76631)
 - Add a changelog for x.py [#76626](https://github.com/rust-lang/rust/pull/76626)
 - Optionally, download LLVM from CI on Linux and NixOS
@@ -21,7 +30,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
  [#77120](https://github.com/rust-lang/rust/pull/77120).
 
 
-## [Version 0] - 2020-09-11
+## [Version 1] - 2020-09-11
 
 This is the first changelog entry, and it does not attempt to be an exhaustive list of features in x.py.
 Instead, this documents the changes to bootstrap in the past 2 months.

src/bootstrap/bin/main.rs

@@ -40,7 +40,7 @@ fn main() {
 }
 
 fn check_version(config: &Config) -> Option<String> {
- const VERSION: usize = 1;
+ const VERSION: usize = 2;
 
  let mut msg = String::new();
 

src/bootstrap/builder.rs

@@ -172,15 +172,7 @@ impl StepDescription {
  }
 
  // Determine the targets participating in this rule.
- let targets = if self.only_hosts {
- if builder.config.skip_only_host_steps {
- return; // don't run anything
- } else {
- &builder.hosts
- }
- } else {
- &builder.targets
- };
+ let targets = if self.only_hosts { &builder.hosts } else { &builder.targets };
 
  for target in targets {
  let run = RunConfig { builder, path: pathset.path(builder), target: *target };

src/bootstrap/builder/tests.rs

@@ -6,7 +6,6 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config {
  let mut config = Config::parse(&[cmd.to_owned()]);
  // don't save toolstates
  config.save_toolstates = None;
- config.skip_only_host_steps = false;
  config.dry_run = true;
  config.ninja_in_file = false;
  // try to avoid spurious failures in dist where we create/delete each others file
@@ -20,16 +19,8 @@ fn configure(cmd: &str, host: &[&str], target: &[&str]) -> Config {
  t!(fs::create_dir_all(&dir));
  config.out = dir;
  config.build = TargetSelection::from_user("A");
- config.hosts = vec![config.build]
- .into_iter()
- .chain(host.iter().map(|s| TargetSelection::from_user(s)))
- .collect::<Vec<_>>();
- config.targets = config
- .hosts
- .clone()
- .into_iter()
- .chain(target.iter().map(|s| TargetSelection::from_user(s)))
- .collect::<Vec<_>>();
+ config.hosts = host.iter().map(|s| TargetSelection::from_user(s)).collect();
+ config.targets = target.iter().map(|s| TargetSelection::from_user(s)).collect();
  config
 }
 
@@ -45,7 +36,7 @@ mod defaults {
 
  #[test]
  fn build_default() {
- let build = Build::new(configure("build", &[], &[]));
+ let build = Build::new(configure("build", &["A"], &["A"]));
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]);
 
@@ -73,7 +64,7 @@ mod defaults {
 
  #[test]
  fn build_stage_0() {
- let config = Config { stage: 0, ..configure("build", &[], &[]) };
+ let config = Config { stage: 0, ..configure("build", &["A"], &["A"]) };
  let build = Build::new(config);
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]);
@@ -95,7 +86,7 @@ mod defaults {
 
  #[test]
  fn build_cross_compile() {
- let config = Config { stage: 1, ..configure("build", &["B"], &["B"]) };
+ let config = Config { stage: 1, ..configure("build", &["A", "B"], &["A", "B"]) };
  let build = Build::new(config);
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]);
@@ -143,7 +134,7 @@ mod defaults {
 
  #[test]
  fn doc_default() {
- let mut config = configure("doc", &[], &[]);
+ let mut config = configure("doc", &["A"], &["A"]);
  config.compiler_docs = true;
  config.cmd = Subcommand::Doc { paths: Vec::new(), open: false };
  let build = Build::new(config);
@@ -182,7 +173,7 @@ mod dist {
 
  #[test]
  fn dist_baseline() {
- let build = Build::new(configure(&[], &[]));
+ let build = Build::new(configure(&["A"], &["A"]));
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]);
 
@@ -208,7 +199,7 @@ mod dist {
 
  #[test]
  fn dist_with_targets() {
- let build = Build::new(configure(&[], &["B"]));
+ let build = Build::new(configure(&["A"], &["A", "B"]));
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]);
 
@@ -239,7 +230,7 @@ mod dist {
 
  #[test]
  fn dist_with_hosts() {
- let build = Build::new(configure(&["B"], &[]));
+ let build = Build::new(configure(&["A", "B"], &["A", "B"]));
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]);
 
@@ -285,7 +276,7 @@ mod dist {
  fn dist_only_cross_host() {
  let a = TargetSelection::from_user("A");
  let b = TargetSelection::from_user("B");
- let mut build = Build::new(configure(&["B"], &[]));
+ let mut build = Build::new(configure(&["A", "B"], &["A", "B"]));
  build.config.docs = false;
  build.config.extended = true;
  build.hosts = vec![b];
@@ -307,7 +298,7 @@ mod dist {
 
  #[test]
  fn dist_with_targets_and_hosts() {
- let build = Build::new(configure(&["B"], &["C"]));
+ let build = Build::new(configure(&["A", "B"], &["A", "B", "C"]));
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]);
 
@@ -342,40 +333,26 @@ mod dist {
  }
 
  #[test]
- fn dist_with_target_flag() {
- let mut config = configure(&["B"], &["C"]);
- config.skip_only_host_steps = true; // as-if --target=C was passed
+ fn dist_with_empty_host() {
+ let config = configure(&[], &["C"]);
  let build = Build::new(config);
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]);
 
  let a = TargetSelection::from_user("A");
- let b = TargetSelection::from_user("B");
  let c = TargetSelection::from_user("C");
 
- assert_eq!(
- first(builder.cache.all::<dist::Docs>()),
- &[dist::Docs { host: a }, dist::Docs { host: b }, dist::Docs { host: c },]
- );
- assert_eq!(
- first(builder.cache.all::<dist::Mingw>()),
- &[dist::Mingw { host: a }, dist::Mingw { host: b }, dist::Mingw { host: c },]
- );
- assert_eq!(first(builder.cache.all::<dist::Rustc>()), &[]);
+ assert_eq!(first(builder.cache.all::<dist::Docs>()), &[dist::Docs { host: c },]);
+ assert_eq!(first(builder.cache.all::<dist::Mingw>()), &[dist::Mingw { host: c },]);
  assert_eq!(
  first(builder.cache.all::<dist::Std>()),
- &[
- dist::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
- dist::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
- dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },
- ]
+ &[dist::Std { compiler: Compiler { host: a, stage: 2 }, target: c },]
  );
- assert_eq!(first(builder.cache.all::<dist::Src>()), &[]);
  }
 
  #[test]
  fn dist_with_same_targets_and_hosts() {
- let build = Build::new(configure(&["B"], &["B"]));
+ let build = Build::new(configure(&["A", "B"], &["A", "B"]));
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Dist), &[]);
 
@@ -428,7 +405,7 @@ mod dist {
 
  #[test]
  fn build_all() {
- let build = Build::new(configure(&["B"], &["C"]));
+ let build = Build::new(configure(&["A", "B"], &["A", "B", "C"]));
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(
  &Builder::get_step_descriptions(Kind::Build),
@@ -464,25 +441,20 @@ mod dist {
  }
 
  #[test]
- fn build_with_target_flag() {
- let mut config = configure(&["B"], &["C"]);
- config.skip_only_host_steps = true;
+ fn build_with_empty_host() {
+ let config = configure(&[], &["C"]);
  let build = Build::new(config);
  let mut builder = Builder::new(&build);
  builder.run_step_descriptions(&Builder::get_step_descriptions(Kind::Build), &[]);
 
  let a = TargetSelection::from_user("A");
- let b = TargetSelection::from_user("B");
  let c = TargetSelection::from_user("C");
 
  assert_eq!(
  first(builder.cache.all::<compile::Std>()),
  &[
  compile::Std { compiler: Compiler { host: a, stage: 0 }, target: a },
  compile::Std { compiler: Compiler { host: a, stage: 1 }, target: a },
- compile::Std { compiler: Compiler { host: a, stage: 2 }, target: a },
- compile::Std { compiler: Compiler { host: a, stage: 1 }, target: b },
- compile::Std { compiler: Compiler { host: a, stage: 2 }, target: b },
  compile::Std { compiler: Compiler { host: a, stage: 2 }, target: c },
  ]
  );
@@ -505,7 +477,7 @@ mod dist {
 
  #[test]
  fn test_with_no_doc_stage0() {
- let mut config = configure(&[], &[]);
+ let mut config = configure(&["A"], &["A"]);
  config.stage = 0;
  config.cmd = Subcommand::Test {
  paths: vec!["library/std".into()],
@@ -545,7 +517,7 @@ mod dist {
 
  #[test]
  fn test_exclude() {
- let mut config = configure(&[], &[]);
+ let mut config = configure(&["A"], &["A"]);
  config.exclude = vec!["src/tools/tidy".into()];
  config.cmd = Subcommand::Test {
  paths: Vec::new(),
@@ -572,7 +544,7 @@ mod dist {
 
  #[test]
  fn doc_ci() {
- let mut config = configure(&[], &[]);
+ let mut config = configure(&["A"], &["A"]);
  config.compiler_docs = true;
  config.cmd = Subcommand::Doc { paths: Vec::new(), open: false };
  let build = Build::new(config);
@@ -601,7 +573,7 @@ mod dist {
  #[test]
  fn test_docs() {
  // Behavior of `x.py test` doing various documentation tests.
- let mut config = configure(&[], &[]);
+ let mut config = configure(&["A"], &["A"]);
  config.cmd = Subcommand::Test {
  paths: vec![],
  test_args: vec![],

src/bootstrap/config.rs

@@ -66,8 +66,6 @@ pub struct Config {
  pub test_compare_mode: bool,
  pub llvm_libunwind: bool,
 
- pub skip_only_host_steps: bool,
-
  pub on_fail: Option<String>,
  pub stage: u32,
  pub keep_stage: Vec<u32>,
@@ -586,11 +584,6 @@ impl Config {
 
  let build = toml.build.unwrap_or_default();
 
- // If --target was specified but --host wasn't specified, don't run any host-only tests.
- let has_hosts = build.host.is_some() || flags.host.is_some();
- let has_targets = build.target.is_some() || flags.target.is_some();
- config.skip_only_host_steps = !has_hosts && has_targets;
-
  config.hosts = if let Some(arg_host) = flags.host {
  arg_host
  } else if let Some(file_host) = build.host {

src/bootstrap/flags.rs

@@ -679,7 +679,7 @@ impl Subcommand {
 }
 
 fn split(s: &[String]) -> Vec<String> {
- s.iter().flat_map(|s| s.split(',')).map(|s| s.to_string()).collect()
+ s.iter().flat_map(|s| s.split(',')).filter(|s| !s.is_empty()).map(|s| s.to_string()).collect()
 }
 
 fn parse_deny_warnings(matches: &getopts::Matches) -> Option<bool> {

src/ci/docker/host-x86_64/arm-android/Dockerfile

@@ -31,7 +31,7 @@ ENV TARGETS=arm-linux-androideabi
 
 ENV RUST_CONFIGURE_ARGS --arm-linux-androideabi-ndk=/android/ndk/arm-14
 
-ENV SCRIPT python3 ../x.py --stage 2 test --target $TARGETS
+ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target $TARGETS
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh

src/ci/docker/host-x86_64/armhf-gnu/Dockerfile

@@ -79,6 +79,6 @@ COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV RUST_CONFIGURE_ARGS --qemu-armhf-rootfs=/tmp/rootfs
-ENV SCRIPT python3 ../x.py --stage 2 test --target arm-unknown-linux-gnueabihf
+ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target arm-unknown-linux-gnueabihf
 
 ENV NO_CHANGE_USER=1

src/ci/docker/host-x86_64/disabled/asmjs/Dockerfile

@@ -34,7 +34,7 @@ ENV EMCC_CFLAGS=-O1
 # Emscripten installation is user-specific
 ENV NO_CHANGE_USER=1
 
-ENV SCRIPT python3 ../x.py --stage 2 test --target $TARGETS
+ENV SCRIPT python3 ../x.py --stage 2 test --host='' --target $TARGETS
 
 # This is almost identical to the wasm32-unknown-emscripten target, so
 # running with assertions again is not useful

src/ci/docker/host-x86_64/disabled/dist-x86_64-redox/Dockerfile

@@ -19,4 +19,4 @@ ENV \
  CXX_x86_64_unknown_redox=x86_64-unknown-redox-g++
 
 ENV RUST_CONFIGURE_ARGS --enable-extended
-ENV SCRIPT python3 ../x.py dist --target x86_64-unknown-redox
+ENV SCRIPT python3 ../x.py dist --host='' --target x86_64-unknown-redox

src/ci/docker/host-x86_64/dist-android/Dockerfile

@@ -32,7 +32,7 @@ ENV RUST_CONFIGURE_ARGS \
  --x86_64-linux-android-ndk=/android/ndk/x86_64-21 \
  --disable-docs
 
-ENV SCRIPT python3 ../x.py dist --target $TARGETS
+ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh

src/ci/docker/host-x86_64/dist-i586-gnu-i586-i686-musl/Dockerfile

@@ -47,5 +47,5 @@ ENV CFLAGS_i586_unknown_linux_musl=-Wa,-mrelax-relocations=no
 ENV TARGETS=i586-unknown-linux-gnu,i686-unknown-linux-musl
 
 ENV SCRIPT \
- python3 ../x.py --stage 2 test --target $TARGETS && \
- python3 ../x.py dist --target $TARGETS,i586-unknown-linux-musl
+ python3 ../x.py --stage 2 test --host='' --target $TARGETS && \
+ python3 ../x.py dist --host='' --target $TARGETS,i586-unknown-linux-musl

src/ci/docker/host-x86_64/dist-various-1/Dockerfile

@@ -187,8 +187,8 @@ ENV RUST_CONFIGURE_ARGS \
  --disable-docs
 
 ENV SCRIPT \
- python3 ../x.py --stage 2 test --target $RUN_MAKE_TARGETS src/test/run-make && \
- python3 ../x.py dist --target $TARGETS
+ python3 ../x.py --stage 2 test --host='' --target $RUN_MAKE_TARGETS src/test/run-make && \
+ python3 ../x.py dist --host='' --target $TARGETS
 
 # sccache
 COPY scripts/sccache.sh /scripts/

src/ci/docker/host-x86_64/dist-various-2/Dockerfile

@@ -110,4 +110,4 @@ ENV RUST_CONFIGURE_ARGS --enable-extended --enable-lld --disable-docs \
  --set target.wasm32-wasi.wasi-root=/wasm32-wasi \
  --musl-root-armv7=/musl-armv7
 
-ENV SCRIPT python3 ../x.py dist --target $TARGETS
+ENV SCRIPT python3 ../x.py dist --host='' --target $TARGETS

src/ci/docker/host-x86_64/test-various/Dockerfile

@@ -41,7 +41,7 @@ ENV RUST_CONFIGURE_ARGS \
 ENV NO_DEBUG_ASSERTIONS=1
 
 ENV WASM_TARGETS=wasm32-unknown-unknown
-ENV WASM_SCRIPT python3 /checkout/x.py --stage 2 test --target $WASM_TARGETS \
+ENV WASM_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $WASM_TARGETS \
  src/test/run-make \
  src/test/ui \
  src/test/compile-fail \
@@ -50,13 +50,13 @@ ENV WASM_SCRIPT python3 /checkout/x.py --stage 2 test --target $WASM_TARGETS \
  library/core
 
 ENV NVPTX_TARGETS=nvptx64-nvidia-cuda
-ENV NVPTX_SCRIPT python3 /checkout/x.py --stage 2 test --target $NVPTX_TARGETS \
+ENV NVPTX_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $NVPTX_TARGETS \
  src/test/run-make \
  src/test/assembly
 
 ENV MUSL_TARGETS=x86_64-unknown-linux-musl \
  CC_x86_64_unknown_linux_musl=x86_64-linux-musl-gcc \
  CXX_x86_64_unknown_linux_musl=x86_64-linux-musl-g++
-ENV MUSL_SCRIPT python3 /checkout/x.py --stage 2 test --target $MUSL_TARGETS
+ENV MUSL_SCRIPT python3 /checkout/x.py --stage 2 test --host='' --target $MUSL_TARGETS
 
 ENV SCRIPT $WASM_SCRIPT && $NVPTX_SCRIPT && $MUSL_SCRIPT