Make sure to clear out the stageN-{rustc,std,tools} directories.

We copy built tool binaries into a dedicated directory to avoid deleting
them, stageN-tools-bin. These aren't ever cleared out by code, since
there should be no reason to do so, and we'll simply overwrite them as
necessary.

When clearing out the stageN-{std,rustc,tools} directories, make sure to
delete both Cargo directories -- per-target and build scripts. This
ensures that changing libstd doesn't cause problems due to build scripts
not being rebuilt, even though they should be.

Author: Mark-Simulacrum

Diff for: src/bootstrap/builder.rs

@@ -612,7 +612,7 @@ impl<'a> Builder<'a> {
         // Set this for all builds to make sure doc builds also get it.
         cargo.env("CFG_RELEASE_CHANNEL", &self.build.config.channel);
 
-        if self.is_verbose() {
+        if self.is_very_verbose() {
             cargo.arg("-v");
         }
         // FIXME: cargo bench does not accept `--release`

Diff for: src/bootstrap/compile.rs

@@ -29,7 +29,7 @@ use build_helper::{output, mtime, up_to_date};
 use filetime::FileTime;
 use serde_json;
 
-use util::{exe, libdir, is_dylib, copy};
+use util::{exe, libdir, is_dylib, copy, read_stamp_file};
 use {Build, Compiler, Mode};
 use native;
 use tool;
@@ -102,7 +102,7 @@ impl Step for Std {
             copy_musl_third_party_objects(build, target, &libdir);
         }
 
-        let out_dir = build.cargo_out(compiler, Mode::Libstd, target);
+        let out_dir = build.stage_out(compiler, Mode::Libstd);
         build.clear_if_dirty(&out_dir, &builder.rustc(compiler));
         let mut cargo = builder.cargo(compiler, Mode::Libstd, target, "build");
         std_cargo(build, &compiler, target, &mut cargo);
@@ -354,7 +354,7 @@ impl Step for Test {
         let _folder = build.fold_output(|| format!("stage{}-test", compiler.stage));
         println!("Building stage{} test artifacts ({} -> {})", compiler.stage,
                 &compiler.host, target);
-        let out_dir = build.cargo_out(compiler, Mode::Libtest, target);
+        let out_dir = build.stage_out(compiler, Mode::Libtest);
         build.clear_if_dirty(&out_dir, &libstd_stamp(build, compiler, target));
         let mut cargo = builder.cargo(compiler, Mode::Libtest, target, "build");
         test_cargo(build, &compiler, target, &mut cargo);
@@ -480,8 +480,9 @@ impl Step for Rustc {
         println!("Building stage{} compiler artifacts ({} -> {})",
                  compiler.stage, &compiler.host, target);
 
-        let out_dir = build.cargo_out(compiler, Mode::Librustc, target);
-        build.clear_if_dirty(&out_dir, &libtest_stamp(build, compiler, target));
+        let stage_out = builder.stage_out(compiler, Mode::Librustc);
+        build.clear_if_dirty(&stage_out, &libstd_stamp(build, compiler, target));
+        build.clear_if_dirty(&stage_out, &libtest_stamp(build, compiler, target));
 
         let mut cargo = builder.cargo(compiler, Mode::Librustc, target, "build");
         rustc_cargo(build, &compiler, target, &mut cargo);
@@ -757,15 +758,7 @@ impl Step for Assemble {
 /// `sysroot_dst` provided.
 fn add_to_sysroot(sysroot_dst: &Path, stamp: &Path) {
     t!(fs::create_dir_all(&sysroot_dst));
-    let mut contents = Vec::new();
-    t!(t!(File::open(stamp)).read_to_end(&mut contents));
-    // This is the method we use for extracting paths from the stamp file passed to us. See
-    // run_cargo for more information (in this file).
-    for part in contents.split(|b| *b == 0) {
-        if part.is_empty() {
-            continue
-        }
-        let path = Path::new(t!(str::from_utf8(part)));
+    for path in read_stamp_file(stamp) {
         copy(&path, &sysroot_dst.join(path.file_name().unwrap()));
     }
 }
@@ -938,6 +931,8 @@ fn run_cargo(build: &Build, cargo: &mut Command, stamp: &Path) {
     let max = max.unwrap();
     let max_path = max_path.unwrap();
     if stamp_contents == new_contents && max <= stamp_mtime {
+        build.verbose(&format!("not updating {:?}; contents equal and {} <= {}",
+                stamp, max, stamp_mtime));
         return
     }
     if max > stamp_mtime {

Diff for: src/bootstrap/lib.rs

@@ -385,16 +385,19 @@ impl Build {
     /// Clear out `dir` if `input` is newer.
     ///
     /// After this executes, it will also ensure that `dir` exists.
-    fn clear_if_dirty(&self, dir: &Path, input: &Path) {
+    fn clear_if_dirty(&self, dir: &Path, input: &Path) -> bool {
         let stamp = dir.join(".stamp");
+        let mut cleared = false;
         if mtime(&stamp) < mtime(input) {
             self.verbose(&format!("Dirty - {}", dir.display()));
             let _ = fs::remove_dir_all(dir);
+            cleared = true;
         } else if stamp.exists() {
-            return
+            return cleared;
         }
         t!(fs::create_dir_all(dir));
         t!(File::create(stamp));
+        cleared
     }
 
     /// Get the space-separated set of activated features for the standard
@@ -435,6 +438,12 @@ impl Build {
         if self.config.rust_optimize {"release"} else {"debug"}
     }
 
+    fn tools_dir(&self, compiler: Compiler) -> PathBuf {
+        let out = self.out.join(&*compiler.host).join(format!("stage{}-tools-bin", compiler.stage));
+        t!(fs::create_dir_all(&out));
+        out
+    }
+
     /// Get the directory for incremental by-products when using the
     /// given compiler.
     fn incremental_dir(&self, compiler: Compiler) -> PathBuf {

Diff for: src/bootstrap/tool.rs

@@ -38,24 +38,40 @@ impl Step for CleanTools {
         run.never()
     }
 
-    /// Build a tool in `src/tools`
-    ///
-    /// This will build the specified tool with the specified `host` compiler in
-    /// `stage` into the normal cargo output directory.
     fn run(self, builder: &Builder) {
         let build = builder.build;
         let compiler = self.compiler;
         let target = self.target;
         let mode = self.mode;
 
-        let stamp = match mode {
-            Mode::Libstd => libstd_stamp(build, compiler, target),
-            Mode::Libtest => libtest_stamp(build, compiler, target),
-            Mode::Librustc => librustc_stamp(build, compiler, target),
-            _ => panic!(),
+        // This is for the original compiler, but if we're forced to use stage 1, then
+        // std/test/rustc stamps won't exist in stage 2, so we need to get those from stage 1, since
+        // we copy the libs forward.
+        let tools_dir = build.stage_out(compiler, Mode::Tool);
+        let compiler = if builder.force_use_stage1(compiler, target) {
+            builder.compiler(1, compiler.host)
+        } else {
+            compiler
         };
-        let out_dir = build.cargo_out(compiler, Mode::Tool, target);
-        build.clear_if_dirty(&out_dir, &stamp);
+
+        for &cur_mode in &[Mode::Libstd, Mode::Libtest, Mode::Librustc] {
+            let stamp = match cur_mode {
+                Mode::Libstd => libstd_stamp(build, compiler, target),
+                Mode::Libtest => libtest_stamp(build, compiler, target),
+                Mode::Librustc => librustc_stamp(build, compiler, target),
+                _ => panic!(),
+            };
+
+            if build.clear_if_dirty(&tools_dir, &stamp) {
+                break;
+            }
+
+            // If we are a rustc tool, and std changed, we also need to clear ourselves out -- our
+            // dependencies depend on std. Therefore, we iterate up until our own mode.
+            if mode == cur_mode {
+                break;
+            }
+        }
     }
 }
 
@@ -100,7 +116,11 @@ impl Step for ToolBuild {
 
         let mut cargo = prepare_tool_cargo(builder, compiler, target, "build", path);
         build.run_expecting(&mut cargo, expectation);
-        build.cargo_out(compiler, Mode::Tool, target).join(exe(tool, &compiler.host))
+        let cargo_out = build.cargo_out(compiler, Mode::Tool, target)
+            .join(exe(tool, &compiler.host));
+        let bin = build.tools_dir(compiler).join(exe(tool, &compiler.host));
+        copy(&cargo_out, &bin);
+        bin
     }
 }
 

Diff for: src/bootstrap/util.rs

@@ -14,8 +14,9 @@
 //! not a lot of interesting happenings here unfortunately.
 
 use std::env;
-use std::fs;
-use std::io::{self, Write};
+use std::str;
+use std::fs::{self, File};
+use std::io::{self, Read, Write};
 use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time::{SystemTime, Instant};
@@ -50,6 +51,22 @@ pub fn copy(src: &Path, dst: &Path) {
     t!(filetime::set_file_times(dst, atime, mtime));
 }
 
+pub fn read_stamp_file(stamp: &Path) -> Vec<PathBuf> {
+    let mut paths = Vec::new();
+    let mut contents = Vec::new();
+    t!(t!(File::open(stamp)).read_to_end(&mut contents));
+    // This is the method we use for extracting paths from the stamp file passed to us. See
+    // run_cargo for more information (in compile.rs).
+    for part in contents.split(|b| *b == 0) {
+        if part.is_empty() {
+            continue
+        }
+        let path = PathBuf::from(t!(str::from_utf8(part)));
+        paths.push(path);
+    }
+    paths
+}
+
 /// Copies the `src` directory recursively to `dst`. Both are assumed to exist
 /// when this function is called.
 pub fn cp_r(src: &Path, dst: &Path) {