Rollup of 10 pull requests

.gitmodules

@@ -22,15 +22,14 @@
 	path = src/doc/nomicon
 	url = https://github.com/rust-lang-nursery/nomicon.git
 [submodule "src/tools/cargo"]
-	path = cargo
-	url = https://github.com/rust-lang/cargo.git
+	path = src/tools/cargo
+	url = https://github.com/rust-lang/cargo
 [submodule "reference"]
 	path = src/doc/reference
 	url = https://github.com/rust-lang-nursery/reference.git
 [submodule "book"]
 	path = src/doc/book
 	url = https://github.com/rust-lang/book.git
-[submodule "rls"]
-	path = rls
-	url = https://github.com/rust-lang-nursery/rls.git
-
+[submodule "src/tools/rls"]
+	path = src/tools/rls
+	url = https://github.com/rust-lang-nursery/rls

.travis.yml

@@ -63,7 +63,7 @@ matrix:
       os: osx
       osx_image: xcode8.2
       install: &osx_install_sccache >
-        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-04-04-sccache-x86_64-apple-darwin &&
+        travis_retry curl -o /usr/local/bin/sccache https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-04-19-sccache-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/sccache &&
         travis_retry curl -o /usr/local/bin/stamp https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-03-17-stamp-x86_64-apple-darwin &&
           chmod +x /usr/local/bin/stamp

appveyor.yml

@@ -32,14 +32,14 @@ environment:
   # came from the mingw-w64 SourceForge download site. Unfortunately
   # SourceForge is notoriously flaky, so we mirror it on our own infrastructure.
   - MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-ninja
+    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
     SCRIPT: python x.py test
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror
     MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
     MINGW_DIR: mingw32
   - MSYS_BITS: 64
     SCRIPT: python x.py test
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-ninja
+    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror
     MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
     MINGW_DIR: mingw64
@@ -57,15 +57,15 @@ environment:
     SCRIPT: python x.py dist
     DEPLOY: 1
   - MSYS_BITS: 32
-    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-extended --enable-ninja
+    RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-extended
     SCRIPT: python x.py dist
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror
     MINGW_ARCHIVE: i686-6.3.0-release-posix-dwarf-rt_v5-rev2.7z
     MINGW_DIR: mingw32
     DEPLOY: 1
   - MSYS_BITS: 64
     SCRIPT: python x.py dist
-    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-extended --enable-ninja
+    RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu --enable-extended
     MINGW_URL: https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror
     MINGW_ARCHIVE: x86_64-6.3.0-release-posix-seh-rt_v5-rev2.7z
     MINGW_DIR: mingw64
@@ -124,15 +124,16 @@ install:
   - set PATH=C:\Python27;%PATH%
 
   # Download and install sccache
-  - appveyor-retry appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-04-04-sccache-x86_64-pc-windows-msvc
-  - mv 2017-04-04-sccache-x86_64-pc-windows-msvc sccache.exe
+  - appveyor-retry appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-04-19-sccache-x86_64-pc-windows-msvc
+  - mv 2017-04-19-sccache-x86_64-pc-windows-msvc sccache.exe
   - set PATH=%PATH%;%CD%
 
   # Download and install ninja
   #
   # Note that this is originally from the github releases patch of Ninja
   - appveyor-retry appveyor DownloadFile https://s3.amazonaws.com/rust-lang-ci/rust-ci-mirror/2017-03-15-ninja-win.zip
   - 7z x 2017-03-15-ninja-win.zip
+  - set RUST_CONFIGURE_ARGS=%RUST_CONFIGURE_ARGS% --enable-ninja
   # - set PATH=%PATH%;%CD% -- this already happens above for sccache
 
   # Install InnoSetup to get `iscc` used to produce installers
@@ -159,12 +160,6 @@ test_script:
 on_failure:
   - cat %CD%\sccache.log || exit 0
 
-cache:
-  - "build/i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
-  - "build/x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
-  - "i686-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
-  - "x86_64-pc-windows-msvc/llvm -> src/rustllvm/llvm-rebuild-trigger"
-
 branches:
   only:
     - auto

src/Cargo.toml

@@ -15,6 +15,12 @@ members = [
   "tools/qemu-test-server",
 ]
 
+# These projects have their own Cargo.lock
+exclude = [
+  "tools/cargo",
+  "tools/rls",
+]
+
 # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit
 # MSVC when running the compile-fail test suite when a should-fail test panics.
 # But hey if this is removed and it gets past the bots, sounds good to me.

src/bootstrap/Cargo.toml

@@ -23,6 +23,11 @@ name = "rustdoc"
 path = "bin/rustdoc.rs"
 test = false
 
+[[bin]]
+name = "sccache-plus-cl"
+path = "bin/sccache-plus-cl.rs"
+test = false
+
 [dependencies]
 build_helper = { path = "../build_helper" }
 cmake = "0.1.17"

src/bootstrap/bin/sccache-plus-cl.rs

@@ -0,0 +1,43 @@
+// Copyright 2017 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+extern crate gcc;
+
+use std::env;
+use std::process::{self, Command};
+
+fn main() {
+    let target = env::var("SCCACHE_TARGET").unwrap();
+    // Locate the actual compiler that we're invoking
+    env::remove_var("CC");
+    env::remove_var("CXX");
+    let mut cfg = gcc::Config::new();
+    cfg.cargo_metadata(false)
+       .out_dir("/")
+       .target(&target)
+       .host(&target)
+       .opt_level(0)
+       .debug(false);
+    let compiler = cfg.get_compiler();
+
+    // Invoke sccache with said compiler
+    let sccache_path = env::var_os("SCCACHE_PATH").unwrap();
+    let mut cmd = Command::new(&sccache_path);
+    cmd.arg(compiler.path());
+    for &(ref k, ref v) in compiler.env() {
+        cmd.env(k, v);
+    }
+    for arg in env::args().skip(1) {
+        cmd.arg(arg);
+    }
+
+    let status = cmd.status().expect("failed to spawn");
+    process::exit(status.code().unwrap_or(2))
+}

src/bootstrap/bootstrap.py

@@ -159,40 +159,41 @@ def format_build_time(duration):
 class RustBuild(object):
     def download_stage0(self):
         cache_dst = os.path.join(self.build_dir, "cache")
-        rustc_cache = os.path.join(cache_dst, self.stage0_rustc_date())
+        rustc_cache = os.path.join(cache_dst, self.stage0_date())
         if not os.path.exists(rustc_cache):
             os.makedirs(rustc_cache)
 
-        channel = self.stage0_rustc_channel()
+        rustc_channel = self.stage0_rustc_channel()
+        cargo_channel = self.stage0_cargo_channel()
 
         if self.rustc().startswith(self.bin_root()) and \
                 (not os.path.exists(self.rustc()) or self.rustc_out_of_date()):
             self.print_what_it_means_to_bootstrap()
             if os.path.exists(self.bin_root()):
                 shutil.rmtree(self.bin_root())
-            filename = "rust-std-{}-{}.tar.gz".format(channel, self.build)
-            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
+            filename = "rust-std-{}-{}.tar.gz".format(rustc_channel, self.build)
+            url = self._download_url + "/dist/" + self.stage0_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(),
                    match="rust-std-" + self.build,
                    verbose=self.verbose)
 
-            filename = "rustc-{}-{}.tar.gz".format(channel, self.build)
-            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
+            filename = "rustc-{}-{}.tar.gz".format(rustc_channel, self.build)
+            url = self._download_url + "/dist/" + self.stage0_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(), match="rustc", verbose=self.verbose)
             self.fix_executable(self.bin_root() + "/bin/rustc")
             self.fix_executable(self.bin_root() + "/bin/rustdoc")
             with open(self.rustc_stamp(), 'w') as f:
-                f.write(self.stage0_rustc_date())
+                f.write(self.stage0_date())
 
             if "pc-windows-gnu" in self.build:
-                filename = "rust-mingw-{}-{}.tar.gz".format(channel, self.build)
-                url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
+                filename = "rust-mingw-{}-{}.tar.gz".format(rustc_channel, self.build)
+                url = self._download_url + "/dist/" + self.stage0_date()
                 tarball = os.path.join(rustc_cache, filename)
                 if not os.path.exists(tarball):
                     get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
@@ -201,15 +202,15 @@ def download_stage0(self):
         if self.cargo().startswith(self.bin_root()) and \
                 (not os.path.exists(self.cargo()) or self.cargo_out_of_date()):
             self.print_what_it_means_to_bootstrap()
-            filename = "cargo-{}-{}.tar.gz".format(channel, self.build)
-            url = "https://static.rust-lang.org/dist/" + self.stage0_rustc_date()
+            filename = "cargo-{}-{}.tar.gz".format(cargo_channel, self.build)
+            url = self._download_url + "/dist/" + self.stage0_date()
             tarball = os.path.join(rustc_cache, filename)
             if not os.path.exists(tarball):
                 get("{}/{}".format(url, filename), tarball, verbose=self.verbose)
             unpack(tarball, self.bin_root(), match="cargo", verbose=self.verbose)
             self.fix_executable(self.bin_root() + "/bin/cargo")
             with open(self.cargo_stamp(), 'w') as f:
-                f.write(self.stage0_rustc_date())
+                f.write(self.stage0_date())
 
     def fix_executable(self, fname):
         # If we're on NixOS we need to change the path to the dynamic loader
@@ -264,12 +265,15 @@ def fix_executable(self, fname):
             print("warning: failed to call patchelf: %s" % e)
             return
 
-    def stage0_rustc_date(self):
-        return self._rustc_date
+    def stage0_date(self):
+        return self._date
 
     def stage0_rustc_channel(self):
         return self._rustc_channel
 
+    def stage0_cargo_channel(self):
+        return self._cargo_channel
+
     def rustc_stamp(self):
         return os.path.join(self.bin_root(), '.rustc-stamp')
 
@@ -280,13 +284,13 @@ def rustc_out_of_date(self):
         if not os.path.exists(self.rustc_stamp()) or self.clean:
             return True
         with open(self.rustc_stamp(), 'r') as f:
-            return self.stage0_rustc_date() != f.read()
+            return self.stage0_date() != f.read()
 
     def cargo_out_of_date(self):
         if not os.path.exists(self.cargo_stamp()) or self.clean:
             return True
         with open(self.cargo_stamp(), 'r') as f:
-            return self.stage0_rustc_date() != f.read()
+            return self.stage0_date() != f.read()
 
     def bin_root(self):
         return os.path.join(self.build_dir, self.build, "stage0")
@@ -585,7 +589,13 @@ def bootstrap():
             shutil.rmtree('.cargo')
 
     data = stage0_data(rb.rust_root)
-    rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
+    rb._date = data['date']
+    rb._rustc_channel = data['rustc']
+    rb._cargo_channel = data['cargo']
+    if 'dev' in data:
+        rb._download_url = 'https://dev-static.rust-lang.org'
+    else:
+        rb._download_url = 'https://static.rust-lang.org'
 
     # Fetch/build the bootstrap
     rb.build = rb.build_triple()

src/bootstrap/channel.rs

@@ -23,7 +23,7 @@ use build_helper::output;
 use Build;
 
 // The version number
-pub const CFG_RELEASE_NUM: &'static str = "1.18.0";
+pub const CFG_RELEASE_NUM: &'static str = "1.19.0";
 
 // An optional number to put after the label, e.g. '.2' -> '-beta.2'
 // Be sure to make this starts with a dot to conform to semver pre-release

src/bootstrap/compile.rs

@@ -461,10 +461,7 @@ pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) {
     let compiler = Compiler::new(stage, &build.config.build);
 
     let mut cargo = build.cargo(&compiler, Mode::Tool, target, "build");
-    let mut dir = build.src.join(tool);
-    if !dir.exists() {
-        dir = build.src.join("src/tools").join(tool);
-    }
+    let dir = build.src.join("src/tools").join(tool);
     cargo.arg("--manifest-path").arg(dir.join("Cargo.toml"));
 
     // We don't want to build tools dynamically as they'll be running across

src/bootstrap/lib.rs

@@ -234,8 +234,8 @@ impl Build {
             None => false,
         };
         let rust_info = channel::GitInfo::new(&src);
-        let cargo_info = channel::GitInfo::new(&src.join("cargo"));
-        let rls_info = channel::GitInfo::new(&src.join("rls"));
+        let cargo_info = channel::GitInfo::new(&src.join("src/tools/cargo"));
+        let rls_info = channel::GitInfo::new(&src.join("src/tools/rls"));
         let src_is_git = src.join(".git").exists();
 
         Build {

src/bootstrap/native.rs

@@ -19,6 +19,7 @@
 //! ensure that they're always in place if needed.
 
 use std::env;
+use std::ffi::OsString;
 use std::fs::{self, File};
 use std::io::{Read, Write};
 use std::path::Path;
@@ -129,22 +130,56 @@ pub fn llvm(build: &Build, target: &str) {
            .define("LLVM_TABLEGEN", &host);
     }
 
-    // MSVC handles compiler business itself
-    if !target.contains("msvc") {
-        if let Some(ref ccache) = build.config.ccache {
+    let sanitize_cc = |cc: &Path| {
+        if target.contains("msvc") {
+            OsString::from(cc.to_str().unwrap().replace("\\", "/"))
+        } else {
+            cc.as_os_str().to_owned()
+        }
+    };
+
+    let configure_compilers = |cfg: &mut cmake::Config| {
+        // MSVC with CMake uses msbuild by default which doesn't respect these
+        // vars that we'd otherwise configure. In that case we just skip this
+        // entirely.
+        if target.contains("msvc") && !build.config.ninja {
+            return
+        }
+
+        let cc = build.cc(target);
+        let cxx = build.cxx(target);
+
+        // Handle msvc + ninja + ccache specially (this is what the bots use)
+        if target.contains("msvc") &&
+           build.config.ninja &&
+           build.config.ccache.is_some() {
+            let mut cc = env::current_exe().expect("failed to get cwd");
+            cc.set_file_name("sccache-plus-cl.exe");
+
+           cfg.define("CMAKE_C_COMPILER", sanitize_cc(&cc))
+              .define("CMAKE_CXX_COMPILER", sanitize_cc(&cc));
+           cfg.env("SCCACHE_PATH",
+                   build.config.ccache.as_ref().unwrap())
+              .env("SCCACHE_TARGET", target);
+
+        // If ccache is configured we inform the build a little differently hwo
+        // to invoke ccache while also invoking our compilers.
+        } else if let Some(ref ccache) = build.config.ccache {
            cfg.define("CMAKE_C_COMPILER", ccache)
-              .define("CMAKE_C_COMPILER_ARG1", build.cc(target))
+              .define("CMAKE_C_COMPILER_ARG1", sanitize_cc(cc))
               .define("CMAKE_CXX_COMPILER", ccache)
-              .define("CMAKE_CXX_COMPILER_ARG1", build.cxx(target));
+              .define("CMAKE_CXX_COMPILER_ARG1", sanitize_cc(cxx));
         } else {
-           cfg.define("CMAKE_C_COMPILER", build.cc(target))
-              .define("CMAKE_CXX_COMPILER", build.cxx(target));
+           cfg.define("CMAKE_C_COMPILER", sanitize_cc(cc))
+              .define("CMAKE_CXX_COMPILER", sanitize_cc(cxx));
         }
-        cfg.build_arg("-j").build_arg(build.jobs().to_string());
 
+        cfg.build_arg("-j").build_arg(build.jobs().to_string());
         cfg.define("CMAKE_C_FLAGS", build.cflags(target).join(" "));
         cfg.define("CMAKE_CXX_FLAGS", build.cflags(target).join(" "));
-    }
+    };
+
+    configure_compilers(&mut cfg);
 
     if env::var_os("SCCACHE_ERROR_LOG").is_some() {
         cfg.env("RUST_LOG", "sccache=info");