rustbuild: Add support for compiling Cargo

This commit adds support to rustbuild for compiling Cargo as part of the release
process. Previously rustbuild would simply download a Cargo snapshot and
repackage it. With this change we should be able to turn off artifacts from the
rust-lang/cargo repository and purely rely on the artifacts Cargo produces here.

The infrastructure added here is intended to be extensible to other components,
such as the RLS. It won't exactly be a one-line addition, but the addition of
Cargo didn't require too much hooplah anyway.

The process for release Cargo will now look like:

* The rust-lang/rust repository has a Cargo submodule which is used to build a
  Cargo to pair with the rust-lang/rust release
* Periodically we'll update the cargo submodule as necessary on rust-lang/rust's
  master branch
* When branching beta we'll create a new branch of Cargo (as we do today), and
  the first commit to the beta branch will be to update the Cargo submodule to
  this exact revision.
* When branching stable, we'll ensure that the Cargo submodule is updated and
  then make a stable release.

Backports to Cargo will look like:

* Send a PR to cargo's master branch
* Send a PR to cargo's release branch (e.g. rust-1.16.0)
* Send a PR to rust-lang/rust's beta branch updating the submodule
* Eventually send a PR to rust-lang/rust's master branch updating the submodule

For reference, the process to add a new component to the rust-lang/rust release
would look like:

* Add `$foo` as a submodule in `src/tools`
* Add a `tool-$foo` step which compiles `$foo` with the specified compiler,
  likely mirroring what Cargo does.
* Add a `dist-$foo` step which uses `src/tools/$foo` and the `tool-$foo` output
  to create a rust-installer package for `$foo` likely mirroring what Cargo
  does.
* Update the `dist-extended` step with a new dependency on `dist-$foo`
* Update `src/tools/build-manifest` for the new component.

configure

@@ -651,6 +651,7 @@ opt locked-deps 0 "force Cargo.lock to be up to date"
 opt vendor 0 "enable usage of vendored Rust crates"
 opt sanitizers 0 "build the sanitizer runtimes (asan, lsan, msan, tsan)"
 opt dist-src 1 "when building tarballs enables building a source tarball"
+opt cargo-openssl-static 0 "static openssl in cargo"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"

src/bootstrap/channel.rs

@@ -15,72 +15,77 @@
 //! `package_vers`, and otherwise indicating to the compiler what it should
 //! print out as part of its version information.
 
+use std::path::Path;
 use std::process::Command;
 
 use build_helper::output;
 
 use Build;
 
 // The version number
-const CFG_RELEASE_NUM: &'static str = "1.17.0";
+pub const CFG_RELEASE_NUM: &'static str = "1.17.0";
 
 // An optional number to put after the label, e.g. '.2' -> '-beta.2'
 // Be sure to make this starts with a dot to conform to semver pre-release
 // versions (section 9)
-const CFG_PRERELEASE_VERSION: &'static str = ".1";
+pub const CFG_PRERELEASE_VERSION: &'static str = ".1";
 
-pub fn collect(build: &mut Build) {
-    build.release_num = CFG_RELEASE_NUM.to_string();
-    build.prerelease_version = CFG_RELEASE_NUM.to_string();
+pub struct GitInfo {
+    inner: Option<Info>,
+}
 
-    // Depending on the channel, passed in `./configure --release-channel`,
-    // determine various properties of the build.
-    match &build.config.channel[..] {
-        "stable" => {
-            build.release = CFG_RELEASE_NUM.to_string();
-            build.package_vers = build.release.clone();
-            build.unstable_features = false;
-        }
-        "beta" => {
-            build.release = format!("{}-beta{}", CFG_RELEASE_NUM,
-                                   CFG_PRERELEASE_VERSION);
-            build.package_vers = "beta".to_string();
-            build.unstable_features = false;
-        }
-        "nightly" => {
-            build.release = format!("{}-nightly", CFG_RELEASE_NUM);
-            build.package_vers = "nightly".to_string();
-            build.unstable_features = true;
-        }
-        _ => {
-            build.release = format!("{}-dev", CFG_RELEASE_NUM);
-            build.package_vers = build.release.clone();
-            build.unstable_features = true;
-        }
-    }
-    build.version = build.release.clone();
+struct Info {
+    commit_date: String,
+    sha: String,
+    short_sha: String,
+}
 
-    // If we have a git directory, add in some various SHA information of what
-    // commit this compiler was compiled from.
-    if build.src.join(".git").is_dir() {
-        let ver_date = output(Command::new("git").current_dir(&build.src)
+impl GitInfo {
+    pub fn new(dir: &Path) -> GitInfo {
+        if !dir.join(".git").exists() {
+            return GitInfo { inner: None }
+        }
+        let ver_date = output(Command::new("git").current_dir(dir)
                                       .arg("log").arg("-1")
                                       .arg("--date=short")
                                       .arg("--pretty=format:%cd"));
-        let ver_hash = output(Command::new("git").current_dir(&build.src)
+        let ver_hash = output(Command::new("git").current_dir(dir)
                                       .arg("rev-parse").arg("HEAD"));
         let short_ver_hash = output(Command::new("git")
-                                            .current_dir(&build.src)
+                                            .current_dir(dir)
                                             .arg("rev-parse")
                                             .arg("--short=9")
                                             .arg("HEAD"));
-        let ver_date = ver_date.trim().to_string();
-        let ver_hash = ver_hash.trim().to_string();
-        let short_ver_hash = short_ver_hash.trim().to_string();
-        build.version.push_str(&format!(" ({} {})", short_ver_hash,
-                                       ver_date));
-        build.ver_date = Some(ver_date.to_string());
-        build.ver_hash = Some(ver_hash);
-        build.short_ver_hash = Some(short_ver_hash);
+        GitInfo {
+            inner: Some(Info {
+                commit_date: ver_date.trim().to_string(),
+                sha: ver_hash.trim().to_string(),
+                short_sha: short_ver_hash.trim().to_string(),
+            }),
+        }
+    }
+
+    pub fn sha(&self) -> Option<&str> {
+        self.inner.as_ref().map(|s| &s.sha[..])
+    }
+
+    pub fn sha_short(&self) -> Option<&str> {
+        self.inner.as_ref().map(|s| &s.short_sha[..])
+    }
+
+    pub fn commit_date(&self) -> Option<&str> {
+        self.inner.as_ref().map(|s| &s.commit_date[..])
+    }
+
+    pub fn version(&self, build: &Build, num: &str) -> String {
+        let mut version = build.release(num);
+        if let Some(ref inner) = self.inner {
+            version.push_str(" (");
+            version.push_str(&inner.short_sha);
+            version.push_str(" ");
+            version.push_str(&inner.commit_date);
+            version.push_str(")");
+        }
+        return version
     }
 }

src/bootstrap/compile.rs

@@ -24,6 +24,7 @@ use std::process::Command;
 use build_helper::{output, mtime, up_to_date};
 use filetime::FileTime;
 
+use channel::GitInfo;
 use util::{exe, libdir, is_dylib, copy};
 use {Build, Compiler, Mode};
 
@@ -210,9 +211,9 @@ pub fn rustc(build: &Build, target: &str, compiler: &Compiler) {
 
     // Set some configuration variables picked up by build scripts and
     // the compiler alike
-    cargo.env("CFG_RELEASE", &build.release)
+    cargo.env("CFG_RELEASE", build.rust_release())
          .env("CFG_RELEASE_CHANNEL", &build.config.channel)
-         .env("CFG_VERSION", &build.version)
+         .env("CFG_VERSION", build.rust_version())
          .env("CFG_PREFIX", build.config.prefix.clone().unwrap_or(PathBuf::new()));
 
     if compiler.stage == 0 {
@@ -229,13 +230,13 @@ pub fn rustc(build: &Build, target: &str, compiler: &Compiler) {
         cargo.env_remove("RUSTC_DEBUGINFO_LINES");
     }
 
-    if let Some(ref ver_date) = build.ver_date {
+    if let Some(ref ver_date) = build.rust_info.commit_date() {
         cargo.env("CFG_VER_DATE", ver_date);
     }
-    if let Some(ref ver_hash) = build.ver_hash {
+    if let Some(ref ver_hash) = build.rust_info.sha() {
         cargo.env("CFG_VER_HASH", ver_hash);
     }
-    if !build.unstable_features {
+    if !build.unstable_features() {
         cargo.env("CFG_DISABLE_UNSTABLE_FEATURES", "1");
     }
     // Flag that rust llvm is in use
@@ -416,13 +417,32 @@ pub fn tool(build: &Build, stage: u32, target: &str, tool: &str) {
     // build.clear_if_dirty(&out_dir, &libstd_stamp(build, stage, &host, target));
 
     let mut cargo = build.cargo(&compiler, Mode::Tool, target, "build");
-    cargo.arg("--manifest-path")
-         .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool)));
+    let dir = build.src.join("src/tools").join(tool);
+    cargo.arg("--manifest-path").arg(dir.join("Cargo.toml"));
 
     // We don't want to build tools dynamically as they'll be running across
     // stages and such and it's just easier if they're not dynamically linked.
     cargo.env("RUSTC_NO_PREFER_DYNAMIC", "1");
 
+    if let Some(dir) = build.openssl_install_dir(target) {
+        cargo.env("OPENSSL_STATIC", "1");
+        cargo.env("OPENSSL_DIR", dir);
+        cargo.env("LIBZ_SYS_STATIC", "1");
+    }
+
+    cargo.env("CFG_RELEASE_CHANNEL", &build.config.channel);
+
+    let info = GitInfo::new(&dir);
+    if let Some(sha) = info.sha() {
+        cargo.env("CFG_COMMIT_HASH", sha);
+    }
+    if let Some(sha_short) = info.sha_short() {
+        cargo.env("CFG_SHORT_COMMIT_HASH", sha_short);
+    }
+    if let Some(date) = info.commit_date() {
+        cargo.env("CFG_COMMIT_DATE", date);
+    }
+
     build.run(&mut cargo);
 }
 

src/bootstrap/config.rs

@@ -106,6 +106,7 @@ pub struct Config {
     pub gdb: Option<PathBuf>,
     pub python: Option<PathBuf>,
     pub configure_args: Vec<String>,
+    pub openssl_static: bool,
 }
 
 /// Per-target configuration stored in the global configuration structure.
@@ -155,6 +156,7 @@ struct Build {
     extended: Option<bool>,
     verbose: Option<usize>,
     sanitizers: Option<bool>,
+    openssl_static: Option<bool>,
 }
 
 /// TOML representation of various global install decisions.
@@ -305,6 +307,7 @@ impl Config {
         set(&mut config.extended, build.extended);
         set(&mut config.verbose, build.verbose);
         set(&mut config.sanitizers, build.sanitizers);
+        set(&mut config.openssl_static, build.openssl_static);
 
         if let Some(ref install) = toml.install {
             config.prefix = install.prefix.clone().map(PathBuf::from);
@@ -453,6 +456,7 @@ impl Config {
                 ("EXTENDED", self.extended),
                 ("SANITIZERS", self.sanitizers),
                 ("DIST_SRC", self.rust_dist_src),
+                ("CARGO_OPENSSL_STATIC", self.openssl_static),
             }
 
             match key {

src/bootstrap/config.toml.example

@@ -134,6 +134,11 @@
 # Build the sanitizer runtimes
 #sanitizers = false
 
+# Indicates whether the OpenSSL linked into Cargo will be statically linked or
+# not. If static linkage is specified then the build system will download a
+# known-good version of OpenSSL, compile it, and link it to Cargo.
+#openssl-static = false
+
 # =============================================================================
 # General install configuration options
 # =============================================================================