diff --git a/src/Cargo.lock b/src/Cargo.lock index 7db243c5eb9d0..93bbf0f227b1b 100644 --- a/src/Cargo.lock +++ b/src/Cargo.lock @@ -50,6 +50,14 @@ dependencies = [ "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "build-manifest" +version = "0.1.0" +dependencies = [ + "rustc-serialize 0.3.19 (registry+https://github.com/rust-lang/crates.io-index)", + "toml 0.1.30 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "build_helper" version = "0.1.0" diff --git a/src/Cargo.toml b/src/Cargo.toml index 8fb5c70c41bf1..0db26ea5ae021 100644 --- a/src/Cargo.toml +++ b/src/Cargo.toml @@ -10,6 +10,7 @@ members = [ "tools/linkchecker", "tools/rustbook", "tools/tidy", + "tools/build-manifest", ] # Curiously, compiletest will segfault if compiled with opt-level=3 on 64-bit diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 7d1abcfa6f6e7..e035f8157ffde 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -78,6 +78,11 @@ pub struct Config { pub cargo: Option, pub local_rebuild: bool, + // dist misc + pub dist_sign_folder: Option, + pub dist_upload_addr: Option, + pub dist_gpg_password_file: Option, + // libstd features pub debug_jemalloc: bool, pub use_jemalloc: bool, @@ -123,6 +128,7 @@ struct TomlConfig { llvm: Option, rust: Option, target: Option>, + dist: Option, } /// TOML representation of various global build decisions. @@ -166,6 +172,13 @@ struct Llvm { targets: Option, } +#[derive(RustcDecodable, Default, Clone)] +struct Dist { + sign_folder: Option, + gpg_password_file: Option, + upload_addr: Option, +} + #[derive(RustcDecodable)] enum StringOrBool { String(String), @@ -352,6 +365,12 @@ impl Config { } } + if let Some(ref t) = toml.dist { + config.dist_sign_folder = t.sign_folder.clone().map(PathBuf::from); + config.dist_gpg_password_file = t.gpg_password_file.clone().map(PathBuf::from); + config.dist_upload_addr = t.upload_addr.clone(); + } + return config } diff --git a/src/bootstrap/config.toml.example b/src/bootstrap/config.toml.example index 4b859482562d2..a53419ad7fd78 100644 --- a/src/bootstrap/config.toml.example +++ b/src/bootstrap/config.toml.example @@ -242,3 +242,33 @@ # that this option only makes sense for MUSL targets that produce statically # linked binaries #musl-root = "..." + +# ============================================================================= +# Distribution options +# +# These options are related to distribution, mostly for the Rust project itself. +# You probably won't need to concern yourself with any of these options +# ============================================================================= +[dist] + +# This is the folder of artifacts that the build system will sign. All files in +# this directory will be signed with the default gpg key using the system `gpg` +# binary. The `asc` and `sha256` files will all be output into the standard dist +# output folder (currently `build/dist`) +# +# This folder should be populated ahead of time before the build system is +# invoked. +#sign-folder = "path/to/folder/to/sign" + +# This is a file which contains the password of the default gpg key. This will +# be passed to `gpg` down the road when signing all files in `sign-folder` +# above. This should be stored in plaintext. +#gpg-password-file = "path/to/gpg/password" + +# The remote address that all artifacts will eventually be uploaded to. The +# build system generates manifests which will point to these urls, and for the +# manifests to be correct they'll have to have the right URLs encoded. +# +# Note that this address should not contain a trailing slash as file names will +# be appended to it. +#upload-addr = "https://example.com/folder" diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index e5f0505952318..71a5f313bbd26 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -22,7 +22,7 @@ use std::env; use std::fs::{self, File}; use std::io::{Read, Write}; use std::path::{PathBuf, Path}; -use std::process::Command; +use std::process::{Command, Stdio}; use build_helper::output; @@ -876,3 +876,34 @@ fn add_env(build: &Build, cmd: &mut Command, target: &str) { cmd.env("CFG_PLATFORM", "x86"); } } + +pub fn hash_and_sign(build: &Build) { + let compiler = Compiler::new(0, &build.config.build); + let mut cmd = build.tool_cmd(&compiler, "build-manifest"); + let sign = build.config.dist_sign_folder.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.sign-folder` in `config.toml`\n\n") + }); + let addr = build.config.dist_upload_addr.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.upload-addr` in `config.toml`\n\n") + }); + let file = build.config.dist_gpg_password_file.as_ref().unwrap_or_else(|| { + panic!("\n\nfailed to specify `dist.gpg-password-file` in `config.toml`\n\n") + }); + let mut pass = String::new(); + t!(t!(File::open(&file)).read_to_string(&mut pass)); + + let today = output(Command::new("date").arg("+%Y-%m-%d")); + + cmd.arg(sign); + cmd.arg(distdir(build)); + cmd.arg(today.trim()); + cmd.arg(package_vers(build)); + cmd.arg(addr); + + t!(fs::create_dir_all(distdir(build))); + + let mut child = t!(cmd.stdin(Stdio::piped()).spawn()); + t!(child.stdin.take().unwrap().write_all(pass.as_bytes())); + let status = t!(child.wait()); + assert!(status.success()); +} diff --git a/src/bootstrap/step.rs b/src/bootstrap/step.rs index 697b14c6050cc..3932a7cf8c563 100644 --- a/src/bootstrap/step.rs +++ b/src/bootstrap/step.rs @@ -513,6 +513,9 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { rules.build("tool-compiletest", "src/tools/compiletest") .dep(|s| s.name("libtest")) .run(move |s| compile::tool(build, s.stage, s.target, "compiletest")); + rules.build("tool-build-manifest", "src/tools/build-manifest") + .dep(|s| s.name("libstd")) + .run(move |s| compile::tool(build, s.stage, s.target, "build-manifest")); // ======================================================================== // Documentation targets @@ -633,6 +636,13 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules { .dep(|d| d.name("dist-cargo")) .run(move |s| dist::extended(build, s.stage, s.target)); + rules.dist("dist-sign", "hash-and-sign") + .host(true) + .only_build(true) + .only_host_build(true) + .dep(move |s| s.name("tool-build-manifest").target(&build.config.build).stage(0)) + .run(move |_| dist::hash_and_sign(build)); + rules.verify(); return rules; } diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml new file mode 100644 index 0000000000000..4b876753b1fea --- /dev/null +++ b/src/tools/build-manifest/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "build-manifest" +version = "0.1.0" +authors = ["Alex Crichton "] + +[dependencies] +toml = "0.1" +rustc-serialize = "0.3" diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs new file mode 100644 index 0000000000000..8c15a6630a33c --- /dev/null +++ b/src/tools/build-manifest/src/main.rs @@ -0,0 +1,404 @@ +// Copyright 2017 The Rust Project Developers. See the COPYRIGHT +// file at the top-level directory of this distribution and at +// http://rust-lang.org/COPYRIGHT. +// +// Licensed under the Apache License, Version 2.0 or the MIT license +// , at your +// option. This file may not be copied, modified, or distributed +// except according to those terms. + +extern crate toml; +extern crate rustc_serialize; + +use std::collections::HashMap; +use std::env; +use std::fs::File; +use std::io::{self, Read, Write}; +use std::path::{PathBuf, Path}; +use std::process::{Command, Stdio}; + +static HOSTS: &'static [&'static str] = &[ + "aarch64-unknown-linux-gnu", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "armv7-unknown-linux-gnueabihf", + "i686-apple-darwin", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-linux-gnu", + "mips-unknown-linux-gnu", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "x86_64-apple-darwin", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-netbsd", +]; + +static TARGETS: &'static [&'static str] = &[ + "aarch64-apple-ios", + "aarch64-linux-android", + "aarch64-unknown-linux-gnu", + "arm-linux-androideabi", + "arm-unknown-linux-gnueabi", + "arm-unknown-linux-gnueabihf", + "arm-unknown-linux-musleabi", + "arm-unknown-linux-musleabihf", + "armv7-apple-ios", + "armv7-linux-androideabi", + "armv7-unknown-linux-gnueabihf", + "armv7-unknown-linux-musleabihf", + "armv7s-apple-ios", + "asmjs-unknown-emscripten", + "i386-apple-ios", + "i586-pc-windows-msvc", + "i586-unknown-linux-gnu", + "i686-apple-darwin", + "i686-linux-android", + "i686-pc-windows-gnu", + "i686-pc-windows-msvc", + "i686-unknown-freebsd", + "i686-unknown-linux-gnu", + "i686-unknown-linux-musl", + "mips-unknown-linux-gnu", + "mips-unknown-linux-musl", + "mips64-unknown-linux-gnuabi64", + "mips64el-unknown-linux-gnuabi64", + "mipsel-unknown-linux-gnu", + "mipsel-unknown-linux-musl", + "powerpc-unknown-linux-gnu", + "powerpc64-unknown-linux-gnu", + "powerpc64le-unknown-linux-gnu", + "s390x-unknown-linux-gnu", + "wasm32-unknown-emscripten", + "x86_64-apple-darwin", + "x86_64-apple-ios", + "x86_64-pc-windows-gnu", + "x86_64-pc-windows-msvc", + "x86_64-rumprun-netbsd", + "x86_64-unknown-freebsd", + "x86_64-unknown-linux-gnu", + "x86_64-unknown-linux-musl", + "x86_64-unknown-netbsd", +]; + +static MINGW: &'static [&'static str] = &[ + "i686-pc-windows-gnu", + "x86_64-pc-windows-gnu", +]; + +#[derive(RustcEncodable)] +struct Manifest { + manifest_version: String, + date: String, + pkg: HashMap, +} + +#[derive(RustcEncodable)] +struct Package { + version: String, + target: HashMap, +} + +#[derive(RustcEncodable)] +struct Target { + available: bool, + url: Option, + hash: Option, + components: Option>, + extensions: Option>, +} + +#[derive(RustcEncodable)] +struct Component { + pkg: String, + target: String, +} + +macro_rules! t { + ($e:expr) => (match $e { + Ok(e) => e, + Err(e) => panic!("{} failed with {}", stringify!($e), e), + }) +} + +struct Builder { + channel: String, + input: PathBuf, + output: PathBuf, + gpg_passphrase: String, + digests: HashMap, + s3_address: String, + date: String, + rust_version: String, + cargo_version: String, +} + +fn main() { + let mut args = env::args().skip(1); + let input = PathBuf::from(args.next().unwrap()); + let output = PathBuf::from(args.next().unwrap()); + let date = args.next().unwrap(); + let channel = args.next().unwrap(); + let s3_address = args.next().unwrap(); + let mut passphrase = String::new(); + t!(io::stdin().read_to_string(&mut passphrase)); + + Builder { + channel: channel, + input: input, + output: output, + gpg_passphrase: passphrase, + digests: HashMap::new(), + s3_address: s3_address, + date: date, + rust_version: String::new(), + cargo_version: String::new(), + }.build(); +} + +impl Builder { + fn build(&mut self) { + self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu"); + self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu"); + + self.digest_and_sign(); + let manifest = self.build_manifest(); + let manifest = toml::encode(&manifest).to_string(); + + let filename = format!("channel-rust-{}.toml", self.channel); + self.write_manifest(&manifest, &filename); + + if self.channel != "beta" && self.channel != "nightly" { + self.write_manifest(&manifest, "channel-rust-stable.toml"); + } + } + + fn digest_and_sign(&mut self) { + for file in t!(self.input.read_dir()).map(|e| t!(e).path()) { + let filename = file.file_name().unwrap().to_str().unwrap(); + let digest = self.hash(&file); + self.sign(&file); + assert!(self.digests.insert(filename.to_string(), digest).is_none()); + } + } + + fn build_manifest(&mut self) -> Manifest { + let mut manifest = Manifest { + manifest_version: "2".to_string(), + date: self.date.to_string(), + pkg: HashMap::new(), + }; + + self.package("rustc", &mut manifest.pkg, HOSTS); + self.package("cargo", &mut manifest.pkg, HOSTS); + self.package("rust-mingw", &mut manifest.pkg, MINGW); + self.package("rust-std", &mut manifest.pkg, TARGETS); + self.package("rust-docs", &mut manifest.pkg, TARGETS); + self.package("rust-src", &mut manifest.pkg, &["*"]); + + let mut pkg = Package { + version: self.cached_version("rust").to_string(), + target: HashMap::new(), + }; + for host in HOSTS { + let filename = self.filename("rust", host); + let digest = match self.digests.remove(&filename) { + Some(digest) => digest, + None => { + pkg.target.insert(host.to_string(), Target { + available: false, + url: None, + hash: None, + components: None, + extensions: None, + }); + continue + } + }; + let mut components = Vec::new(); + let mut extensions = Vec::new(); + + // rustc/rust-std/cargo are all required, and so is rust-mingw if it's + // available for the target. + components.extend(vec![ + Component { pkg: "rustc".to_string(), target: host.to_string() }, + Component { pkg: "rust-std".to_string(), target: host.to_string() }, + Component { pkg: "cargo".to_string(), target: host.to_string() }, + ]); + if host.contains("pc-windows-gnu") { + components.push(Component { + pkg: "rust-mingw".to_string(), + target: host.to_string(), + }); + } + + // Docs, other standard libraries, and the source package are all + // optional. + extensions.push(Component { + pkg: "rust-docs".to_string(), + target: host.to_string(), + }); + for target in TARGETS { + if target != host { + extensions.push(Component { + pkg: "rust-std".to_string(), + target: target.to_string(), + }); + } + } + extensions.push(Component { + pkg: "rust-src".to_string(), + target: "*".to_string(), + }); + + pkg.target.insert(host.to_string(), Target { + available: true, + url: Some(self.url("rust", host)), + hash: Some(to_hex(digest.as_ref())), + components: Some(components), + extensions: Some(extensions), + }); + } + manifest.pkg.insert("rust".to_string(), pkg); + + return manifest + } + + fn package(&mut self, + pkgname: &str, + dst: &mut HashMap, + targets: &[&str]) { + let targets = targets.iter().map(|name| { + let filename = self.filename(pkgname, name); + let digest = match self.digests.remove(&filename) { + Some(digest) => digest, + None => { + return (name.to_string(), Target { + available: false, + url: None, + hash: None, + components: None, + extensions: None, + }) + } + }; + + (name.to_string(), Target { + available: true, + url: Some(self.url(pkgname, name)), + hash: Some(digest), + components: None, + extensions: None, + }) + }).collect(); + + dst.insert(pkgname.to_string(), Package { + version: self.cached_version(pkgname).to_string(), + target: targets, + }); + } + + fn url(&self, component: &str, target: &str) -> String { + format!("{}/{}/{}", + self.s3_address, + self.date, + self.filename(component, target)) + } + + fn filename(&self, component: &str, target: &str) -> String { + if component == "rust-src" { + format!("rust-src-{}.tar.gz", self.channel) + } else { + format!("{}-{}-{}.tar.gz", component, self.channel, target) + } + } + + fn cached_version(&self, component: &str) -> &str { + if component == "cargo" { + &self.cargo_version + } else { + &self.rust_version + } + } + + fn version(&self, component: &str, target: &str) -> String { + let mut cmd = Command::new("tar"); + let filename = self.filename(component, target); + cmd.arg("xf") + .arg(self.input.join(&filename)) + .arg(format!("{}/version", filename.replace(".tar.gz", ""))) + .arg("-O"); + let version = t!(cmd.output()); + if !version.status.success() { + panic!("failed to learn version:\n\n{:?}\n\n{}\n\n{}", + cmd, + String::from_utf8_lossy(&version.stdout), + String::from_utf8_lossy(&version.stderr)); + } + String::from_utf8_lossy(&version.stdout).trim().to_string() + } + + fn hash(&self, path: &Path) -> String { + let sha = t!(Command::new("shasum") + .arg("-a").arg("256") + .arg(path) + .output()); + assert!(sha.status.success()); + + let filename = path.file_name().unwrap().to_str().unwrap(); + let sha256 = self.output.join(format!("{}.sha256", filename)); + t!(t!(File::create(&sha256)).write_all(&sha.stdout)); + + let stdout = String::from_utf8_lossy(&sha.stdout); + stdout.split_whitespace().next().unwrap().to_string() + } + + fn sign(&self, path: &Path) { + let filename = path.file_name().unwrap().to_str().unwrap(); + let asc = self.output.join(format!("{}.asc", filename)); + println!("signing: {:?}", path); + let mut cmd = Command::new("gpg"); + cmd.arg("--no-tty") + .arg("--yes") + .arg("--passphrase-fd").arg("0") + .arg("--armor") + .arg("--output").arg(&asc) + .arg("--detach-sign").arg(path) + .stdin(Stdio::piped()); + let mut child = t!(cmd.spawn()); + t!(child.stdin.take().unwrap().write_all(self.gpg_passphrase.as_bytes())); + assert!(t!(child.wait()).success()); + } + + fn write_manifest(&self, manifest: &str, name: &str) { + let dst = self.output.join(name); + t!(t!(File::create(&dst)).write_all(manifest.as_bytes())); + self.hash(&dst); + self.sign(&dst); + } +} + +fn to_hex(digest: &[u8]) -> String { + let mut ret = String::new(); + for byte in digest { + ret.push(hex((byte & 0xf0) >> 4)); + ret.push(hex(byte & 0xf)); + } + return ret; + + fn hex(b: u8) -> char { + match b { + 0...9 => (b'0' + b) as char, + _ => (b'a' + b - 10) as char, + } + } +}