diff --git a/Cargo.lock b/Cargo.lock index 26a9e64b85af4..277608a9cc99f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -231,8 +231,11 @@ dependencies = [ name = "build-manifest" version = "0.1.0" dependencies = [ + "anyhow", + "flate2", "serde", "serde_json", + "tar", "toml", ] diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index c119ae38fc391..b9bc7d246dbd4 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2368,15 +2368,9 @@ impl Step for HashSign { cmd.arg(sign); cmd.arg(distdir(builder)); cmd.arg(today.trim()); - cmd.arg(builder.rust_package_vers()); cmd.arg(addr); - cmd.arg(builder.package_vers(&builder.release_num("cargo"))); - cmd.arg(builder.package_vers(&builder.release_num("rls"))); - cmd.arg(builder.package_vers(&builder.release_num("rust-analyzer/crates/rust-analyzer"))); - cmd.arg(builder.package_vers(&builder.release_num("clippy"))); - cmd.arg(builder.package_vers(&builder.release_num("miri"))); - cmd.arg(builder.package_vers(&builder.release_num("rustfmt"))); - cmd.arg(builder.llvm_tools_package_vers()); + cmd.arg(&builder.config.channel); + cmd.arg(&builder.src); builder.create_dir(&distdir(builder)); diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml index 0bbbabd29989e..4f89c31936dda 100644 --- a/src/tools/build-manifest/Cargo.toml +++ b/src/tools/build-manifest/Cargo.toml @@ -8,3 +8,6 @@ edition = "2018" toml = "0.5" serde = { version = "1.0", features = ["derive"] } serde_json = "1.0" +anyhow = "1.0.32" +flate2 = "1.0.16" +tar = "0.4.29" diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index a80f36d496998..4d7d9f7da1874 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -21,10 +21,9 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to ``` $ BUILD_MANIFEST_DISABLE_SIGNING=1 cargo +nightly run \ - path/to/dist path/to/output 1970-01-01 \ - nightly nightly nightly nightly nightly nightly nightly nightly \ - http://example.com + path/to/dist path/to/output 1970-01-01 http://example.com \ + CHANNEL path/to/rust/repo ``` -In the future, if the tool complains about missing arguments just add more -`nightly`s in the middle. +Remember to replace `CHANNEL` with the channel you produced dist artifacts of +and `path/to/rust/repo` with the path to your checkout of the Rust repository. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 125f2d1f76cce..be3e862e7aecd 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -4,8 +4,10 @@ //! via `x.py dist hash-and-sign`; the cmdline arguments are set up //! by rustbuild (in `src/bootstrap/dist.rs`). -use serde::Serialize; +mod versions; +use crate::versions::{PkgType, Versions}; +use serde::Serialize; use std::collections::BTreeMap; use std::collections::HashMap; use std::env; @@ -226,14 +228,7 @@ macro_rules! t { } struct Builder { - rust_release: String, - cargo_release: String, - rls_release: String, - rust_analyzer_release: String, - clippy_release: String, - rustfmt_release: String, - llvm_tools_release: String, - miri_release: String, + versions: Versions, input: PathBuf, output: PathBuf, @@ -242,24 +237,6 @@ struct Builder { s3_address: String, date: String, - rust_version: Option, - cargo_version: Option, - rls_version: Option, - rust_analyzer_version: Option, - clippy_version: Option, - rustfmt_version: Option, - llvm_tools_version: Option, - miri_version: Option, - - rust_git_commit_hash: Option, - cargo_git_commit_hash: Option, - rls_git_commit_hash: Option, - rust_analyzer_git_commit_hash: Option, - clippy_git_commit_hash: Option, - rustfmt_git_commit_hash: Option, - llvm_tools_git_commit_hash: Option, - miri_git_commit_hash: Option, - should_sign: bool, } @@ -280,15 +257,9 @@ fn main() { let input = PathBuf::from(args.next().unwrap()); let output = PathBuf::from(args.next().unwrap()); let date = args.next().unwrap(); - let rust_release = args.next().unwrap(); let s3_address = args.next().unwrap(); - let cargo_release = args.next().unwrap(); - let rls_release = args.next().unwrap(); - let rust_analyzer_release = args.next().unwrap(); - let clippy_release = args.next().unwrap(); - let miri_release = args.next().unwrap(); - let rustfmt_release = args.next().unwrap(); - let llvm_tools_release = args.next().unwrap(); + let channel = args.next().unwrap(); + let monorepo_path = args.next().unwrap(); // Do not ask for a passphrase while manually testing let mut passphrase = String::new(); @@ -298,14 +269,7 @@ fn main() { } Builder { - rust_release, - cargo_release, - rls_release, - rust_analyzer_release, - clippy_release, - rustfmt_release, - llvm_tools_release, - miri_release, + versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(), input, output, @@ -314,87 +278,21 @@ fn main() { s3_address, date, - rust_version: None, - cargo_version: None, - rls_version: None, - rust_analyzer_version: None, - clippy_version: None, - rustfmt_version: None, - llvm_tools_version: None, - miri_version: None, - - rust_git_commit_hash: None, - cargo_git_commit_hash: None, - rls_git_commit_hash: None, - rust_analyzer_git_commit_hash: None, - clippy_git_commit_hash: None, - rustfmt_git_commit_hash: None, - llvm_tools_git_commit_hash: None, - miri_git_commit_hash: None, - should_sign, } .build(); } -enum PkgType { - RustSrc, - Cargo, - Rls, - RustAnalyzer, - Clippy, - Rustfmt, - LlvmTools, - Miri, - Other, -} - -impl PkgType { - fn from_component(component: &str) -> Self { - use PkgType::*; - match component { - "rust-src" => RustSrc, - "cargo" => Cargo, - "rls" | "rls-preview" => Rls, - "rust-analyzer" | "rust-analyzer-preview" => RustAnalyzer, - "clippy" | "clippy-preview" => Clippy, - "rustfmt" | "rustfmt-preview" => Rustfmt, - "llvm-tools" | "llvm-tools-preview" => LlvmTools, - "miri" | "miri-preview" => Miri, - _ => Other, - } - } -} - impl Builder { fn build(&mut self) { - self.rust_version = self.version("rust", "x86_64-unknown-linux-gnu"); - self.cargo_version = self.version("cargo", "x86_64-unknown-linux-gnu"); - self.rls_version = self.version("rls", "x86_64-unknown-linux-gnu"); - self.rust_analyzer_version = self.version("rust-analyzer", "x86_64-unknown-linux-gnu"); - self.clippy_version = self.version("clippy", "x86_64-unknown-linux-gnu"); - self.rustfmt_version = self.version("rustfmt", "x86_64-unknown-linux-gnu"); - self.llvm_tools_version = self.version("llvm-tools", "x86_64-unknown-linux-gnu"); - self.miri_version = self.version("miri", "x86_64-unknown-linux-gnu"); - - self.rust_git_commit_hash = self.git_commit_hash("rust", "x86_64-unknown-linux-gnu"); - self.cargo_git_commit_hash = self.git_commit_hash("cargo", "x86_64-unknown-linux-gnu"); - self.rls_git_commit_hash = self.git_commit_hash("rls", "x86_64-unknown-linux-gnu"); - self.rust_analyzer_git_commit_hash = - self.git_commit_hash("rust-analyzer", "x86_64-unknown-linux-gnu"); - self.clippy_git_commit_hash = self.git_commit_hash("clippy", "x86_64-unknown-linux-gnu"); - self.rustfmt_git_commit_hash = self.git_commit_hash("rustfmt", "x86_64-unknown-linux-gnu"); - self.llvm_tools_git_commit_hash = - self.git_commit_hash("llvm-tools", "x86_64-unknown-linux-gnu"); - self.miri_git_commit_hash = self.git_commit_hash("miri", "x86_64-unknown-linux-gnu"); - self.check_toolstate(); self.digest_and_sign(); let manifest = self.build_manifest(); - self.write_channel_files(&self.rust_release, &manifest); - if self.rust_release != "beta" && self.rust_release != "nightly" { - self.write_channel_files("stable", &manifest); + let rust_version = self.versions.package_version(&PkgType::Rust).unwrap(); + self.write_channel_files(self.versions.channel(), &manifest); + if self.versions.channel() != rust_version { + self.write_channel_files(&rust_version, &manifest); } } @@ -415,8 +313,7 @@ impl Builder { // Mark some tools as missing based on toolstate. if toolstates.get("miri").map(|s| &*s as &str) != Some("test-pass") { println!("Miri tests are not passing, removing component"); - self.miri_version = None; - self.miri_git_commit_hash = None; + self.versions.disable_version(&PkgType::Miri); } } @@ -501,7 +398,7 @@ impl Builder { // The compiler libraries are not stable for end users, and they're also huge, so we only // `rustc-dev` for nightly users, and only in the "complete" profile. It's still possible // for users to install the additional component manually, if needed. - if self.rust_release == "nightly" { + if self.versions.channel() == "nightly" { self.extend_profile("complete", &mut manifest.profiles, &["rustc-dev"]); self.extend_profile("complete", &mut manifest.profiles, &["rustc-docs"]); } @@ -518,13 +415,10 @@ impl Builder { } fn rust_package(&mut self, manifest: &Manifest) -> Package { + let version_info = self.versions.version(&PkgType::Rust).expect("missing Rust tarball"); let mut pkg = Package { - version: self - .cached_version("rust") - .as_ref() - .expect("Couldn't find Rust version") - .clone(), - git_commit_hash: self.cached_git_commit_hash("rust").clone(), + version: version_info.version.expect("missing Rust version"), + git_commit_hash: version_info.git_commit, target: BTreeMap::new(), }; for host in HOSTS { @@ -539,7 +433,7 @@ impl Builder { } fn target_host_combination(&mut self, host: &str, manifest: &Manifest) -> Option { - let filename = self.filename("rust", host); + let filename = self.versions.tarball_name(&PkgType::Rust, host).unwrap(); let digest = self.digests.remove(&filename)?; let xz_filename = filename.replace(".tar.gz", ".tar.xz"); let xz_digest = self.digests.remove(&xz_filename); @@ -630,15 +524,14 @@ impl Builder { } fn package(&mut self, pkgname: &str, dst: &mut BTreeMap, targets: &[&str]) { - let (version, mut is_present) = self - .cached_version(pkgname) - .as_ref() - .cloned() - .map(|version| (version, true)) - .unwrap_or_default(); // `is_present` defaults to `false` here. + let version_info = self + .versions + .version(&PkgType::from_component(pkgname)) + .expect("failed to load package version"); + let mut is_present = version_info.present; // Never ship nightly-only components for other trains. - if self.rust_release != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { + if self.versions.channel() != "nightly" && NIGHTLY_ONLY_COMPONENTS.contains(&pkgname) { is_present = false; // Pretend the component is entirely missing. } @@ -647,7 +540,10 @@ impl Builder { .map(|name| { if is_present { // The component generally exists, but it might still be missing for this target. - let filename = self.filename(pkgname, name); + let filename = self + .versions + .tarball_name(&PkgType::from_component(pkgname), name) + .unwrap(); let digest = match self.digests.remove(&filename) { Some(digest) => digest, // This component does not exist for this target -- skip it. @@ -679,8 +575,8 @@ impl Builder { dst.insert( pkgname.to_string(), Package { - version, - git_commit_hash: self.cached_git_commit_hash(pkgname).clone(), + version: version_info.version.unwrap_or_default(), + git_commit_hash: version_info.git_commit, target: targets, }, ); @@ -690,77 +586,6 @@ impl Builder { format!("{}/{}/{}", self.s3_address, self.date, filename) } - fn filename(&self, component: &str, target: &str) -> String { - use PkgType::*; - match PkgType::from_component(component) { - RustSrc => format!("rust-src-{}.tar.gz", self.rust_release), - Cargo => format!("cargo-{}-{}.tar.gz", self.cargo_release, target), - Rls => format!("rls-{}-{}.tar.gz", self.rls_release, target), - RustAnalyzer => { - format!("rust-analyzer-{}-{}.tar.gz", self.rust_analyzer_release, target) - } - Clippy => format!("clippy-{}-{}.tar.gz", self.clippy_release, target), - Rustfmt => format!("rustfmt-{}-{}.tar.gz", self.rustfmt_release, target), - LlvmTools => format!("llvm-tools-{}-{}.tar.gz", self.llvm_tools_release, target), - Miri => format!("miri-{}-{}.tar.gz", self.miri_release, target), - Other => format!("{}-{}-{}.tar.gz", component, self.rust_release, target), - } - } - - fn cached_version(&self, component: &str) -> &Option { - use PkgType::*; - match PkgType::from_component(component) { - Cargo => &self.cargo_version, - Rls => &self.rls_version, - RustAnalyzer => &self.rust_analyzer_version, - Clippy => &self.clippy_version, - Rustfmt => &self.rustfmt_version, - LlvmTools => &self.llvm_tools_version, - Miri => &self.miri_version, - _ => &self.rust_version, - } - } - - fn cached_git_commit_hash(&self, component: &str) -> &Option { - use PkgType::*; - match PkgType::from_component(component) { - Cargo => &self.cargo_git_commit_hash, - Rls => &self.rls_git_commit_hash, - RustAnalyzer => &self.rust_analyzer_git_commit_hash, - Clippy => &self.clippy_git_commit_hash, - Rustfmt => &self.rustfmt_git_commit_hash, - LlvmTools => &self.llvm_tools_git_commit_hash, - Miri => &self.miri_git_commit_hash, - _ => &self.rust_git_commit_hash, - } - } - - fn version(&self, component: &str, target: &str) -> Option { - self.untar(component, target, |filename| format!("{}/version", filename)) - } - - fn git_commit_hash(&self, component: &str, target: &str) -> Option { - self.untar(component, target, |filename| format!("{}/git-commit-hash", filename)) - } - - fn untar(&self, component: &str, target: &str, dir: F) -> Option - where - F: FnOnce(String) -> String, - { - let mut cmd = Command::new("tar"); - let filename = self.filename(component, target); - cmd.arg("xf") - .arg(self.input.join(&filename)) - .arg(dir(filename.replace(".tar.gz", ""))) - .arg("-O"); - let output = t!(cmd.output()); - if output.status.success() { - Some(String::from_utf8_lossy(&output.stdout).trim().to_string()) - } else { - None - } - } - fn hash(&self, path: &Path) -> String { let sha = t!(Command::new("shasum") .arg("-a") diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs new file mode 100644 index 0000000000000..0721e6a336eda --- /dev/null +++ b/src/tools/build-manifest/src/versions.rs @@ -0,0 +1,248 @@ +use anyhow::{Context, Error}; +use flate2::read::GzDecoder; +use std::collections::HashMap; +use std::fs::File; +use std::io::Read; +use std::path::{Path, PathBuf}; +use tar::Archive; + +const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; + +#[derive(Debug, Hash, Eq, PartialEq, Clone)] +pub(crate) enum PkgType { + Rust, + RustSrc, + Cargo, + Rls, + RustAnalyzer, + Clippy, + Rustfmt, + LlvmTools, + Miri, + Other(String), +} + +impl PkgType { + pub(crate) fn from_component(component: &str) -> Self { + match component { + "rust" => PkgType::Rust, + "rust-src" => PkgType::RustSrc, + "cargo" => PkgType::Cargo, + "rls" | "rls-preview" => PkgType::Rls, + "rust-analyzer" | "rust-analyzer-preview" => PkgType::RustAnalyzer, + "clippy" | "clippy-preview" => PkgType::Clippy, + "rustfmt" | "rustfmt-preview" => PkgType::Rustfmt, + "llvm-tools" | "llvm-tools-preview" => PkgType::LlvmTools, + "miri" | "miri-preview" => PkgType::Miri, + other => PkgType::Other(other.into()), + } + } + + /// The directory containing the `Cargo.toml` of this component inside the monorepo, to + /// retrieve the source code version. If `None` is returned Rust's version will be used. + fn rust_monorepo_path(&self) -> Option<&'static str> { + match self { + PkgType::Cargo => Some("src/tools/cargo"), + PkgType::Rls => Some("src/tools/rls"), + PkgType::RustAnalyzer => Some("src/tools/rust-analyzer/crates/rust-analyzer"), + PkgType::Clippy => Some("src/tools/clippy"), + PkgType::Rustfmt => Some("src/tools/rustfmt"), + PkgType::Miri => Some("src/tools/miri"), + PkgType::Rust => None, + PkgType::RustSrc => None, + PkgType::LlvmTools => None, + PkgType::Other(_) => None, + } + } + + /// First part of the tarball name. + fn tarball_component_name(&self) -> &str { + match self { + PkgType::Rust => "rust", + PkgType::RustSrc => "rust-src", + PkgType::Cargo => "cargo", + PkgType::Rls => "rls", + PkgType::RustAnalyzer => "rust-analyzer", + PkgType::Clippy => "clippy", + PkgType::Rustfmt => "rustfmt", + PkgType::LlvmTools => "llvm-tools", + PkgType::Miri => "miri", + PkgType::Other(component) => component, + } + } + + /// Whether this package has the same version as Rust itself, or has its own `version` and + /// `git-commit-hash` files inside the tarball. + fn should_use_rust_version(&self) -> bool { + match self { + PkgType::Cargo => false, + PkgType::Rls => false, + PkgType::RustAnalyzer => false, + PkgType::Clippy => false, + PkgType::Rustfmt => false, + PkgType::LlvmTools => false, + PkgType::Miri => false, + + PkgType::Rust => true, + PkgType::RustSrc => true, + PkgType::Other(_) => true, + } + } +} + +#[derive(Debug, Default, Clone)] +pub(crate) struct VersionInfo { + pub(crate) version: Option, + pub(crate) git_commit: Option, + pub(crate) present: bool, +} + +pub(crate) struct Versions { + channel: String, + rustc_version: String, + monorepo_root: PathBuf, + dist_path: PathBuf, + package_versions: HashMap, + versions: HashMap, +} + +impl Versions { + pub(crate) fn new( + channel: &str, + dist_path: &Path, + monorepo_root: &Path, + ) -> Result { + Ok(Self { + channel: channel.into(), + rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version")) + .context("failed to read the rustc version from src/version")? + .trim() + .to_string(), + monorepo_root: monorepo_root.into(), + dist_path: dist_path.into(), + package_versions: HashMap::new(), + versions: HashMap::new(), + }) + } + + pub(crate) fn channel(&self) -> &str { + &self.channel + } + + pub(crate) fn version(&mut self, mut package: &PkgType) -> Result { + if package.should_use_rust_version() { + package = &PkgType::Rust; + } + + match self.versions.get(package) { + Some(version) => Ok(version.clone()), + None => { + let version_info = self.load_version_from_tarball(package)?; + self.versions.insert(package.clone(), version_info.clone()); + println!("{:?} => {:?}", package, &version_info); + Ok(version_info) + } + } + } + + fn load_version_from_tarball(&mut self, package: &PkgType) -> Result { + let tarball_name = self.tarball_name(package, DEFAULT_TARGET)?; + let tarball = self.dist_path.join(tarball_name); + + let file = match File::open(&tarball) { + Ok(file) => file, + Err(err) if err.kind() == std::io::ErrorKind::NotFound => { + // Missing tarballs do not return an error, but return empty data. + return Ok(VersionInfo::default()); + } + Err(err) => return Err(err.into()), + }; + let mut tar = Archive::new(GzDecoder::new(file)); + + let mut version = None; + let mut git_commit = None; + for entry in tar.entries()? { + let mut entry = entry?; + + let dest; + match entry.path()?.components().nth(1).and_then(|c| c.as_os_str().to_str()) { + Some("version") => dest = &mut version, + Some("git-commit-hash") => dest = &mut git_commit, + _ => continue, + } + let mut buf = String::new(); + entry.read_to_string(&mut buf)?; + *dest = Some(buf); + + // Short circuit to avoid reading the whole tar file if not necessary. + if version.is_some() && git_commit.is_some() { + break; + } + } + + Ok(VersionInfo { version, git_commit, present: true }) + } + + pub(crate) fn disable_version(&mut self, package: &PkgType) { + match self.versions.get_mut(package) { + Some(version) => { + *version = VersionInfo::default(); + } + None => { + self.versions.insert(package.clone(), VersionInfo::default()); + } + } + } + + pub(crate) fn tarball_name( + &mut self, + package: &PkgType, + target: &str, + ) -> Result { + Ok(format!( + "{}-{}-{}.tar.gz", + package.tarball_component_name(), + self.package_version(package).with_context(|| format!( + "failed to get the package version for component {:?}", + package, + ))?, + target + )) + } + + pub(crate) fn package_version(&mut self, package: &PkgType) -> Result { + match self.package_versions.get(package) { + Some(release) => Ok(release.clone()), + None => { + let version = match package.rust_monorepo_path() { + Some(path) => { + let path = self.monorepo_root.join(path).join("Cargo.toml"); + let cargo_toml: CargoToml = toml::from_slice(&std::fs::read(path)?)?; + cargo_toml.package.version + } + None => self.rustc_version.clone(), + }; + + let release = match self.channel.as_str() { + "stable" => version, + "beta" => "beta".into(), + "nightly" => "nightly".into(), + _ => format!("{}-dev", version), + }; + + self.package_versions.insert(package.clone(), release.clone()); + Ok(release) + } + } + } +} + +#[derive(serde::Deserialize)] +struct CargoToml { + package: CargoTomlPackage, +} + +#[derive(serde::Deserialize)] +struct CargoTomlPackage { + version: String, +}