From 7715af57e12d10b743ab1445782769444aa74b34 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 29 Sep 2020 15:33:25 +0200 Subject: [PATCH 01/12] bootstrap: always use the Rust version in package names The format of the tarballs produced by CI is roughly the following: {component}-{release}-{target}.{ext} While on the beta and nightly channels `{release}` is just the channel name, on the stable channel is either the Rust version or the version of the component we're shipping: cargo-0.47.0-x86_64-unknown-linux-gnu.tar.xz clippy-0.0.212-x86_64-unknown-linux-gnu.tar.xz llvm-tools-1.46.0-x86_64-unknown-linux-gnu.tar.xz miri-0.1.0-x86_64-unknown-linux-gnu.tar.xz rls-1.41.0-x86_64-unknown-linux-gnu.tar.xz rust-1.46.0-x86_64-unknown-linux-gnu.tar.xz ... This makes it really hard to get the package URL without having access to the manifest (and there is no manifest on ci-artifacts.rlo), as there is no consistent version number to use. This commit addresses the problem by always using the Rust version number as `{release}` for the stable channel, regardless of the version number of the component we're shipping. I chose that instead of "stable" to avoid breaking the URL scheme *that* much. Rustup should not be affected by this change, as it fetches the URLs from the manifest. Unfortunately we don't have a way to test other clients before making a stable release, as this change only affects the stable channel. --- src/bootstrap/dist.rs | 19 +------ src/bootstrap/lib.rs | 34 ------------ src/tools/build-manifest/src/main.rs | 2 +- src/tools/build-manifest/src/versions.rs | 67 ++++-------------------- 4 files changed, 11 insertions(+), 111 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 857e06d846de4..3a0743da7a415 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -26,24 +26,7 @@ use crate::{Compiler, DependencyType, Mode, LLVM_TOOLS}; use time::{self, Timespec}; pub fn pkgname(builder: &Builder<'_>, component: &str) -> String { - if component == "cargo" { - format!("{}-{}", component, builder.cargo_package_vers()) - } else if component == "rls" { - format!("{}-{}", component, builder.rls_package_vers()) - } else if component == "rust-analyzer" { - format!("{}-{}", component, builder.rust_analyzer_package_vers()) - } else if component == "clippy" { - format!("{}-{}", component, builder.clippy_package_vers()) - } else if component == "miri" { - format!("{}-{}", component, builder.miri_package_vers()) - } else if component == "rustfmt" { - format!("{}-{}", component, builder.rustfmt_package_vers()) - } else if component == "llvm-tools" { - format!("{}-{}", component, builder.llvm_tools_package_vers()) - } else { - assert!(component.starts_with("rust")); - format!("{}-{}", component, builder.rust_package_vers()) - } + format!("{}-{}", component, builder.rust_package_vers()) } pub(crate) fn distdir(builder: &Builder<'_>) -> PathBuf { diff --git a/src/bootstrap/lib.rs b/src/bootstrap/lib.rs index 4cc72f5f39c97..852316a8d8dc4 100644 --- a/src/bootstrap/lib.rs +++ b/src/bootstrap/lib.rs @@ -1051,40 +1051,6 @@ impl Build { self.package_vers(&self.version) } - /// Returns the value of `package_vers` above for Cargo - fn cargo_package_vers(&self) -> String { - self.package_vers(&self.release_num("cargo")) - } - - /// Returns the value of `package_vers` above for rls - fn rls_package_vers(&self) -> String { - self.package_vers(&self.release_num("rls")) - } - - /// Returns the value of `package_vers` above for rust-analyzer - fn rust_analyzer_package_vers(&self) -> String { - self.package_vers(&self.release_num("rust-analyzer/crates/rust-analyzer")) - } - - /// Returns the value of `package_vers` above for clippy - fn clippy_package_vers(&self) -> String { - self.package_vers(&self.release_num("clippy")) - } - - /// Returns the value of `package_vers` above for miri - fn miri_package_vers(&self) -> String { - self.package_vers(&self.release_num("miri")) - } - - /// Returns the value of `package_vers` above for rustfmt - fn rustfmt_package_vers(&self) -> String { - self.package_vers(&self.release_num("rustfmt")) - } - - fn llvm_tools_package_vers(&self) -> String { - self.package_vers(&self.version) - } - fn llvm_tools_vers(&self) -> String { self.rust_version() } diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index c7e7d88c68fa8..b35f3a595fb38 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -252,7 +252,7 @@ impl Builder { } let manifest = self.build_manifest(); - let rust_version = self.versions.package_version(&PkgType::Rust).unwrap(); + let rust_version = self.versions.rustc_version(); self.write_channel_files(self.versions.channel(), &manifest); if self.versions.channel() != rust_version { self.write_channel_files(&rust_version, &manifest); diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index d949dff72798b..75b6979b54a78 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -38,23 +38,6 @@ impl PkgType { } } - /// The directory containing the `Cargo.toml` of this component inside the monorepo, to - /// retrieve the source code version. If `None` is returned Rust's version will be used. - fn rust_monorepo_path(&self) -> Option<&'static str> { - match self { - PkgType::Cargo => Some("src/tools/cargo"), - PkgType::Rls => Some("src/tools/rls"), - PkgType::RustAnalyzer => Some("src/tools/rust-analyzer/crates/rust-analyzer"), - PkgType::Clippy => Some("src/tools/clippy"), - PkgType::Rustfmt => Some("src/tools/rustfmt"), - PkgType::Miri => Some("src/tools/miri"), - PkgType::Rust => None, - PkgType::RustSrc => None, - PkgType::LlvmTools => None, - PkgType::Other(_) => None, - } - } - /// First part of the tarball name. fn tarball_component_name(&self) -> &str { match self { @@ -105,9 +88,7 @@ pub(crate) struct VersionInfo { pub(crate) struct Versions { channel: String, rustc_version: String, - monorepo_root: PathBuf, dist_path: PathBuf, - package_versions: HashMap, versions: HashMap, } @@ -123,9 +104,7 @@ impl Versions { .context("failed to read the rustc version from src/version")? .trim() .to_string(), - monorepo_root: monorepo_root.into(), dist_path: dist_path.into(), - package_versions: HashMap::new(), versions: HashMap::new(), }) } @@ -204,9 +183,13 @@ impl Versions { target: &str, ) -> Result { let component_name = package.tarball_component_name(); - let version = self.package_version(package).with_context(|| { - format!("failed to get the package version for component {:?}", package,) - })?; + let version = match self.channel.as_str() { + "stable" => self.rustc_version.clone(), + "beta" => "beta".into(), + "nightly" => "nightly".into(), + _ => format!("{}-dev", self.rustc_version), + }; + if package.target_independent() { Ok(format!("{}-{}.tar.gz", component_name, version)) } else { @@ -214,39 +197,7 @@ impl Versions { } } - pub(crate) fn package_version(&mut self, package: &PkgType) -> Result { - match self.package_versions.get(package) { - Some(release) => Ok(release.clone()), - None => { - let version = match package.rust_monorepo_path() { - Some(path) => { - let path = self.monorepo_root.join(path).join("Cargo.toml"); - let cargo_toml: CargoToml = toml::from_slice(&std::fs::read(path)?)?; - cargo_toml.package.version - } - None => self.rustc_version.clone(), - }; - - let release = match self.channel.as_str() { - "stable" => version, - "beta" => "beta".into(), - "nightly" => "nightly".into(), - _ => format!("{}-dev", version), - }; - - self.package_versions.insert(package.clone(), release.clone()); - Ok(release) - } - } + pub(crate) fn rustc_version(&self) -> &str { + &self.rustc_version } } - -#[derive(serde::Deserialize)] -struct CargoToml { - package: CargoTomlPackage, -} - -#[derive(serde::Deserialize)] -struct CargoTomlPackage { - version: String, -} From e1e1c0e2008b5e584d61dc5e225d4422d3ffc121 Mon Sep 17 00:00:00 2001 From: Vadim Petrochenkov Date: Sat, 10 Oct 2020 02:01:44 +0300 Subject: [PATCH 02/12] rustc_parse: More precise spans for `tuple.0.0` --- compiler/rustc_parse/src/parser/expr.rs | 40 ++++++++++++++++++++----- src/test/ui/parser/float-field.stderr | 10 +++---- src/test/ui/tuple/index-invalid.stderr | 8 ++--- src/test/ui/union/union-deref.stderr | 6 ++-- 4 files changed, 45 insertions(+), 19 deletions(-) diff --git a/compiler/rustc_parse/src/parser/expr.rs b/compiler/rustc_parse/src/parser/expr.rs index 69d13b5cf53a2..a11ad6e89c084 100644 --- a/compiler/rustc_parse/src/parser/expr.rs +++ b/compiler/rustc_parse/src/parser/expr.rs @@ -16,6 +16,7 @@ use rustc_ast_pretty::pprust; use rustc_errors::{Applicability, DiagnosticBuilder, PResult}; use rustc_span::source_map::{self, Span, Spanned}; use rustc_span::symbol::{kw, sym, Ident, Symbol}; +use rustc_span::{BytePos, Pos}; use std::mem; use tracing::debug; @@ -839,9 +840,10 @@ impl<'a> Parser<'a> { } use FloatComponent::*; + let float_str = float.as_str(); let mut components = Vec::new(); let mut ident_like = String::new(); - for c in float.as_str().chars() { + for c in float_str.chars() { if c == '_' || c.is_ascii_alphanumeric() { ident_like.push(c); } else if matches!(c, '.' | '+' | '-') { @@ -857,8 +859,13 @@ impl<'a> Parser<'a> { components.push(IdentLike(ident_like)); } - // FIXME: Make the span more precise. + // With proc macros the span can refer to anything, the source may be too short, + // or too long, or non-ASCII. It only makes sense to break our span into components + // if its underlying text is identical to our float literal. let span = self.token.span; + let can_take_span_apart = + || self.span_to_snippet(span).as_deref() == Ok(float_str).as_deref(); + match &*components { // 1e2 [IdentLike(i)] => { @@ -866,21 +873,40 @@ impl<'a> Parser<'a> { } // 1. [IdentLike(i), Punct('.')] => { + let (ident_span, dot_span) = if can_take_span_apart() { + let (span, ident_len) = (span.data(), BytePos::from_usize(i.len())); + let ident_span = span.with_hi(span.lo + ident_len); + let dot_span = span.with_lo(span.lo + ident_len); + (ident_span, dot_span) + } else { + (span, span) + }; assert!(suffix.is_none()); let symbol = Symbol::intern(&i); - self.token = Token::new(token::Ident(symbol, false), span); - let next_token = Token::new(token::Dot, span); + self.token = Token::new(token::Ident(symbol, false), ident_span); + let next_token = Token::new(token::Dot, dot_span); self.parse_tuple_field_access_expr(lo, base, symbol, None, Some(next_token)) } // 1.2 | 1.2e3 [IdentLike(i1), Punct('.'), IdentLike(i2)] => { + let (ident1_span, dot_span, ident2_span) = if can_take_span_apart() { + let (span, ident1_len) = (span.data(), BytePos::from_usize(i1.len())); + let ident1_span = span.with_hi(span.lo + ident1_len); + let dot_span = span + .with_lo(span.lo + ident1_len) + .with_hi(span.lo + ident1_len + BytePos(1)); + let ident2_span = self.token.span.with_lo(span.lo + ident1_len + BytePos(1)); + (ident1_span, dot_span, ident2_span) + } else { + (span, span, span) + }; let symbol1 = Symbol::intern(&i1); - self.token = Token::new(token::Ident(symbol1, false), span); - let next_token1 = Token::new(token::Dot, span); + self.token = Token::new(token::Ident(symbol1, false), ident1_span); + let next_token1 = Token::new(token::Dot, dot_span); let base1 = self.parse_tuple_field_access_expr(lo, base, symbol1, None, Some(next_token1)); let symbol2 = Symbol::intern(&i2); - let next_token2 = Token::new(token::Ident(symbol2, false), span); + let next_token2 = Token::new(token::Ident(symbol2, false), ident2_span); self.bump_with(next_token2); // `.` self.parse_tuple_field_access_expr(lo, base1, symbol2, suffix, None) } diff --git a/src/test/ui/parser/float-field.stderr b/src/test/ui/parser/float-field.stderr index 62202b999648d..7090efc5014b3 100644 --- a/src/test/ui/parser/float-field.stderr +++ b/src/test/ui/parser/float-field.stderr @@ -271,10 +271,10 @@ LL | s.1e1; = note: available fields are: `0`, `1` error[E0609]: no field `1e1` on type `(u8, u8)` - --> $DIR/float-field.rs:9:7 + --> $DIR/float-field.rs:9:9 | LL | s.1.1e1; - | ^^^^^ + | ^^^ error[E0609]: no field `0x1e1` on type `S` --> $DIR/float-field.rs:24:7 @@ -288,7 +288,7 @@ error[E0609]: no field `0x1` on type `S` --> $DIR/float-field.rs:25:7 | LL | s.0x1.; - | ^^^^ unknown field + | ^^^ unknown field | = note: available fields are: `0`, `1` @@ -296,7 +296,7 @@ error[E0609]: no field `0x1` on type `S` --> $DIR/float-field.rs:28:7 | LL | s.0x1.1; - | ^^^^^ unknown field + | ^^^ unknown field | = note: available fields are: `0`, `1` @@ -304,7 +304,7 @@ error[E0609]: no field `0x1` on type `S` --> $DIR/float-field.rs:30:7 | LL | s.0x1.1e1; - | ^^^^^^^ unknown field + | ^^^ unknown field | = note: available fields are: `0`, `1` diff --git a/src/test/ui/tuple/index-invalid.stderr b/src/test/ui/tuple/index-invalid.stderr index 800b5a31d98ab..8d22f458a6c6e 100644 --- a/src/test/ui/tuple/index-invalid.stderr +++ b/src/test/ui/tuple/index-invalid.stderr @@ -2,19 +2,19 @@ error[E0609]: no field `1` on type `(((),),)` --> $DIR/index-invalid.rs:2:22 | LL | let _ = (((),),).1.0; - | ^^^ + | ^ error[E0609]: no field `1` on type `((),)` - --> $DIR/index-invalid.rs:4:22 + --> $DIR/index-invalid.rs:4:24 | LL | let _ = (((),),).0.1; - | ^^^ + | ^ error[E0609]: no field `000` on type `(((),),)` --> $DIR/index-invalid.rs:6:22 | LL | let _ = (((),),).000.000; - | ^^^^^^^ + | ^^^ error: aborting due to 3 previous errors diff --git a/src/test/ui/union/union-deref.stderr b/src/test/ui/union/union-deref.stderr index fb16649767fb7..f7722764cd3f0 100644 --- a/src/test/ui/union/union-deref.stderr +++ b/src/test/ui/union/union-deref.stderr @@ -29,7 +29,7 @@ error: not automatically applying `DerefMut` on `ManuallyDrop` union field --> $DIR/union-deref.rs:23:14 | LL | unsafe { u.f.0.0 = Vec::new() }; - | ^^^^^^^ + | ^^^^^ | = help: writing to this reference calls the destructor for the old value = help: add an explicit `*` if that is desired, or call `ptr::write` to not run the destructor @@ -38,7 +38,7 @@ error: not automatically applying `DerefMut` on `ManuallyDrop` union field --> $DIR/union-deref.rs:25:19 | LL | unsafe { &mut u.f.0.0 }; - | ^^^^^^^ + | ^^^^^ | = help: writing to this reference calls the destructor for the old value = help: add an explicit `*` if that is desired, or call `ptr::write` to not run the destructor @@ -47,7 +47,7 @@ error: not automatically applying `DerefMut` on `ManuallyDrop` union field --> $DIR/union-deref.rs:27:14 | LL | unsafe { u.f.0.0.push(0) }; - | ^^^^^^^ + | ^^^^^ | = help: writing to this reference calls the destructor for the old value = help: add an explicit `*` if that is desired, or call `ptr::write` to not run the destructor From 99613177ff4c5b01469a56917d3e30810fb922c2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mateusz=20Miku=C5=82a?= Date: Sun, 11 Oct 2020 16:19:35 +0200 Subject: [PATCH 03/12] Update crossbeam-channel to avoid UB --- Cargo.lock | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 9111e6ccf0fb0..275630fd6d511 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -728,12 +728,12 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.4.3" +version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09ee0cc8804d5393478d743b035099520087a5186f3b93fa58cec08fa62407b6" +checksum = "b153fe7cbef478c567df0f972e02e6d736db11affe43dfc9c56a9374d1adfb87" dependencies = [ - "cfg-if", "crossbeam-utils 0.7.2", + "maybe-uninit", ] [[package]] From 7701b0bf6d0d3b5b0c74d8122e329b9f1927f974 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 12 Oct 2020 17:39:13 +0200 Subject: [PATCH 04/12] build-manifest: stop generating numbered channel names except for stable This fixes numbered channel names being created for the nightly channel, and once the root cause of this rides the trains, for beta. --- src/tools/build-manifest/src/main.rs | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index b35f3a595fb38..7ee28cd6e5d95 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -252,12 +252,13 @@ impl Builder { } let manifest = self.build_manifest(); - let rust_version = self.versions.rustc_version(); self.write_channel_files(self.versions.channel(), &manifest); - if self.versions.channel() != rust_version { - self.write_channel_files(&rust_version, &manifest); - } if self.versions.channel() == "stable" { + // channel-rust-1.XX.YY.toml + let rust_version = self.versions.rustc_version(); + self.write_channel_files(rust_version, &manifest); + + // channel-rust-1.XX.toml let major_minor = rust_version.split('.').take(2).collect::>().join("."); self.write_channel_files(&major_minor, &manifest); } From c83642c3327ebb49d0aa06c249dfeefa4a55c4dc Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 9 Oct 2020 15:34:57 +0200 Subject: [PATCH 05/12] build-manifest: accept the Rust version instead of the monorepo path This commit changes the way build-manifest is invoked, to let it accept the Rust version directly instead of requiring the path of the Rust monorepo and letting build-manifest figure out the path on its own. This allows to run build-manifest without a clone of the monorepo. --- src/bootstrap/dist.rs | 2 +- src/bootstrap/run.rs | 2 +- src/tools/build-manifest/README.md | 4 ++-- src/tools/build-manifest/src/main.rs | 4 ++-- src/tools/build-manifest/src/versions.rs | 13 +++---------- 5 files changed, 9 insertions(+), 16 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 3a0743da7a415..8f4968d5ac69f 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2353,7 +2353,7 @@ impl Step for HashSign { cmd.arg(today.trim()); cmd.arg(addr); cmd.arg(&builder.config.channel); - cmd.arg(&builder.src); + cmd.arg(&builder.version); cmd.env("BUILD_MANIFEST_LEGACY", "1"); builder.create_dir(&distdir(builder)); diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index 80c093e713eff..d8169549c07ba 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -77,7 +77,7 @@ impl Step for BuildManifest { cmd.arg(today.trim()); cmd.arg(addr); cmd.arg(&builder.config.channel); - cmd.arg(&builder.src); + cmd.arg(&builder.version); builder.create_dir(&distdir(builder)); builder.run(&mut cmd); diff --git a/src/tools/build-manifest/README.md b/src/tools/build-manifest/README.md index 26e96c9fd8fda..b77c5a907c118 100644 --- a/src/tools/build-manifest/README.md +++ b/src/tools/build-manifest/README.md @@ -21,8 +21,8 @@ Then, you can generate the manifest and all the packages from `path/to/dist` to ``` $ cargo +nightly run path/to/dist path/to/output 1970-01-01 http://example.com \ - CHANNEL path/to/rust/repo + CHANNEL VERSION ``` Remember to replace `CHANNEL` with the channel you produced dist artifacts of -and `path/to/rust/repo` with the path to your checkout of the Rust repository. +and `VERSION` with the current Rust version. diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 7ee28cd6e5d95..0efebe363f88f 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -221,7 +221,7 @@ fn main() { let date = args.next().unwrap(); let s3_address = args.next().unwrap(); let channel = args.next().unwrap(); - let monorepo_path = args.next().unwrap(); + let rustc_version = args.next().unwrap(); // Do not ask for a passphrase while manually testing let mut passphrase = String::new(); @@ -231,7 +231,7 @@ fn main() { } Builder { - versions: Versions::new(&channel, &input, Path::new(&monorepo_path)).unwrap(), + versions: Versions::new(&channel, &rustc_version, &input).unwrap(), input, output, diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index 75b6979b54a78..c2bce0f1cf39c 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -1,4 +1,4 @@ -use anyhow::{Context, Error}; +use anyhow::Error; use flate2::read::GzDecoder; use std::collections::HashMap; use std::fs::File; @@ -93,17 +93,10 @@ pub(crate) struct Versions { } impl Versions { - pub(crate) fn new( - channel: &str, - dist_path: &Path, - monorepo_root: &Path, - ) -> Result { + pub(crate) fn new(channel: &str, rustc_version: &str, dist_path: &Path) -> Result { Ok(Self { channel: channel.into(), - rustc_version: std::fs::read_to_string(monorepo_root.join("src").join("version")) - .context("failed to read the rustc version from src/version")? - .trim() - .to_string(), + rustc_version: rustc_version.into(), dist_path: dist_path.into(), versions: HashMap::new(), }) From 380f541818b1ce75ff8d1528c34a027c9f52c8eb Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 9 Oct 2020 19:49:13 +0200 Subject: [PATCH 06/12] bootstrap: add disabled by default build-manifest dist component --- src/bootstrap/builder.rs | 1 + src/bootstrap/dist.rs | 67 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 3de5797180cf2..b5ca7c712ab8e 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -462,6 +462,7 @@ impl<'a> Builder<'a> { dist::LlvmTools, dist::RustDev, dist::Extended, + dist::BuildManifest, dist::HashSign ), Kind::Install => describe!( diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 8f4968d5ac69f..12533d29809cf 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2584,3 +2584,70 @@ impl Step for RustDev { Some(distdir(builder).join(format!("{}-{}.tar.gz", name, target.triple))) } } + +/// Tarball containing a prebuilt version of the build-manifest tool, intented to be used by the +/// release process to avoid cloning the monorepo and building stuff. +/// +/// Should not be considered stable by end users. +#[derive(Clone, Debug, Eq, Hash, PartialEq)] +pub struct BuildManifest { + pub target: TargetSelection, +} + +impl Step for BuildManifest { + type Output = PathBuf; + const DEFAULT: bool = false; + const ONLY_HOSTS: bool = true; + + fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> { + run.path("src/tools/build-manifest") + } + + fn make_run(run: RunConfig<'_>) { + run.builder.ensure(BuildManifest { target: run.target }); + } + + fn run(self, builder: &Builder<'_>) -> PathBuf { + let build_manifest = builder.tool_exe(Tool::BuildManifest); + + let name = pkgname(builder, "build-manifest"); + let tmp = tmpdir(builder); + + // Prepare the image. + let image = tmp.join("build-manifest-image"); + let image_bin = image.join("bin"); + let _ = fs::remove_dir_all(&image); + t!(fs::create_dir_all(&image_bin)); + builder.install(&build_manifest, &image_bin.join("build-manifest"), 0o755); + + // Prepare the overlay. + let overlay = tmp.join("build-manifest-overlay"); + let _ = fs::remove_dir_all(&overlay); + builder.create_dir(&overlay); + builder.create(&overlay.join("version"), &builder.rust_version()); + for file in &["COPYRIGHT", "LICENSE-APACHE", "LICENSE-MIT", "README.md"] { + builder.install(&builder.src.join(file), &overlay, 0o644); + } + + // Create the final tarball. + let mut cmd = rust_installer(builder); + cmd.arg("generate") + .arg("--product-name=Rust") + .arg("--rel-manifest-dir=rustlib") + .arg("--success-message=build-manifest installed.") + .arg("--image-dir") + .arg(&image) + .arg("--work-dir") + .arg(&tmpdir(builder)) + .arg("--output-dir") + .arg(&distdir(builder)) + .arg("--non-installed-overlay") + .arg(&overlay) + .arg(format!("--package-name={}-{}", name, self.target.triple)) + .arg("--legacy-manifest-dirs=rustlib,cargo") + .arg("--component-name=build-manifest"); + + builder.run(&mut cmd); + distdir(builder).join(format!("{}-{}.tar.gz", name, self.target.triple)) + } +} From e1a033ac67afc716e1ead633b228e6fac6ede94f Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 9 Oct 2020 19:51:07 +0200 Subject: [PATCH 07/12] bootstrap: add --include-default-paths to ./x.py --- src/bootstrap/builder.rs | 42 ++++++++++++++++++++-------------------- src/bootstrap/config.rs | 2 ++ src/bootstrap/flags.rs | 7 +++++++ 3 files changed, 30 insertions(+), 21 deletions(-) diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index b5ca7c712ab8e..3fa77982a9ca3 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -193,37 +193,37 @@ impl StepDescription { ); } - if paths.is_empty() { - for (desc, should_run) in v.iter().zip(should_runs) { + if paths.is_empty() || builder.config.include_default_paths { + for (desc, should_run) in v.iter().zip(&should_runs) { if desc.default && should_run.is_really_default { for pathset in &should_run.paths { desc.maybe_run(builder, pathset); } } } - } else { - for path in paths { - // strip CurDir prefix if present - let path = match path.strip_prefix(".") { - Ok(p) => p, - Err(_) => path, - }; + } - let mut attempted_run = false; - for (desc, should_run) in v.iter().zip(&should_runs) { - if let Some(suite) = should_run.is_suite_path(path) { - attempted_run = true; - desc.maybe_run(builder, suite); - } else if let Some(pathset) = should_run.pathset_for_path(path) { - attempted_run = true; - desc.maybe_run(builder, pathset); - } - } + for path in paths { + // strip CurDir prefix if present + let path = match path.strip_prefix(".") { + Ok(p) => p, + Err(_) => path, + }; - if !attempted_run { - panic!("error: no rules matched {}", path.display()); + let mut attempted_run = false; + for (desc, should_run) in v.iter().zip(&should_runs) { + if let Some(suite) = should_run.is_suite_path(path) { + attempted_run = true; + desc.maybe_run(builder, suite); + } else if let Some(pathset) = should_run.pathset_for_path(path) { + attempted_run = true; + desc.maybe_run(builder, pathset); } } + + if !attempted_run { + panic!("error: no rules matched {}", path.display()); + } } } } diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 5314398ce9a22..8985d1fbd0365 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -61,6 +61,7 @@ pub struct Config { pub profiler: bool, pub ignore_git: bool, pub exclude: Vec, + pub include_default_paths: bool, pub rustc_error_format: Option, pub json_output: bool, pub test_compare_mode: bool, @@ -532,6 +533,7 @@ impl Config { let mut config = Config::default_opts(); config.exclude = flags.exclude; + config.include_default_paths = flags.include_default_paths; config.rustc_error_format = flags.rustc_error_format; config.json_output = flags.json_output; config.on_fail = flags.on_fail; diff --git a/src/bootstrap/flags.rs b/src/bootstrap/flags.rs index 795244e7cd4a4..4bf4a193630d2 100644 --- a/src/bootstrap/flags.rs +++ b/src/bootstrap/flags.rs @@ -29,6 +29,7 @@ pub struct Flags { pub cmd: Subcommand, pub incremental: bool, pub exclude: Vec, + pub include_default_paths: bool, pub rustc_error_format: Option, pub json_output: bool, pub dry_run: bool, @@ -133,6 +134,11 @@ To learn more about a subcommand, run `./x.py -h`", opts.optmulti("", "host", "host targets to build", "HOST"); opts.optmulti("", "target", "target targets to build", "TARGET"); opts.optmulti("", "exclude", "build paths to exclude", "PATH"); + opts.optflag( + "", + "include-default-paths", + "include default paths in addition to the provided ones", + ); opts.optopt("", "on-fail", "command to run on failure", "CMD"); opts.optflag("", "dry-run", "dry run; don't build anything"); opts.optopt( @@ -601,6 +607,7 @@ Arguments: .into_iter() .map(|p| p.into()) .collect::>(), + include_default_paths: matches.opt_present("include-default-paths"), deny_warnings: parse_deny_warnings(&matches), llvm_skip_rebuild: matches.opt_str("llvm-skip-rebuild").map(|s| s.to_lowercase()).map( |s| s.parse::().expect("`llvm-skip-rebuild` should be either true or false"), From 168174303db4165a9271e0c0bbabd9db5295d8e0 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Fri, 9 Oct 2020 19:53:48 +0200 Subject: [PATCH 08/12] ci: also build the build-manifest component on dist-x86_64-linux --- src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile index 58e2567a58f08..14700aeea05af 100644 --- a/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile +++ b/src/ci/docker/host-x86_64/dist-x86_64-linux/Dockerfile @@ -98,7 +98,9 @@ ENV RUST_CONFIGURE_ARGS \ --set llvm.thin-lto=true \ --set llvm.ninja=false \ --set rust.jemalloc -ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS +ENV SCRIPT python2.7 ../x.py dist --host $HOSTS --target $HOSTS \ + --include-default-paths \ + src/tools/build-manifest ENV CARGO_TARGET_X86_64_UNKNOWN_LINUX_GNU_LINKER=clang # This is the only builder which will create source tarballs From bf81d422cba34e25ab7babd138169f3f0362c141 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 12 Oct 2020 17:47:37 +0200 Subject: [PATCH 09/12] build-manifest: allow configuring the number of threads --- Cargo.lock | 1 + src/tools/build-manifest/Cargo.toml | 1 + src/tools/build-manifest/src/main.rs | 19 ++++++++++++------- 3 files changed, 14 insertions(+), 7 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 275630fd6d511..ecd42d6d4a36a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -243,6 +243,7 @@ dependencies = [ "anyhow", "flate2", "hex 0.4.2", + "num_cpus", "rayon", "serde", "serde_json", diff --git a/src/tools/build-manifest/Cargo.toml b/src/tools/build-manifest/Cargo.toml index 4ae4dbfc06ede..4a2c710811f61 100644 --- a/src/tools/build-manifest/Cargo.toml +++ b/src/tools/build-manifest/Cargo.toml @@ -14,3 +14,4 @@ tar = "0.4.29" sha2 = "0.9.1" rayon = "1.3.1" hex = "0.4.2" +num_cpus = "1.13.0" diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 0efebe363f88f..6fda9f4e59f52 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -207,13 +207,18 @@ fn main() { // related code in this tool and ./x.py dist hash-and-sign can be removed. let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok(); - // Avoid overloading the old server in legacy mode. - if legacy { - rayon::ThreadPoolBuilder::new() - .num_threads(1) - .build_global() - .expect("failed to initialize Rayon"); - } + let num_threads = if legacy { + // Avoid overloading the old server in legacy mode. + 1 + } else if let Ok(num) = env::var("BUILD_MANIFEST_NUM_THREADS") { + num.parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS") + } else { + num_cpus::get() + }; + rayon::ThreadPoolBuilder::new() + .num_threads(num_threads) + .build_global() + .expect("failed to initialize Rayon"); let mut args = env::args().skip(1); let input = PathBuf::from(args.next().unwrap()); From 45b5f50e25165a6d2fcc2970146fdc706d75bf7c Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 12 Oct 2020 19:34:01 +0200 Subject: [PATCH 10/12] build-manifest: use var_os instead of var to check if vars exist This will prevent the tool mistakenly ignoring the variables if they happen to contain non-utf8 data. --- src/tools/build-manifest/src/main.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 6fda9f4e59f52..1515f46e03ac2 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -205,13 +205,13 @@ fn main() { // // Once the old release process is fully decommissioned, the environment variable, all the // related code in this tool and ./x.py dist hash-and-sign can be removed. - let legacy = env::var("BUILD_MANIFEST_LEGACY").is_ok(); + let legacy = env::var_os("BUILD_MANIFEST_LEGACY").is_some(); let num_threads = if legacy { // Avoid overloading the old server in legacy mode. 1 - } else if let Ok(num) = env::var("BUILD_MANIFEST_NUM_THREADS") { - num.parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS") + } else if let Some(num) = env::var_os("BUILD_MANIFEST_NUM_THREADS") { + num.to_str().unwrap().parse().expect("invalid number for BUILD_MANIFEST_NUM_THREADS") } else { num_cpus::get() }; From 0c294e29cea3faa98e11d7d82b1d0a8e6083e8e0 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Mon, 12 Oct 2020 19:40:35 +0200 Subject: [PATCH 11/12] build-manifest: bundle the rustc version in the binary --- src/bootstrap/dist.rs | 1 - src/bootstrap/run.rs | 1 - src/tools/build-manifest/src/main.rs | 3 +-- src/tools/build-manifest/src/versions.rs | 17 ++++++----------- 4 files changed, 7 insertions(+), 15 deletions(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index 12533d29809cf..dd4cf9d595323 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2353,7 +2353,6 @@ impl Step for HashSign { cmd.arg(today.trim()); cmd.arg(addr); cmd.arg(&builder.config.channel); - cmd.arg(&builder.version); cmd.env("BUILD_MANIFEST_LEGACY", "1"); builder.create_dir(&distdir(builder)); diff --git a/src/bootstrap/run.rs b/src/bootstrap/run.rs index d8169549c07ba..7c64e5a0aadc8 100644 --- a/src/bootstrap/run.rs +++ b/src/bootstrap/run.rs @@ -77,7 +77,6 @@ impl Step for BuildManifest { cmd.arg(today.trim()); cmd.arg(addr); cmd.arg(&builder.config.channel); - cmd.arg(&builder.version); builder.create_dir(&distdir(builder)); builder.run(&mut cmd); diff --git a/src/tools/build-manifest/src/main.rs b/src/tools/build-manifest/src/main.rs index 1515f46e03ac2..cb04900c737e0 100644 --- a/src/tools/build-manifest/src/main.rs +++ b/src/tools/build-manifest/src/main.rs @@ -226,7 +226,6 @@ fn main() { let date = args.next().unwrap(); let s3_address = args.next().unwrap(); let channel = args.next().unwrap(); - let rustc_version = args.next().unwrap(); // Do not ask for a passphrase while manually testing let mut passphrase = String::new(); @@ -236,7 +235,7 @@ fn main() { } Builder { - versions: Versions::new(&channel, &rustc_version, &input).unwrap(), + versions: Versions::new(&channel, &input).unwrap(), input, output, diff --git a/src/tools/build-manifest/src/versions.rs b/src/tools/build-manifest/src/versions.rs index c2bce0f1cf39c..79f2ef8dfc450 100644 --- a/src/tools/build-manifest/src/versions.rs +++ b/src/tools/build-manifest/src/versions.rs @@ -7,6 +7,7 @@ use std::path::{Path, PathBuf}; use tar::Archive; const DEFAULT_TARGET: &str = "x86_64-unknown-linux-gnu"; +const RUSTC_VERSION: &str = include_str!("../../../version"); #[derive(Debug, Hash, Eq, PartialEq, Clone)] pub(crate) enum PkgType { @@ -87,19 +88,13 @@ pub(crate) struct VersionInfo { pub(crate) struct Versions { channel: String, - rustc_version: String, dist_path: PathBuf, versions: HashMap, } impl Versions { - pub(crate) fn new(channel: &str, rustc_version: &str, dist_path: &Path) -> Result { - Ok(Self { - channel: channel.into(), - rustc_version: rustc_version.into(), - dist_path: dist_path.into(), - versions: HashMap::new(), - }) + pub(crate) fn new(channel: &str, dist_path: &Path) -> Result { + Ok(Self { channel: channel.into(), dist_path: dist_path.into(), versions: HashMap::new() }) } pub(crate) fn channel(&self) -> &str { @@ -177,10 +172,10 @@ impl Versions { ) -> Result { let component_name = package.tarball_component_name(); let version = match self.channel.as_str() { - "stable" => self.rustc_version.clone(), + "stable" => RUSTC_VERSION.into(), "beta" => "beta".into(), "nightly" => "nightly".into(), - _ => format!("{}-dev", self.rustc_version), + _ => format!("{}-dev", RUSTC_VERSION), }; if package.target_independent() { @@ -191,6 +186,6 @@ impl Versions { } pub(crate) fn rustc_version(&self) -> &str { - &self.rustc_version + RUSTC_VERSION } } From 89184152b9c99721be54f1f9b49bc84f4f31fd40 Mon Sep 17 00:00:00 2001 From: Pietro Albini Date: Tue, 13 Oct 2020 20:37:38 +0200 Subject: [PATCH 12/12] bootstrap: set correct path for the build-manifest binary --- src/bootstrap/dist.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/bootstrap/dist.rs b/src/bootstrap/dist.rs index dd4cf9d595323..1887b805da19b 100644 --- a/src/bootstrap/dist.rs +++ b/src/bootstrap/dist.rs @@ -2617,7 +2617,7 @@ impl Step for BuildManifest { let image_bin = image.join("bin"); let _ = fs::remove_dir_all(&image); t!(fs::create_dir_all(&image_bin)); - builder.install(&build_manifest, &image_bin.join("build-manifest"), 0o755); + builder.install(&build_manifest, &image_bin, 0o755); // Prepare the overlay. let overlay = tmp.join("build-manifest-overlay");