diff --git a/Cargo.lock b/Cargo.lock index 4a9b35fe4aba9..b6077574ee600 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -216,6 +216,7 @@ dependencies = [ "cmake", "filetime", "getopts", + "hex 0.4.2", "ignore", "libc", "once_cell", @@ -223,6 +224,7 @@ dependencies = [ "pretty_assertions 0.7.2", "serde", "serde_json", + "sha2", "sysinfo", "tar", "toml", diff --git a/src/bootstrap/Cargo.toml b/src/bootstrap/Cargo.toml index 5027a45e0ada0..0e54837610a4b 100644 --- a/src/bootstrap/Cargo.toml +++ b/src/bootstrap/Cargo.toml @@ -40,8 +40,10 @@ filetime = "0.2" getopts = "0.2.19" cc = "1.0.69" libc = "0.2" +hex = "0.4" serde = { version = "1.0.8", features = ["derive"] } serde_json = "1.0.2" +sha2 = "0.10" tar = "0.4" toml = "0.5" ignore = "0.4.10" diff --git a/src/bootstrap/builder.rs b/src/bootstrap/builder.rs index 47551c5082e4b..38d4f15d3c858 100644 --- a/src/bootstrap/builder.rs +++ b/src/bootstrap/builder.rs @@ -879,7 +879,6 @@ impl<'a> Builder<'a> { ) { // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/. let tempfile = self.tempdir().join(dest_path.file_name().unwrap()); - // FIXME: support `do_verify` (only really needed for nightly rustfmt) self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error); t!(std::fs::rename(&tempfile, dest_path)); } @@ -971,6 +970,28 @@ impl<'a> Builder<'a> { t!(fs::remove_dir_all(dst.join(directory_prefix))); } + /// Returns whether the SHA256 checksum of `path` matches `expected`. + pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool { + use sha2::Digest; + + self.verbose(&format!("verifying {}", path.display())); + let mut hasher = sha2::Sha256::new(); + // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components. + // Consider using streaming IO instead? + let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) }; + hasher.update(&contents); + let found = hex::encode(hasher.finalize().as_slice()); + let verified = found == expected; + if !verified && !self.config.dry_run { + println!( + "invalid checksum: \n\ + found: {found}\n\ + expected: {expected}", + ); + } + return verified; + } + /// Obtain a compiler at a given stage and for a given host. Explicitly does /// not take `Compiler` since all `Compiler` instances are meant to be /// obtained through this function, since it ensures that they are valid diff --git a/src/bootstrap/config.rs b/src/bootstrap/config.rs index 1beb198340711..70bd7473c872d 100644 --- a/src/bootstrap/config.rs +++ b/src/bootstrap/config.rs @@ -1486,6 +1486,7 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { #[derive(Deserialize)] struct Stage0Metadata { dist_server: String, + checksums_sha256: HashMap, rustfmt: Option, } #[derive(Deserialize)] @@ -1495,10 +1496,11 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { } let stage0_json = builder.read(&builder.src.join("src").join("stage0.json")); - let metadata = t!(serde_json::from_str::(&stage0_json)); - let RustfmtMetadata { date, version } = metadata.rustfmt?; + let Stage0Metadata { dist_server, checksums_sha256, rustfmt } = + t!(serde_json::from_str::(&stage0_json)); + let RustfmtMetadata { date, version } = rustfmt?; let channel = format!("{version}-{date}"); - let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(metadata.dist_server); + let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(dist_server); dist_server.push_str("/dist"); let host = builder.config.build; @@ -1510,8 +1512,15 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option { } let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple); - download_component(builder, &dist_server, filename, "rustfmt-preview", &date, "stage0"); - assert!(rustfmt_path.exists()); + download_component( + builder, + &dist_server, + filename, + "rustfmt-preview", + &date, + "stage0", + Some(checksums_sha256), + ); builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt")); builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt")); @@ -1564,6 +1573,7 @@ fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str, prefix, commit, "ci-rustc", + None, ) } @@ -1574,6 +1584,7 @@ fn download_component( prefix: &str, key: &str, destination: &str, + checksums: Option>, ) { let cache_dst = builder.out.join("cache"); let cache_dir = cache_dst.join(key); @@ -1581,10 +1592,47 @@ fn download_component( t!(fs::create_dir_all(&cache_dir)); } + let bin_root = builder.out.join(builder.config.build.triple).join(destination); let tarball = cache_dir.join(&filename); - if !tarball.exists() { - builder.download_component(base_url, &format!("{key}/{filename}"), &tarball, ""); + let url = format!("{key}/{filename}"); + + // For the beta compiler, put special effort into ensuring the checksums are valid. + // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update + // this on each and every nightly ... + let checksum = if let Some(checksums) = &checksums { + let error = format!( + "src/stage0.json doesn't contain a checksum for {url}. \ + Pre-built artifacts might not be available for this \ + target at this time, see https://doc.rust-lang.org/nightly\ + /rustc/platform-support.html for more information." + ); + // TODO: add an enum { Commit, Published } so we don't have to hardcode `dist` in two places + let sha256 = checksums.get(&format!("dist/{url}")).expect(&error); + if tarball.exists() { + if builder.verify(&tarball, sha256) { + builder.unpack(&tarball, &bin_root, prefix); + return; + } else { + builder.verbose(&format!( + "ignoring cached file {} due to failed verification", + tarball.display() + )); + builder.remove(&tarball); + } + } + Some(sha256) + } else if tarball.exists() { + return; + } else { + None + }; + + builder.download_component(base_url, &url, &tarball, ""); + if let Some(sha256) = checksum { + if !builder.verify(&tarball, sha256) { + panic!("failed to verify {}", tarball.display()); + } } - let bin_root = builder.out.join(builder.config.build.triple).join(destination); - builder.unpack(&tarball, &bin_root, prefix) + + builder.unpack(&tarball, &bin_root, prefix); }