Add checksum verification for rustfmt downloads

Cargo.lock

@@ -216,13 +216,15 @@ dependencies = [
  "cmake",
  "filetime",
  "getopts",
+ "hex 0.4.2",
  "ignore",
  "libc",
  "once_cell",
  "opener",
  "pretty_assertions 0.7.2",
  "serde",
  "serde_json",
+ "sha2",
  "sysinfo",
  "tar",
  "toml",

src/bootstrap/Cargo.toml

@@ -40,8 +40,10 @@ filetime = "0.2"
 getopts = "0.2.19"
 cc = "1.0.69"
 libc = "0.2"
+hex = "0.4"
 serde = { version = "1.0.8", features = ["derive"] }
 serde_json = "1.0.2"
+sha2 = "0.10"
 tar = "0.4"
 toml = "0.5"
 ignore = "0.4.10"

src/bootstrap/builder.rs

@@ -879,7 +879,6 @@ impl<'a> Builder<'a> {
  ) {
  // Use a temporary file in case we crash while downloading, to avoid a corrupt download in cache/.
  let tempfile = self.tempdir().join(dest_path.file_name().unwrap());
- // FIXME: support `do_verify` (only really needed for nightly rustfmt)
  self.download_with_retries(&tempfile, &format!("{}/{}", base, url), help_on_error);
  t!(std::fs::rename(&tempfile, dest_path));
  }
@@ -971,6 +970,28 @@ impl<'a> Builder<'a> {
  t!(fs::remove_dir_all(dst.join(directory_prefix)));
  }
 
+ /// Returns whether the SHA256 checksum of `path` matches `expected`.
+ pub(crate) fn verify(&self, path: &Path, expected: &str) -> bool {
+ use sha2::Digest;
+
+ self.verbose(&format!("verifying {}", path.display()));
+ let mut hasher = sha2::Sha256::new();
+ // FIXME: this is ok for rustfmt (4.1 MB large at time of writing), but it seems memory-intensive for rustc and larger components.
+ // Consider using streaming IO instead?
+ let contents = if self.config.dry_run { vec![] } else { t!(fs::read(path)) };
+ hasher.update(&contents);
+ let found = hex::encode(hasher.finalize().as_slice());
+ let verified = found == expected;
+ if !verified && !self.config.dry_run {
+ println!(
+ "invalid checksum: \n\
+ found: {found}\n\
+ expected: {expected}",
+ );
+ }
+ return verified;
+ }
+
  /// Obtain a compiler at a given stage and for a given host. Explicitly does
  /// not take `Compiler` since all `Compiler` instances are meant to be
  /// obtained through this function, since it ensures that they are valid

src/bootstrap/config.rs

@@ -1486,6 +1486,7 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
  #[derive(Deserialize)]
  struct Stage0Metadata {
  dist_server: String,
+ checksums_sha256: HashMap<String, String>,
  rustfmt: Option<RustfmtMetadata>,
  }
  #[derive(Deserialize)]
@@ -1495,10 +1496,11 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
  }
 
  let stage0_json = builder.read(&builder.src.join("src").join("stage0.json"));
- let metadata = t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
- let RustfmtMetadata { date, version } = metadata.rustfmt?;
+ let Stage0Metadata { dist_server, checksums_sha256, rustfmt } =
+ t!(serde_json::from_str::<Stage0Metadata>(&stage0_json));
+ let RustfmtMetadata { date, version } = rustfmt?;
  let channel = format!("{version}-{date}");
- let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(metadata.dist_server);
+ let mut dist_server = env::var("RUSTUP_DIST_SERVER").unwrap_or(dist_server);
  dist_server.push_str("/dist");
 
  let host = builder.config.build;
@@ -1510,8 +1512,15 @@ fn maybe_download_rustfmt(builder: &Builder<'_>) -> Option<PathBuf> {
  }
 
  let filename = format!("rustfmt-{version}-{build}.tar.xz", build = host.triple);
- download_component(builder, &dist_server, filename, "rustfmt-preview", &date, "stage0");
- assert!(rustfmt_path.exists());
+ download_component(
+ builder,
+ &dist_server,
+ filename,
+ "rustfmt-preview",
+ &date,
+ "stage0",
+ Some(checksums_sha256),
+ );
 
  builder.fix_bin_or_dylib(&bin_root.join("bin").join("rustfmt"));
  builder.fix_bin_or_dylib(&bin_root.join("bin").join("cargo-fmt"));
@@ -1564,6 +1573,7 @@ fn download_ci_component(builder: &Builder<'_>, filename: String, prefix: &str,
  prefix,
  commit,
  "ci-rustc",
+ None,
  )
 }
 
@@ -1574,17 +1584,55 @@ fn download_component(
  prefix: &str,
  key: &str,
  destination: &str,
+ checksums: Option<HashMap<String, String>>,
 ) {
  let cache_dst = builder.out.join("cache");
  let cache_dir = cache_dst.join(key);
  if !cache_dir.exists() {
  t!(fs::create_dir_all(&cache_dir));
  }
 
+ let bin_root = builder.out.join(builder.config.build.triple).join(destination);
  let tarball = cache_dir.join(&filename);
- if !tarball.exists() {
- builder.download_component(base_url, &format!("{key}/{filename}"), &tarball, "");
+ let url = format!("{key}/{filename}");
+
+ // For the beta compiler, put special effort into ensuring the checksums are valid.
+ // FIXME: maybe we should do this for download-rustc as well? but it would be a pain to update
+ // this on each and every nightly ...
+ let checksum = if let Some(checksums) = &checksums {
+ let error = format!(
+ "src/stage0.json doesn't contain a checksum for {url}. \
+ Pre-built artifacts might not be available for this \
+ target at this time, see https://doc.rust-lang.org/nightly\
+ /rustc/platform-support.html for more information."
+ );
+ // TODO: add an enum { Commit, Published } so we don't have to hardcode `dist` in two places
+ let sha256 = checksums.get(&format!("dist/{url}")).expect(&error);
+ if tarball.exists() {
+ if builder.verify(&tarball, sha256) {
+ builder.unpack(&tarball, &bin_root, prefix);
+ return;
+ } else {
+ builder.verbose(&format!(
+ "ignoring cached file {} due to failed verification",
+ tarball.display()
+ ));
+ builder.remove(&tarball);
+ }
+ }
+ Some(sha256)
+ } else if tarball.exists() {
+ return;
+ } else {
+ None
+ };
+
+ builder.download_component(base_url, &url, &tarball, "");
+ if let Some(sha256) = checksum {
+ if !builder.verify(&tarball, sha256) {
+ panic!("failed to verify {}", tarball.display());
+ }
  }
- let bin_root = builder.out.join(builder.config.build.triple).join(destination);
- builder.unpack(&tarball, &bin_root, prefix)
+
+ builder.unpack(&tarball, &bin_root, prefix);
 }