Fix checksum failures with new self-update process

This changes the update process in the following ways: the current
version is read from the server at /rustup/stable-release.toml; if the
version is different from the running version then rustup downloads
the new release from the archives at /rustup/archive/$version/.

Fixes #524

Author: brson

ci/sync-dist.py

@@ -10,12 +10,18 @@
 # * Sync local bins to dev archives
 #   python sync-dist.py local-to-dev-archives 0.2.0
 #
+# * Update dev release number
+#   python sync-dist.py update-dev-release 0.2.0
+#
 # * Sync local bins to prod archives
 #   python sync-dist.py local-to-prod-archives 0.2.0
 #
 # * Sync local bins to prod
 #   python sync-dist.py local-to-prod
 #
+# * Update prod release number
+#   python sync-dist.py update-prod-release 0.2.0
+#
 # Don't forget to tag the release, dummy!
 
 import sys
@@ -26,8 +32,10 @@
 def usage():
     print ("usage: sync-dist dev-to-local [--live-run]\n"
            "       sync-dist local-to-dev-archives $version [--live-run]\n"
+           "       sync-dist update-dev-release $version [--live-run]\n"
            "       sync-dist local-to-prod-archives $version [--live-run]\n"
-           "       sync-dist local-to-prod [--live-run]\n")
+           "       sync-dist local-to-prod [--live-run]\n"
+           "       sync-dist update-prod-release $version [--live-run]\n")
     sys.exit(1)
 
 command = None
@@ -41,11 +49,13 @@ def usage():
 
 if not command in ["dev-to-local",
                    "local-to-dev-archives",
+                   "update-dev-release",
                    "local-to-prod-archives",
-                   "local-to-prod"]:
+                   "local-to-prod",
+                   "update-prod-release"]:
     usage()
 
-if "archives" in command:
+if "archives" in command or "release" in command:
     if len(sys.argv) < 3:
         usage()
     archive_version = sys.argv[2]
@@ -77,12 +87,22 @@ def usage():
     s3cmd = "s3cmd sync ./local-rustup/dist/ s3://{}/rustup/archive/{}/".format(s3_bucket, archive_version)
 elif command == "local-to-prod":
     s3cmd = "s3cmd sync ./local-rustup/dist/ s3://{}/rustup/dist/".format(s3_bucket)
+elif command == "update-dev-release" \
+     or command == "update-prod-release":
+    s3cmd = "s3cmd put ./local-rustup/release-stable.toml s3://{}/rustup/release-stable.toml".format(s3_bucket)
 else:
     sys.exit(1)
 
 print "s3 command: {}".format(s3cmd)
 print
 
+# Create the release information
+if command == "update-dev-release" \
+   or command == "update-prod-release":
+    with open("./local-rustup/release-stable.toml", "w") as f:
+        f.write("schema-version = '1'\n")
+        f.write("version = '{}'\n".format(archive_version))
+
 def run_s3cmd(command):
     s3cmd = command.split(" ")
 

src/rustup-cli/main.rs

@@ -19,6 +19,7 @@ extern crate scopeguard;
 extern crate tempdir;
 extern crate sha2;
 extern crate markdown;
+extern crate toml;
 
 #[cfg(windows)]
 extern crate winapi;

src/rustup-cli/self_update.rs

@@ -34,13 +34,11 @@ use common::{self, Confirm};
 use errors::*;
 use rustup_dist::dist;
 use rustup_utils::utils;
-use sha2::{Sha256, Digest};
 use std::env;
 use std::env::consts::EXE_SUFFIX;
 use std::path::{Path, PathBuf};
 use std::process::{self, Command};
-use std::fs::{self, File};
-use std::io::Read;
+use std::fs;
 use tempdir::TempDir;
 use term2;
 use regex::Regex;
@@ -1210,6 +1208,8 @@ fn parse_new_rustup_version(version: String) -> String {
 }
 
 pub fn prepare_update() -> Result<Option<PathBuf>> {
+    use toml;
+
     let ref cargo_home = try!(utils::cargo_home());
     let ref rustup_path = cargo_home.join(&format!("bin/rustup{}", EXE_SUFFIX));
     let ref setup_path = cargo_home.join(&format!("bin/rustup-init{}", EXE_SUFFIX));
@@ -1231,53 +1231,49 @@ pub fn prepare_update() -> Result<Option<PathBuf>> {
     let tempdir = try!(TempDir::new("rustup-update")
         .chain_err(|| "error creating temp directory"));
 
-    // Get download URL
-    let url = format!("{}/{}/rustup-init{}", update_root, triple, EXE_SUFFIX);
-
-    // Calculate own hash
-    let mut hasher = Sha256::new();
-    let mut self_exe = try!(File::open(rustup_path)
-                        .chain_err(|| "can't open self exe to calculate hash"));
-    let ref mut buf = [0; 4096];
-    loop {
-        let bytes = try!(self_exe.read(buf)
-                         .chain_err(|| "failed to read from self exe while calculating hash"));
-        if bytes == 0 { break; }
-        hasher.input(&buf[0..bytes]);
-    }
-    let current_hash = hasher.result_str();
-    drop(self_exe);
+    // Get current version
+    let current_version = env!("CARGO_PKG_VERSION");
 
-    // Download latest hash
+    // Download available version
     info!("checking for self-updates");
-    let hash_url = try!(utils::parse_url(&(url.clone() + ".sha256")));
-    let hash_file = tempdir.path().join("hash");
-    try!(utils::download_file(&hash_url, &hash_file, None, &|_| ()));
-    let mut latest_hash = try!(utils::read_file("hash", &hash_file));
-    latest_hash.truncate(64);
+    let release_file_url = format!("{}/release-stable.toml", update_root);
+    let release_file_url = try!(utils::parse_url(&release_file_url));
+    let release_file = tempdir.path().join("release-stable.toml");
+    try!(utils::download_file(&release_file_url, &release_file, None, &|_| ()));
+    let release_toml_str = try!(utils::read_file("rustup release", &release_file));
+    let release_toml = try!(toml::Parser::new(&release_toml_str).parse()
+                            .ok_or(Error::from("unable to parse rustup release file")));
+    let schema = try!(release_toml.get("schema-version")
+                      .ok_or(Error::from("no schema key in rustup release file")));
+    let schema = try!(schema.as_str()
+                      .ok_or(Error::from("invalid schema key in rustup release file")));
+    let available_version = try!(release_toml.get("version")
+                                 .ok_or(Error::from("no version key in rustup release file")));
+    let available_version = try!(available_version.as_str()
+                                 .ok_or(Error::from("invalid version key in rustup release file")));
+
+    if schema != "1" {
+        return Err(Error::from(&*format!("unknown schema version '{}' in rustup release file", schema)));
+    }
 
     // If up-to-date
-    if latest_hash == current_hash {
+    if available_version == current_version {
         return Ok(None);
     }
 
+    // Get download URL
+    let url = format!("{}/archive/{}/{}/rustup-init{}", update_root,
+                      available_version, triple, EXE_SUFFIX);
+
     // Get download path
     let download_url = try!(utils::parse_url(&url));
 
     // Download new version
     info!("downloading self-update");
-    let mut hasher = Sha256::new();
     try!(utils::download_file(&download_url,
                               &setup_path,
-                              Some(&mut hasher),
+                              None,
                               &|_| ()));
-    let download_hash = hasher.result_str();
-
-    // Check that hash is correct
-    if latest_hash != download_hash {
-        info!("update not yet available. bug #364");
-        return Ok(None);
-    }
 
     // Mark as executable
     try!(utils::make_executable(setup_path));

tests/cli-self-upd.rs

@@ -29,12 +29,14 @@ use rustup_mock::clitools::{self, Config, Scenario,
                                expect_stderr_ok,
                                expect_err, expect_err_ex,
                                this_host_triple};
-use rustup_mock::dist::{create_hash, calc_hash};
+use rustup_mock::dist::{calc_hash};
 use rustup_mock::{get_path, restore_path};
-use rustup_utils::raw;
+use rustup_utils::{utils, raw};
 
 macro_rules! for_host { ($s: expr) => (&format!($s, this_host_triple())) }
 
+const TEST_VERSION: &'static str = "1.1.1";
+
 pub fn setup(f: &Fn(&Config)) {
     clitools::setup(Scenario::SimpleV2, &|config| {
         // Lock protects environment variables
@@ -60,16 +62,15 @@ pub fn update_setup(f: &Fn(&Config, &Path)) {
         let ref self_dist = self_dist_tmp.path();
 
         let ref trip = this_host_triple();
-        let ref dist_dir = self_dist.join(&format!("{}", trip));
+        let ref dist_dir = self_dist.join(&format!("archive/{}/{}", TEST_VERSION, trip));
         let ref dist_exe = dist_dir.join(&format!("rustup-init{}", EXE_SUFFIX));
-        let ref dist_hash = dist_dir.join(&format!("rustup-init{}.sha256", EXE_SUFFIX));
         let ref rustup_bin = config.exedir.join(&format!("rustup-init{}", EXE_SUFFIX));
 
         fs::create_dir_all(dist_dir).unwrap();
+        output_release_file(self_dist, "1", TEST_VERSION);
         fs::copy(rustup_bin, dist_exe).unwrap();
         // Modify the exe so it hashes different
         raw::append_file(dist_exe, "").unwrap();
-        create_hash(dist_exe, dist_hash);
 
         let ref root_url = format!("file://{}", self_dist.display());
         env::set_var("RUSTUP_UPDATE_ROOT", root_url);
@@ -78,6 +79,15 @@ pub fn update_setup(f: &Fn(&Config, &Path)) {
     });
 }
 
+fn output_release_file(dist_dir: &Path, schema: &str, version: &str) {
+    let contents = format!(r#"
+schema-version = "{}"
+version = "{}"
+"#, schema, version);
+    let file = dist_dir.join("release-stable.toml");
+    utils::write_file("release", &file, &contents).unwrap();
+}
+
 #[test]
 fn install_bins_to_cargo_home() {
     setup(&|config| {
@@ -503,88 +513,13 @@ fn update_but_delete_existing_updater_first() {
     });
 }
 
-#[test]
-fn update_no_change() {
-    update_setup(&|config, self_dist| {
-        expect_ok(config, &["rustup-init", "-y"]);
-
-        let ref trip = this_host_triple();
-        let ref dist_dir = self_dist.join(&format!("{}", trip));
-        let ref dist_exe = dist_dir.join(&format!("rustup-init{}", EXE_SUFFIX));
-        let ref dist_hash = dist_dir.join(&format!("rustup-init{}.sha256", EXE_SUFFIX));
-        let ref rustup_bin = config.exedir.join(&format!("rustup{}", EXE_SUFFIX));
-        fs::copy(rustup_bin, dist_exe).unwrap();
-        create_hash(dist_exe, dist_hash);
-
-        expect_ok_ex(config, &["rustup", "self", "update"],
-r"",
-r"info: checking for self-updates
-");
-
-    });
-}
-
-#[test]
-#[ignore] // Workaround for #346
-fn update_bad_hash() {
-    update_setup(&|config, self_dist| {
-        expect_ok(config, &["rustup-init", "-y"]);
-
-        let ref trip = this_host_triple();
-        let ref dist_dir = self_dist.join(&format!("{}", trip));
-        let ref dist_hash = dist_dir.join(&format!("rustup-init{}.sha256", EXE_SUFFIX));
-
-        let ref some_other_file = config.distdir.join("dist/channel-rust-nightly.toml");
-
-        create_hash(some_other_file, dist_hash);
-
-        expect_err(config, &["rustup", "self", "update"],
-                   "checksum failed");
-    });
-}
-
-// Workaround for #346
-#[test]
-fn update_hash_drift() {
-    update_setup(&|config, self_dist| {
-        expect_ok(config, &["rustup-init", "-y"]);
-
-        let ref trip = this_host_triple();
-        let ref dist_dir = self_dist.join(&format!("{}", trip));
-        let ref dist_hash = dist_dir.join(&format!("rustup-init{}.sha256", EXE_SUFFIX));
-
-        let ref some_other_file = config.distdir.join("dist/channel-rust-nightly.toml");
-
-        create_hash(some_other_file, dist_hash);
-
-        expect_stderr_ok(config, &["rustup", "self", "update"],
-                         "update not yet available");
-    });
-}
-
-#[test]
-fn update_hash_file_404() {
-    update_setup(&|config, self_dist| {
-        expect_ok(config, &["rustup-init", "-y"]);
-
-        let ref trip = this_host_triple();
-        let ref dist_dir = self_dist.join(&format!("{}", trip));
-        let ref dist_hash = dist_dir.join(&format!("rustup-init{}.sha256", EXE_SUFFIX));
-
-        fs::remove_file(dist_hash).unwrap();
-
-        expect_err(config, &["rustup", "self", "update"],
-                   "could not download file");
-    });
-}
-
 #[test]
 fn update_download_404() {
     update_setup(&|config, self_dist| {
         expect_ok(config, &["rustup-init", "-y"]);
 
         let ref trip = this_host_triple();
-        let ref dist_dir = self_dist.join(&format!("{}", trip));
+        let ref dist_dir = self_dist.join(&format!("archive/{}/{}", TEST_VERSION, trip));
         let ref dist_exe = dist_dir.join(&format!("rustup-init{}", EXE_SUFFIX));
 
         fs::remove_file(dist_exe).unwrap();
@@ -632,6 +567,29 @@ fn update_updates_rustup_bin() {
     });
 }
 
+#[test]
+fn update_bad_schema() {
+    update_setup(&|config, self_dist| {
+        expect_ok(config, &["rustup-init", "-y"]);
+        output_release_file(dist_dir, "17", "1.1.1");
+        expect_err(config, &["rustup", "self", "update"],
+                     "unknown schema version");
+    });
+}
+
+#[test]
+fn update_no_change() {
+    let version = env!("CARGO_PKG_VERSION");
+    update_setup(&|config, self_dist| {
+        expect_ok(config, &["rustup-init", "-y"]);
+        output_release_file(dist_dir, "1", version);
+        expect_ok_ex(config, &["rustup", "self", "update"],
+r"",
+r"info: checking for self-updates
+");
+    });
+}
+
 #[test]
 fn rustup_self_updates() {
     update_setup(&|config, _| {