auto merge of #12533 : alexcrichton/rust/svh, r=brson

These hashes are used to detect changes to upstream crates and generate errors which mention that crates possibly need recompilation.

More details can be found in the respective commit messages below. This change is also accompanied with a much needed refactoring of some of the crate loading code to focus more on crate ids instead of name/version pairs.

Closes #12601

Author: bors

Diff for: mk/clean.mk

@@ -58,6 +58,7 @@ clean-generic-$(2)-$(1):
          -name '*.[odasS]' -o \
          -name '*.so' -o      \
          -name '*.dylib' -o   \
+         -name '*.rlib' -o   \
          -name 'stamp.*' -o   \
          -name '*.lib' -o     \
          -name '*.dll' -o     \

Diff for: src/librustc/back/link.rs

@@ -8,9 +8,9 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-
 use back::archive::{Archive, METADATA_FILENAME};
 use back::rpath;
+use back::svh::Svh;
 use driver::driver::{CrateTranslation, OutputFilenames};
 use driver::session::Session;
 use driver::session;
@@ -499,28 +499,31 @@ pub mod write {
  *    system linkers understand.
  */
 
-pub fn build_link_meta(attrs: &[ast::Attribute],
-                       output: &OutputFilenames,
-                       symbol_hasher: &mut Sha256)
-                       -> LinkMeta {
-    // This calculates CMH as defined above
-    fn crate_hash(symbol_hasher: &mut Sha256, crateid: &CrateId) -> ~str {
-        symbol_hasher.reset();
-        symbol_hasher.input_str(crateid.to_str());
-        truncated_hash_result(symbol_hasher)
-    }
-
-    let crateid = match attr::find_crateid(attrs) {
+pub fn find_crate_id(attrs: &[ast::Attribute],
+                     output: &OutputFilenames) -> CrateId {
+    match attr::find_crateid(attrs) {
         None => from_str(output.out_filestem).unwrap(),
         Some(s) => s,
-    };
+    }
+}
 
-    let hash = crate_hash(symbol_hasher, &crateid);
+pub fn crate_id_hash(crate_id: &CrateId) -> ~str {
+    // This calculates CMH as defined above. Note that we don't use the path of
+    // the crate id in the hash because lookups are only done by (name/vers),
+    // not by path.
+    let mut s = Sha256::new();
+    s.input_str(crate_id.short_name_with_version());
+    truncated_hash_result(&mut s).slice_to(8).to_owned()
+}
 
-    LinkMeta {
-        crateid: crateid,
-        crate_hash: hash,
-    }
+pub fn build_link_meta(krate: &ast::Crate,
+                       output: &OutputFilenames) -> LinkMeta {
+    let r = LinkMeta {
+        crateid: find_crate_id(krate.attrs, output),
+        crate_hash: Svh::calculate(krate),
+    };
+    info!("{}", r);
+    return r;
 }
 
 fn truncated_hash_result(symbol_hasher: &mut Sha256) -> ~str {
@@ -539,7 +542,7 @@ fn symbol_hash(tcx: ty::ctxt, symbol_hasher: &mut Sha256,
     symbol_hasher.reset();
     symbol_hasher.input_str(link_meta.crateid.name);
     symbol_hasher.input_str("-");
-    symbol_hasher.input_str(link_meta.crate_hash);
+    symbol_hasher.input_str(link_meta.crate_hash.as_str());
     symbol_hasher.input_str("-");
     symbol_hasher.input_str(encoder::encoded_ty(tcx, t));
     let mut hash = truncated_hash_result(symbol_hasher);
@@ -712,11 +715,8 @@ pub fn mangle_internal_name_by_path_and_seq(path: PathElems, flav: &str) -> ~str
     mangle(path.chain(Some(gensym_name(flav)).move_iter()), None, None)
 }
 
-pub fn output_lib_filename(lm: &LinkMeta) -> ~str {
-    format!("{}-{}-{}",
-            lm.crateid.name,
-            lm.crate_hash.slice_chars(0, 8),
-            lm.crateid.version_or_default())
+pub fn output_lib_filename(id: &CrateId) -> ~str {
+    format!("{}-{}-{}", id.name, crate_id_hash(id), id.version_or_default())
 }
 
 pub fn get_cc_prog(sess: Session) -> ~str {
@@ -779,11 +779,11 @@ fn remove(sess: Session, path: &Path) {
 pub fn link_binary(sess: Session,
                    trans: &CrateTranslation,
                    outputs: &OutputFilenames,
-                   lm: &LinkMeta) -> ~[Path] {
+                   id: &CrateId) -> ~[Path] {
     let mut out_filenames = ~[];
     let crate_types = sess.crate_types.borrow();
     for &crate_type in crate_types.get().iter() {
-        let out_file = link_binary_output(sess, trans, crate_type, outputs, lm);
+        let out_file = link_binary_output(sess, trans, crate_type, outputs, id);
         out_filenames.push(out_file);
     }
 
@@ -807,8 +807,8 @@ fn is_writeable(p: &Path) -> bool {
 }
 
 pub fn filename_for_input(sess: &Session, crate_type: session::CrateType,
-                          lm: &LinkMeta, out_filename: &Path) -> Path {
-    let libname = output_lib_filename(lm);
+                          id: &CrateId, out_filename: &Path) -> Path {
+    let libname = output_lib_filename(id);
     match crate_type {
         session::CrateTypeRlib => {
             out_filename.with_filename(format!("lib{}.rlib", libname))
@@ -834,13 +834,13 @@ fn link_binary_output(sess: Session,
                       trans: &CrateTranslation,
                       crate_type: session::CrateType,
                       outputs: &OutputFilenames,
-                      lm: &LinkMeta) -> Path {
+                      id: &CrateId) -> Path {
     let obj_filename = outputs.temp_path(OutputTypeObject);
     let out_filename = match outputs.single_output_file {
         Some(ref file) => file.clone(),
         None => {
             let out_filename = outputs.path(OutputTypeExe);
-            filename_for_input(&sess, crate_type, lm, &out_filename)
+            filename_for_input(&sess, crate_type, id, &out_filename)
         }
     };
 

Diff for: src/librustc/back/svh.rs

@@ -0,0 +1,112 @@
+// Copyright 2012-2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Calculation and management of a Strict Version Hash for crates
+//!
+//! # Today's ABI problem
+//!
+//! In today's implementation of rustc, it is incredibly difficult to achieve
+//! forward binary compatibility without resorting to C-like interfaces. Within
+//! rust code itself, abi details such as symbol names suffer from a variety of
+//! unrelated factors to code changing such as the "def id drift" problem. This
+//! ends up yielding confusing error messages about metadata mismatches and
+//! such.
+//!
+//! The core of this problem is when when an upstream dependency changes and
+//! downstream dependants are not recompiled. This causes compile errors because
+//! the upstream crate's metadata has changed but the downstream crates are
+//! still referencing the older crate's metadata.
+//!
+//! This problem exists for many reasons, the primary of which is that rust does
+//! not currently support forwards ABI compatibility (in place upgrades of a
+//! crate).
+//!
+//! # SVH and how it alleviates the problem
+//!
+//! With all of this knowledge on hand, this module contains the implementation
+//! of a notion of a "Strict Version Hash" for a crate. This is essentially a
+//! hash of all contents of a crate which can somehow be exposed to downstream
+//! crates.
+//!
+//! This hash is currently calculated by just hashing the AST, but this is
+//! obviously wrong (doc changes should not result in an incompatible ABI).
+//! Implementation-wise, this is required at this moment in time.
+//!
+//! By encoding this strict version hash into all crate's metadata, stale crates
+//! can be detected immediately and error'd about by rustc itself.
+//!
+//! # Relevant links
+//!
+//! Original issue: https://github.com/mozilla/rust/issues/10207
+
+use std::fmt;
+use std::hash::Hash;
+use std::hash::sip::SipState;
+use std::iter::range_step;
+use syntax::ast;
+
+#[deriving(Clone, Eq)]
+pub struct Svh {
+    priv hash: ~str,
+}
+
+impl Svh {
+    pub fn new(hash: &str) -> Svh {
+        assert!(hash.len() == 16);
+        Svh { hash: hash.to_owned() }
+    }
+
+    pub fn as_str<'a>(&'a self) -> &'a str {
+        self.hash.as_slice()
+    }
+
+    pub fn calculate(krate: &ast::Crate) -> Svh {
+        // FIXME: see above for why this is wrong, it shouldn't just hash the
+        //        crate.  Fixing this would require more in-depth analysis in
+        //        this function about what portions of the crate are reachable
+        //        in tandem with bug fixes throughout the rest of the compiler.
+        //
+        //        Note that for now we actually exclude some top-level things
+        //        from the crate like the CrateConfig/span. The CrateConfig
+        //        contains command-line `--cfg` flags, so this means that the
+        //        stage1/stage2 AST for libstd and such is different hash-wise
+        //        when it's actually the exact same representation-wise.
+        //
+        //        As a first stab at only hashing the relevant parts of the
+        //        AST, this only hashes the module/attrs, not the CrateConfig
+        //        field.
+        //
+        // FIXME: this should use SHA1, not SipHash. SipHash is not built to
+        //        avoid collisions.
+        let mut state = SipState::new();
+        krate.module.hash(&mut state);
+        krate.attrs.hash(&mut state);
+
+        let hash = state.result();
+        return Svh {
+            hash: range_step(0, 64, 4).map(|i| hex(hash >> i)).collect()
+        };
+
+        fn hex(b: u64) -> char {
+            let b = (b & 0xf) as u8;
+            let b = match b {
+                0 .. 9 => '0' as u8 + b,
+                _ => 'a' as u8 + b - 10,
+            };
+            b as char
+        }
+    }
+}
+
+impl fmt::Show for Svh {
+    fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
+        f.pad(self.as_str())
+    }
+}

Diff for: src/librustc/driver/driver.rs

@@ -433,7 +433,7 @@ pub fn phase_6_link_output(sess: Session,
          link::link_binary(sess,
                            trans,
                            outputs,
-                           &trans.link));
+                           &trans.link.crateid));
 }
 
 pub fn stop_after_phase_3(sess: Session) -> bool {
@@ -472,8 +472,7 @@ fn write_out_deps(sess: Session,
                   input: &Input,
                   outputs: &OutputFilenames,
                   krate: &ast::Crate) -> io::IoResult<()> {
-    let lm = link::build_link_meta(krate.attrs, outputs,
-                                   &mut ::util::sha2::Sha256::new());
+    let id = link::find_crate_id(krate.attrs, outputs);
 
     let mut out_filenames = ~[];
     for output_type in sess.opts.output_types.iter() {
@@ -482,7 +481,7 @@ fn write_out_deps(sess: Session,
             link::OutputTypeExe => {
                 let crate_types = sess.crate_types.borrow();
                 for output in crate_types.get().iter() {
-                    let p = link::filename_for_input(&sess, *output, &lm, &file);
+                    let p = link::filename_for_input(&sess, *output, &id, &file);
                     out_filenames.push(p);
                 }
             }

Diff for: src/librustc/lib.rs

@@ -55,7 +55,6 @@ use std::str;
 use std::task;
 use std::vec;
 use syntax::ast;
-use syntax::attr;
 use syntax::diagnostic::Emitter;
 use syntax::diagnostic;
 use syntax::parse;
@@ -104,16 +103,17 @@ pub mod front {
 }
 
 pub mod back {
-    pub mod archive;
-    pub mod link;
     pub mod abi;
+    pub mod archive;
     pub mod arm;
+    pub mod link;
+    pub mod lto;
     pub mod mips;
-    pub mod x86;
-    pub mod x86_64;
     pub mod rpath;
+    pub mod svh;
     pub mod target_strs;
-    pub mod lto;
+    pub mod x86;
+    pub mod x86_64;
 }
 
 pub mod metadata;
@@ -312,28 +312,18 @@ pub fn run_compiler(args: &[~str]) {
         let attrs = parse_crate_attrs(sess, &input);
         let t_outputs = d::build_output_filenames(&input, &odir, &ofile,
                                                   attrs, sess);
-        if crate_id || crate_name {
-            let crateid = match attr::find_crateid(attrs) {
-                Some(crateid) => crateid,
-                None => {
-                    sess.fatal("No crate_id and --crate-id or \
-                                --crate-name requested")
-                }
-            };
-            if crate_id {
-                println!("{}", crateid.to_str());
-            }
-            if crate_name {
-                println!("{}", crateid.name);
-            }
-        }
+        let id = link::find_crate_id(attrs, &t_outputs);
 
+        if crate_id {
+            println!("{}", id.to_str());
+        }
+        if crate_name {
+            println!("{}", id.name);
+        }
         if crate_file_name {
-            let lm = link::build_link_meta(attrs, &t_outputs,
-                                           &mut ::util::sha2::Sha256::new());
             let crate_types = session::collect_crate_types(&sess, attrs);
             for &style in crate_types.iter() {
-                let fname = link::filename_for_input(&sess, style, &lm,
+                let fname = link::filename_for_input(&sess, style, &id,
                                                      &t_outputs.with_extension(""));
                 println!("{}", fname.filename_display());
             }

Diff for: src/librustc/metadata/common.rs

@@ -12,6 +12,7 @@
 
 use std::cast;
 use syntax::crateid::CrateId;
+use back::svh::Svh;
 
 // EBML enum definitions and utils shared by the encoder and decoder
 
@@ -70,12 +71,12 @@ pub static tag_crate_deps: uint = 0x18;
 pub static tag_crate_dep: uint = 0x19;
 
 pub static tag_crate_hash: uint = 0x1a;
+pub static tag_crate_crateid: uint = 0x1b;
 
-pub static tag_parent_item: uint = 0x1b;
+pub static tag_parent_item: uint = 0x1c;
 
-pub static tag_crate_dep_name: uint = 0x1c;
-pub static tag_crate_dep_hash: uint = 0x1d;
-pub static tag_crate_dep_vers: uint = 0x1e;
+pub static tag_crate_dep_crateid: uint = 0x1d;
+pub static tag_crate_dep_hash: uint = 0x1e;
 
 pub static tag_mod_impl: uint = 0x1f;
 
@@ -207,8 +208,8 @@ pub static tag_macro_registrar_fn: uint = 0x63;
 pub static tag_exported_macros: uint = 0x64;
 pub static tag_macro_def: uint = 0x65;
 
-#[deriving(Clone)]
+#[deriving(Clone, Show)]
 pub struct LinkMeta {
     crateid: CrateId,
-    crate_hash: ~str,
+    crate_hash: Svh,
 }