Prepare for per-target fingerprints

This commit refactors all related infrastructure for the cargo_rustc module to
move the granularity of fingerprints from packages to targets. This involved
heavily modifying the `JobQueue` structure to understand a finer-grained target
for dirtiness propagation and dependency management, and then dealing with
fallout in the main module.

The fingerprint module has been refactored to support a per-target fingerprint.
A fallout of this change is that each build step has its own fingerprint,
including the custom build command step. This will be implemented in a future
commit.

As fallout of this reorganization, we are now exploiting the maximal parallelism
within packages themselves whereas before we were only partially parallelizing.
No new features were added as part of this commit, so I just ensured that all
the tests ran.

src/cargo/core/manifest.rs

@@ -344,13 +344,13 @@ impl Target {
 
  pub fn lib_target(name: &str, crate_targets: Vec<LibKind>,
  src_path: &Path, profile: &Profile,
- metadata: &Metadata) -> Target {
+ metadata: Metadata) -> Target {
  Target {
  kind: LibTarget(crate_targets),
  name: name.to_string(),
  src_path: src_path.clone(),
  profile: profile.clone(),
- metadata: Some(metadata.clone())
+ metadata: Some(metadata)
  }
  }
 

src/cargo/ops/cargo_rustc/context.rs

@@ -5,6 +5,7 @@ use core::{SourceMap, Package, PackageId, PackageSet, Resolve, Target};
 use util;
 use util::{CargoResult, ChainError, internal, Config, profile};
 
+use super::{Kind, KindPlugin, KindTarget};
 use super::layout::{Layout, LayoutProxy};
 
 #[deriving(Show)]
@@ -155,21 +156,21 @@ impl<'a, 'b> Context<'a, 'b> {
  }
 
  /// Returns the appropriate directory layout for either a plugin or not.
- pub fn layout(&self, plugin: bool) -> LayoutProxy {
- if plugin {
- LayoutProxy::new(&self.host, self.primary)
- } else {
- LayoutProxy::new(self.target.as_ref().unwrap_or(&self.host),
- self.primary)
+ pub fn layout(&self, kind: Kind) -> LayoutProxy {
+ match kind {
+ KindPlugin => LayoutProxy::new(&self.host, self.primary),
+  KindTarget => LayoutProxy::new(self.target.as_ref()
+  .unwrap_or(&self.host),
+  self.primary)
  }
  }
 
  /// Return the (prefix, suffix) pair for dynamic libraries.
  ///
  /// If `plugin` is true, the pair corresponds to the host platform,
  /// otherwise it corresponds to the target platform.
- fn dylib(&self, plugin: bool) -> (&str, &str) {
- let pair = if plugin {&self.host_dylib} else {&self.target_dylib};
+ fn dylib(&self, kind: Kind) -> (&str, &str) {
+ let pair = if kind == KindPlugin {&self.host_dylib} else {&self.target_dylib};
  (pair.ref0().as_slice(), pair.ref1().as_slice())
  }
 
@@ -182,7 +183,9 @@ impl<'a, 'b> Context<'a, 'b> {
  ret.push(format!("{}{}", stem, self.target_exe));
  } else {
  if target.is_dylib() {
- let (prefix, suffix) = self.dylib(target.get_profile().is_plugin());
+ let plugin = target.get_profile().is_plugin();
+ let kind = if plugin {KindPlugin} else {KindTarget};
+ let (prefix, suffix) = self.dylib(kind);
  ret.push(format!("{}{}{}", prefix, stem, suffix));
  }
  if target.is_rlib() {

src/cargo/ops/cargo_rustc/fingerprint.rs

@@ -1,125 +1,196 @@
-use std::hash::Hasher;
+use std::hash::{Hash, Hasher};
 use std::hash::sip::SipHasher;
-use std::io::{fs, File};
+use std::io::{fs, File, UserRWX};
 
 use core::{Package, Target};
 use util;
 use util::hex::short_hash;
 use util::{CargoResult, Fresh, Dirty, Freshness, internal, Require, profile};
 
-use super::job::Job;
+use super::{Kind, KindTarget};
+use super::job::Work;
 use super::context::Context;
 
-/// Calculates the fingerprint of a package's targets and prepares to write a
-/// new fingerprint.
+/// A tuple result of the `prepare_foo` functions in this module.
 ///
-/// This function will first calculate the freshness of the package and return
-/// it as the first part of the return tuple. It will then prepare a job to
-/// update the fingerprint if this package is actually rebuilt as part of
-/// compilation, returning the job as the second part of the tuple.
+/// The first element of the triple is whether the target in question is
+/// currently fresh or not, and the second two elements are work to perform when
+/// the target is dirty or fresh, respectively.
 ///
-/// The third part of the tuple is a job to run when a package is discovered to
-/// be fresh to ensure that all of its artifacts are moved to the correct
-/// location.
-pub fn prepare<'a, 'b>(cx: &mut Context<'a, 'b>, pkg: &'a Package,
- targets: &[&'a Target])
- -> CargoResult<(Freshness, Job, Job)> {
- let _p = profile::start(format!("fingerprint: {}", pkg));
- let filename = format!(".{}.{}.fingerprint", pkg.get_name(),
- short_hash(pkg.get_package_id()));
- let filename = filename.as_slice();
- let (old_fingerprint_loc, new_fingerprint_loc) = {
- let layout = cx.layout(false);
- (layout.old_root().join(filename), layout.root().join(filename))
+/// Both units of work are always generated because a fresh package may still be
+/// rebuilt if some upstream dependency changes.
+pub type Preparation = (Freshness, Work, Work);
+
+/// Prepare the necessary work for the fingerprint for a specific target.
+///
+/// When dealing with fingerprints, cargo gets to choose what granularity
+/// "freshness" is considered at. One option is considering freshness at the
+/// package level. This means that if anything in a package changes, the entire
+/// package is rebuilt, unconditionally. This simplicity comes at a cost,
+/// however, in that test-only changes will cause libraries to be rebuilt, which
+/// is quite unfortunate!
+///
+/// The cost was deemed high enough that fingerprints are now calculated at the
+/// layer of a target rather than a package. Each target can then be kept track
+/// of separately and only rebuilt as necessary. This requires cargo to
+/// understand what the inputs are to a target, so we drive rustc with the
+/// --dep-info flag to learn about all input files to a unit of compilation.
+///
+/// This function will calculate the fingerprint for a target and prepare the
+/// work necessary to either write the fingerprint or copy over all fresh files
+/// from the old directories to their new locations.
+pub fn prepare_target(cx: &mut Context, pkg: &Package, target: &Target,
+ kind: Kind) -> CargoResult<Preparation> {
+ let _p = profile::start(format!("fingerprint: {} / {}",
+ pkg.get_package_id(), target));
+ let (old, new) = dirs(cx, pkg, kind);
+ let filename = if target.is_lib() {
+ format!("lib-{}", target.get_name())
+ } else if target.get_profile().is_doc() {
+ format!("doc-{}", target.get_name())
+ } else {
+ format!("bin-{}", target.get_name())
  };
+ let old_loc = old.join(filename.as_slice());
+ let new_loc = new.join(filename.as_slice());
 
- // First, figure out if the old location exists, and if it does whether it's
- // still fresh or not.
- let (is_fresh, fingerprint) = try!(is_fresh(pkg, &old_fingerprint_loc,
- cx, targets));
-
- // Prepare a job to update the location of the new fingerprint.
- let new_fingerprint_loc2 = new_fingerprint_loc.clone();
- let write_fingerprint = Job::new(proc() {
- let mut f = try!(File::create(&new_fingerprint_loc2));
- try!(f.write_str(fingerprint.as_slice()));
- Ok(Vec::new())
- });
-
- // Prepare a job to copy over all old artifacts into their new destination.
- let mut pairs = Vec::new();
- pairs.push((old_fingerprint_loc, new_fingerprint_loc));
-
- // TODO: this shouldn't explicitly pass false, for more info see
- // cargo_rustc::compile_custom
- if pkg.get_manifest().get_build().len() > 0 {
- let layout = cx.layout(false);
- pairs.push((layout.old_native(pkg), layout.native(pkg)));
- }
+ let new_fingerprint = try!(calculate_target_fingerprint(cx, pkg, target));
+ let new_fingerprint = mk_fingerprint(cx, &new_fingerprint);
 
- for &target in targets.iter() {
- if target.get_profile().is_doc() { continue }
- let target_layout = cx.layout(false);
- let plugin_layout = cx.layout(true);
- let req = cx.get_requirement(pkg, target);
+ let is_fresh = try!(is_fresh(&old_loc, new_fingerprint.as_slice()));
+ let layout = cx.layout(kind);
+ let mut pairs = vec![(old_loc, new_loc.clone())];
 
- for filename in cx.target_filenames(target).iter() {
+ if !target.get_profile().is_doc() {
+ pairs.extend(cx.target_filenames(target).iter().map(|filename| {
  let filename = filename.as_slice();
- if req.is_target() {
- pairs.push((target_layout.old_root().join(filename),
- target_layout.root().join(filename)));
- }
- if req.is_plugin() && plugin_layout.root() != target_layout.root() {
- pairs.push((plugin_layout.old_root().join(filename),
- plugin_layout.root().join(filename)));
- }
- }
+ ((layout.old_root().join(filename), layout.root().join(filename)))
+ }));
  }
- let move_old = Job::new(proc() {
- for &(ref src, ref dst) in pairs.iter() {
+
+ Ok(prepare(is_fresh, new_loc, new_fingerprint, pairs))
+}
+
+/// Prepare the necessary work for the fingerprint of a build command.
+///
+/// Build commands are located on packages, not on targets. Additionally, we
+/// don't have --dep-info to drive calculation of the fingerprint of a build
+/// command. This brings up an interesting predicament which gives us a few
+/// options to figure out whether a build command is dirty or not:
+///
+/// 1. A build command is dirty if *any* file in a package changes. In theory
+/// all files are candidate for being used by the build command.
+/// 2. A build command is dirty if any file in a *specific directory* changes.
+/// This may lose information as it may require files outside of the specific
+/// directory.
+/// 3. A build command must itself provide a dep-info-like file stating how it
+/// should be considered dirty or not.
+///
+/// The currently implemented solution is option (1), although it is planned to
+/// migrate to option (2) in the near future.
+pub fn prepare_build_cmd(cx: &mut Context, pkg: &Package)
+ -> CargoResult<Preparation> {
+ let _p = profile::start(format!("fingerprint build cmd: {}",
+ pkg.get_package_id()));
+
+ // TODO: this should not explicitly pass KindTarget
+ let kind = KindTarget;
+
+ if pkg.get_manifest().get_build().len() == 0 {
+ return Ok((Fresh, proc() Ok(()), proc() Ok(())))
+ }
+ let (old, new) = dirs(cx, pkg, kind);
+ let old_loc = old.join("build");
+ let new_loc = new.join("build");
+
+ let new_fingerprint = try!(calculate_build_cmd_fingerprint(cx, pkg));
+ let new_fingerprint = mk_fingerprint(cx, &new_fingerprint);
+
+ let is_fresh = try!(is_fresh(&old_loc, new_fingerprint.as_slice()));
+ let layout = cx.layout(kind);
+ let pairs = vec![(old_loc, new_loc.clone()),
+ (layout.old_native(pkg), layout.native(pkg))];
+
+ Ok(prepare(is_fresh, new_loc, new_fingerprint, pairs))
+}
+
+/// Prepare work for when a package starts to build
+pub fn prepare_init(cx: &mut Context, pkg: &Package, kind: Kind)
+ -> (Work, Work) {
+ let (_, new1) = dirs(cx, pkg, kind);
+ let new2 = new1.clone();
+
+ let work1 = proc() { try!(fs::mkdir(&new1, UserRWX)); Ok(()) };
+ let work2 = proc() { try!(fs::mkdir(&new2, UserRWX)); Ok(()) };
+
+ (work1, work2)
+}
+
+/// Given the data to build and write a fingerprint, generate some Work
+/// instances to actually perform the necessary work.
+fn prepare(is_fresh: bool, loc: Path, fingerprint: String,
+ to_copy: Vec<(Path, Path)>) -> Preparation {
+ let write_fingerprint = proc() {
+ try!(File::create(&loc).write_str(fingerprint.as_slice()));
+ Ok(())
+ };
+
+ let move_old = proc() {
+ for &(ref src, ref dst) in to_copy.iter() {
  try!(fs::rename(src, dst));
  }
- Ok(Vec::new())
- });
+ Ok(())
+ };
 
- Ok((if is_fresh {Fresh} else {Dirty}, write_fingerprint, move_old))
+ (if is_fresh {Fresh} else {Dirty}, write_fingerprint, move_old)
 }
 
-fn is_fresh(dep: &Package, loc: &Path, cx: &mut Context, targets: &[&Target])
- -> CargoResult<(bool, String)> {
- let dep_fingerprint = try!(get_fingerprint(dep, cx));
- let new_pkg_fingerprint = format!("{}{}", cx.rustc_version, dep_fingerprint);
-
- let new_fingerprint = fingerprint(new_pkg_fingerprint, hash_targets(targets));
+/// Return the (old, new) location for fingerprints for a package
+pub fn dirs(cx: &mut Context, pkg: &Package, kind: Kind) -> (Path, Path) {
+ let dirname = format!("{}-{}", pkg.get_name(),
+ short_hash(pkg.get_package_id()));
+ let dirname = dirname.as_slice();
+ let layout = cx.layout(kind);
+ let layout = layout.proxy();
+ (layout.old_fingerprint().join(dirname), layout.fingerprint().join(dirname))
+}
 
+fn is_fresh(loc: &Path, new_fingerprint: &str) -> CargoResult<bool> {
  let mut file = match File::open(loc) {
  Ok(file) => file,
- Err(..) => return Ok((false, new_fingerprint)),
+ Err(..) => return Ok(false),
  };
 
  let old_fingerprint = try!(file.read_to_string());
 
  log!(5, "old fingerprint: {}", old_fingerprint);
  log!(5, "new fingerprint: {}", new_fingerprint);
 
- Ok((old_fingerprint == new_fingerprint, new_fingerprint))
+ Ok(old_fingerprint.as_slice() == new_fingerprint)
 }
 
-fn get_fingerprint(pkg: &Package, cx: &Context) -> CargoResult<String> {
- let id = pkg.get_package_id().get_source_id();
- let source = try!(cx.sources.get(id).require(|| {
- internal(format!("Missing package source for: {}", id))
- }));
- source.fingerprint(pkg)
+/// Frob in the necessary data from the context to generate the real
+/// fingerprint.
+fn mk_fingerprint<T: Hash>(cx: &Context, data: &T) -> String {
+ let hasher = SipHasher::new_with_keys(0,0);
+ util::to_hex(hasher.hash(&(&cx.rustc_version, data)))
 }
 
-fn hash_targets(targets: &[&Target]) -> u64 {
- let hasher = SipHasher::new_with_keys(0,0);
- let targets = targets.iter().map(|t| (*t).clone()).collect::<Vec<Target>>();
- hasher.hash(&targets)
+fn calculate_target_fingerprint(cx: &Context, pkg: &Package, target: &Target)
+ -> CargoResult<String> {
+ let source = cx.sources
+ .get(pkg.get_package_id().get_source_id())
+ .expect("BUG: Missing package source");
+
+ let pkg_fingerprint = try!(source.fingerprint(pkg));
+ Ok(pkg_fingerprint + short_hash(target))
 }
 
-fn fingerprint(package: String, profiles: u64) -> String {
- let hasher = SipHasher::new_with_keys(0,0);
- util::to_hex(hasher.hash(&(package, profiles)))
+fn calculate_build_cmd_fingerprint(cx: &Context, pkg: &Package)
+ -> CargoResult<String> {
+ let source = cx.sources
+ .get(pkg.get_package_id().get_source_id())
+ .expect("BUG: Missing package source");
+
+ source.fingerprint(pkg)
 }