Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Only allow one of each links attribute in resolver #4978

Merged
merged 13 commits into from
Feb 24, 2018
47 changes: 28 additions & 19 deletions src/cargo/core/resolver/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,9 @@
//! * Never try to activate a crate version which is incompatible. This means we
//! only try crates which will actually satisfy a dependency and we won't ever
//! try to activate a crate that's semver compatible with something else
//! activated (as we're only allowed to have one).
//! activated (as we're only allowed to have one) nor try to activate a crate
//! that has the same links attribute as something else
//! activated.
//! * Always try to activate the highest version crate first. The default
//! dependency in Cargo (e.g. when you write `foo = "0.1.2"`) is
//! semver-compatible, so selecting the highest version possible will allow us
Expand Down Expand Up @@ -345,11 +347,12 @@ enum GraphNode {
// possible.
#[derive(Clone)]
struct Context<'a> {
// TODO: Both this and the map below are super expensive to clone. We should
// TODO: Both this and the map two below are super expensive to clone. We should
// switch to persistent hash maps if we can at some point or otherwise
// make these much cheaper to clone in general.
activations: Activations,
resolve_features: HashMap<PackageId, HashSet<String>>,
links: HashSet<String>,

// These are two cheaply-cloneable lists (O(1) clone) which are effectively
// hash maps but are built up as "construction lists". We'll iterate these
Expand All @@ -374,9 +377,10 @@ pub fn resolve(summaries: &[(Summary, Method)],
let cx = Context {
resolve_graph: RcList::new(),
resolve_features: HashMap::new(),
links: HashSet::new(),
resolve_replacements: RcList::new(),
activations: HashMap::new(),
replacements: replacements,
replacements,
warnings: RcList::new(),
};
let _p = profile::start("resolving");
Expand Down Expand Up @@ -476,7 +480,7 @@ impl<T> RcVecIter<T> {
fn new(vec: Rc<Vec<T>>) -> RcVecIter<T> {
RcVecIter {
rest: 0..vec.len(),
vec: vec,
vec,
}
}
}
Expand Down Expand Up @@ -564,19 +568,21 @@ struct RemainingCandidates {
}

impl RemainingCandidates {
fn next(&mut self, prev_active: &[Summary]) -> Option<Candidate> {
fn next(&mut self, prev_active: &[Summary], links: &HashSet<String>) -> Option<Candidate> {
// Filter the set of candidates based on the previously activated
// versions for this dependency. We can actually use a version if it
// precisely matches an activated version or if it is otherwise
// incompatible with all other activated versions. Note that we
// define "compatible" here in terms of the semver sense where if
// the left-most nonzero digit is the same they're considered
// compatible.
// compatible unless we have a `*-sys` crate (defined by having a
// linked attribute) then we can only have one version.
self.remaining.by_ref().map(|p| p.1).find(|b| {
prev_active.iter().any(|a| *a == b.summary) ||
prev_active.iter().all(|a| {
!compatible(a.version(), b.summary.version())
})
prev_active.iter().any(|a| *a == b.summary)
|| (b.summary.links().map(|l| !links.contains(l)).unwrap_or(true)
&& prev_active
.iter()
.all(|a| !compatible(a.version(), b.summary.version())))
})
}
}
Expand Down Expand Up @@ -671,8 +677,8 @@ fn activate_deps_loop<'a>(mut cx: Context<'a>,
let mut candidates = RemainingCandidates {
remaining: RcVecIter::new(Rc::clone(&candidates)),
};
(candidates.next(prev_active),
candidates.clone().next(prev_active).is_some(),
(candidates.next(prev_active, &cx.links),
candidates.clone().next(prev_active, &cx.links).is_some(),
candidates)
};

Expand All @@ -682,7 +688,7 @@ fn activate_deps_loop<'a>(mut cx: Context<'a>,
// 1. The version matches the dependency requirement listed for this
// package
// 2. There are no activated versions for this package which are
// semver-compatible, or there's an activated version which is
// semver/links-compatible, or there's an activated version which is
// precisely equal to `candidate`.
//
// This means that we're going to attempt to activate each candidate in
Expand All @@ -697,7 +703,7 @@ fn activate_deps_loop<'a>(mut cx: Context<'a>,
cur,
context_backup: Context::clone(&cx),
deps_backup: <BinaryHeap<DepsFrame>>::clone(&remaining_deps),
remaining_candidates: remaining_candidates,
remaining_candidates,
parent: Summary::clone(&parent),
dep: Dependency::clone(&dep),
features: Rc::clone(&features),
Expand Down Expand Up @@ -758,8 +764,8 @@ fn find_candidate<'a>(backtrack_stack: &mut Vec<BacktrackFrame<'a>>,
while let Some(mut frame) = backtrack_stack.pop() {
let (next, has_another) = {
let prev_active = frame.context_backup.prev_active(&frame.dep);
(frame.remaining_candidates.next(prev_active),
frame.remaining_candidates.clone().next(prev_active).is_some())
(frame.remaining_candidates.next(prev_active, &frame.context_backup.links),
frame.remaining_candidates.clone().next(prev_active, &frame.context_backup.links).is_some())
};
if let Some(candidate) = next {
*cur = frame.cur;
Expand Down Expand Up @@ -1033,9 +1039,9 @@ fn build_requirements<'a, 'b: 'a>(s: &'a Summary, method: &'b Method)
}

impl<'a> Context<'a> {
// Activate this summary by inserting it into our list of known activations.
//
// Returns if this summary with the given method is already activated.
/// Activate this summary by inserting it into our list of known activations.
///
/// Returns true if this summary with the given method is already activated.
fn flag_activated(&mut self,
summary: &Summary,
method: &Method) -> bool {
Expand All @@ -1047,6 +1053,9 @@ impl<'a> Context<'a> {
.or_insert(Vec::new());
if !prev.iter().any(|c| c == summary) {
self.resolve_graph.push(GraphNode::Add(id.clone()));
if let Some(link) = summary.links() {
self.links.insert(link.to_owned());
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could this assert that we're not replacing somethign that was already there?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I could make it assert!(self.links.insert(link.to_owned())); but I think this can get triggered if updating a lock file that already has multiple crates with the same links attribute. Is this acceptable? Is there some way to get better error messages?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Hm that would indeed be bad! Thinking through how this can happen I think we could trigger this if an older cargo created a lock file with a bunch of new crates (that all have links in the manifest) and then a newer cargo tries to use that lock file.

In that case, sounds like we should skip the assert for now!

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Er well, but if we do actually run into that case the previous project surely couldn't build previously becuase it would have been caught later...

How about this, could we return an error from here to entirely abort resolution?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think I have that working now.

}
prev.push(summary.clone());
return false
}
Expand Down
14 changes: 10 additions & 4 deletions src/cargo/core/summary.rs
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ struct Inner {
dependencies: Vec<Dependency>,
features: BTreeMap<String, Vec<String>>,
checksum: Option<String>,
links: Option<String>,
}

impl Summary {
pub fn new(pkg_id: PackageId,
dependencies: Vec<Dependency>,
features: BTreeMap<String, Vec<String>>) -> CargoResult<Summary> {
features: BTreeMap<String, Vec<String>>,
links: Option<String>) -> CargoResult<Summary> {
for dep in dependencies.iter() {
if features.get(dep.name()).is_some() {
bail!("Features and dependencies cannot have the \
Expand Down Expand Up @@ -66,9 +68,10 @@ impl Summary {
Ok(Summary {
inner: Rc::new(Inner {
package_id: pkg_id,
dependencies: dependencies,
features: features,
dependencies,
features,
checksum: None,
links,
}),
})
}
Expand All @@ -82,6 +85,9 @@ impl Summary {
pub fn checksum(&self) -> Option<&str> {
self.inner.checksum.as_ref().map(|s| &s[..])
}
pub fn links(&self) -> Option<&str> {
self.inner.links.as_ref().map(|s| &s[..])
}

pub fn override_id(mut self, id: PackageId) -> Summary {
Rc::make_mut(&mut self.inner).package_id = id;
Expand All @@ -94,7 +100,7 @@ impl Summary {
}

pub fn map_dependencies<F>(mut self, f: F) -> Summary
where F: FnMut(Dependency) -> Dependency {
where F: FnMut(Dependency) -> Dependency {
{
let slot = &mut Rc::make_mut(&mut self.inner).dependencies;
let deps = mem::replace(slot, Vec::new());
Expand Down
4 changes: 2 additions & 2 deletions src/cargo/sources/registry/index.rs
Original file line number Diff line number Diff line change
Expand Up @@ -146,12 +146,12 @@ impl<'cfg> RegistryIndex<'cfg> {
fn parse_registry_package(&mut self, line: &str)
-> CargoResult<(Summary, bool)> {
let RegistryPackage {
name, vers, cksum, deps, features, yanked
name, vers, cksum, deps, features, yanked, links
} = super::DEFAULT_ID.set(&self.source_id, || {
serde_json::from_str::<RegistryPackage>(line)
})?;
let pkgid = PackageId::new(&name, &vers, &self.source_id)?;
let summary = Summary::new(pkgid, deps.inner, features)?;
let summary = Summary::new(pkgid, deps.inner, features, links)?;
let summary = summary.set_checksum(cksum.clone());
if self.hashes.contains_key(&name[..]) {
self.hashes.get_mut(&name[..]).unwrap().insert(vers, cksum);
Expand Down
2 changes: 2 additions & 0 deletions src/cargo/sources/registry/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,8 @@ struct RegistryPackage<'a> {
features: BTreeMap<String, Vec<String>>,
cksum: String,
yanked: Option<bool>,
#[serde(default)]
links: Option<String>,
}

struct DependencyList {
Expand Down
2 changes: 1 addition & 1 deletion src/cargo/util/toml/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -672,7 +672,7 @@ impl TomlManifest {
let include = project.include.clone().unwrap_or_default();

let summary = Summary::new(pkgid, deps, me.features.clone()
.unwrap_or_else(BTreeMap::new))?;
.unwrap_or_else(BTreeMap::new), project.links.clone())?;
let metadata = ManifestMetadata {
description: project.description.clone(),
homepage: project.homepage.clone(),
Expand Down
42 changes: 12 additions & 30 deletions tests/build-script.rs
Original file line number Diff line number Diff line change
Expand Up @@ -227,6 +227,7 @@ not have a custom build script

#[test]
fn links_duplicates() {
// this tests that the links_duplicates are caught at resolver time
let p = project("foo")
.file("Cargo.toml", r#"
[project]
Expand Down Expand Up @@ -256,20 +257,15 @@ fn links_duplicates() {
assert_that(p.cargo("build"),
execs().with_status(101)
.with_stderr("\
[ERROR] multiple packages link to native library `a`, but a native library can \
be linked only once

package `foo v0.5.0 ([..])`
links to native library `a`

package `a-sys v0.5.0 ([..])`
... which is depended on by `foo v0.5.0 ([..])`
also links to native library `a`
error: failed to select a version for `a-sys` (required by `foo`):
all possible versions conflict with previously selected versions of `a-sys`
possible versions to select: 0.5.0
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I wonder if we could perhaps tidy up the wording around this sort of error? I found this sort of hard to follow as there's a lot of mentions of versions and such. Maybe something like:

error: failed to select a version for `a-sys`
  ... required by `foo v0.5.0`

the package `a-sys` links to the native library `a`, but it conflicts
with a previous package which links to `a` as well:

  foo v0.5.0
     .. required by ...

failed to select a version for `a-sys` which could resolve this conflict

I do realize that most of this is fitting in the mold of the previous error messages, but perhaps we can just tidy up the wording around them?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I will see what I can do. :-)

One thing to keep in mind is that different versions of a-sys may have different errors we need to report. For example only some versions have the link attribute. So we may, at least in theory, have to coherently report on a link error with 'a-fork-sys' and a semver error with 'a-sys' from before it had a link attribute.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yeah I'm not entirely sure what to do about "given all these errors present the most reasonable one". I'm ok punting on it for now though or otherwise maybe adding a heuristic to print errors where everything is at maximal versions or something like that

"));
}

#[test]
fn links_duplicates_deep_dependency() {
// this tests that the links_duplicates are caught at resolver time
let p = project("foo")
.file("Cargo.toml", r#"
[project]
Expand Down Expand Up @@ -311,16 +307,9 @@ fn links_duplicates_deep_dependency() {
assert_that(p.cargo("build"),
execs().with_status(101)
.with_stderr("\
[ERROR] multiple packages link to native library `a`, but a native library can \
be linked only once

package `foo v0.5.0 ([..])`
links to native library `a`

package `a-sys v0.5.0 ([..])`
... which is depended on by `a v0.5.0 ([..])`
... which is depended on by `foo v0.5.0 ([..])`
also links to native library `a`
error: failed to select a version for `a-sys` (required by `a`):
all possible versions conflict with previously selected versions of `a-sys`
possible versions to select: 0.5.0
"));
}

Expand Down Expand Up @@ -2741,6 +2730,7 @@ fn deterministic_rustc_dependency_flags() {

#[test]
fn links_duplicates_with_cycle() {
// this tests that the links_duplicates are caught at resolver time
let p = project("foo")
.file("Cargo.toml", r#"
[project]
Expand Down Expand Up @@ -2783,17 +2773,9 @@ fn links_duplicates_with_cycle() {
assert_that(p.cargo("build"),
execs().with_status(101)
.with_stderr("\
[ERROR] multiple packages link to native library `a`, but a native library can \
be linked only once

package `foo v0.5.0 ([..])`
... which is depended on by `b v0.5.0 ([..])`
links to native library `a`

package `a v0.5.0 (file://[..])`
... which is depended on by `foo v0.5.0 ([..])`
... which is depended on by `b v0.5.0 ([..])`
also links to native library `a`
error: failed to select a version for `a` (required by `foo`):
all possible versions conflict with previously selected versions of `a`
possible versions to select: 0.5.0
"));
}

Expand Down
10 changes: 5 additions & 5 deletions tests/resolve.rs
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ fn resolve(pkg: &PackageId, deps: Vec<Dependency>, registry: &[Summary])
fn requires_precise(&self) -> bool { false }
}
let mut registry = MyRegistry(registry);
let summary = Summary::new(pkg.clone(), deps, BTreeMap::new()).unwrap();
let summary = Summary::new(pkg.clone(), deps, BTreeMap::new(), None).unwrap();
let method = Method::Everything;
let resolve = resolver::resolve(&[(summary, method)], &[], &mut registry, None, false)?;
let res = resolve.iter().cloned().collect();
Expand Down Expand Up @@ -78,11 +78,11 @@ macro_rules! pkg {
($pkgid:expr => [$($deps:expr),+]) => ({
let d: Vec<Dependency> = vec![$($deps.to_dep()),+];

Summary::new($pkgid.to_pkgid(), d, BTreeMap::new()).unwrap()
Summary::new($pkgid.to_pkgid(), d, BTreeMap::new(), None).unwrap()
});

($pkgid:expr) => (
Summary::new($pkgid.to_pkgid(), Vec::new(), BTreeMap::new()).unwrap()
Summary::new($pkgid.to_pkgid(), Vec::new(), BTreeMap::new(), None).unwrap()
)
}

Expand All @@ -92,7 +92,7 @@ fn registry_loc() -> SourceId {
}

fn pkg(name: &str) -> Summary {
Summary::new(pkg_id(name), Vec::new(), BTreeMap::new()).unwrap()
Summary::new(pkg_id(name), Vec::new(), BTreeMap::new(), None).unwrap()
}

fn pkg_id(name: &str) -> PackageId {
Expand All @@ -108,7 +108,7 @@ fn pkg_id_loc(name: &str, loc: &str) -> PackageId {
}

fn pkg_loc(name: &str, loc: &str) -> Summary {
Summary::new(pkg_id_loc(name, loc), Vec::new(), BTreeMap::new()).unwrap()
Summary::new(pkg_id_loc(name, loc), Vec::new(), BTreeMap::new(), None).unwrap()
}

fn dep(name: &str) -> Dependency { dep_req(name, "1.0.0") }
Expand Down