diff --git a/news/113.feature.rst b/news/113.feature.rst new file mode 100644 index 0000000..9dddaa8 --- /dev/null +++ b/news/113.feature.rst @@ -0,0 +1 @@ +Implement backjumping to significantly speed up the resolution process by skipping over irrelevant parts of the resolution search space. diff --git a/src/resolvelib/resolvers.py b/src/resolvelib/resolvers.py index 49e30c7..1d110b8 100644 --- a/src/resolvelib/resolvers.py +++ b/src/resolvelib/resolvers.py @@ -266,8 +266,8 @@ def _attempt_to_pin_criterion(self, name): # end, signal for backtracking. return causes - def _backtrack(self): - """Perform backtracking. + def _backjump(self, causes): + """Perform backjumping. When we enter here, the stack is like this:: @@ -283,22 +283,44 @@ def _backtrack(self): Each iteration of the loop will: - 1. Discard Z. - 2. Discard Y but remember its incompatibility information gathered + 1. Identify Z. The incompatibility is not always caused by the latest state. + For example, given three requirements A, B and C, with dependencies + A1, B1 and C1, where A1 and B1 are incompatible: the last state + might be related to C, so we want to discard the previous state. + 2. Discard Z. + 3. Discard Y but remember its incompatibility information gathered previously, and the failure we're dealing with right now. - 3. Push a new state Y' based on X, and apply the incompatibility + 4. Push a new state Y' based on X, and apply the incompatibility information from Y to Y'. - 4a. If this causes Y' to conflict, we need to backtrack again. Make Y' + 5a. If this causes Y' to conflict, we need to backtrack again. Make Y' the new Z and go back to step 2. - 4b. If the incompatibilities apply cleanly, end backtracking. + 5b. If the incompatibilities apply cleanly, end backtracking. """ + incompatible_deps = set( + [c.parent.name for c in causes if c.parent is not None] + + [c.requirement.name for c in causes] + ) while len(self._states) >= 3: # Remove the state that triggered backtracking. del self._states[-1] - # Retrieve the last candidate pin and known incompatibilities. - broken_state = self._states.pop() - name, candidate = broken_state.mapping.popitem() + # Ensure to backtrack to a state that caused the incompatibility + incompatible_state = False + while not incompatible_state: + # Retrieve the last candidate pin and known incompatibilities. + try: + broken_state = self._states.pop() + name, candidate = broken_state.mapping.popitem() + except (IndexError, KeyError): + raise ResolutionImpossible(causes) + current_dependencies = { + self._p.identify(d) + for d in self._p.get_dependencies(candidate) + } + incompatible_state = not current_dependencies.isdisjoint( + incompatible_deps + ) + incompatibilities_from_broken = [ (k, list(v.incompatibilities)) for k, v in broken_state.criteria.items() @@ -403,10 +425,10 @@ def resolve(self, requirements, max_rounds): if failure_causes: causes = [i for c in failure_causes for i in c.information] - # Backtrack if pinning fails. The backtrack process puts us in + # Backjump if pinning fails. The backjump process puts us in # an unpinned state, so we can work on it in the next round. self._r.resolving_conflicts(causes=causes) - success = self._backtrack() + success = self._backjump(causes) self.state.backtrack_causes[:] = causes # Dead ends everywhere. Give up. diff --git a/tests/test_resolvers.py b/tests/test_resolvers.py index 8a1349a..fe95e4b 100644 --- a/tests/test_resolvers.py +++ b/tests/test_resolvers.py @@ -1,14 +1,17 @@ -from typing import ( - Any, - Iterable, - Iterator, - List, - Mapping, - Sequence, - Set, - Tuple, - Union, -) +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import ( + Any, + Iterable, + Iterator, + List, + Mapping, + Sequence, + Set, + Tuple, + Union, + ) import pytest from packaging.requirements import Requirement @@ -21,12 +24,17 @@ ResolutionImpossible, Resolver, ) -from resolvelib.resolvers import ( - Criterion, - RequirementInformation, - RequirementsConflicted, - Resolution, -) + +if TYPE_CHECKING: + from resolvelib.resolvers import ( + Criterion, + RequirementInformation, + RequirementsConflicted, + ) + +from collections import namedtuple + +from resolvelib.resolvers import Resolution def test_candidate_inconsistent_error(): @@ -115,10 +123,21 @@ def is_satisfied_by(self, requirement, candidate): def test_resolving_conflicts(): + Candidate = namedtuple( + "Candidate", ["name", "version", "requirements"] + ) # name, version, requirements + _Requirement = namedtuple( + "Requirement", ["name", "versions"] + ) # name, versions + a1 = Candidate("a", 1, [_Requirement("q", {1})]) + a2 = Candidate("a", 2, [_Requirement("q", {2})]) + b = Candidate("b", 1, [_Requirement("q", {1})]) + q1 = Candidate("q", 1, []) + q2 = Candidate("q", 2, []) all_candidates = { - "a": [("a", 1, [("q", {1})]), ("a", 2, [("q", {2})])], - "b": [("b", 1, [("q", {1})])], - "q": [("q", 1, []), ("q", 2, [])], + "a": [a1, a2], + "b": [b], + "q": [q1, q2], } class Reporter(BaseReporter): @@ -136,20 +155,22 @@ def get_preference(self, **_): return 0 def get_dependencies(self, candidate): - return candidate[2] + return candidate.requirements def find_matches(self, identifier, requirements, incompatibilities): - bad_versions = {c[1] for c in incompatibilities[identifier]} + bad_versions = {c.version for c in incompatibilities[identifier]} candidates = [ c for c in all_candidates[identifier] - if all(c[1] in r[1] for r in requirements[identifier]) - and c[1] not in bad_versions + if all( + c.version in r.versions for r in requirements[identifier] + ) + and c.version not in bad_versions ] - return sorted(candidates, key=lambda c: c[1], reverse=True) + return sorted(candidates, key=lambda c: c.version, reverse=True) def is_satisfied_by(self, requirement, candidate): - return candidate[1] in requirement[1] + return candidate.version in requirement.versions def run_resolver(*args): reporter = Reporter() @@ -160,8 +181,12 @@ def run_resolver(*args): except ResolutionImpossible as e: return e.causes - backtracking_causes = run_resolver([("a", {1, 2}), ("b", {1})]) - exception_causes = run_resolver([("a", {2}), ("b", {1})]) + backtracking_causes = run_resolver( + [_Requirement("a", {1, 2}), _Requirement("b", {1})] + ) + exception_causes = run_resolver( + [_Requirement("a", {2}), _Requirement("b", {1})] + ) assert exception_causes == backtracking_causes @@ -171,9 +196,10 @@ def test_pin_conflict_with_self(monkeypatch, reporter): Verify correct behavior of attempting to pin a candidate version that conflicts with a previously pinned (now invalidated) version for that same candidate (#91). """ - Candidate = Tuple[ # noqa: F841 - str, Version, Sequence[str] - ] # name, version, requirements + if TYPE_CHECKING: + Candidate = Tuple[ # noqa: F841 + str, Version, Sequence[str] + ] # name, version, requirements all_candidates = { "parent": [("parent", Version("1"), ["child<2"])], "child": [