From 0ac13f141aa437e0ec1587855d3a67d46a6999b7 Mon Sep 17 00:00:00 2001 From: Stefano Bennati Date: Tue, 3 Jan 2023 17:13:38 +0100 Subject: [PATCH] Implement backjumping and skip unrelated states The current backtracking logic assumes that the last state in the stack is the state that caused the incompatibility. This is not always the case. For example, given three requirements A, B and C, with dependencies A1, B1 and C1, where A1 and B1 are incompatible. The requirements are processed one after the other, so the last state is related to C, while the incompatibility is caused by B. The current behaviour causes significant slowdowns in case there are many candidates for B and C, as all their combination are evaluated before a compatible version of B can be found. The new backjumping behaviour discards states iteratively "backward" until any of the candidates in the state depends on any of the packages that caused the incompatibility in the current state. In our example, this causes the state related to C to be dropped without evaluating any of its candidates, until a compatible candidate of B is found. If the backjumping logic is not able to find a compatible candidate, it means that there is no solution for the requirements and a ResolutionImpossible exception is raised. Co-authored-by: Frost Ming Co-authored-by: Pradyun Gedam Signed-off-by: Bennati, Stefano --- news/113.feature.rst | 1 + src/resolvelib/resolvers.py | 46 ++++++++++++++----- tests/test_resolvers.py | 88 ++++++++++++++++++++++++------------- 3 files changed, 92 insertions(+), 43 deletions(-) create mode 100644 news/113.feature.rst diff --git a/news/113.feature.rst b/news/113.feature.rst new file mode 100644 index 0000000..9dddaa8 --- /dev/null +++ b/news/113.feature.rst @@ -0,0 +1 @@ +Implement backjumping to significantly speed up the resolution process by skipping over irrelevant parts of the resolution search space. diff --git a/src/resolvelib/resolvers.py b/src/resolvelib/resolvers.py index 49e30c7..1d110b8 100644 --- a/src/resolvelib/resolvers.py +++ b/src/resolvelib/resolvers.py @@ -266,8 +266,8 @@ def _attempt_to_pin_criterion(self, name): # end, signal for backtracking. return causes - def _backtrack(self): - """Perform backtracking. + def _backjump(self, causes): + """Perform backjumping. When we enter here, the stack is like this:: @@ -283,22 +283,44 @@ def _backtrack(self): Each iteration of the loop will: - 1. Discard Z. - 2. Discard Y but remember its incompatibility information gathered + 1. Identify Z. The incompatibility is not always caused by the latest state. + For example, given three requirements A, B and C, with dependencies + A1, B1 and C1, where A1 and B1 are incompatible: the last state + might be related to C, so we want to discard the previous state. + 2. Discard Z. + 3. Discard Y but remember its incompatibility information gathered previously, and the failure we're dealing with right now. - 3. Push a new state Y' based on X, and apply the incompatibility + 4. Push a new state Y' based on X, and apply the incompatibility information from Y to Y'. - 4a. If this causes Y' to conflict, we need to backtrack again. Make Y' + 5a. If this causes Y' to conflict, we need to backtrack again. Make Y' the new Z and go back to step 2. - 4b. If the incompatibilities apply cleanly, end backtracking. + 5b. If the incompatibilities apply cleanly, end backtracking. """ + incompatible_deps = set( + [c.parent.name for c in causes if c.parent is not None] + + [c.requirement.name for c in causes] + ) while len(self._states) >= 3: # Remove the state that triggered backtracking. del self._states[-1] - # Retrieve the last candidate pin and known incompatibilities. - broken_state = self._states.pop() - name, candidate = broken_state.mapping.popitem() + # Ensure to backtrack to a state that caused the incompatibility + incompatible_state = False + while not incompatible_state: + # Retrieve the last candidate pin and known incompatibilities. + try: + broken_state = self._states.pop() + name, candidate = broken_state.mapping.popitem() + except (IndexError, KeyError): + raise ResolutionImpossible(causes) + current_dependencies = { + self._p.identify(d) + for d in self._p.get_dependencies(candidate) + } + incompatible_state = not current_dependencies.isdisjoint( + incompatible_deps + ) + incompatibilities_from_broken = [ (k, list(v.incompatibilities)) for k, v in broken_state.criteria.items() @@ -403,10 +425,10 @@ def resolve(self, requirements, max_rounds): if failure_causes: causes = [i for c in failure_causes for i in c.information] - # Backtrack if pinning fails. The backtrack process puts us in + # Backjump if pinning fails. The backjump process puts us in # an unpinned state, so we can work on it in the next round. self._r.resolving_conflicts(causes=causes) - success = self._backtrack() + success = self._backjump(causes) self.state.backtrack_causes[:] = causes # Dead ends everywhere. Give up. diff --git a/tests/test_resolvers.py b/tests/test_resolvers.py index 8a1349a..fe95e4b 100644 --- a/tests/test_resolvers.py +++ b/tests/test_resolvers.py @@ -1,14 +1,17 @@ -from typing import ( - Any, - Iterable, - Iterator, - List, - Mapping, - Sequence, - Set, - Tuple, - Union, -) +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + from typing import ( + Any, + Iterable, + Iterator, + List, + Mapping, + Sequence, + Set, + Tuple, + Union, + ) import pytest from packaging.requirements import Requirement @@ -21,12 +24,17 @@ ResolutionImpossible, Resolver, ) -from resolvelib.resolvers import ( - Criterion, - RequirementInformation, - RequirementsConflicted, - Resolution, -) + +if TYPE_CHECKING: + from resolvelib.resolvers import ( + Criterion, + RequirementInformation, + RequirementsConflicted, + ) + +from collections import namedtuple + +from resolvelib.resolvers import Resolution def test_candidate_inconsistent_error(): @@ -115,10 +123,21 @@ def is_satisfied_by(self, requirement, candidate): def test_resolving_conflicts(): + Candidate = namedtuple( + "Candidate", ["name", "version", "requirements"] + ) # name, version, requirements + _Requirement = namedtuple( + "Requirement", ["name", "versions"] + ) # name, versions + a1 = Candidate("a", 1, [_Requirement("q", {1})]) + a2 = Candidate("a", 2, [_Requirement("q", {2})]) + b = Candidate("b", 1, [_Requirement("q", {1})]) + q1 = Candidate("q", 1, []) + q2 = Candidate("q", 2, []) all_candidates = { - "a": [("a", 1, [("q", {1})]), ("a", 2, [("q", {2})])], - "b": [("b", 1, [("q", {1})])], - "q": [("q", 1, []), ("q", 2, [])], + "a": [a1, a2], + "b": [b], + "q": [q1, q2], } class Reporter(BaseReporter): @@ -136,20 +155,22 @@ def get_preference(self, **_): return 0 def get_dependencies(self, candidate): - return candidate[2] + return candidate.requirements def find_matches(self, identifier, requirements, incompatibilities): - bad_versions = {c[1] for c in incompatibilities[identifier]} + bad_versions = {c.version for c in incompatibilities[identifier]} candidates = [ c for c in all_candidates[identifier] - if all(c[1] in r[1] for r in requirements[identifier]) - and c[1] not in bad_versions + if all( + c.version in r.versions for r in requirements[identifier] + ) + and c.version not in bad_versions ] - return sorted(candidates, key=lambda c: c[1], reverse=True) + return sorted(candidates, key=lambda c: c.version, reverse=True) def is_satisfied_by(self, requirement, candidate): - return candidate[1] in requirement[1] + return candidate.version in requirement.versions def run_resolver(*args): reporter = Reporter() @@ -160,8 +181,12 @@ def run_resolver(*args): except ResolutionImpossible as e: return e.causes - backtracking_causes = run_resolver([("a", {1, 2}), ("b", {1})]) - exception_causes = run_resolver([("a", {2}), ("b", {1})]) + backtracking_causes = run_resolver( + [_Requirement("a", {1, 2}), _Requirement("b", {1})] + ) + exception_causes = run_resolver( + [_Requirement("a", {2}), _Requirement("b", {1})] + ) assert exception_causes == backtracking_causes @@ -171,9 +196,10 @@ def test_pin_conflict_with_self(monkeypatch, reporter): Verify correct behavior of attempting to pin a candidate version that conflicts with a previously pinned (now invalidated) version for that same candidate (#91). """ - Candidate = Tuple[ # noqa: F841 - str, Version, Sequence[str] - ] # name, version, requirements + if TYPE_CHECKING: + Candidate = Tuple[ # noqa: F841 + str, Version, Sequence[str] + ] # name, version, requirements all_candidates = { "parent": [("parent", Version("1"), ["child<2"])], "child": [