diff --git a/news/8792.bugfix b/news/8792.bugfix
new file mode 100644
index 00000000000..e83bdb09cfe
--- /dev/null
+++ b/news/8792.bugfix
@@ -0,0 +1,2 @@
+New resolver: Pick up hash declarations in constraints files and use them to
+filter available distributions.
diff --git a/news/8839.bugfix b/news/8839.bugfix
new file mode 100644
index 00000000000..987b801e932
--- /dev/null
+++ b/news/8839.bugfix
@@ -0,0 +1,3 @@
+New resolver: If a package appears multiple times in user specification with
+different ``--hash`` options, only hashes that present in all specifications
+should be allowed.
diff --git a/src/pip/_internal/resolution/resolvelib/base.py b/src/pip/_internal/resolution/resolvelib/base.py
index 9245747bf2b..7c09cd70b8d 100644
--- a/src/pip/_internal/resolution/resolvelib/base.py
+++ b/src/pip/_internal/resolution/resolvelib/base.py
@@ -1,5 +1,8 @@
+from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.packaging.utils import canonicalize_name
+from pip._internal.req.req_install import InstallRequirement
+from pip._internal.utils.hashes import Hashes
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
if MYPY_CHECK_RUNNING:
@@ -8,7 +11,6 @@
from pip._vendor.packaging.version import _BaseVersion
from pip._internal.models.link import Link
- from pip._internal.req.req_install import InstallRequirement
CandidateLookup = Tuple[
Optional["Candidate"],
@@ -24,6 +26,39 @@ def format_name(project, extras):
return "{}[{}]".format(project, ",".join(canonical_extras))
+class Constraint(object):
+ def __init__(self, specifier, hashes):
+ # type: (SpecifierSet, Hashes) -> None
+ self.specifier = specifier
+ self.hashes = hashes
+
+ @classmethod
+ def empty(cls):
+ # type: () -> Constraint
+ return Constraint(SpecifierSet(), Hashes())
+
+ @classmethod
+ def from_ireq(cls, ireq):
+ # type: (InstallRequirement) -> Constraint
+ return Constraint(ireq.specifier, ireq.hashes(trust_internet=False))
+
+ def __nonzero__(self):
+ # type: () -> bool
+ return bool(self.specifier) or bool(self.hashes)
+
+ def __bool__(self):
+ # type: () -> bool
+ return self.__nonzero__()
+
+ def __and__(self, other):
+ # type: (InstallRequirement) -> Constraint
+ if not isinstance(other, InstallRequirement):
+ return NotImplemented
+ specifier = self.specifier & other.specifier
+ hashes = self.hashes & other.hashes(trust_internet=False)
+ return Constraint(specifier, hashes)
+
+
class Requirement(object):
@property
def name(self):
diff --git a/src/pip/_internal/resolution/resolvelib/factory.py b/src/pip/_internal/resolution/resolvelib/factory.py
index dab23aa09d1..172f054fa72 100644
--- a/src/pip/_internal/resolution/resolvelib/factory.py
+++ b/src/pip/_internal/resolution/resolvelib/factory.py
@@ -22,6 +22,7 @@
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
from pip._internal.utils.virtualenv import running_under_virtualenv
+from .base import Constraint
from .candidates import (
AlreadyInstalledCandidate,
EditableCandidate,
@@ -154,6 +155,7 @@ def _iter_found_candidates(
self,
ireqs, # type: Sequence[InstallRequirement]
specifier, # type: SpecifierSet
+ hashes, # type: Hashes
):
# type: (...) -> Iterable[Candidate]
if not ireqs:
@@ -166,11 +168,10 @@ def _iter_found_candidates(
template = ireqs[0]
name = canonicalize_name(template.req.name)
- hashes = Hashes()
extras = frozenset() # type: FrozenSet[str]
for ireq in ireqs:
specifier &= ireq.req.specifier
- hashes |= ireq.hashes(trust_internet=False)
+ hashes &= ireq.hashes(trust_internet=False)
extras |= frozenset(ireq.extras)
# We use this to ensure that we only yield a single candidate for
@@ -220,7 +221,7 @@ def _iter_found_candidates(
return six.itervalues(candidates)
def find_candidates(self, requirements, constraint):
- # type: (Sequence[Requirement], SpecifierSet) -> Iterable[Candidate]
+ # type: (Sequence[Requirement], Constraint) -> Iterable[Candidate]
explicit_candidates = set() # type: Set[Candidate]
ireqs = [] # type: List[InstallRequirement]
for req in requirements:
@@ -233,7 +234,11 @@ def find_candidates(self, requirements, constraint):
# If none of the requirements want an explicit candidate, we can ask
# the finder for candidates.
if not explicit_candidates:
- return self._iter_found_candidates(ireqs, constraint)
+ return self._iter_found_candidates(
+ ireqs,
+ constraint.specifier,
+ constraint.hashes,
+ )
if constraint:
name = explicit_candidates.pop().name
diff --git a/src/pip/_internal/resolution/resolvelib/provider.py b/src/pip/_internal/resolution/resolvelib/provider.py
index bf704db3726..19c4d543f52 100644
--- a/src/pip/_internal/resolution/resolvelib/provider.py
+++ b/src/pip/_internal/resolution/resolvelib/provider.py
@@ -1,8 +1,9 @@
-from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.resolvelib.providers import AbstractProvider
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from .base import Constraint
+
if MYPY_CHECK_RUNNING:
from typing import (
Any,
@@ -41,7 +42,7 @@ class PipProvider(AbstractProvider):
def __init__(
self,
factory, # type: Factory
- constraints, # type: Dict[str, SpecifierSet]
+ constraints, # type: Dict[str, Constraint]
ignore_dependencies, # type: bool
upgrade_strategy, # type: str
user_requested, # type: Set[str]
@@ -141,7 +142,7 @@ def find_matches(self, requirements):
if not requirements:
return []
constraint = self._constraints.get(
- requirements[0].name, SpecifierSet(),
+ requirements[0].name, Constraint.empty(),
)
candidates = self._factory.find_candidates(requirements, constraint)
return reversed(self._sort_matches(candidates))
diff --git a/src/pip/_internal/resolution/resolvelib/resolver.py b/src/pip/_internal/resolution/resolvelib/resolver.py
index aecddb1138c..449cfea287a 100644
--- a/src/pip/_internal/resolution/resolvelib/resolver.py
+++ b/src/pip/_internal/resolution/resolvelib/resolver.py
@@ -14,12 +14,12 @@
from pip._internal.utils.misc import dist_is_editable
from pip._internal.utils.typing import MYPY_CHECK_RUNNING
+from .base import Constraint
from .factory import Factory
if MYPY_CHECK_RUNNING:
from typing import Dict, List, Optional, Set, Tuple
- from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.resolvelib.resolvers import Result
from pip._vendor.resolvelib.structs import Graph
@@ -81,7 +81,7 @@ def __init__(
def resolve(self, root_reqs, check_supported_wheels):
# type: (List[InstallRequirement], bool) -> RequirementSet
- constraints = {} # type: Dict[str, SpecifierSet]
+ constraints = {} # type: Dict[str, Constraint]
user_requested = set() # type: Set[str]
requirements = []
for req in root_reqs:
@@ -94,9 +94,9 @@ def resolve(self, root_reqs, check_supported_wheels):
continue
name = canonicalize_name(req.name)
if name in constraints:
- constraints[name] = constraints[name] & req.specifier
+ constraints[name] &= req
else:
- constraints[name] = req.specifier
+ constraints[name] = Constraint.from_ireq(req)
else:
if req.user_supplied and req.name:
user_requested.add(canonicalize_name(req.name))
diff --git a/src/pip/_internal/utils/hashes.py b/src/pip/_internal/utils/hashes.py
index d1b062fedf6..d9f74a64083 100644
--- a/src/pip/_internal/utils/hashes.py
+++ b/src/pip/_internal/utils/hashes.py
@@ -46,16 +46,24 @@ def __init__(self, hashes=None):
"""
self._allowed = {} if hashes is None else hashes
- def __or__(self, other):
+ def __and__(self, other):
# type: (Hashes) -> Hashes
if not isinstance(other, Hashes):
return NotImplemented
- new = self._allowed.copy()
+
+ # If either of the Hashes object is entirely empty (i.e. no hash
+ # specified at all), all hashes from the other object are allowed.
+ if not other:
+ return self
+ if not self:
+ return other
+
+ # Otherwise only hashes that present in both objects are allowed.
+ new = {}
for alg, values in iteritems(other._allowed):
- try:
- new[alg] += values
- except KeyError:
- new[alg] = values
+ if alg not in self._allowed:
+ continue
+ new[alg] = [v for v in values if v in self._allowed[alg]]
return Hashes(new)
@property
diff --git a/tests/functional/test_new_resolver_hashes.py b/tests/functional/test_new_resolver_hashes.py
new file mode 100644
index 00000000000..4b13ebc307d
--- /dev/null
+++ b/tests/functional/test_new_resolver_hashes.py
@@ -0,0 +1,213 @@
+import collections
+import hashlib
+
+import pytest
+
+from pip._internal.utils.urls import path_to_url
+from tests.lib import (
+ create_basic_sdist_for_package,
+ create_basic_wheel_for_package,
+)
+
+_FindLinks = collections.namedtuple(
+ "_FindLinks", "index_html sdist_hash wheel_hash",
+)
+
+
+def _create_find_links(script):
+ sdist_path = create_basic_sdist_for_package(script, "base", "0.1.0")
+ wheel_path = create_basic_wheel_for_package(script, "base", "0.1.0")
+
+ sdist_hash = hashlib.sha256(sdist_path.read_bytes()).hexdigest()
+ wheel_hash = hashlib.sha256(wheel_path.read_bytes()).hexdigest()
+
+ index_html = script.scratch_path / "index.html"
+ index_html.write_text(
+ """
+ {sdist_path.stem}
+ {wheel_path.stem}
+ """.format(
+ sdist_url=path_to_url(sdist_path),
+ sdist_hash=sdist_hash,
+ sdist_path=sdist_path,
+ wheel_url=path_to_url(wheel_path),
+ wheel_hash=wheel_hash,
+ wheel_path=wheel_path,
+ )
+ )
+
+ return _FindLinks(index_html, sdist_hash, wheel_hash)
+
+
+@pytest.mark.parametrize(
+ "requirements_template, message",
+ [
+ (
+ """
+ base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash}
+ base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash}
+ """,
+ "Checked 2 links for project {name!r} against 2 hashes "
+ "(2 matches, 0 no digest): discarding no candidates",
+ ),
+ (
+ # Different hash lists are intersected.
+ """
+ base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash}
+ base==0.1.0 --hash=sha256:{sdist_hash}
+ """,
+ "Checked 2 links for project {name!r} against 1 hashes "
+ "(1 matches, 0 no digest): discarding 1 non-matches",
+ ),
+ ],
+ ids=["identical", "intersect"],
+)
+def test_new_resolver_hash_intersect(script, requirements_template, message):
+ find_links = _create_find_links(script)
+
+ requirements_txt = script.scratch_path / "requirements.txt"
+ requirements_txt.write_text(
+ requirements_template.format(
+ sdist_hash=find_links.sdist_hash,
+ wheel_hash=find_links.wheel_hash,
+ ),
+ )
+
+ result = script.pip(
+ "install",
+ "--use-feature=2020-resolver",
+ "--no-cache-dir",
+ "--no-deps",
+ "--no-index",
+ "--find-links", find_links.index_html,
+ "--verbose",
+ "--requirement", requirements_txt,
+ )
+
+ assert message.format(name=u"base") in result.stdout, str(result)
+
+
+def test_new_resolver_hash_intersect_from_constraint(script):
+ find_links = _create_find_links(script)
+
+ constraints_txt = script.scratch_path / "constraints.txt"
+ constraints_txt.write_text(
+ "base==0.1.0 --hash=sha256:{sdist_hash}".format(
+ sdist_hash=find_links.sdist_hash,
+ ),
+ )
+ requirements_txt = script.scratch_path / "requirements.txt"
+ requirements_txt.write_text(
+ """
+ base==0.1.0 --hash=sha256:{sdist_hash} --hash=sha256:{wheel_hash}
+ """.format(
+ sdist_hash=find_links.sdist_hash,
+ wheel_hash=find_links.wheel_hash,
+ ),
+ )
+
+ result = script.pip(
+ "install",
+ "--use-feature=2020-resolver",
+ "--no-cache-dir",
+ "--no-deps",
+ "--no-index",
+ "--find-links", find_links.index_html,
+ "--verbose",
+ "--constraint", constraints_txt,
+ "--requirement", requirements_txt,
+ )
+
+ message = (
+ "Checked 2 links for project {name!r} against 1 hashes "
+ "(1 matches, 0 no digest): discarding 1 non-matches"
+ ).format(name=u"base")
+ assert message in result.stdout, str(result)
+
+
+@pytest.mark.parametrize(
+ "requirements_template, constraints_template",
+ [
+ (
+ """
+ base==0.1.0 --hash=sha256:{sdist_hash}
+ base==0.1.0 --hash=sha256:{wheel_hash}
+ """,
+ "",
+ ),
+ (
+ "base==0.1.0 --hash=sha256:{sdist_hash}",
+ "base==0.1.0 --hash=sha256:{wheel_hash}",
+ ),
+ ],
+ ids=["both-requirements", "one-each"],
+)
+def test_new_resolver_hash_intersect_empty(
+ script, requirements_template, constraints_template,
+):
+ find_links = _create_find_links(script)
+
+ constraints_txt = script.scratch_path / "constraints.txt"
+ constraints_txt.write_text(
+ constraints_template.format(
+ sdist_hash=find_links.sdist_hash,
+ wheel_hash=find_links.wheel_hash,
+ ),
+ )
+
+ requirements_txt = script.scratch_path / "requirements.txt"
+ requirements_txt.write_text(
+ requirements_template.format(
+ sdist_hash=find_links.sdist_hash,
+ wheel_hash=find_links.wheel_hash,
+ ),
+ )
+
+ result = script.pip(
+ "install",
+ "--use-feature=2020-resolver",
+ "--no-cache-dir",
+ "--no-deps",
+ "--no-index",
+ "--find-links", find_links.index_html,
+ "--constraint", constraints_txt,
+ "--requirement", requirements_txt,
+ expect_error=True,
+ )
+
+ assert (
+ "THESE PACKAGES DO NOT MATCH THE HASHES FROM THE REQUIREMENTS FILE."
+ ) in result.stderr, str(result)
+
+
+def test_new_resolver_hash_intersect_empty_from_constraint(script):
+ find_links = _create_find_links(script)
+
+ constraints_txt = script.scratch_path / "constraints.txt"
+ constraints_txt.write_text(
+ """
+ base==0.1.0 --hash=sha256:{sdist_hash}
+ base==0.1.0 --hash=sha256:{wheel_hash}
+ """.format(
+ sdist_hash=find_links.sdist_hash,
+ wheel_hash=find_links.wheel_hash,
+ ),
+ )
+
+ result = script.pip(
+ "install",
+ "--use-feature=2020-resolver",
+ "--no-cache-dir",
+ "--no-deps",
+ "--no-index",
+ "--find-links", find_links.index_html,
+ "--constraint", constraints_txt,
+ "base==0.1.0",
+ expect_error=True,
+ )
+
+ message = (
+ "Hashes are required in --require-hashes mode, but they are missing "
+ "from some requirements."
+ )
+ assert message in result.stderr, str(result)
diff --git a/tests/unit/resolution_resolvelib/test_requirement.py b/tests/unit/resolution_resolvelib/test_requirement.py
index 21de3df4a4f..a03edb6f7c2 100644
--- a/tests/unit/resolution_resolvelib/test_requirement.py
+++ b/tests/unit/resolution_resolvelib/test_requirement.py
@@ -1,8 +1,7 @@
import pytest
-from pip._vendor.packaging.specifiers import SpecifierSet
from pip._vendor.resolvelib import BaseReporter, Resolver
-from pip._internal.resolution.resolvelib.base import Candidate
+from pip._internal.resolution.resolvelib.base import Candidate, Constraint
from pip._internal.utils.urls import path_to_url
# NOTE: All tests are prefixed `test_rlr` (for "test resolvelib resolver").
@@ -59,7 +58,7 @@ def test_new_resolver_correct_number_of_matches(test_cases, factory):
"""Requirements should return the correct number of candidates"""
for spec, _, match_count in test_cases:
req = factory.make_requirement_from_spec(spec, comes_from=None)
- matches = factory.find_candidates([req], SpecifierSet())
+ matches = factory.find_candidates([req], Constraint.empty())
assert len(list(matches)) == match_count
@@ -68,7 +67,7 @@ def test_new_resolver_candidates_match_requirement(test_cases, factory):
"""
for spec, _, _ in test_cases:
req = factory.make_requirement_from_spec(spec, comes_from=None)
- for c in factory.find_candidates([req], SpecifierSet()):
+ for c in factory.find_candidates([req], Constraint.empty()):
assert isinstance(c, Candidate)
assert req.is_satisfied_by(c)