Merge pull request #2643 from pypa/fix-find-links

Fix resolution using `dependency_links` with ssh

news/2434.bugfix

@@ -0,0 +1 @@
+Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled.

news/2643.bugfix

@@ -0,0 +1 @@
+Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``.

news/2643.feature

@@ -0,0 +1 @@
+Enhanced resolution of editable and VCS dependencies.

pipenv/patched/piptools/repositories/pypi.py

@@ -1,7 +1,7 @@
 # coding: utf-8
 from __future__ import (absolute_import, division, print_function,
  unicode_literals)
-
+import copy
 import hashlib
 import os
 import sys
@@ -21,18 +21,16 @@
  SafeFileCache,
 )
 
-from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
-from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
-from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
-from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable
-from pipenv.patched.notpip._vendor.pyparsing import ParseException
+from pipenv.patched.notpip._vendor.packaging.requirements import Requirement
+from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, Specifier
+from pipenv.patched.notpip._vendor.packaging.markers import Op, Value, Variable
 from pipenv.patched.notpip._internal.exceptions import InstallationError
+from pipenv.patched.notpip._internal.vcs import VcsSupport
 
-from ..cache import CACHE_DIR
 from pipenv.environments import PIPENV_CACHE_DIR
 from ..exceptions import NoCandidateFound
-from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
- make_install_requirement, format_requirement, dedup, clean_requires_python)
+from ..utils import (fs_str, is_pinned_requirement, lookup_table,
+ make_install_requirement, clean_requires_python)
 
 from .base import BaseRepository
 
@@ -64,15 +62,20 @@ def __init__(self, *args, **kwargs):
  def get_hash(self, location):
  # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
  hash_value = None
- can_hash = location.hash
+ vcs = VcsSupport()
+ orig_scheme = location.scheme
+ new_location = copy.deepcopy(location)
+ if orig_scheme in vcs.all_schemes:
+ new_location.url = new_location.url.split("+", 1)[-1]
+ can_hash = new_location.hash
  if can_hash:
  # hash url WITH fragment
- hash_value = self.get(location.url)
+ hash_value = self.get(new_location.url)
  if not hash_value:
- hash_value = self._get_file_hash(location)
+ hash_value = self._get_file_hash(new_location)
  hash_value = hash_value.encode('utf8')
  if can_hash:
- self.set(location.url, hash_value)
+ self.set(new_location.url, hash_value)
  return hash_value.decode('utf8')
 
  def _get_file_hash(self, location):
@@ -276,6 +279,13 @@ def get_legacy_dependencies(self, ireq):
  setup_requires = {}
  dist = None
  if ireq.editable:
+ try:
+ from pipenv.utils import chdir
+ with chdir(ireq.setup_py_dir):
+ from setuptools.dist import distutils
+ distutils.core.run_setup(ireq.setup_py)
+ except (ImportError, InstallationError, TypeError, AttributeError):
+ pass
  try:
  dist = ireq.get_dist()
  except InstallationError:
@@ -425,6 +435,10 @@ def get_hashes(self, ireq):
  if ireq.editable:
  return set()
 
+ vcs = VcsSupport()
+ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme:
+ return set()
+
  if not is_pinned_requirement(ireq):
  raise TypeError(
  "Expected pinned requirement, got {}".format(ireq))

pipenv/utils.py

@@ -1361,3 +1361,19 @@ def is_virtual_environment(path):
  if python_like.is_file() and os.access(str(python_like), os.X_OK):
  return True
  return False
+
+
+@contextmanager
+def chdir(path):
+ """Context manager to change working directories."""
+ from ._compat import Path
+ if not path:
+ return
+ prev_cwd = Path.cwd().as_posix()
+ if isinstance(path, Path):
+ path = path.as_posix()
+ os.chdir(str(path))
+ try:
+ yield
+ finally:
+ os.chdir(prev_cwd)

tasks/vendoring/patches/patched/piptools.patch

@@ -19,18 +19,22 @@ index 4e6174c..75f9b49 100644
  # NOTE
  # We used to store the cache dir under ~/.pip-tools, which is not the
 diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
-index 1c4b943..c922be1 100644
+index 1c4b943..91902dc 100644
 --- a/pipenv/patched/piptools/repositories/pypi.py
 +++ b/pipenv/patched/piptools/repositories/pypi.py
-@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
-
+@@ -1,9 +1,10 @@
+ # coding: utf-8
+ from __future__ import (absolute_import, division, print_function,
+ unicode_literals)
+-
++import copy
  import hashlib
  import os
 +import sys
  from contextlib import contextmanager
  from shutil import rmtree
 
-@@ -15,13 +16,24 @@ from .._compat import (
+@@ -15,13 +16,22 @@ from .._compat import (
  Wheel,
  FAVORITE_HASH,
  TemporaryDirectory,
@@ -40,25 +44,23 @@ index 1c4b943..c922be1 100644
 + SafeFileCache,
  )
 
-+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
-+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
-+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
-+from pip._vendor.packaging.markers import Marker, Op, Value, Variable
-+from pip._vendor.pyparsing import ParseException
+-from ..cache import CACHE_DIR
++from pip._vendor.packaging.requirements import Requirement
++from pip._vendor.packaging.specifiers import SpecifierSet, Specifier
++from pip._vendor.packaging.markers import Op, Value, Variable
 +from pip._internal.exceptions import InstallationError
++from pip._internal.vcs import VcsSupport
 +
- from ..cache import CACHE_DIR
 +from pipenv.environments import PIPENV_CACHE_DIR
  from ..exceptions import NoCandidateFound
--from ..utils import (fs_str, is_pinned_requirement, lookup_table,
+ from ..utils import (fs_str, is_pinned_requirement, lookup_table,
 - make_install_requirement)
-+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
-+ make_install_requirement, format_requirement, dedup, clean_requires_python)
++ make_install_requirement, clean_requires_python)
 +
  from .base import BaseRepository
 
 
-@@ -37,6 +49,40 @@ except ImportError:
+@@ -37,6 +47,45 @@ except ImportError:
  from pip.wheel import WheelCache
 
 
@@ -77,15 +79,20 @@ index 1c4b943..c922be1 100644
 + def get_hash(self, location):
 + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it
 + hash_value = None
-+ can_hash = location.hash
++ vcs = VcsSupport()
++ orig_scheme = location.scheme
++ new_location = copy.deepcopy(location)
++ if orig_scheme in vcs.all_schemes:
++ new_location.url = new_location.url.split("+", 1)[-1]
++ can_hash = new_location.hash
 + if can_hash:
 + # hash url WITH fragment
-+ hash_value = self.get(location.url)
++ hash_value = self.get(new_location.url)
 + if not hash_value:
-+ hash_value = self._get_file_hash(location)
++ hash_value = self._get_file_hash(new_location)
 + hash_value = hash_value.encode('utf8')
 + if can_hash:
-+ self.set(location.url, hash_value)
++ self.set(new_location.url, hash_value)
 + return hash_value.decode('utf8')
 +
 + def _get_file_hash(self, location):
@@ -99,7 +106,7 @@ index 1c4b943..c922be1 100644
  class PyPIRepository(BaseRepository):
  DEFAULT_INDEX_URL = PyPI.simple_url
 
-@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository):
+@@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository):
  config), but any other PyPI mirror can be used if index_urls is
  changed/configured on the Finder.
  """
@@ -113,7 +120,7 @@ index 1c4b943..c922be1 100644
 
  index_urls = [pip_options.index_url] + pip_options.extra_index_urls
  if pip_options.no_index:
-@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository):
+@@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository):
  # of all secondary dependencies for the given requirement, so we
  # only have to go to disk once for each requirement
  self._dependencies_cache = {}
@@ -131,7 +138,7 @@ index 1c4b943..c922be1 100644
 
  def freshen_build_caches(self):
  """
-@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository):
+@@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository):
  if ireq.editable:
  return ireq # return itself as the best match
 
@@ -148,7 +155,7 @@ index 1c4b943..c922be1 100644
 
  # Reuses pip's internal candidate sort key to sort
  matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
-@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository):
+@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository):
  best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
 
  # Turn the candidate into a pinned InstallRequirement
@@ -223,7 +230,7 @@ index 1c4b943..c922be1 100644
  """
  Given a pinned or an editable InstallRequirement, returns a set of
  dependencies (also InstallRequirements, but not necessarily pinned).
-@@ -155,20 +270,40 @@ class PyPIRepository(BaseRepository):
+@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository):
  os.makedirs(download_dir)
  if not os.path.isdir(self._wheel_download_dir):
  os.makedirs(self._wheel_download_dir)
@@ -235,6 +242,13 @@ index 1c4b943..c922be1 100644
 + dist = None
 + if ireq.editable:
 + try:
++ from pipenv.utils import chdir
++ with chdir(ireq.setup_py_dir):
++ from setuptools.dist import distutils
++ distutils.core.run_setup(ireq.setup_py)
++ except (ImportError, InstallationError, TypeError, AttributeError):
++ pass
++ try:
 + dist = ireq.get_dist()
 + except InstallationError:
 + ireq.run_egg_info()
@@ -268,7 +282,7 @@ index 1c4b943..c922be1 100644
  )
  except TypeError:
  # Pip >= 10 (new resolver!)
-@@ -188,17 +323,97 @@ class PyPIRepository(BaseRepository):
+@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository):
  finder=self.finder,
  session=self.session,
  upgrade_strategy="to-satisfy-only",
@@ -369,7 +383,18 @@ index 1c4b943..c922be1 100644
  return set(self._dependencies_cache[ireq])
 
  def get_hashes(self, ireq):
-@@ -217,24 +432,22 @@ class PyPIRepository(BaseRepository):
+@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository):
+ if ireq.editable:
+ return set()
+
++ vcs = VcsSupport()
++ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme:
++ return set()
++
+ if not is_pinned_requirement(ireq):
+ raise TypeError(
+ "Expected pinned requirement, got {}".format(ireq))
+@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository):
  # We need to get all of the candidates that match our current version
  # pin, these will represent all of the files that could possibly
  # satisfy this constraint.