diff --git a/news/2434.bugfix b/news/2434.bugfix new file mode 100644 index 0000000000..0a9603e102 --- /dev/null +++ b/news/2434.bugfix @@ -0,0 +1 @@ +Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled. diff --git a/news/2643.bugfix b/news/2643.bugfix new file mode 100644 index 0000000000..566879afbc --- /dev/null +++ b/news/2643.bugfix @@ -0,0 +1 @@ +Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``. diff --git a/news/2643.feature b/news/2643.feature new file mode 100644 index 0000000000..052398c7d9 --- /dev/null +++ b/news/2643.feature @@ -0,0 +1 @@ +Enhanced resolution of editable and VCS dependencies. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index f09ff372e1..bf7ebd2da6 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import (absolute_import, division, print_function, unicode_literals) - +import copy import hashlib import os import sys @@ -21,18 +21,16 @@ SafeFileCache, ) -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement -from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable -from pipenv.patched.notpip._vendor.pyparsing import ParseException +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, Specifier +from pipenv.patched.notpip._vendor.packaging.markers import Op, Value, Variable from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.vcs import VcsSupport -from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, - make_install_requirement, format_requirement, dedup, clean_requires_python) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, + make_install_requirement, clean_requires_python) from .base import BaseRepository @@ -64,15 +62,20 @@ def __init__(self, *args, **kwargs): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - can_hash = location.hash + vcs = VcsSupport() + orig_scheme = location.scheme + new_location = copy.deepcopy(location) + if orig_scheme in vcs.all_schemes: + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash if can_hash: # hash url WITH fragment - hash_value = self.get(location.url) + hash_value = self.get(new_location.url) if not hash_value: - hash_value = self._get_file_hash(location) + hash_value = self._get_file_hash(new_location) hash_value = hash_value.encode('utf8') if can_hash: - self.set(location.url, hash_value) + self.set(new_location.url, hash_value) return hash_value.decode('utf8') def _get_file_hash(self, location): @@ -276,6 +279,13 @@ def get_legacy_dependencies(self, ireq): setup_requires = {} dist = None if ireq.editable: + try: + from pipenv.utils import chdir + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils + distutils.core.run_setup(ireq.setup_py) + except (ImportError, InstallationError, TypeError, AttributeError): + pass try: dist = ireq.get_dist() except InstallationError: @@ -425,6 +435,10 @@ def get_hashes(self, ireq): if ireq.editable: return set() + vcs = VcsSupport() + if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() + if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) diff --git a/pipenv/utils.py b/pipenv/utils.py index 08c08381cf..286b18665e 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1361,3 +1361,19 @@ def is_virtual_environment(path): if python_like.is_file() and os.access(str(python_like), os.X_OK): return True return False + + +@contextmanager +def chdir(path): + """Context manager to change working directories.""" + from ._compat import Path + if not path: + return + prev_cwd = Path.cwd().as_posix() + if isinstance(path, Path): + path = path.as_posix() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(prev_cwd) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index f5fb822f4a..7d9b64d8ee 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,18 +19,22 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..c922be1 100644 +index 1c4b943..91902dc 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, - +@@ -1,9 +1,10 @@ + # coding: utf-8 + from __future__ import (absolute_import, division, print_function, + unicode_literals) +- ++import copy import hashlib import os +import sys from contextlib import contextmanager from shutil import rmtree -@@ -15,13 +16,24 @@ from .._compat import ( +@@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -40,25 +44,23 @@ index 1c4b943..c922be1 100644 + SafeFileCache, ) -+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement -+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -+from pip._vendor.packaging.markers import Marker, Op, Value, Variable -+from pip._vendor.pyparsing import ParseException +-from ..cache import CACHE_DIR ++from pip._vendor.packaging.requirements import Requirement ++from pip._vendor.packaging.specifiers import SpecifierSet, Specifier ++from pip._vendor.packaging.markers import Op, Value, Variable +from pip._internal.exceptions import InstallationError ++from pip._internal.vcs import VcsSupport + - from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound --from ..utils import (fs_str, is_pinned_requirement, lookup_table, + from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) -+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, -+ make_install_requirement, format_requirement, dedup, clean_requires_python) ++ make_install_requirement, clean_requires_python) + from .base import BaseRepository -@@ -37,6 +49,40 @@ except ImportError: +@@ -37,6 +47,45 @@ except ImportError: from pip.wheel import WheelCache @@ -77,15 +79,20 @@ index 1c4b943..c922be1 100644 + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None -+ can_hash = location.hash ++ vcs = VcsSupport() ++ orig_scheme = location.scheme ++ new_location = copy.deepcopy(location) ++ if orig_scheme in vcs.all_schemes: ++ new_location.url = new_location.url.split("+", 1)[-1] ++ can_hash = new_location.hash + if can_hash: + # hash url WITH fragment -+ hash_value = self.get(location.url) ++ hash_value = self.get(new_location.url) + if not hash_value: -+ hash_value = self._get_file_hash(location) ++ hash_value = self._get_file_hash(new_location) + hash_value = hash_value.encode('utf8') + if can_hash: -+ self.set(location.url, hash_value) ++ self.set(new_location.url, hash_value) + return hash_value.decode('utf8') + + def _get_file_hash(self, location): @@ -99,7 +106,7 @@ index 1c4b943..c922be1 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -113,7 +120,7 @@ index 1c4b943..c922be1 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -131,7 +138,7 @@ index 1c4b943..c922be1 100644 def freshen_build_caches(self): """ -@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -148,7 +155,7 @@ index 1c4b943..c922be1 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -223,7 +230,7 @@ index 1c4b943..c922be1 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +270,40 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -235,6 +242,13 @@ index 1c4b943..c922be1 100644 + dist = None + if ireq.editable: + try: ++ from pipenv.utils import chdir ++ with chdir(ireq.setup_py_dir): ++ from setuptools.dist import distutils ++ distutils.core.run_setup(ireq.setup_py) ++ except (ImportError, InstallationError, TypeError, AttributeError): ++ pass ++ try: + dist = ireq.get_dist() + except InstallationError: + ireq.run_egg_info() @@ -268,7 +282,7 @@ index 1c4b943..c922be1 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +323,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -369,7 +383,18 @@ index 1c4b943..c922be1 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +432,22 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository): + if ireq.editable: + return set() + ++ vcs = VcsSupport() ++ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: ++ return set() ++ + if not is_pinned_requirement(ireq): + raise TypeError( + "Expected pinned requirement, got {}".format(ireq)) +@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint.