From 00dd84537244262e98db1fa45319d33254f383e7 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 23 Jul 2018 12:38:50 -0400 Subject: [PATCH 01/69] Enable parsing of dependency links in setup.py - Fixed errors with url parsing during hashing Signed-off-by: Dan Ryan --- news/2434.bugfix | 1 + pipenv/patched/piptools/repositories/pypi.py | 14 ++++--- .../vendoring/patches/patched/piptools.patch | 38 +++++++++++-------- 3 files changed, 33 insertions(+), 20 deletions(-) create mode 100644 news/2434.bugfix diff --git a/news/2434.bugfix b/news/2434.bugfix new file mode 100644 index 0000000000..0a9603e102 --- /dev/null +++ b/news/2434.bugfix @@ -0,0 +1 @@ +Fixed the ability of pipenv to parse ``dependency_links`` from ``setup.py`` when ``PIP_PROCESS_DEPENDENCY_LINKS`` is enabled. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index f09ff372e1..2f746094be 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,7 +1,7 @@ # coding: utf-8 from __future__ import (absolute_import, division, print_function, unicode_literals) - +import copy import hashlib import os import sys @@ -64,15 +64,19 @@ def __init__(self, *args, **kwargs): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - can_hash = location.hash + vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') + new_location = copy.deepcopy(location) + if any(new_location.url.startswith(vcs) for vcs in vcs_uris): + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash if can_hash: # hash url WITH fragment - hash_value = self.get(location.url) + hash_value = self.get(new_location.url) if not hash_value: - hash_value = self._get_file_hash(location) + hash_value = self._get_file_hash(new_location) hash_value = hash_value.encode('utf8') if can_hash: - self.set(location.url, hash_value) + self.set(new_location.url, hash_value) return hash_value.decode('utf8') def _get_file_hash(self, location): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index f5fb822f4a..56696e0889 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,11 +19,15 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..c922be1 100644 +index 1c4b943..245e9ce 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py -@@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, - +@@ -1,9 +1,10 @@ + # coding: utf-8 + from __future__ import (absolute_import, division, print_function, + unicode_literals) +- ++import copy import hashlib import os +import sys @@ -58,7 +62,7 @@ index 1c4b943..c922be1 100644 from .base import BaseRepository -@@ -37,6 +49,40 @@ except ImportError: +@@ -37,6 +49,44 @@ except ImportError: from pip.wheel import WheelCache @@ -77,15 +81,19 @@ index 1c4b943..c922be1 100644 + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None -+ can_hash = location.hash ++ vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') ++ new_location = copy.deepcopy(location) ++ if any(new_location.url.startswith(vcs) for vcs in vcs_uris): ++ new_location.url = new_location.url.split("+", 1)[-1] ++ can_hash = new_location.hash + if can_hash: + # hash url WITH fragment -+ hash_value = self.get(location.url) ++ hash_value = self.get(new_location.url) + if not hash_value: -+ hash_value = self._get_file_hash(location) ++ hash_value = self._get_file_hash(new_location) + hash_value = hash_value.encode('utf8') + if can_hash: -+ self.set(location.url, hash_value) ++ self.set(new_location.url, hash_value) + return hash_value.decode('utf8') + + def _get_file_hash(self, location): @@ -99,7 +107,7 @@ index 1c4b943..c922be1 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +96,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -113,7 +121,7 @@ index 1c4b943..c922be1 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +125,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -131,7 +139,7 @@ index 1c4b943..c922be1 100644 def freshen_build_caches(self): """ -@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +169,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -148,7 +156,7 @@ index 1c4b943..c922be1 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -223,7 +231,7 @@ index 1c4b943..c922be1 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +270,40 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +274,40 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -268,7 +276,7 @@ index 1c4b943..c922be1 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +323,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +327,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -369,7 +377,7 @@ index 1c4b943..c922be1 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +432,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +436,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From f3e4e73cff7ac91fc13da87f367b2bdbfb2b9d31 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 18:31:55 -0400 Subject: [PATCH 02/69] Fix resolution using `dependency_links` with ssh - Exclude VCS SSH uris from hashing - Add additional resilience to the piptools resolver - Fixes #2613 Signed-off-by: Dan Ryan --- news/2643.bugfix | 1 + pipenv/patched/piptools/repositories/pypi.py | 28 +++++---- .../vendoring/patches/patched/piptools.patch | 57 ++++++++++++------- 3 files changed, 55 insertions(+), 31 deletions(-) create mode 100644 news/2643.bugfix diff --git a/news/2643.bugfix b/news/2643.bugfix new file mode 100644 index 0000000000..566879afbc --- /dev/null +++ b/news/2643.bugfix @@ -0,0 +1 @@ +Dependency links to private repositories defined via ``ssh://`` schemes will now install correctly and skip hashing as long as ``PIP_PROCESS_DEPENDENCY_LINKS=1``. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 2f746094be..feab21b53c 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -21,18 +21,16 @@ SafeFileCache, ) -from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement -from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable -from pipenv.patched.notpip._vendor.pyparsing import ParseException +from pipenv.patched.notpip._vendor.packaging.requirements import Requirement +from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, Specifier +from pipenv.patched.notpip._vendor.packaging.markers import Op, Value, Variable from pipenv.patched.notpip._internal.exceptions import InstallationError +from pipenv.patched.notpip._internal.vcs import VcsSupport -from ..cache import CACHE_DIR from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound -from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, - make_install_requirement, format_requirement, dedup, clean_requires_python) +from ..utils import (fs_str, is_pinned_requirement, lookup_table, + make_install_requirement, clean_requires_python) from .base import BaseRepository @@ -64,9 +62,10 @@ def __init__(self, *args, **kwargs): def get_hash(self, location): # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it hash_value = None - vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') + vcs = VcsSupport() + orig_scheme = location.scheme new_location = copy.deepcopy(location) - if any(new_location.url.startswith(vcs) for vcs in vcs_uris): + if orig_scheme in vcs.all_schemes: new_location.url = new_location.url.split("+", 1)[-1] can_hash = new_location.hash if can_hash: @@ -280,6 +279,11 @@ def get_legacy_dependencies(self, ireq): setup_requires = {} dist = None if ireq.editable: + try: + from setuptools.build_meta import _run_setup + _run_setup(ireq.setup_py) + except (ImportError, InstallationError): + pass try: dist = ireq.get_dist() except InstallationError: @@ -429,6 +433,10 @@ def get_hashes(self, ireq): if ireq.editable: return set() + vcs = VcsSupport() + if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() + if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 56696e0889..b527470bdf 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..245e9ce 100644 +index 1c4b943..9461709 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -34,7 +34,7 @@ index 1c4b943..245e9ce 100644 from contextlib import contextmanager from shutil import rmtree -@@ -15,13 +16,24 @@ from .._compat import ( +@@ -15,13 +16,22 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -44,25 +44,23 @@ index 1c4b943..245e9ce 100644 + SafeFileCache, ) -+from pip._vendor.packaging.requirements import InvalidRequirement, Requirement -+from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version -+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier -+from pip._vendor.packaging.markers import Marker, Op, Value, Variable -+from pip._vendor.pyparsing import ParseException +-from ..cache import CACHE_DIR ++from pip._vendor.packaging.requirements import Requirement ++from pip._vendor.packaging.specifiers import SpecifierSet, Specifier ++from pip._vendor.packaging.markers import Op, Value, Variable +from pip._internal.exceptions import InstallationError ++from pip._internal.vcs import VcsSupport + - from ..cache import CACHE_DIR +from pipenv.environments import PIPENV_CACHE_DIR from ..exceptions import NoCandidateFound --from ..utils import (fs_str, is_pinned_requirement, lookup_table, + from ..utils import (fs_str, is_pinned_requirement, lookup_table, - make_install_requirement) -+from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req, -+ make_install_requirement, format_requirement, dedup, clean_requires_python) ++ make_install_requirement, clean_requires_python) + from .base import BaseRepository -@@ -37,6 +49,44 @@ except ImportError: +@@ -37,6 +47,45 @@ except ImportError: from pip.wheel import WheelCache @@ -81,9 +79,10 @@ index 1c4b943..245e9ce 100644 + def get_hash(self, location): + # if there is no location hash (i.e., md5 / sha256 / etc) we on't want to store it + hash_value = None -+ vcs_uris = ('git+', 'bzr+', 'hg+', 'svn+') ++ vcs = VcsSupport() ++ orig_scheme = location.scheme + new_location = copy.deepcopy(location) -+ if any(new_location.url.startswith(vcs) for vcs in vcs_uris): ++ if orig_scheme in vcs.all_schemes: + new_location.url = new_location.url.split("+", 1)[-1] + can_hash = new_location.hash + if can_hash: @@ -107,7 +106,7 @@ index 1c4b943..245e9ce 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +96,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +95,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -121,7 +120,7 @@ index 1c4b943..245e9ce 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +125,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +124,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -139,7 +138,7 @@ index 1c4b943..245e9ce 100644 def freshen_build_caches(self): """ -@@ -114,10 +169,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +168,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -156,7 +155,7 @@ index 1c4b943..245e9ce 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +185,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +184,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -231,7 +230,7 @@ index 1c4b943..245e9ce 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +274,40 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,45 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -243,6 +242,11 @@ index 1c4b943..245e9ce 100644 + dist = None + if ireq.editable: + try: ++ from setuptools.build_meta import _run_setup ++ _run_setup(ireq.setup_py) ++ except (ImportError, InstallationError): ++ pass ++ try: + dist = ireq.get_dist() + except InstallationError: + ireq.run_egg_info() @@ -276,7 +280,7 @@ index 1c4b943..245e9ce 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +327,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +331,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -377,7 +381,18 @@ index 1c4b943..245e9ce 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +436,22 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +433,10 @@ class PyPIRepository(BaseRepository): + if ireq.editable: + return set() + ++ vcs = VcsSupport() ++ if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: ++ return set() ++ + if not is_pinned_requirement(ireq): + raise TypeError( + "Expected pinned requirement, got {}".format(ireq)) +@@ -217,24 +444,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From 5334183c436c6a1572645fbd7de7720eb51f419f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 19:11:36 -0400 Subject: [PATCH 03/69] Make sure there is a link to check against Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index feab21b53c..daaa590287 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -434,7 +434,7 @@ def get_hashes(self, ireq): return set() vcs = VcsSupport() - if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: return set() if not is_pinned_requirement(ireq): diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index b527470bdf..8dd58066d2 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -386,7 +386,7 @@ index 1c4b943..9461709 100644 return set() + vcs = VcsSupport() -+ if ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: ++ if ireq.link and ireq.link.scheme in vcs.all_schemes and 'ssh' in ireq.link.scheme: + return set() + if not is_pinned_requirement(ireq): From 6e38560a7f8026caeeaa4a3b486115ff430a24c9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 19:35:44 -0400 Subject: [PATCH 04/69] Add chdir context manager and properly run setup.py files during resolution Signed-off-by: Dan Ryan --- news/2643.feature | 1 + pipenv/patched/piptools/repositories/pypi.py | 6 ++++-- pipenv/utils.py | 12 ++++++++++++ tasks/vendoring/patches/patched/piptools.patch | 16 +++++++++------- 4 files changed, 26 insertions(+), 9 deletions(-) create mode 100644 news/2643.feature diff --git a/news/2643.feature b/news/2643.feature new file mode 100644 index 0000000000..052398c7d9 --- /dev/null +++ b/news/2643.feature @@ -0,0 +1 @@ +Enhanced resolution of editable and VCS dependencies. diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index daaa590287..94879f2ff5 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -280,8 +280,10 @@ def get_legacy_dependencies(self, ireq): dist = None if ireq.editable: try: - from setuptools.build_meta import _run_setup - _run_setup(ireq.setup_py) + from pipenv.utils import chdir + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils + distutils.core.run_setup(ireq.setup_py) except (ImportError, InstallationError): pass try: diff --git a/pipenv/utils.py b/pipenv/utils.py index 08c08381cf..b45081fe11 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1361,3 +1361,15 @@ def is_virtual_environment(path): if python_like.is_file() and os.access(str(python_like), os.X_OK): return True return False + + +@contextmanager +def chdir(path): + """Context manager to change working directories.""" + from ._compat import Path + prev_cwd = Path.cwd() + os.chdir(str(path)) + try: + yield + finally: + os.chdir(prev_cwd) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 8dd58066d2..17f2e2b56f 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..9461709 100644 +index 1c4b943..91902dc 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -1,9 +1,10 @@ @@ -230,7 +230,7 @@ index 1c4b943..9461709 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,20 +273,45 @@ class PyPIRepository(BaseRepository): +@@ -155,20 +273,47 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -242,8 +242,10 @@ index 1c4b943..9461709 100644 + dist = None + if ireq.editable: + try: -+ from setuptools.build_meta import _run_setup -+ _run_setup(ireq.setup_py) ++ from pipenv.utils import chdir ++ with chdir(ireq.setup_py_dir): ++ from setuptools.dist import distutils ++ distutils.core.run_setup(ireq.setup_py) + except (ImportError, InstallationError): + pass + try: @@ -280,7 +282,7 @@ index 1c4b943..9461709 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +331,97 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +333,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -381,7 +383,7 @@ index 1c4b943..9461709 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -210,6 +433,10 @@ class PyPIRepository(BaseRepository): +@@ -210,6 +435,10 @@ class PyPIRepository(BaseRepository): if ireq.editable: return set() @@ -392,7 +394,7 @@ index 1c4b943..9461709 100644 if not is_pinned_requirement(ireq): raise TypeError( "Expected pinned requirement, got {}".format(ireq)) -@@ -217,24 +444,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +446,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. From 1c6acfecb5db81575ea3af8266dbf08bb812324c Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 20:24:06 -0400 Subject: [PATCH 05/69] Ensure that we use posix style strings instead of Path objects for chdir context manager Signed-off-by: Dan Ryan --- pipenv/utils.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pipenv/utils.py b/pipenv/utils.py index b45081fe11..be1240ac05 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1367,7 +1367,9 @@ def is_virtual_environment(path): def chdir(path): """Context manager to change working directories.""" from ._compat import Path - prev_cwd = Path.cwd() + prev_cwd = Path.cwd().as_posix() + if isinstance(path, Path): + path = path.as_posix() os.chdir(str(path)) try: yield From c584739b417c52d06c4b14a0e58fba620e01dc00 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 20:31:27 -0400 Subject: [PATCH 06/69] Handle NoneType paths Signed-off-by: Dan Ryan --- pipenv/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pipenv/utils.py b/pipenv/utils.py index be1240ac05..286b18665e 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1367,6 +1367,8 @@ def is_virtual_environment(path): def chdir(path): """Context manager to change working directories.""" from ._compat import Path + if not path: + return prev_cwd = Path.cwd().as_posix() if isinstance(path, Path): path = path.as_posix() From dc97d73e62bf89a8a57d9efb25c24676c0e39df3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 24 Jul 2018 23:37:28 -0400 Subject: [PATCH 07/69] Add exception handling for non-existent setup_py_dir Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index 94879f2ff5..f92bd97e0b 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -284,7 +284,7 @@ def get_legacy_dependencies(self, ireq): with chdir(ireq.setup_py_dir): from setuptools.dist import distutils distutils.core.run_setup(ireq.setup_py) - except (ImportError, InstallationError): + except (ImportError, InstallationError, TypeError): pass try: dist = ireq.get_dist() diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 17f2e2b56f..273955a1f2 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -246,7 +246,7 @@ index 1c4b943..91902dc 100644 + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils + distutils.core.run_setup(ireq.setup_py) -+ except (ImportError, InstallationError): ++ except (ImportError, InstallationError, TypeError): + pass + try: + dist = ireq.get_dist() From d33c4e86e18489e0ac51b1d225fe26e1d68d1d69 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 00:14:44 -0400 Subject: [PATCH 08/69] Fix exception handling Signed-off-by: Dan Ryan --- pipenv/patched/piptools/repositories/pypi.py | 2 +- tasks/vendoring/patches/patched/piptools.patch | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index f92bd97e0b..bf7ebd2da6 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -284,7 +284,7 @@ def get_legacy_dependencies(self, ireq): with chdir(ireq.setup_py_dir): from setuptools.dist import distutils distutils.core.run_setup(ireq.setup_py) - except (ImportError, InstallationError, TypeError): + except (ImportError, InstallationError, TypeError, AttributeError): pass try: dist = ireq.get_dist() diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 273955a1f2..7d9b64d8ee 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -246,7 +246,7 @@ index 1c4b943..91902dc 100644 + with chdir(ireq.setup_py_dir): + from setuptools.dist import distutils + distutils.core.run_setup(ireq.setup_py) -+ except (ImportError, InstallationError, TypeError): ++ except (ImportError, InstallationError, TypeError, AttributeError): + pass + try: + dist = ireq.get_dist() From 0d2ceb63128625f5f552d5ac8e2f7b0ab0370ac3 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 18:43:07 +0800 Subject: [PATCH 09/69] Pull in PythonFinder updates --- pipenv/vendor/pythonfinder/models/path.py | 18 +++++++++++---- pipenv/vendor/pythonfinder/pythonfinder.py | 27 ++++++++++++++++++++-- 2 files changed, 39 insertions(+), 6 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 2c6ea95098..63c8abdce2 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -26,6 +26,7 @@ @attr.s class SystemPath(object): + global_search = attr.ib(default=True) paths = attr.ib(default=attr.Factory(defaultdict)) _executables = attr.ib(default=attr.Factory(list)) _python_executables = attr.ib(default=attr.Factory(list)) @@ -94,7 +95,10 @@ def _setup_pyenv(self): (p for p in reversed(self.path_order) if PYENV_ROOT.lower() in p.lower()), None, ) - pyenv_index = self.path_order.index(last_pyenv) + try: + pyenv_index = self.path_order.index(last_pyenv) + except ValueError: + return self.pyenv_finder = PyenvFinder.create(root=PYENV_ROOT) # paths = (v.paths.values() for v in self.pyenv_finder.versions.values()) root_paths = ( @@ -200,7 +204,7 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) ) @classmethod - def create(cls, path=None, system=False, only_python=False): + def create(cls, path=None, system=False, only_python=False, global_search=True): """Create a new :class:`pythonfinder.models.SystemPath` instance. :param path: Search path to prepend when searching, defaults to None @@ -214,7 +218,9 @@ def create(cls, path=None, system=False, only_python=False): """ path_entries = defaultdict(PathEntry) - paths = os.environ.get("PATH").split(os.pathsep) + paths = [] + if global_search: + paths = os.environ.get("PATH").split(os.pathsep) if path: paths = [path] + paths _path_objects = [ensure_path(p.strip('"')) for p in paths] @@ -227,7 +233,11 @@ def create(cls, path=None, system=False, only_python=False): for p in _path_objects } ) - return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system) + return cls( + paths=path_entries, path_order=paths, + only_python=only_python, + system=system, global_search=global_search, + ) @attr.s diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 50e1b8d89d..7e5813bb9b 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -6,8 +6,30 @@ class Finder(object): - def __init__(self, path=None, system=False): + def __init__(self, path=None, system=False, global_search=True): + """Cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in `path`, + if supplied, but searching the bin path of `sys.executable` if + `system=True`, and then searching in the `os.environ['PATH']` if + `global_search=True`. + + When `global_search` is `False`, this search operation is restricted to + the allowed locations of `path` and `system`. + + :param path: A bin-directory search location, or None to ignore. + :type path: str or None + :param system: Whether to include the bin-dir of `sys.executable`, + defaults to False + :type system: bool + :param global_search: Whether to search the global path from + os.environ, defaults to True. + :type global_search: bool + :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. + """ + self.path_prepend = path + self.global_search = global_search self.system = system self._system_path = None self._windows_finder = None @@ -16,7 +38,8 @@ def __init__(self, path=None, system=False): def system_path(self): if not self._system_path: self._system_path = SystemPath.create( - path=self.path_prepend, system=self.system + path=self.path_prepend, system=self.system, + global_search=self.global_search, ) return self._system_path From 6d7b26ae6c40379f91745d5ef6da868c739c0469 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:08:51 +0800 Subject: [PATCH 10/69] Minimal Pythonfinder intergration --- pipenv/core.py | 110 ++++--------------------------------------------- 1 file changed, 9 insertions(+), 101 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 25bd0b7e45..40b2ef401f 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -317,76 +317,15 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): project.write_toml(p) -def find_python_from_py(python): - """Find a Python executable from on Windows. - - Ask py.exe for its opinion. - """ - py = system_which("py") - if not py: - return None - - version_args = ["-{0}".format(python[0])] - if len(python) >= 2: - version_args.append("-{0}.{1}".format(python[0], python[2])) - import subprocess - - for ver_arg in reversed(version_args): - try: - python_exe = subprocess.check_output( - [py, ver_arg, "-c", "import sys; print(sys.executable)"] - ) - except subprocess.CalledProcessError: - continue - - if not isinstance(python_exe, str): - python_exe = python_exe.decode(sys.getdefaultencoding()) - python_exe = python_exe.strip() - version = python_version(python_exe) - if (version or "").startswith(python): - return python_exe - - -def find_python_in_path(python): - """Find a Python executable from a version number. - - This uses the PATH environment variable to locate an appropriate Python. - """ - possibilities = ["python", "python{0}".format(python[0])] - if len(python) >= 2: - possibilities.extend( - [ - "python{0}{1}".format(python[0], python[2]), - "python{0}.{1}".format(python[0], python[2]), - "python{0}.{1}m".format(python[0], python[2]), - ] - ) - # Reverse the list, so we find specific ones first. - possibilities = reversed(possibilities) - for possibility in possibilities: - # Windows compatibility. - if os.name == "nt": - possibility = "{0}.exe".format(possibility) - pythons = system_which(possibility, mult=True) - for p in pythons: - version = python_version(p) - if (version or "").startswith(python): - return p - - def find_a_system_python(python): - """Finds a system python, given a version (e.g. 2 / 2.7 / 3.6.2), or a full path.""" - if python.startswith("py"): - return system_which(python) - - elif os.path.isabs(python): - return python - - python_from_py = find_python_from_py(python) - if python_from_py: - return python_from_py - - return find_python_in_path(python) + if not python: + return None + from .vendor import pythonfinder + finder = pythonfinder.Finder() + python_entry = finder.find_python_version(python) + if python_entry: + return str(python_entry.path) + return None def ensure_python(three=None, python=None): @@ -409,35 +348,7 @@ def abort(): ) sys.exit(1) - def activate_pyenv(): - from notpip._vendor.packaging.version import parse as parse_version - - """Adds all pyenv installations to the PATH.""" - if PYENV_INSTALLED: - if PYENV_ROOT: - pyenv_paths = {} - for found in glob("{0}{1}versions{1}*".format(PYENV_ROOT, os.sep)): - pyenv_paths[os.path.split(found)[1]] = "{0}{1}bin".format( - found, os.sep - ) - for version_str, pyenv_path in pyenv_paths.items(): - version = parse_version(version_str) - if version.is_prerelease and pyenv_paths.get(version.base_version): - continue - - add_to_path(pyenv_path) - else: - click.echo( - "{0}: PYENV_ROOT is not set. New python paths will " - "probably not be exported properly after installation." - "".format(crayons.red("Warning", bold=True)), - err=True, - ) - global USING_DEFAULT_PYTHON - # Add pyenv paths to PATH. - activate_pyenv() - path_to_python = None USING_DEFAULT_PYTHON = three is None and not python # Find out which python is desired. if not python: @@ -446,8 +357,7 @@ def activate_pyenv(): python = project.required_python_version if not python: python = PIPENV_DEFAULT_PYTHON_VERSION - if python: - path_to_python = find_a_system_python(python) + path_to_python = find_a_system_python(python) if not path_to_python and python is not None: # We need to install Python. click.echo( @@ -501,8 +411,6 @@ def activate_pyenv(): click.echo(crayons.blue(e.err), err=True) # Print the results, in a beautiful blue… click.echo(crayons.blue(c.out), err=True) - # Add new paths to PATH. - activate_pyenv() # Find the newly installed Python, hopefully. version = str(version) path_to_python = find_a_system_python(version) From ac41cdcf4e3fe6d351f9dfe635cc90ddd4421535 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:17:23 +0800 Subject: [PATCH 11/69] Remove duplicate and unused code --- pipenv/core.py | 4 +--- pipenv/environments.py | 7 ------- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 40b2ef401f..1379333166 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -6,7 +6,6 @@ import shutil import time import tempfile -from glob import glob import json as simplejson import click import click_completion @@ -51,8 +50,6 @@ PIPENV_SKIP_VALIDATION, PIPENV_HIDE_EMOJIS, PIPENV_INSTALL_TIMEOUT, - PYENV_ROOT, - PYENV_INSTALLED, PIPENV_YES, PIPENV_DONT_LOAD_ENV, PIPENV_DEFAULT_PYTHON_VERSION, @@ -369,6 +366,7 @@ def abort(): err=True, ) # Pyenv is installed + from .vendor.pythonfinder.environment import PYENV_INSTALLED if not PYENV_INSTALLED: abort() else: diff --git a/pipenv/environments.py b/pipenv/environments.py index afe82988e1..037e75a3c1 100644 --- a/pipenv/environments.py +++ b/pipenv/environments.py @@ -209,12 +209,5 @@ # Internal, the default shell to use if shell detection fails. PIPENV_SHELL = os.environ.get("SHELL") or os.environ.get("PYENV_SHELL") -# Internal, to tell if pyenv is installed. -PYENV_ROOT = os.environ.get("PYENV_ROOT", os.path.expanduser("~/.pyenv")) -PYENV_INSTALLED = ( - bool(os.environ.get("PYENV_SHELL")) or - bool(os.environ.get("PYENV_ROOT")) -) - # Internal, to tell whether the command line session is interactive. SESSION_IS_INTERACTIVE = bool(os.isatty(sys.stdout.fileno())) From 59b974bfc19131b07daa7de131ab9aa644e4e180 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:25:04 +0800 Subject: [PATCH 12/69] Use Pythonfinder in help --- pipenv/help.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index 32ffa7c05b..42a4df3f62 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -5,7 +5,7 @@ from pprint import pprint from .__version__ import __version__ -from .core import project, system_which, find_python_in_path, python_version +from .core import project, system_which, find_a_system_python, python_version from .pep508checker import lookup @@ -27,14 +27,10 @@ def get_pipenv_diagnostics(): print("") print("Other Python installations in `PATH`:") print("") - for python_v in ("2.5", "2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): - found = find_python_in_path(python_v) - if found: - print(" - `{0}`: `{1}`".format(python_v, found)) - found = system_which("python{0}".format(python_v), mult=True) - if found: - for f in found: - print(" - `{0}`: `{1}`".format(python_v, f)) + for python_v in ("2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): + entry = find_a_system_python(python_v) + if entry: + print(" - `{0}`: `{1}`".format(python_v, entry.path)) print("") for p in ("python", "python2", "python3", "py"): found = system_which(p, mult=True) From d5cebedee8146cd36acd8025f29571ace9f0dd79 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:29:05 +0800 Subject: [PATCH 13/69] Keep logic of using actual Python commands --- pipenv/core.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pipenv/core.py b/pipenv/core.py index 1379333166..b059ddc98e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -317,6 +317,10 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): if not python: return None + if python.startswith("py"): + return system_which(python) + if os.path.isabs(python): + return python from .vendor import pythonfinder finder = pythonfinder.Finder() python_entry = finder.find_python_version(python) From b82d5587e369ab2da0894d9c6e654aa38f175fdc Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Sat, 14 Jul 2018 19:43:24 +0800 Subject: [PATCH 14/69] Tape directly into Pythonfinder instead --- pipenv/help.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index 42a4df3f62..b6e6b084c8 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -5,8 +5,9 @@ from pprint import pprint from .__version__ import __version__ -from .core import project, system_which, find_a_system_python, python_version +from .core import project, system_which, python_version from .pep508checker import lookup +from .vendor import pythonfinder def print_utf(line): @@ -27,8 +28,9 @@ def get_pipenv_diagnostics(): print("") print("Other Python installations in `PATH`:") print("") + finder = pythonfinder.Finder() for python_v in ("2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): - entry = find_a_system_python(python_v) + entry = finder.find_python_version(python_v) if entry: print(" - `{0}`: `{1}`".format(python_v, entry.path)) print("") From d3d5d563e7ab9cba1f7b4220b6111252ecd7c941 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 16:27:51 -0400 Subject: [PATCH 15/69] Update pythonfinder and make full use of it in support calls Signed-off-by: Dan Ryan --- pipenv/help.py | 21 +++++++------ pipenv/vendor/pythonfinder/models/path.py | 8 ++--- pipenv/vendor/pythonfinder/pythonfinder.py | 35 ++++++++++------------ 3 files changed, 27 insertions(+), 37 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index b6e6b084c8..118a2272f1 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -3,9 +3,10 @@ import sys import pipenv +from itertools import chain from pprint import pprint from .__version__ import __version__ -from .core import project, system_which, python_version +from .core import project, system_which from .pep508checker import lookup from .vendor import pythonfinder @@ -28,16 +29,14 @@ def get_pipenv_diagnostics(): print("") print("Other Python installations in `PATH`:") print("") - finder = pythonfinder.Finder() - for python_v in ("2.6", "2.7", "3.4", "3.5", "3.6", "3.7"): - entry = finder.find_python_version(python_v) - if entry: - print(" - `{0}`: `{1}`".format(python_v, entry.path)) - print("") - for p in ("python", "python2", "python3", "py"): - found = system_which(p, mult=True) - for f in found: - print(" - `{0}`: `{1}`".format(python_version(f), f)) + finder = pythonfinder.Finder(system=False, global_search=True) + python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) + python_paths = list(chain(*python_versions)) + for python in python_paths: + python_version = python.py_version.version + python_path = python.path.as_posix() + print(" - `{0}`: `{1}`".format(python_version, python_path)) + print("") print("PEP 508 Information:") print("") diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 63c8abdce2..c394fb22da 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -72,7 +72,7 @@ def __attrs_post_init__(self): bin_dir = 'Scripts' else: bin_dir = 'bin' - if venv: + if venv and (self.system or self.global_search): p = Path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order self.paths[p] = PathEntry.create( @@ -233,11 +233,7 @@ def create(cls, path=None, system=False, only_python=False, global_search=True): for p in _path_objects } ) - return cls( - paths=path_entries, path_order=paths, - only_python=only_python, - system=system, global_search=global_search, - ) + return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system, global_search=global_search) @attr.s diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 7e5813bb9b..32e62b188a 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -7,24 +7,20 @@ class Finder(object): def __init__(self, path=None, system=False, global_search=True): - """Cross-platform Finder for locating python and other executables. - - Searches for python and other specified binaries starting in `path`, - if supplied, but searching the bin path of `sys.executable` if - `system=True`, and then searching in the `os.environ['PATH']` if - `global_search=True`. - - When `global_search` is `False`, this search operation is restricted to - the allowed locations of `path` and `system`. - - :param path: A bin-directory search location, or None to ignore. - :type path: str or None - :param system: Whether to include the bin-dir of `sys.executable`, - defaults to False - :type system: bool - :param global_search: Whether to search the global path from - os.environ, defaults to True. - :type global_search: bool + """Finder A cross-platform Finder for locating python and other executables. + + Searches for python and other specified binaries starting in `path`, if supplied, + but searching the bin path of `sys.executable` if `system=True`, and then + searching in the `os.environ['PATH']` if `global_search=True`. When `global_search` + is `False`, this search operation is restricted to the allowed locations of + `path` and `system`. + + :param path: A bin-directory search location, defaults to None + :param path: str, optional + :param system: Whether to include the bin-dir of `sys.executable`, defaults to False + :param system: bool, optional + :param global_search: Whether to search the global path from os.environ, defaults to True + :param global_search: bool, optional :returns: a :class:`~pythonfinder.pythonfinder.Finder` object. """ @@ -38,8 +34,7 @@ def __init__(self, path=None, system=False, global_search=True): def system_path(self): if not self._system_path: self._system_path = SystemPath.create( - path=self.path_prepend, system=self.system, - global_search=self.global_search, + path=self.path_prepend, system=self.system, global_search=self.global_search ) return self._system_path From 33328c435ab028faae36709cbeb930ee5602d774 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:00:25 -0400 Subject: [PATCH 16/69] Fully integrate pythonfinder for system pythons Signed-off-by: Dan Ryan --- pipenv/core.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index b059ddc98e..38cb5b7bd4 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -315,17 +315,24 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): + from .vendor.pythonfinder import Finder + # system always refers to sys.executable, which could point at a virtualenv + # for global searches we most likely want to turn that off + finder = Finder(system=False, global_search=True) if not python: return None + # when using the python launcher on windows we can find the versions ourselves + if os.name == 'nt' and python.startswith("py -"): + python = python[len("py -"):] if python.startswith("py"): - return system_which(python) + python_entry = finder.which(python) + if python_entry: + return python_entry.path.as_posix() if os.path.isabs(python): return python - from .vendor import pythonfinder - finder = pythonfinder.Finder() python_entry = finder.find_python_version(python) if python_entry: - return str(python_entry.path) + return python_entry.path.as_posix() return None From aeb07cdec03d20dcf98914aa124510049a7615c4 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:40:29 -0400 Subject: [PATCH 17/69] Fix search path Signed-off-by: Dan Ryan --- pipenv/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 38cb5b7bd4..1a002dcdd5 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -318,7 +318,7 @@ def find_a_system_python(python): from .vendor.pythonfinder import Finder # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off - finder = Finder(system=False, global_search=True) + finder = Finder(system=True, global_search=True) if not python: return None # when using the python launcher on windows we can find the versions ourselves @@ -330,6 +330,7 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python + python_entry = finder.find_python_version(python) if python_entry: return python_entry.path.as_posix() From 26c34641d148369cb4fad8b31d1c5f5da8251543 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:54:17 -0400 Subject: [PATCH 18/69] Updated buildkite to echo out the path Signed-off-by: Dan Ryan --- run-tests.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/run-tests.sh b/run-tests.sh index b71fd4fb83..692a69ea4f 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -17,6 +17,7 @@ fi export PATH="$HOME/.local/bin:$PATH" # pip uninstall -y pipenv +echo "Path: $PATH" echo "Installing Pipenv…" pip install -e "$(pwd)" --upgrade pipenv install --deploy --dev From 5da85d34b3981e161e6bee65197453a6c11b930b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 17:58:39 -0400 Subject: [PATCH 19/69] Add some debugging Signed-off-by: Dan Ryan --- pipenv/core.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 1a002dcdd5..4f7a08a2c5 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -330,7 +330,7 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - + python_entry = finder.find_python_version(python) if python_entry: return python_entry.path.as_posix() @@ -367,6 +367,7 @@ def abort(): if not python: python = PIPENV_DEFAULT_PYTHON_VERSION path_to_python = find_a_system_python(python) + click.echo("Found path to python: %s" % path_to_python) if not path_to_python and python is not None: # We need to install Python. click.echo( From c6a3284d307b973059071071b6aba637688fdb16 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 18:07:31 -0400 Subject: [PATCH 20/69] Updated buildkite to set home properly Signed-off-by: Dan Ryan --- run-tests.sh | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/run-tests.sh b/run-tests.sh index 692a69ea4f..1243e03341 100755 --- a/run-tests.sh +++ b/run-tests.sh @@ -15,7 +15,15 @@ if [[ ! -z "$TEST_SUITE" ]]; then echo "Using TEST_SUITE=$TEST_SUITE" fi -export PATH="$HOME/.local/bin:$PATH" +HOME=$(readlink -f ~/) +if [[ -z "$HOME" ]]; then + if [[ "$USER" == "root" ]]; then + HOME="/root" + fi +fi +if [[ ! -z "$HOME" ]]; then + export PATH="${HOME}/.local/bin:${PATH}" +fi # pip uninstall -y pipenv echo "Path: $PATH" echo "Installing Pipenv…" From dd06342428046978e685b03a5ace1b732af153cb Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 18:13:37 -0400 Subject: [PATCH 21/69] More debug info for buildkite Signed-off-by: Dan Ryan --- pipenv/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4f7a08a2c5..a488c72e0a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -330,7 +330,10 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - + click.echo("Path order: %s" % finder.system_path.path_order) + version = python.split('.')[0] + versions_avail = [v.path.as_posix() for v in finder.system_path.find_all_python_versions(version)] + click.echo("Python versions found: %s" % versions_avail) python_entry = finder.find_python_version(python) if python_entry: return python_entry.path.as_posix() From 684475e2c60ee9a4ba6304559c6a0c3fc0b08876 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 19:41:32 -0400 Subject: [PATCH 22/69] More debugging Signed-off-by: Dan Ryan --- pipenv/core.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index a488c72e0a..0658a0968c 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -316,6 +316,7 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): from .vendor.pythonfinder import Finder + from itertools import chain # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off finder = Finder(system=True, global_search=True) @@ -332,7 +333,12 @@ def find_a_system_python(python): return python click.echo("Path order: %s" % finder.system_path.path_order) version = python.split('.')[0] - versions_avail = [v.path.as_posix() for v in finder.system_path.find_all_python_versions(version)] + python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) + python_paths = list(chain(*python_versions)) + for python in python_paths: + python_version = python.py_version.version + python_path = python.path.as_posix() + click.echo(" - `{0}`: `{1}`".format(python_version, python_path)) click.echo("Python versions found: %s" % versions_avail) python_entry = finder.find_python_version(python) if python_entry: From c6fb1ea31f4099f78c71d5a818a46f818f5f980e Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 19:59:27 -0400 Subject: [PATCH 23/69] Fix path search to include prereleases Signed-off-by: Dan Ryan --- pipenv/core.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 0658a0968c..23982adbb8 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -316,7 +316,6 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): def find_a_system_python(python): from .vendor.pythonfinder import Finder - from itertools import chain # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off finder = Finder(system=True, global_search=True) @@ -331,16 +330,9 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - click.echo("Path order: %s" % finder.system_path.path_order) - version = python.split('.')[0] - python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) - python_paths = list(chain(*python_versions)) - for python in python_paths: - python_version = python.py_version.version - python_path = python.path.as_posix() - click.echo(" - `{0}`: `{1}`".format(python_version, python_path)) - click.echo("Python versions found: %s" % versions_avail) python_entry = finder.find_python_version(python) + if not python_entry: + python_entry = finder.find_python_version(python, pre=True) if python_entry: return python_entry.path.as_posix() return None From 5366253f39c70cfc44e57683deacc2427dbfb832 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Sun, 15 Jul 2018 20:19:17 -0400 Subject: [PATCH 24/69] More robust path searching Signed-off-by: Dan Ryan --- pipenv/core.py | 26 +++++++++++++++++++++++--- 1 file changed, 23 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 23982adbb8..c55cc73453 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -12,6 +12,7 @@ import crayons import dotenv import delegator +from first import first import pipfile from blindspin import spinner import six @@ -319,6 +320,7 @@ def find_a_system_python(python): # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off finder = Finder(system=True, global_search=True) + python_entry = None if not python: return None # when using the python launcher on windows we can find the versions ourselves @@ -330,14 +332,32 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - python_entry = finder.find_python_version(python) + version = [int(v) for v in python.split('.')] + minor = None + patch = None + if len(version) > 1: + minor = version[1] + if len(version) > 2: + patch = version[2] + python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) + python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] + if minor and patch: + _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) + python_entry = _py[1] if _py else None + elif minor: + _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) + python_entry = _py[1] if _py else None + else: + _py = first(python_entries) + python_entry = _py[1] if _py else None if not python_entry: - python_entry = finder.find_python_version(python, pre=True) + version_str = "{0}.{1}".format(version[0], minor) if minor else "{0}".format(version[0]) + exe_name = "python{0}".format(version_str) + python_entry = finder.which(exe_name) if python_entry: return python_entry.path.as_posix() return None - def ensure_python(three=None, python=None): # Support for the PIPENV_PYTHON environment variable. from .environments import PIPENV_PYTHON From 50aff8e330893db5a1e6ac9fe34343b405bfb5a3 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 01:38:09 -0400 Subject: [PATCH 25/69] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 10 +++---- pipenv/vendor/pythonfinder/models/path.py | 8 ++--- pipenv/vendor/pythonfinder/models/python.py | 30 +++++++++++++++++-- pipenv/vendor/pythonfinder/pythonfinder.py | 29 ++++-------------- 4 files changed, 42 insertions(+), 35 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index e455ffb0e3..0c6f0134c1 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -42,7 +42,7 @@ def which(self, name): found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) return found - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -55,21 +55,21 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) """ version_matcher = operator.methodcaller( - "matches", major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python.matches(major, minor=minor, patch=patch, pre=pre, dev=dev): + if self.is_python and self.as_python.matches(major=major, minor=minor, patch=patch, pre=pre, dev=dev): return self return finder = ((child, child.as_python) for child in self.children.values() if child.is_python and child.as_python) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) - version_sort = operator.attrgetter("version") + version_sort = operator.attrgetter("version_sort") return next( - (c[0] for c in sorted(py_filter, key=lambda child: child[1].version, reverse=True)), None + (c[0] for c in sorted(py_filter, key=lambda child: child[1].version_sort, reverse=True)), None ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index c394fb22da..893b6af09b 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -151,7 +151,7 @@ def which(self, executable): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -173,10 +173,10 @@ def find_all_python_versions(self, major, minor=None, patch=None, pre=None, dev= return windows_finder_version paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version") + version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. :param major: Major python version to search for. @@ -198,7 +198,7 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) return windows_finder_version paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) - version_sort = operator.attrgetter("as_python.version") + version_sort = operator.attrgetter("as_python.version_sort") return next( (c for c in sorted(path_filter, key=version_sort, reverse=True)), None ) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 1f68176704..4ff364faba 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -27,6 +27,30 @@ class PythonVersion(object): comes_from = attr.ib(default=None) executable = attr.ib(default=None) + @property + def version_sort(self): + """version_sort tuple for sorting against other instances of the same class. + + Returns a tuple of the python version but includes a point for non-dev, + and a point for non-prerelease versions. So released versions will have 2 points + for this value. E.g. `(3, 6, 6, 2)` is a release, `(3, 6, 6, 1)` is a prerelease, + `(3, 6, 6, 0)` is a dev release, and `(3, 6, 6, 3)` is a postrelease. + """ + release_sort = 2 + if self.is_postrelease: + release_sort = 3 + elif self.is_prerelease: + release_sort = 1 + elif self.is_devrelease: + release_sort = 0 + return ( + self.major, + self.minor, + self.patch, + release_sort + ) + + @property def version_tuple(self): """Provides a version tuple for using as a dictionary key. @@ -43,9 +67,9 @@ def version_tuple(self): self.is_devrelease, ) - def matches(self, major, minor=None, patch=None, pre=False, dev=False): + def matches(self, major=None, minor=None, patch=None, pre=False, dev=False): return ( - self.major == major + (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) @@ -77,7 +101,7 @@ def parse(cls, version): """ try: - version = parse_version(version) + version = parse_version(str(version)) except TypeError: raise ValueError("Unable to parse version: %s" % version) if not version or not version.release: diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 32e62b188a..eff73e221c 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -50,31 +50,14 @@ def which(self, exe): return self.system_path.which(exe) def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): - if ( - major - and not minor - and not patch - and not pre - and not dev - and isinstance(major, six.string_types) - ): - from .models import PythonVersion - version_dict = {} - if "." in major: - version_dict = PythonVersion.parse(major) - elif len(major) == 1: - version_dict = { - 'major': int(major), - 'minor': None, - 'patch': None, - 'is_prerelease': False, - 'is_devrelease': False - } + from .models import PythonVersion + if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: + version_dict = PythonVersion.parse(major) major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) - dev = version_dict.get("is_devrelease", dev) + pre = version_dict.get("is_prerelease", pre) if pre is not None else pre + dev = version_dict.get("is_devrelease", dev) if dev is not None else dev if os.name == "nt": match = self.windows_finder.find_python_version( major, minor=minor, patch=patch, pre=pre, dev=dev @@ -82,5 +65,5 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) if match: return match return self.system_path.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev ) From b93b287fc9594c174460c37f919507763320e24f Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 01:43:13 -0400 Subject: [PATCH 26/69] Try the new code Signed-off-by: Dan Ryan --- pipenv/core.py | 42 ++++++++++++++++++++++-------------------- 1 file changed, 22 insertions(+), 20 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index c55cc73453..de26da9bcb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -332,32 +332,34 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - version = [int(v) for v in python.split('.')] - minor = None - patch = None - if len(version) > 1: - minor = version[1] - if len(version) > 2: - patch = version[2] - python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) - python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] - if minor and patch: - _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) - python_entry = _py[1] if _py else None - elif minor: - _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) - python_entry = _py[1] if _py else None - else: - _py = first(python_entries) - python_entry = _py[1] if _py else None + # version = [int(v) for v in python.split('.')] + # minor = None + # patch = None + # if len(version) > 1: + # minor = version[1] + # if len(version) > 2: + # patch = version[2] + # python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) + # python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] + python_entry = finder.find_python_version(python) + # if minor and patch: + # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) + # python_entry = _py[1] if _py else None + # elif minor: + # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) + # python_entry = _py[1] if _py else None + # else: + # _py = first(python_entries) + # python_entry = _py[1] if _py else None if not python_entry: - version_str = "{0}.{1}".format(version[0], minor) if minor else "{0}".format(version[0]) - exe_name = "python{0}".format(version_str) + exe_name = "python{0}".format(python) python_entry = finder.which(exe_name) if python_entry: + click.echo("Found python: {0} => {1}".format(python_entry.as_python.version, python_entry.path.as_posix())) return python_entry.path.as_posix() return None + def ensure_python(three=None, python=None): # Support for the PIPENV_PYTHON environment variable. from .environments import PIPENV_PYTHON From 597cd2d917f7ec6d9a32dc3f5ba0f1da5cf3b7f0 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 01:46:39 -0400 Subject: [PATCH 27/69] Cleanup core and help to use new finder Signed-off-by: Dan Ryan --- pipenv/core.py | 19 ------------------- pipenv/help.py | 3 +-- 2 files changed, 1 insertion(+), 21 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index de26da9bcb..4178b4f276 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -332,30 +332,11 @@ def find_a_system_python(python): return python_entry.path.as_posix() if os.path.isabs(python): return python - # version = [int(v) for v in python.split('.')] - # minor = None - # patch = None - # if len(version) > 1: - # minor = version[1] - # if len(version) > 2: - # patch = version[2] - # python_entries = getattr(finder, 'system_path').find_all_python_versions(version[0]) - # python_entries = [(entry.py_version.version_tuple, entry) for entry in python_entries] python_entry = finder.find_python_version(python) - # if minor and patch: - # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor, patch)) - # python_entry = _py[1] if _py else None - # elif minor: - # _py = first(entry for entry in python_entries if entry[0][:3] == (version[0], minor)) - # python_entry = _py[1] if _py else None - # else: - # _py = first(python_entries) - # python_entry = _py[1] if _py else None if not python_entry: exe_name = "python{0}".format(python) python_entry = finder.which(exe_name) if python_entry: - click.echo("Found python: {0} => {1}".format(python_entry.as_python.version, python_entry.path.as_posix())) return python_entry.path.as_posix() return None diff --git a/pipenv/help.py b/pipenv/help.py index 118a2272f1..93f8f3ff8b 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -30,8 +30,7 @@ def get_pipenv_diagnostics(): print("Other Python installations in `PATH`:") print("") finder = pythonfinder.Finder(system=False, global_search=True) - python_versions = (getattr(finder, 'system_path').find_all_python_versions(major) for major in (2, 3)) - python_paths = list(chain(*python_versions)) + python_paths = getattr(finder, 'system_path').find_all_python_versions() for python in python_paths: python_version = python.py_version.version python_path = python.path.as_posix() From 0d7afadd63cfaabec1ecd224a7f731ae7bd20543 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:17:00 -0400 Subject: [PATCH 28/69] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 2 +- pipenv/vendor/pythonfinder/models/windows.py | 21 ++++++++++++++++---- pipenv/vendor/pythonfinder/pythonfinder.py | 9 +++++++++ 3 files changed, 27 insertions(+), 5 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 893b6af09b..a725504931 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -174,7 +174,7 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) version_sort = operator.attrgetter("as_python.version_sort") - return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + return (c for c in sorted(path_filter, key=version_sort, reverse=True)) def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index b6998823de..4e5b5b9868 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -16,16 +16,29 @@ class WindowsFinder(BaseFinder): version_list = attr.ib(default=attr.Factory(list)) versions = attr.ib() - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): version_matcher = operator.methodcaller( - "matches", major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) ) - version_sort = operator.attrgetter("version") + version_sort = operator.attrgetter("version_sort") + for c in sorted(py_filter, key=version_sort, reverse=True): + yield c.comes_from + + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + # version_matcher = operator.methodcaller( + # "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + # ) + # py_filter = filter( + # None, filter(lambda c: version_matcher(c), self.version_list) + # ) + # version_sort = operator.attrgetter("version_sort") return next( - (c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)), None + self.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev + ), None ) @versions.default diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index eff73e221c..fa7578d2c7 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -2,6 +2,7 @@ from __future__ import print_function, absolute_import import os import six +import operator from .models import SystemPath @@ -67,3 +68,11 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) return self.system_path.find_python_version( major=major, minor=minor, patch=patch, pre=pre, dev=dev ) + + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + version_sort = operator.attrgetter("as_python.version_sort") + versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + if os.name == 'nt': + windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + versions = list(versions) + list(windows_versions) + return sorted(versions, key=version_sort, reverse=True) From 08cb159e22d8b670daefc09c88feaff8188d8d00 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:21:42 -0400 Subject: [PATCH 29/69] Windows fix Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/windows.py | 16 ++++------------ pipenv/vendor/pythonfinder/pythonfinder.py | 5 +++-- 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 4e5b5b9868..f33a4807f7 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -24,21 +24,13 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, None, filter(lambda c: version_matcher(c), self.version_list) ) version_sort = operator.attrgetter("version_sort") - for c in sorted(py_filter, key=version_sort, reverse=True): - yield c.comes_from + return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): - # version_matcher = operator.methodcaller( - # "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev - # ) - # py_filter = filter( - # None, filter(lambda c: version_matcher(c), self.version_list) - # ) - # version_sort = operator.attrgetter("version_sort") - return next( - self.find_all_python_versions( + return next(( + v for v in self.find_all_python_versions( major=major, minor=minor, patch=patch, pre=pre, dev=dev - ), None + )), None ) @versions.default diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index fa7578d2c7..e40405fd74 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -71,8 +71,9 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): version_sort = operator.attrgetter("as_python.version_sort") - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + versions = [] + versions.extend([p for p in self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev)]) if os.name == 'nt': windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) - versions = list(versions) + list(windows_versions) + versions = versions + list(windows_versions) return sorted(versions, key=version_sort, reverse=True) From 2de9505668b40cf3a847bedf6604a7de8bc73c05 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:31:22 -0400 Subject: [PATCH 30/69] Fix system path iterator Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index a725504931..893b6af09b 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -174,7 +174,7 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) version_sort = operator.attrgetter("as_python.version_sort") - return (c for c in sorted(path_filter, key=version_sort, reverse=True)) + return [c for c in sorted(path_filter, key=version_sort, reverse=True)] def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): """Search for a specific python version on the path. From 8bb39a412add8db4092484806b9afb43c0aefa56 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:41:57 -0400 Subject: [PATCH 31/69] Fix version iteration in pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/pythonfinder.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index e40405fd74..891a3a1757 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -71,8 +71,9 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): version_sort = operator.attrgetter("as_python.version_sort") - versions = [] - versions.extend([p for p in self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev)]) + versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + if not isinstance(versions, list): + versions = [versions,] if os.name == 'nt': windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) versions = versions + list(windows_versions) From 861c075876f40d24b09cff087a205bb59c84962b Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 16 Jul 2018 02:42:45 -0400 Subject: [PATCH 32/69] Use new api in help methods Signed-off-by: Dan Ryan --- pipenv/help.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/help.py b/pipenv/help.py index 93f8f3ff8b..ac8d70bd88 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -30,7 +30,7 @@ def get_pipenv_diagnostics(): print("Other Python installations in `PATH`:") print("") finder = pythonfinder.Finder(system=False, global_search=True) - python_paths = getattr(finder, 'system_path').find_all_python_versions() + python_paths = finder.find_all_python_versions() for python in python_paths: python_version = python.py_version.version python_path = python.path.as_posix() From 27028e1942969339e9a065dae62cc789dacb2a0f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 03:22:44 +0800 Subject: [PATCH 33/69] Possibly stray echo --- pipenv/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4178b4f276..7d7a855b38 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -371,7 +371,6 @@ def abort(): if not python: python = PIPENV_DEFAULT_PYTHON_VERSION path_to_python = find_a_system_python(python) - click.echo("Found path to python: %s" % path_to_python) if not path_to_python and python is not None: # We need to install Python. click.echo( From 9cb3f970bd1eaffb01aecb990a022931294c1573 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 03:53:36 +0800 Subject: [PATCH 34/69] Clean up help module --- pipenv/help.py | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/pipenv/help.py b/pipenv/help.py index ac8d70bd88..e8fbf1c83a 100644 --- a/pipenv/help.py +++ b/pipenv/help.py @@ -1,12 +1,11 @@ # coding: utf-8 import os +import pprint import sys + import pipenv -from itertools import chain -from pprint import pprint -from .__version__ import __version__ -from .core import project, system_which +from .core import project from .pep508checker import lookup from .vendor import pythonfinder @@ -21,26 +20,25 @@ def print_utf(line): def get_pipenv_diagnostics(): print("
$ pipenv --support") print("") - print("Pipenv version: `{0!r}`".format(__version__)) + print("Pipenv version: `{0!r}`".format(pipenv.__version__)) print("") print("Pipenv location: `{0!r}`".format(os.path.dirname(pipenv.__file__))) print("") print("Python location: `{0!r}`".format(sys.executable)) print("") - print("Other Python installations in `PATH`:") + print("Python installations found:") print("") + finder = pythonfinder.Finder(system=False, global_search=True) python_paths = finder.find_all_python_versions() for python in python_paths: - python_version = python.py_version.version - python_path = python.path.as_posix() - print(" - `{0}`: `{1}`".format(python_version, python_path)) + print(" - `{}`: `{}`".format(python.py_version.version, python.path)) print("") print("PEP 508 Information:") print("") print("```") - pprint(lookup) + pprint.pprint(lookup) print("```") print("") print("System environment variables:") From a2f84421791c12ca9743f4037546da740bd141a7 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 03:53:52 +0800 Subject: [PATCH 35/69] We DONT want sys.executable when searching sys.executable should be the last resort if the user gives us nothing. If we are given ANYTHING, we should avoid using sys.executable. --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7d7a855b38..873d1ff137 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -319,7 +319,7 @@ def find_a_system_python(python): from .vendor.pythonfinder import Finder # system always refers to sys.executable, which could point at a virtualenv # for global searches we most likely want to turn that off - finder = Finder(system=True, global_search=True) + finder = Finder(system=False, global_search=True) python_entry = None if not python: return None From 0b7b0c983a46e3519c4538c6755e64d83d6d3a4f Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:17:15 +0800 Subject: [PATCH 36/69] Refactor and docstring for find_a_system_python --- pipenv/core.py | 42 ++++++++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 873d1ff137..dba192bcc6 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -315,27 +315,37 @@ def ensure_pipfile(validate=True, skip_requirements=False, system=False): project.write_toml(p) -def find_a_system_python(python): +def find_a_system_python(line): + """Find a Python installation from a given line. + + This tries to parse the line in various of ways: + + * Looks like an absolute path? Use it directly. + * Looks like a py.exe call? Use py.exe to get the executable. + * Starts with "py" something? Looks like a python command. Try to find it + in PATH, and use it directly. + * Search for "python" and "pythonX.Y" executables in PATH to find a match. + * Nothing fits, return None. + """ + if not line: + return None + if os.path.isabs(line): + return line from .vendor.pythonfinder import Finder - # system always refers to sys.executable, which could point at a virtualenv - # for global searches we most likely want to turn that off finder = Finder(system=False, global_search=True) - python_entry = None - if not python: - return None - # when using the python launcher on windows we can find the versions ourselves - if os.name == 'nt' and python.startswith("py -"): - python = python[len("py -"):] - if python.startswith("py"): - python_entry = finder.which(python) + if ((line.startswith("py ") or line.startswith("py.exe ")) + and finder.which("py.exe")): + import subprocess + return subprocess.check_output( + '{} -c "import sys; print(sys.executable)"'.format(line), + ) + if line.startswith("py"): + python_entry = finder.which(line) if python_entry: return python_entry.path.as_posix() - if os.path.isabs(python): - return python - python_entry = finder.find_python_version(python) + python_entry = finder.find_python_version(line) if not python_entry: - exe_name = "python{0}".format(python) - python_entry = finder.which(exe_name) + python_entry = finder.which("python{0}".format(line)) if python_entry: return python_entry.path.as_posix() return None From 07a64777b4f0394392cce425b304ece59b27228e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:19:23 +0800 Subject: [PATCH 37/69] Unused --- pipenv/core.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index dba192bcc6..710483cbc7 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -12,7 +12,6 @@ import crayons import dotenv import delegator -from first import first import pipfile from blindspin import spinner import six From 3a734f1ef0de69b2a4b7d83d6ee9c664470271d0 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:25:16 +0800 Subject: [PATCH 38/69] Proper subprocess decoding --- pipenv/core.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 710483cbc7..0cabc05b56 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -335,9 +335,12 @@ def find_a_system_python(line): if ((line.startswith("py ") or line.startswith("py.exe ")) and finder.which("py.exe")): import subprocess - return subprocess.check_output( + output = subprocess.check_output( '{} -c "import sys; print(sys.executable)"'.format(line), ) + if not isinstance(output, str): + output = output.decode(sys.getdefaultencoding()) + return output.strip() if line.startswith("py"): python_entry = finder.which(line) if python_entry: From 10749cc66cb531101cc1e0e434a86fbd0f0dbb8e Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 04:34:18 +0800 Subject: [PATCH 39/69] Don't fallback if a command is not found --- pipenv/core.py | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 0cabc05b56..b5301c0186 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -345,6 +345,7 @@ def find_a_system_python(line): python_entry = finder.which(line) if python_entry: return python_entry.path.as_posix() + return None python_entry = finder.find_python_version(line) if not python_entry: python_entry = finder.which("python{0}".format(line)) @@ -849,21 +850,16 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): # Actually create the virtualenv. with spinner(): - try: - c = delegator.run(cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config) - except OSError: - click.echo( - "{0}: it looks like {1} is not in your {2}. " - "We cannot continue until this is resolved." - "".format( - crayons.red("Warning", bold=True), - crayons.red(cmd[0]), - crayons.normal("PATH", bold=True), - ), - err=True, - ) - sys.exit(1) + c = delegator.run( + cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, + ) click.echo(crayons.blue(c.out), err=True) + if c.return_code != 0: + click.echo(crayons.blue(c.err), err=True) + click.echo("{0}: Failed to create virtual environment.".format( + crayons.red("Warning", bold=True), + ), err=True) + sys.exit(1) # Associate project directory with the environment. # This mimics Pew's "setproject". From a709bf845f4d0422d6dd0b682aca4c702d039680 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:37:32 -0400 Subject: [PATCH 40/69] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 41 ++++-- pipenv/vendor/pythonfinder/models/path.py | 130 +++++++++++++----- pipenv/vendor/pythonfinder/models/pyenv.py | 16 +++ pipenv/vendor/pythonfinder/models/python.py | 29 +++- pipenv/vendor/pythonfinder/models/windows.py | 19 ++- pipenv/vendor/pythonfinder/pythonfinder.py | 32 +++-- pipenv/vendor/pythonfinder/utils.py | 13 +- 7 files changed, 217 insertions(+), 63 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 0c6f0134c1..b1b56d4320 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -3,6 +3,7 @@ import abc import operator import six +from itertools import chain from ..utils import KNOWN_EXTS @@ -42,28 +43,52 @@ def which(self, name): found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) return found - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + """Search for a specific python version on the path. Return all copies + + :param major: Major python version to search for. + :type major: int + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None + :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. + :rtype: List[:class:`~pythonfinder.models.PathEntry`] + """ + + sub_finder = operator.methodcaller( + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) + if not self.is_dir: + return sub_finder(self) + path_filter = filter(None, (sub_finder(p) for p in self.children.values())) + version_sort = operator.attrgetter("as_python.version_sort") + return [c for c in sorted(path_filter, key=version_sort, reverse=True)] + + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search or self for the specified Python version and return the first match. :param major: Major version number. :type major: int - :param minor: Minor python version, defaults to None - :param minor: int, optional - :param patch: Patch python version, defaults to None - :param patch: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :returns: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. """ version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and self.as_python.matches(major=major, minor=minor, patch=patch, pre=pre, dev=dev): + if self.is_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in self.children.values() if child.is_python and child.as_python) + finder = ((child, child.as_python) for child in chain(*filter(None, self.pythons.values())) if child.as_python) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 893b6af09b..5e1807ab06 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -1,10 +1,12 @@ # -*- coding=utf-8 -*- from __future__ import print_function, absolute_import import attr +import copy import operator import os import sys from collections import defaultdict +from itertools import chain from . import BasePath from .python import PythonVersion from ..environment import PYENV_INSTALLED, PYENV_ROOT @@ -13,7 +15,7 @@ optional_instance_of, filter_pythons, path_is_known_executable, - is_python_name, + looks_like_python, ensure_path, fs_str ) @@ -31,32 +33,49 @@ class SystemPath(object): _executables = attr.ib(default=attr.Factory(list)) _python_executables = attr.ib(default=attr.Factory(list)) path_order = attr.ib(default=attr.Factory(list)) - python_version_dict = attr.ib() + python_version_dict = attr.ib(default=attr.Factory(defaultdict)) only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) system = attr.ib(default=False) + __finders = attr.ib(default=attr.Factory(list)) + + def _register_finder(self, finder): + if not finder in self.__finders: + self.__finders.append(finder) + @property def executables(self): if not self._executables: - self._executables = [p for p in self.paths.values() if p.is_executable] + self._executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] return self._executables @property def python_executables(self): + python_executables = {} if not self._python_executables: - self._python_executables = [p for p in self.paths.values() if p.is_python] + for child in self.paths.values(): + if child.pythons: + python_executables.update(dict(child.pythons)) + for finder in self.__finders: + if finder.pythons: + python_executables.update(dict(finder.pythons)) + self._python_executables = python_executables return self._python_executables - @python_version_dict.default def get_python_version_dict(self): version_dict = defaultdict(list) - for p in self.python_executables: - try: - version_object = PythonVersion.from_path(p) - except (ValueError, InvalidPythonVersion): + for finder in self.__finders: + for version, entry in finder.versions.items(): + if entry not in version_dict[version]: + version_dict[version].append(entry) + for p, entry in self.python_executables.items(): + version = entry.as_python + if not version: continue - version_dict[version_object.version_tuple].append(version_object) + version = version.version_tuple + if version and entry not in version_dict[version]: + version_dict[version].append(entry) return version_dict def __attrs_post_init__(self): @@ -73,7 +92,7 @@ def __attrs_post_init__(self): else: bin_dir = 'bin' if venv and (self.system or self.global_search): - p = Path(venv) + p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order self.paths[p] = PathEntry.create( path=p, is_root=True, only_python=False @@ -84,9 +103,10 @@ def __attrs_post_init__(self): if syspath_bin.name != bin_dir and syspath_bin.joinpath(bin_dir).exists(): syspath_bin = syspath_bin / bin_dir self.path_order = [syspath_bin.as_posix()] + self.path_order - self.paths[syspath_bin.as_posix()] = PathEntry.create( + self.paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) + self.python_version_dict = self.get_python_version_dict() def _setup_pyenv(self): from .pyenv import PyenvFinder @@ -110,6 +130,7 @@ def _setup_pyenv(self): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) + self._register_finder(self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -119,15 +140,17 @@ def _setup_windows(self): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) + self._register_finder(self.windows_finder) def get_path(self, path): - path = Path(path) + path = ensure_path(path) _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: - self.paths[path.as_posix()] = PathEntry.create( - path=path.resolve(), is_root=True, only_python=self.only_python + _path = PathEntry.create( + path=path.absolute(), is_root=True, only_python=self.only_python ) - return self.paths.get(path.as_posix()) + self.paths[path.as_posix()] = _path + return _path def find_all(self, executable): """Search the path for an executable. Return all copies. @@ -151,21 +174,22 @@ def which(self, executable): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. :type major: int - :param minor: Minor python version to search for, defaults to None - :param minor: int, optional - :param path: Patch python version to search for, defaults to None - :param path: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :return: A list of :class:`~pythonfinder.models.PathEntry` instances matching the version requested. :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) @@ -176,22 +200,33 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): """Search for a specific python version on the path. :param major: Major python version to search for. :type major: int - :param minor: Minor python version to search for, defaults to None - :param minor: int, optional - :param path: Patch python version to search for, defaults to None - :param path: int, optional + :param int minor: Minor python version to search for, defaults to None + :param int patch: Patch python version to search for, defaults to None + :param bool pre: Search for prereleases (default None) - prioritize releases if None + :param bool dev: Search for devreleases (default None) - prioritize releases if None + :param str arch: Architecture to include, e.g. '64bit', defaults to None :return: A :class:`~pythonfinder.models.PathEntry` instance matching the version requested. :rtype: :class:`~pythonfinder.models.PathEntry` """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev + "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) + if major and minor and patch: + _tuple_pre = pre if pre is not None else False + _tuple_dev = dev if dev is not None else False + version_tuple = (major, minor_, patch, _tuple_pre, _tuple_dev) + version_tuple_pre = (major, minor, patch, True, False) + version = self.python_version_dict.get(version_tuple) + if not version: + version = self.python_version_dict.get(version_tuple_pre) + if version: + return first(version.comes_from) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: @@ -243,7 +278,7 @@ class PathEntry(BasePath): is_root = attr.ib(default=True) only_python = attr.ib(default=False) py_version = attr.ib(default=None) - pythons = attr.ib(default=None) + pythons = attr.ib() def __str__(self): return fs_str('{0}'.format(self.path.as_posix())) @@ -259,11 +294,27 @@ def _filter_children(self): def children(self): if not self._children and self.is_dir and self.is_root: self._children = { - child.as_posix(): PathEntry(path=child, is_root=False) + child.as_posix(): PathEntry.create(path=child, is_root=False) for child in self._filter_children() } + elif not self.is_dir: + return {self.path.as_posix(): self} return self._children + @pythons.default + def get_pythons(self): + pythons = defaultdict() + if self.is_dir: + for path, entry in self.children.items(): + _path = ensure_path(entry.path) + if entry.is_python: + pythons[_path.as_posix()] = entry + else: + if self.is_python: + _path = ensure_path(self.path) + pythons[_path.as_posix()] = copy.deepcopy(self) + return pythons + @property def as_python(self): if not self.is_dir and self.is_python: @@ -292,9 +343,14 @@ def create(cls, path, is_root=False, only_python=False, pythons=None): """ target = ensure_path(path) - _new = cls( - path=target, is_root=is_root, only_python=only_python, pythons=pythons - ) + creation_args = { + "path": target, + "is_root": is_root, + "only_python": only_python + } + if pythons: + creation_args["pythons"] = pythons + _new = cls(**creation_args) if pythons and only_python: children = {} for pth, python in pythons.items(): @@ -311,7 +367,11 @@ def name(self): @property def is_dir(self): - return self.path.is_dir() + try: + ret_val = self.path.is_dir() + except OSError: + ret_val = False + return ret_val @property def is_executable(self): @@ -320,7 +380,7 @@ def is_executable(self): @property def is_python(self): return self.is_executable and ( - self.py_version or is_python_name(self.path.name) + self.py_version or looks_like_python(self.path.name) ) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 8545ac598c..e61e715353 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -18,6 +18,7 @@ class PyenvFinder(BaseFinder): root = attr.ib(default=None, validator=optional_instance_of(Path)) versions = attr.ib() + pythons = attr.ib() @versions.default def get_versions(self): @@ -34,6 +35,21 @@ def get_versions(self): versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True) return versions + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for v in self.versions.values(): + for p in v.paths.values(): + _path = p.path + try: + _path = _path.resolve() + except OSError: + _path = _path.absolute() + _path = _path.as_posix() + if p.is_python: + pythons[_path] = p + return pythons + @classmethod def create(cls, root): root = ensure_path(root) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4ff364faba..687f1d433b 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -2,6 +2,7 @@ from __future__ import print_function, absolute_import import attr import copy +from collections import defaultdict import platform from packaging.version import parse as parse_version, Version from ..environment import SYSTEM_ARCH @@ -67,13 +68,14 @@ def version_tuple(self): self.is_devrelease, ) - def matches(self, major=None, minor=None, patch=None, pre=False, dev=False): + def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): return ( (major is None or self.major == major) and (minor is None or self.minor == minor) and (patch is None or self.patch == patch) and (pre is None or self.is_prerelease == pre) and (dev is None or self.is_devrelease == dev) + and (arch is None or self.architecture == arch) ) def as_major(self): @@ -143,7 +145,7 @@ def from_path(cls, path): from .path import PathEntry if not isinstance(path, PathEntry): - path = PathEntry(path) + path = PathEntry.create(path, is_root=False, only_python=True) if not path.is_python: raise ValueError("Not a valid python path: %s" % path.path) return @@ -192,3 +194,26 @@ def from_windows_launcher(cls, launcher_entry): @classmethod def create(cls, **kwargs): return cls(**kwargs) + + +@attr.s +class VersionMap(object): + versions = attr.ib(default=attr.Factory(defaultdict(list))) + + def add_entry(self, entry): + version = entry.as_python + if version: + entries = versions[version.version_tuple] + paths = {p.path for p in self.versions.get(version.version_tuple, [])} + if entry.path not in paths: + self.versions[version.version_tuple].append(entry) + + def merge(self, target): + for version, entries in target.versions.items(): + if version not in self.versions: + self.versions[version] = entries + else: + current_entries = {p.path for p in self.versions.get(version)} + new_entries = {p.path for p in entries} + new_entries -= current_entries + self.versions[version].append([e for e in entries if e.path in new_entries]) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index f33a4807f7..90f2d803d7 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -5,7 +5,7 @@ from collections import defaultdict from . import BaseFinder from .path import PathEntry -from .python import PythonVersion +from .python import PythonVersion, VersionMap from ..exceptions import InvalidPythonVersion from ..utils import ensure_path @@ -15,10 +15,11 @@ class WindowsFinder(BaseFinder): paths = attr.ib(default=attr.Factory(list)) version_list = attr.ib(default=attr.Factory(list)) versions = attr.ib() + pythons = attr.ib() - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -26,10 +27,10 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, version_sort = operator.attrgetter("version_sort") return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): return next(( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None )), None ) @@ -57,6 +58,14 @@ def get_versions(self): self.paths.append(base_dir) return versions + @pythons.default + def get_pythons(self): + pythons = defaultdict() + for version in self.version_list: + _path = ensure_path(version.comes_from.path) + pythons[_path.as_posix()] = version.comes_from + return pythons + @classmethod def create(cls): return cls() diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 891a3a1757..c74eadeb61 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -50,15 +50,20 @@ def windows_finder(self): def which(self, exe): return self.system_path.which(exe) - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None): + def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch=None): from .models import PythonVersion if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: + if arch is None and '-' in major: + major, arch = major.rsplit('-', 1) + if not arch.isnumeric(): + major = "{0}-{1}".format(major, arch) version_dict = PythonVersion.parse(major) major = version_dict.get("major", major) minor = version_dict.get("minor", minor) patch = version_dict.get("patch", patch) - pre = version_dict.get("is_prerelease", pre) if pre is not None else pre - dev = version_dict.get("is_devrelease", dev) if dev is not None else dev + pre = version_dict.get("is_prerelease", pre) if pre is None else pre + dev = version_dict.get("is_devrelease", dev) if dev is None else dev + arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( major, minor=minor, patch=patch, pre=pre, dev=dev @@ -66,15 +71,24 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None) if match: return match return self.system_path.find_python_version( - major=major, minor=minor, patch=patch, pre=pre, dev=dev + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None): + def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_sort = operator.attrgetter("as_python.version_sort") - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) + versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) if not isinstance(versions, list): versions = [versions,] if os.name == 'nt': - windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev) - versions = versions + list(windows_versions) - return sorted(versions, key=version_sort, reverse=True) + windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + versions = list(windows_versions) + versions + paths = sorted(versions, key=version_sort, reverse=True) + path_map = {} + for path in paths: + try: + resolved_path = path.path.resolve() + except OSError: + resolved_path = path.path.absolute() + if not path_map.get(resolved_path.as_posix()): + path_map[resolved_path.as_posix()] = path + return list(path_map.values()) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index 0cc5370a3e..df837ea90a 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -72,7 +72,7 @@ def path_is_known_executable(path): ) -def is_python_name(name): +def looks_like_python(name): rules = ["*python", "*python?", "*python?.?", "*python?.?m"] match_rules = [] for rule in rules: @@ -88,7 +88,7 @@ def is_python_name(name): def path_is_python(path): - return path_is_executable(path) and is_python_name(path.name) + return path_is_executable(path) and looks_like_python(path.name) def ensure_path(path): @@ -101,8 +101,13 @@ def ensure_path(path): """ if isinstance(path, Path): - return Path(os.path.expandvars(path.as_posix())) - return Path(os.path.expandvars(path)) + path = path.as_posix() + path = Path(os.path.expandvars(path)) + try: + path = path.resolve() + except OSError: + path = path.absolute() + return path def _filter_none(k, v): From 6518325a3c5be27a8d4661ac9084ef921c2d8759 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:37:55 -0400 Subject: [PATCH 41/69] Stop using subprocess to interface with windows finder Signed-off-by: Dan Ryan --- pipenv/core.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index b5301c0186..2fc60b45c8 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -333,14 +333,8 @@ def find_a_system_python(line): from .vendor.pythonfinder import Finder finder = Finder(system=False, global_search=True) if ((line.startswith("py ") or line.startswith("py.exe ")) - and finder.which("py.exe")): - import subprocess - output = subprocess.check_output( - '{} -c "import sys; print(sys.executable)"'.format(line), - ) - if not isinstance(output, str): - output = output.decode(sys.getdefaultencoding()) - return output.strip() + and os.name == 'nt'): + line = line.split(" ", 1)[1] if line.startswith("py"): python_entry = finder.which(line) if python_entry: From febfc6e907fe24cf20ba7cefc00e83a64079a8f2 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:45:57 -0400 Subject: [PATCH 42/69] Better algorithm for find_all_versions Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/pythonfinder.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index c74eadeb61..792f599196 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -76,6 +76,11 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_sort = operator.attrgetter("as_python.version_sort") + python_version_dict = getattr(self.system_path, 'python_version_dict') + if python_version_dict: + paths = [path for version in python_version_dict.values() for path in version] + paths = sorted(paths, key=version_sort, reverse=True) + return paths versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) if not isinstance(versions, list): versions = [versions,] From cc9c5a3ff8fecd6dbcae466af38a4f288fb78f53 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 02:58:37 -0400 Subject: [PATCH 43/69] Strip dashes from `py -n` commands Signed-off-by: Dan Ryan --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 2fc60b45c8..f32230bc24 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -334,7 +334,7 @@ def find_a_system_python(line): finder = Finder(system=False, global_search=True) if ((line.startswith("py ") or line.startswith("py.exe ")) and os.name == 'nt'): - line = line.split(" ", 1)[1] + line = line.split(" ", 1)[1].lstrip("-") if line.startswith("py"): python_entry = finder.which(line) if python_entry: From 95f0df79e8a51d9fa3d4b956420dce70b64100d9 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:25:18 -0400 Subject: [PATCH 44/69] Fix pythonfinder bug unnesting python versions Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 4 ++-- pipenv/vendor/pythonfinder/utils.py | 7 +++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index b1b56d4320..4d90654427 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -4,7 +4,7 @@ import operator import six from itertools import chain -from ..utils import KNOWN_EXTS +from ..utils import KNOWN_EXTS, unnest @six.add_metaclass(abc.ABCMeta) @@ -88,7 +88,7 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= if self.is_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in chain(*filter(None, self.pythons.values())) if child.as_python) + finder = ((child, child.as_python) for child in unnest(self.pythons.values()) if child.as_python) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index df837ea90a..fdc543811d 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -8,6 +8,7 @@ import sys from fnmatch import fnmatch from .exceptions import InvalidPythonVersion +from itertools import chain try: from pathlib import Path @@ -137,3 +138,9 @@ def fs_str(string): _fs_encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() + + +def unnest(item): + if isinstance(next((i for i in item), None), (list, tuple)): + return chain(*filter(None, item)) + return chain(filter(None, item)) From b5bd420c05e51e891d07d40d2069b234c0273740 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:34:19 -0400 Subject: [PATCH 45/69] Fix click encoding for terminal outputs Signed-off-by: Dan Ryan --- pipenv/core.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index f32230bc24..4718ab99d5 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -847,10 +847,10 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): c = delegator.run( cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) - click.echo(crayons.blue(c.out), err=True) + click.echo(u"{0}".format(crayons.blue(c.out), err=True)) if c.return_code != 0: - click.echo(crayons.blue(c.err), err=True) - click.echo("{0}: Failed to create virtual environment.".format( + click.echo(u"{0}".format(crayons.blue(c.err), err=True)) + click.echo(u"{0}: Failed to create virtual environment.".format( crayons.red("Warning", bold=True), ), err=True) sys.exit(1) From 9e10e72493ee96a8ab53eea250881e77c03862a8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:48:31 -0400 Subject: [PATCH 46/69] Block before getting outputs Signed-off-by: Dan Ryan --- pipenv/core.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index 4718ab99d5..88b4c931cb 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -847,9 +847,10 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): c = delegator.run( cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) - click.echo(u"{0}".format(crayons.blue(c.out), err=True)) + c.block() + click.echo(crayons.blue({0}.format(c.out)), err=True) if c.return_code != 0: - click.echo(u"{0}".format(crayons.blue(c.err), err=True)) + click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo(u"{0}: Failed to create virtual environment.".format( crayons.red("Warning", bold=True), ), err=True) From 2f85e9bcc753143db22dfe8034cb6ecd20a146b8 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 03:54:01 -0400 Subject: [PATCH 47/69] Add quotes Signed-off-by: Dan Ryan --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 88b4c931cb..7992797fc7 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -848,7 +848,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) c.block() - click.echo(crayons.blue({0}.format(c.out)), err=True) + click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.return_code != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) click.echo(u"{0}: Failed to create virtual environment.".format( From 4b26aa8c9f864e55a17dd8f57de6b6f1b34de0f1 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 04:30:42 -0400 Subject: [PATCH 48/69] Fix aggregation for support Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 5 ++++- pipenv/vendor/pythonfinder/models/pyenv.py | 7 +------ pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 3 files changed, 6 insertions(+), 8 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 5e1807ab06..11548f23ca 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -67,7 +67,10 @@ def get_python_version_dict(self): version_dict = defaultdict(list) for finder in self.__finders: for version, entry in finder.versions.items(): - if entry not in version_dict[version]: + if isinstance(entry, VersionPath): + paths = [p for p in entry.paths.values() if p.is_python and p not in version_dict[version]] + version_dict[version].extend(paths) + elif entry not in version_dict[version]: version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index e61e715353..13476b6b7b 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -40,12 +40,7 @@ def get_pythons(self): pythons = defaultdict() for v in self.versions.values(): for p in v.paths.values(): - _path = p.path - try: - _path = _path.resolve() - except OSError: - _path = _path.absolute() - _path = _path.as_posix() + _path = ensure_path(p.path) if p.is_python: pythons[_path] = p return pythons diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 792f599196..2cc82d86a6 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -78,7 +78,7 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, 'python_version_dict') if python_version_dict: - paths = [path for version in python_version_dict.values() for path in version] + paths = filter(None, [path for version in python_version_dict.values() for path in version]) paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) From 216f639ae25fe8e56ce138e079b2b2ace5fff2fd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 22:13:24 +0800 Subject: [PATCH 49/69] Don't process the py line twice --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 7992797fc7..c3bf66123d 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -335,7 +335,7 @@ def find_a_system_python(line): if ((line.startswith("py ") or line.startswith("py.exe ")) and os.name == 'nt'): line = line.split(" ", 1)[1].lstrip("-") - if line.startswith("py"): + elif line.startswith("py"): python_entry = finder.which(line) if python_entry: return python_entry.path.as_posix() From 6e4205a1348e0c53a1b4598074653be9f696cfd7 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Tue, 17 Jul 2018 22:20:48 +0800 Subject: [PATCH 50/69] Pass arch into Windows finder This is still not working though. --- pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 2cc82d86a6..7592507589 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -66,7 +66,7 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev + major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, ) if match: return match From bf8d0b9592a4a27f958c789abcd250b1fdba1d50 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 18:23:55 -0400 Subject: [PATCH 51/69] Update pythonfinder for better windows support Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/python.py | 7 ++++++- pipenv/vendor/pythonfinder/pythonfinder.py | 6 ++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 687f1d433b..1bc34cc09a 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -69,6 +69,8 @@ def version_tuple(self): ) def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): + if arch and arch.isnumeric(): + arch = '{0}bit'.format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) @@ -180,7 +182,7 @@ def from_windows_launcher(cls, launcher_entry): creation_dict.update( { "architecture": getattr( - launcher_entry, "sys_architecture", SYSTEM_ARCH + launcher_entry.info, "sys_architecture", SYSTEM_ARCH ), "executable": exe_path, } @@ -193,6 +195,9 @@ def from_windows_launcher(cls, launcher_entry): @classmethod def create(cls, **kwargs): + if 'architecture' in kwargs: + if kwargs['architecture'].isnumeric(): + kwargs['architecture'] = '{0}bit'.format(kwargs['architecture']) return cls(**kwargs) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 7592507589..8e775ffd63 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -57,6 +57,8 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, major, arch = major.rsplit('-', 1) if not arch.isnumeric(): major = "{0}-{1}".format(major, arch) + else: + arch = "{0}bit".format(arch) version_dict = PythonVersion.parse(major) major = version_dict.get("major", major) minor = version_dict.get("minor", minor) @@ -66,7 +68,7 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch = version_dict.get("architecture", arch) if arch is None else arch if os.name == "nt": match = self.windows_finder.find_python_version( - major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch, + major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if match: return match @@ -78,7 +80,7 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, version_sort = operator.attrgetter("as_python.version_sort") python_version_dict = getattr(self.system_path, 'python_version_dict') if python_version_dict: - paths = filter(None, [path for version in python_version_dict.values() for path in version]) + paths = filter(None, [path for version in python_version_dict.values() for path in version if path.as_python]) paths = sorted(paths, key=version_sort, reverse=True) return paths versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) From 21cbf0c5b8806d6fc7c02e329b6f8788b4b34ddf Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 20:12:41 -0400 Subject: [PATCH 52/69] Update pythonfinder to default patch versions to 0 Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/python.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 1bc34cc09a..4c6aa1f67e 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -19,7 +19,7 @@ class PythonVersion(object): major = attr.ib(default=0) minor = attr.ib(default=None) - patch = attr.ib(default=None) + patch = attr.ib(default=0) is_prerelease = attr.ib(default=False) is_postrelease = attr.ib(default=False) is_devrelease = attr.ib(default=False) @@ -47,7 +47,7 @@ def version_sort(self): return ( self.major, self.minor, - self.patch, + self.patch if self.patch else 0, release_sort ) From b7daa2525b49e2d8c64dddc1b3a68d436677cb06 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 20:27:42 -0400 Subject: [PATCH 53/69] Update pythonfinder Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/path.py | 29 ++++++++++++++-------- pipenv/vendor/pythonfinder/pythonfinder.py | 6 ++--- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 11548f23ca..9315ae237d 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -38,11 +38,11 @@ class SystemPath(object): pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) system = attr.ib(default=False) - __finders = attr.ib(default=attr.Factory(list)) + __finders = attr.ib(default=attr.Factory(dict)) - def _register_finder(self, finder): - if not finder in self.__finders: - self.__finders.append(finder) + def _register_finder(self, finder_name, finder): + if finder_name not in self.__finders: + self.__finders[finder_name] = finder @property def executables(self): @@ -57,7 +57,7 @@ def python_executables(self): for child in self.paths.values(): if child.pythons: python_executables.update(dict(child.pythons)) - for finder in self.__finders: + for finder_name, finder in self.__finders.items(): if finder.pythons: python_executables.update(dict(finder.pythons)) self._python_executables = python_executables @@ -65,12 +65,19 @@ def python_executables(self): def get_python_version_dict(self): version_dict = defaultdict(list) - for finder in self.__finders: + for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): + if finder_name == 'windows': + if entry not in version_dict[version]: + version_dict[version].append(entry) + continue if isinstance(entry, VersionPath): - paths = [p for p in entry.paths.values() if p.is_python and p not in version_dict[version]] - version_dict[version].extend(paths) - elif entry not in version_dict[version]: + for path in entry.paths.values(): + if path not in version_dict[version] and path.is_python: + version_dict[version].append(path) + continue + continue + elif entry not in version_dict[version] and entry.is_python: version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python @@ -133,7 +140,7 @@ def _setup_pyenv(self): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) - self._register_finder(self.pyenv_finder) + self._register_finder('pyenv', self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -143,7 +150,7 @@ def _setup_windows(self): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) - self._register_finder(self.windows_finder) + self._register_finder('windows', self.windows_finder) def get_path(self, path): path = ensure_path(path) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index 8e775ffd63..fa8706f161 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -86,9 +86,9 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) if not isinstance(versions, list): versions = [versions,] - if os.name == 'nt': - windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) - versions = list(windows_versions) + versions + # if os.name == 'nt': + # windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + # versions = list(windows_versions) + versions paths = sorted(versions, key=version_sort, reverse=True) path_map = {} for path in paths: From 1b69089cb16318fdbef048278578b997efc71cfa Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Tue, 17 Jul 2018 23:13:13 -0400 Subject: [PATCH 54/69] Cached properties! Signed-off-by: Dan Ryan --- pipenv/vendor/pythonfinder/models/__init__.py | 5 +- pipenv/vendor/pythonfinder/models/path.py | 84 ++++++++++--------- 2 files changed, 46 insertions(+), 43 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 4d90654427..7cf0fadf29 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -57,8 +57,9 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ + call_method = "find_all_python_versions" if self.is_dir else "find_python_version" sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if not self.is_dir: return sub_finder(self) @@ -85,7 +86,7 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") if not self.is_dir: - if self.is_python and version_matcher(self.as_python): + if self.is_python and self.as_python and version_matcher(self.as_python): return self return finder = ((child, child.as_python) for child in unnest(self.pythons.values()) if child.as_python) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 9315ae237d..6857d17da8 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -6,6 +6,7 @@ import os import sys from collections import defaultdict +from cached_property import cached_property from itertools import chain from . import BasePath from .python import PythonVersion @@ -17,7 +18,8 @@ path_is_known_executable, looks_like_python, ensure_path, - fs_str + fs_str, + unnest, ) try: @@ -37,6 +39,7 @@ class SystemPath(object): only_python = attr.ib(default=False) pyenv_finder = attr.ib(default=None, validator=optional_instance_of("PyenvPath")) system = attr.ib(default=False) + _version_dict = attr.ib(default=attr.Factory(defaultdict)) __finders = attr.ib(default=attr.Factory(dict)) @@ -44,49 +47,48 @@ def _register_finder(self, finder_name, finder): if finder_name not in self.__finders: self.__finders[finder_name] = finder - @property + @cached_property def executables(self): - if not self._executables: - self._executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] - return self._executables + self.executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] + return self.executables - @property + @cached_property def python_executables(self): python_executables = {} - if not self._python_executables: - for child in self.paths.values(): - if child.pythons: - python_executables.update(dict(child.pythons)) - for finder_name, finder in self.__finders.items(): - if finder.pythons: - python_executables.update(dict(finder.pythons)) - self._python_executables = python_executables + for child in self.paths.values(): + if child.pythons: + python_executables.update(dict(child.pythons)) + for finder_name, finder in self.__finders.items(): + if finder.pythons: + python_executables.update(dict(finder.pythons)) + self._python_executables = python_executables return self._python_executables - def get_python_version_dict(self): - version_dict = defaultdict(list) + @cached_property + def version_dict(self): + self._version_dict = defaultdict(list) for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): if finder_name == 'windows': - if entry not in version_dict[version]: - version_dict[version].append(entry) + if entry not in self._version_dict[version]: + self._version_dict[version].append(entry) continue if isinstance(entry, VersionPath): for path in entry.paths.values(): - if path not in version_dict[version] and path.is_python: - version_dict[version].append(path) + if path not in self._version_dict[version] and path.is_python: + self._version_dict[version].append(path) continue continue - elif entry not in version_dict[version] and entry.is_python: - version_dict[version].append(entry) + elif entry not in self._version_dict[version] and entry.is_python: + self._version_dict[version].append(entry) for p, entry in self.python_executables.items(): version = entry.as_python if not version: continue version = version.version_tuple - if version and entry not in version_dict[version]: - version_dict[version].append(entry) - return version_dict + if version and entry not in self._version_dict[version]: + self._version_dict[version].append(entry) + return self._version_dict def __attrs_post_init__(self): #: slice in pyenv @@ -116,7 +118,6 @@ def __attrs_post_init__(self): self.paths[syspath_bin] = PathEntry.create( path=syspath_bin, is_root=True, only_python=False ) - self.python_version_dict = self.get_python_version_dict() def _setup_pyenv(self): from .pyenv import PyenvFinder @@ -199,14 +200,14 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_all_python_versions", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) + path_filter = filter(None, unnest((sub_finder(p) for p in paths if p is not None))) version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] @@ -232,11 +233,6 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= _tuple_dev = dev if dev is not None else False version_tuple = (major, minor_, patch, _tuple_pre, _tuple_dev) version_tuple_pre = (major, minor, patch, True, False) - version = self.python_version_dict.get(version_tuple) - if not version: - version = self.python_version_dict.get(version_tuple_pre) - if version: - return first(version.comes_from) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: @@ -244,9 +240,15 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= paths = (self.get_path(k) for k in self.path_order) path_filter = filter(None, (sub_finder(p) for p in paths if p is not None)) version_sort = operator.attrgetter("as_python.version_sort") - return next( + ver = next( (c for c in sorted(path_filter, key=version_sort, reverse=True)), None ) + if ver: + if ver.as_python.version_tuple[:5] in self.python_version_dict: + self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) + else: + self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver,] + return ver @classmethod def create(cls, path=None, system=False, only_python=False, global_search=True): @@ -300,7 +302,7 @@ def _filter_children(self): children = self.path.iterdir() return children - @property + @cached_property def children(self): if not self._children and self.is_dir and self.is_root: self._children = { @@ -308,7 +310,7 @@ def children(self): for child in self._filter_children() } elif not self.is_dir: - return {self.path.as_posix(): self} + self._children = {self.path.as_posix(): self} return self._children @pythons.default @@ -325,7 +327,7 @@ def get_pythons(self): pythons[_path.as_posix()] = copy.deepcopy(self) return pythons - @property + @cached_property def as_python(self): if not self.is_dir and self.is_python: if not self.py_version: @@ -371,11 +373,11 @@ def create(cls, path, is_root=False, only_python=False, pythons=None): _new._children = children return _new - @property + @cached_property def name(self): return self.path.name - @property + @cached_property def is_dir(self): try: ret_val = self.path.is_dir() @@ -383,11 +385,11 @@ def is_dir(self): ret_val = False return ret_val - @property + @cached_property def is_executable(self): return path_is_known_executable(self.path) - @property + @cached_property def is_python(self): return self.is_executable and ( self.py_version or looks_like_python(self.path.name) From dbcb5a71b389d7c3e31298297f3edf1842572cf6 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 18 Jul 2018 00:16:17 -0400 Subject: [PATCH 55/69] Add cached property to vendored deps Signed-off-by: Dan Ryan --- pipenv/vendor/cached-property.LICENSE | 12 ++ pipenv/vendor/cached_property.py | 152 ++++++++++++++++++++++++++ pipenv/vendor/vendor.txt | 1 + tasks/vendoring/__init__.py | 96 +++++++++++----- 4 files changed, 235 insertions(+), 26 deletions(-) create mode 100644 pipenv/vendor/cached-property.LICENSE create mode 100644 pipenv/vendor/cached_property.py diff --git a/pipenv/vendor/cached-property.LICENSE b/pipenv/vendor/cached-property.LICENSE new file mode 100644 index 0000000000..a181761ca5 --- /dev/null +++ b/pipenv/vendor/cached-property.LICENSE @@ -0,0 +1,12 @@ +Copyright (c) 2015, Daniel Greenfeld +All rights reserved. + +Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met: + +* Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer. + +* Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution. + +* Neither the name of cached-property nor the names of its contributors may be used to endorse or promote products derived from this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/pipenv/vendor/cached_property.py b/pipenv/vendor/cached_property.py new file mode 100644 index 0000000000..a06be97a19 --- /dev/null +++ b/pipenv/vendor/cached_property.py @@ -0,0 +1,152 @@ +# -*- coding: utf-8 -*- + +__author__ = "Daniel Greenfeld" +__email__ = "pydanny@gmail.com" +__version__ = "1.4.3" +__license__ = "BSD" + +from time import time +import threading + +try: + import asyncio +except (ImportError, SyntaxError): + asyncio = None + + +class cached_property(object): + """ + A property that is only computed once per instance and then replaces itself + with an ordinary attribute. Deleting the attribute resets the property. + Source: https://github.com/bottlepy/bottle/commit/fa7733e075da0d790d809aa3d2f53071897e6f76 + """ # noqa + + def __init__(self, func): + self.__doc__ = getattr(func, "__doc__") + self.func = func + + def __get__(self, obj, cls): + if obj is None: + return self + + if asyncio and asyncio.iscoroutinefunction(self.func): + return self._wrap_in_coroutine(obj) + + value = obj.__dict__[self.func.__name__] = self.func(obj) + return value + + def _wrap_in_coroutine(self, obj): + + @asyncio.coroutine + def wrapper(): + future = asyncio.ensure_future(self.func(obj)) + obj.__dict__[self.func.__name__] = future + return future + + return wrapper() + + +class threaded_cached_property(object): + """ + A cached_property version for use in environments where multiple threads + might concurrently try to access the property. + """ + + def __init__(self, func): + self.__doc__ = getattr(func, "__doc__") + self.func = func + self.lock = threading.RLock() + + def __get__(self, obj, cls): + if obj is None: + return self + + obj_dict = obj.__dict__ + name = self.func.__name__ + with self.lock: + try: + # check if the value was computed before the lock was acquired + return obj_dict[name] + + except KeyError: + # if not, do the calculation and release the lock + return obj_dict.setdefault(name, self.func(obj)) + + +class cached_property_with_ttl(object): + """ + A property that is only computed once per instance and then replaces itself + with an ordinary attribute. Setting the ttl to a number expresses how long + the property will last before being timed out. + """ + + def __init__(self, ttl=None): + if callable(ttl): + func = ttl + ttl = None + else: + func = None + self.ttl = ttl + self._prepare_func(func) + + def __call__(self, func): + self._prepare_func(func) + return self + + def __get__(self, obj, cls): + if obj is None: + return self + + now = time() + obj_dict = obj.__dict__ + name = self.__name__ + try: + value, last_updated = obj_dict[name] + except KeyError: + pass + else: + ttl_expired = self.ttl and self.ttl < now - last_updated + if not ttl_expired: + return value + + value = self.func(obj) + obj_dict[name] = (value, now) + return value + + def __delete__(self, obj): + obj.__dict__.pop(self.__name__, None) + + def __set__(self, obj, value): + obj.__dict__[self.__name__] = (value, time()) + + def _prepare_func(self, func): + self.func = func + if func: + self.__doc__ = func.__doc__ + self.__name__ = func.__name__ + self.__module__ = func.__module__ + + +# Aliases to make cached_property_with_ttl easier to use +cached_property_ttl = cached_property_with_ttl +timed_cached_property = cached_property_with_ttl + + +class threaded_cached_property_with_ttl(cached_property_with_ttl): + """ + A cached_property version for use in environments where multiple threads + might concurrently try to access the property. + """ + + def __init__(self, ttl=None): + super(threaded_cached_property_with_ttl, self).__init__(ttl) + self.lock = threading.RLock() + + def __get__(self, obj, cls): + with self.lock: + return super(threaded_cached_property_with_ttl, self).__get__(obj, cls) + + +# Alias to make threaded_cached_property_with_ttl easier to use +threaded_cached_property_ttl = threaded_cached_property_with_ttl +timed_threaded_cached_property = threaded_cached_property_with_ttl diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index 961ad01604..e2c3165c6e 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -39,3 +39,4 @@ six==1.11.0 semver==2.8.0 shutilwhich==1.1.0 toml==0.9.4 +cached-property==1.4.3 diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index 9b8a2c82cf..a93360cecd 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -280,19 +280,47 @@ def write_backport_imports(ctx, vendor_dir): backport_init.write_text('\n'.join(init_py_lines) + '\n') -def vendor(ctx, vendor_dir, rewrite=True): - log('Reinstalling vendored libraries') - is_patched = vendor_dir.name == 'patched' - requirements_file = vendor_dir.name +def _ensure_package_in_requirements(ctx, requirements_file, package): + requirement = None + log('using requirements file: %s' % requirements_file) + req_file_lines = [l for l in requirements_file.read_text().splitlines()] + if package: + match = [r for r in req_file_lines if r.strip().lower().startswith(package)] + matched_req = None + if match: + for m in match: + specifiers = [m.index(s) for s in ['>', '<', '=', '~'] if s in m] + if m.lower() == package or (specifiers and m[:min(specifiers)].lower() == package): + matched_req = "{0}".format(m) + requirement = matched_req + log("Matched req: %r" % matched_req) + if not matched_req: + req_file_lines.append("{0}".format(package)) + log("Writing requirements file: %s" % requirements_file) + requirements_file.write_text('\n'.join(req_file_lines)) + requirement = "{0}".format(package) + return requirement + + + +def install(ctx, vendor_dir, package=None): + requirements_file = vendor_dir / "{0}.txt".format(vendor_dir.name) + requirement = "-r {0}".format(requirements_file.as_posix()) + log('Using requirements file: %s' % requirement) + if package: + requirement = _ensure_package_in_requirements(ctx, requirements_file, package) # We use --no-deps because we want to ensure that all of our dependencies # are added to vendor.txt, this includes all dependencies recursively up # the chain. ctx.run( - 'pip install -t {0} -r {0}/{1}.txt --no-compile --no-deps'.format( - str(vendor_dir), - requirements_file, + 'pip install -t {0} --no-compile --no-deps {1}'.format( + vendor_dir.as_posix(), + requirement, ) ) + + +def post_install_cleanup(ctx, vendor_dir): remove_all(vendor_dir.glob('*.dist-info')) remove_all(vendor_dir.glob('*.egg-info')) @@ -300,6 +328,13 @@ def vendor(ctx, vendor_dir, rewrite=True): drop_dir(vendor_dir / 'bin') drop_dir(vendor_dir / 'tests') + +def vendor(ctx, vendor_dir, package=None, rewrite=True): + log('Reinstalling vendored libraries') + is_patched = vendor_dir.name == 'patched' + install(ctx, vendor_dir, package=package) + log('Running post-install cleanup...') + post_install_cleanup(ctx, vendor_dir) # Detect the vendored packages/modules vendored_libs = detect_vendored_libs(_get_vendor_dir(ctx)) patched_libs = detect_vendored_libs(_get_patched_dir(ctx)) @@ -320,25 +355,26 @@ def vendor(ctx, vendor_dir, rewrite=True): log('Renaming specified libs...') for item in vendor_dir.iterdir(): if item.is_dir(): - if rewrite: + if rewrite and not package or (package and item.name.lower() in package): log('Rewriting imports for %s...' % item) rewrite_imports(item, vendored_libs, vendor_dir) rename_if_needed(ctx, vendor_dir, item) elif item.name not in FILE_WHITE_LIST: - if rewrite: + if rewrite and not package or (package and item.stem.lower() in package): rewrite_file_imports(item, vendored_libs, vendor_dir) write_backport_imports(ctx, vendor_dir) - log('Applying post-patches...') - patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') - for patch in patches: - apply_patch(ctx, patch) - if is_patched: - piptools_vendor = vendor_dir / 'piptools' / '_vendored' - if piptools_vendor.exists(): - drop_dir(piptools_vendor) - msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack' - if msgpack.exists(): - remove_all(msgpack.glob('*.so')) + if not package: + log('Applying post-patches...') + patches = patch_dir.glob('*.patch' if not is_patched else '_post*.patch') + for patch in patches: + apply_patch(ctx, patch) + if is_patched: + piptools_vendor = vendor_dir / 'piptools' / '_vendored' + if piptools_vendor.exists(): + drop_dir(piptools_vendor) + msgpack = vendor_dir / 'notpip' / '_vendor' / 'msgpack' + if msgpack.exists(): + remove_all(msgpack.glob('*.so')) @invoke.task @@ -371,16 +407,19 @@ def rewrite_all_imports(ctx): @invoke.task -def download_licenses(ctx, vendor_dir, requirements_file='vendor.txt'): +def download_licenses(ctx, vendor_dir=None, requirements_file='vendor.txt', package=None): log('Downloading licenses') if not vendor_dir: vendor_dir = _get_vendor_dir(ctx) + requirements_file = vendor_dir / requirements_file + requirement = "-r {0}".format(requirements_file.as_posix()) + if package: + requirement = _ensure_package_in_requirements(ctx, requirements_file, package) tmp_dir = vendor_dir / '__tmp__' ctx.run( - 'pip download -r {0}/{1} --no-binary :all: --no-deps -d {2}'.format( - str(vendor_dir), - requirements_file, - str(tmp_dir), + 'pip download --no-binary :all: --no-deps -d {0} {1}'.format( + tmp_dir.as_posix(), + requirement, ) ) for sdist in tmp_dir.iterdir(): @@ -503,10 +542,15 @@ def generate_patch(ctx, package_path, patch_description, base='HEAD'): @invoke.task(name=TASK_NAME) -def main(ctx): +def main(ctx, package=None): vendor_dir = _get_vendor_dir(ctx) patched_dir = _get_patched_dir(ctx) log('Using vendor dir: %s' % vendor_dir) + if package: + vendor(ctx, vendor_dir, package=package) + download_licenses(ctx, vendor_dir, package=package) + log("Vendored %s" % package) + return clean_vendor(ctx, vendor_dir) clean_vendor(ctx, patched_dir) vendor(ctx, vendor_dir) From b8aa62ac664b8e679c1b7b52dce393aff42b02de Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 18 Jul 2018 14:47:04 +0800 Subject: [PATCH 56/69] Python --- pipenv/vendor/pythonfinder/models/python.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 4c6aa1f67e..f0bcbfd0b8 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -69,7 +69,7 @@ def version_tuple(self): ) def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): - if arch and arch.isnumeric(): + if arch and arch.isdigit(): arch = '{0}bit'.format(arch) return ( (major is None or self.major == major) From 29326d53f6557785fba01a59e692ddf3c77772cd Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 18 Jul 2018 14:47:54 +0800 Subject: [PATCH 57/69] Block inside spinner block --- pipenv/core.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index c3bf66123d..eaea6e41bd 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -847,7 +847,7 @@ def do_create_virtualenv(python=None, site_packages=False, pypi_mirror=None): c = delegator.run( cmd, block=False, timeout=PIPENV_TIMEOUT, env=pip_config, ) - c.block() + c.block() click.echo(crayons.blue("{0}".format(c.out)), err=True) if c.return_code != 0: click.echo(crayons.blue("{0}".format(c.err)), err=True) From c9277c500833e2f435d5258b1f82a10fbfc8561a Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Wed, 18 Jul 2018 15:14:11 +0800 Subject: [PATCH 58/69] More isdigit() fixes --- pipenv/vendor/pythonfinder/models/python.py | 2 +- pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index f0bcbfd0b8..fcb48723af 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -196,7 +196,7 @@ def from_windows_launcher(cls, launcher_entry): @classmethod def create(cls, **kwargs): if 'architecture' in kwargs: - if kwargs['architecture'].isnumeric(): + if kwargs['architecture'].isdigit(): kwargs['architecture'] = '{0}bit'.format(kwargs['architecture']) return cls(**kwargs) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index fa8706f161..c6edbaf96e 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -55,7 +55,7 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: if arch is None and '-' in major: major, arch = major.rsplit('-', 1) - if not arch.isnumeric(): + if not arch.isdigit(): major = "{0}-{1}".format(major, arch) else: arch = "{0}bit".format(arch) From e3f37bebfe50d5248c3330adfd421be4ff93a155 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 26 Jul 2018 02:42:15 +0800 Subject: [PATCH 59/69] Fix missing arch arguments --- pipenv/vendor/pythonfinder/models/windows.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 90f2d803d7..40dd93ede9 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -19,7 +19,7 @@ class WindowsFinder(BaseFinder): def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None + "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -30,7 +30,7 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): return next(( v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=None + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch )), None ) From f9655cbacf5a33115c24e0d49a492ac4eb24c6ac Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 16:56:49 -0400 Subject: [PATCH 60/69] Update vendoring tasks for updating single vendored deps Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 1 + tasks/vendoring/__init__.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index e2c3165c6e..c4f4412324 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -40,3 +40,4 @@ semver==2.8.0 shutilwhich==1.1.0 toml==0.9.4 cached-property==1.4.3 +pythonfinder==1.0.0 \ No newline at end of file diff --git a/tasks/vendoring/__init__.py b/tasks/vendoring/__init__.py index a93360cecd..4deb58b86b 100644 --- a/tasks/vendoring/__init__.py +++ b/tasks/vendoring/__init__.py @@ -313,7 +313,7 @@ def install(ctx, vendor_dir, package=None): # are added to vendor.txt, this includes all dependencies recursively up # the chain. ctx.run( - 'pip install -t {0} --no-compile --no-deps {1}'.format( + 'pip install -t {0} --no-compile --no-deps --upgrade {1}'.format( vendor_dir.as_posix(), requirement, ) From f5df34f85bbb64237b1ce6130a05a14f7e815453 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 17:02:32 -0400 Subject: [PATCH 61/69] Update vendoring instructions Signed-off-by: Dan Ryan --- pipenv/vendor/README.md | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/pipenv/vendor/README.md b/pipenv/vendor/README.md index 11bbbb4b64..0e3ab8bb60 100644 --- a/pipenv/vendor/README.md +++ b/pipenv/vendor/README.md @@ -2,10 +2,21 @@ These packages are copied as-is from upstream to reduce Pipenv dependencies. They should always be kept synced with upstream. DO NOT MODIFY DIRECTLY! If -you need to patch anything, move the package to `patched`. +you need to patch anything, move the package to `patched` and generate a +patch for it using `git diff -p `. This patch belongs +in `./pipenv/tasks/vendoring/patches/patched/.patch`. -Known vendored versions: +To add a vendored dependency or to update a single dependency, use the +vendoring scripts: +``` + pipenv run inv vendoring.update --package="pkgname==versionnum" +``` -- python-dotenv: 0.8.2 +This will automatically pin the package in `./pipenv/vendor/vendor.txt` +or it will update the pin if the package is already present, and it will +then update the package and download any necessary licenses (if available). +Note that this will not download any dependencies, you must add those each +individually. -When updating, update the corresponding LICENSE files as well. +When updating, ensure that the corresponding LICENSE files are still +up-to-date. From ef060c4b9b5c119e60feaba167bb22a273e04d90 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 17:02:45 -0400 Subject: [PATCH 62/69] Update pythonfinder Signed-off-by: Dan Ryan --- news/2582.bugfix | 1 + news/2582.feature | 4 + news/2582.vendor | 1 + pipenv/vendor/pythonfinder/__init__.py | 2 +- pipenv/vendor/pythonfinder/_vendor/Makefile | 14 ---- pipenv/vendor/pythonfinder/_vendor/vendor.txt | 1 - pipenv/vendor/pythonfinder/exceptions.py | 1 + pipenv/vendor/pythonfinder/models/__init__.py | 43 +++++++++-- pipenv/vendor/pythonfinder/models/path.py | 73 ++++++++++++------- pipenv/vendor/pythonfinder/models/pyenv.py | 4 +- pipenv/vendor/pythonfinder/models/python.py | 26 +++---- pipenv/vendor/pythonfinder/models/windows.py | 28 +++++-- pipenv/vendor/pythonfinder/pythonfinder.py | 48 ++++++++---- pipenv/vendor/pythonfinder/utils.py | 5 +- 14 files changed, 163 insertions(+), 88 deletions(-) create mode 100644 news/2582.bugfix create mode 100644 news/2582.feature create mode 100644 news/2582.vendor delete mode 100644 pipenv/vendor/pythonfinder/_vendor/Makefile delete mode 100644 pipenv/vendor/pythonfinder/_vendor/vendor.txt diff --git a/news/2582.bugfix b/news/2582.bugfix new file mode 100644 index 0000000000..f434be812c --- /dev/null +++ b/news/2582.bugfix @@ -0,0 +1 @@ +Fixed multiple issues with finding the correct system python locations. diff --git a/news/2582.feature b/news/2582.feature new file mode 100644 index 0000000000..4d1f7a363b --- /dev/null +++ b/news/2582.feature @@ -0,0 +1,4 @@ +Greatly enhanced python discovery functionality: + +- Added pep514 (windows launcher/finder) support for python discovery. +- Introduced architecture discovery for python installations which support different architectures. diff --git a/news/2582.vendor b/news/2582.vendor new file mode 100644 index 0000000000..1a031eb7c4 --- /dev/null +++ b/news/2582.vendor @@ -0,0 +1 @@ +Update ``pythonfinder`` to major release ``1.0.0`` for integration. diff --git a/pipenv/vendor/pythonfinder/__init__.py b/pipenv/vendor/pythonfinder/__init__.py index eb3d0363b6..9f1628be84 100644 --- a/pipenv/vendor/pythonfinder/__init__.py +++ b/pipenv/vendor/pythonfinder/__init__.py @@ -1,6 +1,6 @@ from __future__ import print_function, absolute_import -__version__ = "0.1.4.dev0" +__version__ = "1.0.0" __all__ = ["Finder", "WindowsFinder", "SystemPath", "InvalidPythonVersion"] from .pythonfinder import Finder diff --git a/pipenv/vendor/pythonfinder/_vendor/Makefile b/pipenv/vendor/pythonfinder/_vendor/Makefile deleted file mode 100644 index 5c44fea4e9..0000000000 --- a/pipenv/vendor/pythonfinder/_vendor/Makefile +++ /dev/null @@ -1,14 +0,0 @@ -# Taken from pip: https://github.com/pypa/pip/blob/95bcf8c5f6394298035a7332c441868f3b0169f4/src/pip/_vendor/Makefile -all: clean vendor - -clean: - @# Delete vendored items - find . -maxdepth 1 -mindepth 1 -type d -exec rm -rf {} \; - -vendor: - @# Install vendored libraries - pip install -t . -r vendor.txt - - @# Cleanup .egg-info directories - rm -rf *.egg-info - rm -rf *.dist-info diff --git a/pipenv/vendor/pythonfinder/_vendor/vendor.txt b/pipenv/vendor/pythonfinder/_vendor/vendor.txt deleted file mode 100644 index 8875249845..0000000000 --- a/pipenv/vendor/pythonfinder/_vendor/vendor.txt +++ /dev/null @@ -1 +0,0 @@ --e git+https://github.com/zooba/pep514tools.git@320e48745660b696e2dcaee888fc2e516b435e48#egg=pep514tools diff --git a/pipenv/vendor/pythonfinder/exceptions.py b/pipenv/vendor/pythonfinder/exceptions.py index 13e56e2cb1..df381daf62 100644 --- a/pipenv/vendor/pythonfinder/exceptions.py +++ b/pipenv/vendor/pythonfinder/exceptions.py @@ -4,4 +4,5 @@ class InvalidPythonVersion(Exception): """Raised when parsing an invalid python version""" + pass diff --git a/pipenv/vendor/pythonfinder/models/__init__.py b/pipenv/vendor/pythonfinder/models/__init__.py index 7cf0fadf29..a38494edb7 100644 --- a/pipenv/vendor/pythonfinder/models/__init__.py +++ b/pipenv/vendor/pythonfinder/models/__init__.py @@ -40,10 +40,19 @@ def which(self, name): for ext in KNOWN_EXTS ] children = self.children - found = next((children[(self.path / child).as_posix()] for child in valid_names if (self.path / child).as_posix() in children), None) + found = next( + ( + children[(self.path / child).as_posix()] + for child in valid_names + if (self.path / child).as_posix() in children + ), + None, + ) return found - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -57,7 +66,9 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, :rtype: List[:class:`~pythonfinder.models.PathEntry`] """ - call_method = "find_all_python_versions" if self.is_dir else "find_python_version" + call_method = ( + "find_all_python_versions" if self.is_dir else "find_python_version" + ) sub_finder = operator.methodcaller( call_method, major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) @@ -67,7 +78,9 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search or self for the specified Python version and return the first match. :param major: Major version number. @@ -81,7 +94,13 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= """ version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) is_py = operator.attrgetter("is_python") py_version = operator.attrgetter("as_python") @@ -89,13 +108,23 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= if self.is_python and self.as_python and version_matcher(self.as_python): return self return - finder = ((child, child.as_python) for child in unnest(self.pythons.values()) if child.as_python) + finder = ( + (child, child.as_python) + for child in unnest(self.pythons.values()) + if child.as_python + ) py_filter = filter( None, filter(lambda child: version_matcher(child[1]), finder) ) version_sort = operator.attrgetter("version_sort") return next( - (c[0] for c in sorted(py_filter, key=lambda child: child[1].version_sort, reverse=True)), None + ( + c[0] + for c in sorted( + py_filter, key=lambda child: child[1].version_sort, reverse=True + ) + ), + None, ) diff --git a/pipenv/vendor/pythonfinder/models/path.py b/pipenv/vendor/pythonfinder/models/path.py index 6857d17da8..af1039157c 100644 --- a/pipenv/vendor/pythonfinder/models/path.py +++ b/pipenv/vendor/pythonfinder/models/path.py @@ -49,7 +49,11 @@ def _register_finder(self, finder_name, finder): @cached_property def executables(self): - self.executables = [p for p in chain(*(child.children.values() for child in self.paths.values())) if p.is_executable] + self.executables = [ + p + for p in chain(*(child.children.values() for child in self.paths.values())) + if p.is_executable + ] return self.executables @cached_property @@ -69,7 +73,7 @@ def version_dict(self): self._version_dict = defaultdict(list) for finder_name, finder in self.__finders.items(): for version, entry in finder.versions.items(): - if finder_name == 'windows': + if finder_name == "windows": if entry not in self._version_dict[version]: self._version_dict[version].append(entry) continue @@ -98,17 +102,15 @@ def __attrs_post_init__(self): self._setup_windows() if PYENV_INSTALLED: self._setup_pyenv() - venv = os.environ.get('VIRTUAL_ENV') - if os.name == 'nt': - bin_dir = 'Scripts' + venv = os.environ.get("VIRTUAL_ENV") + if os.name == "nt": + bin_dir = "Scripts" else: - bin_dir = 'bin' + bin_dir = "bin" if venv and (self.system or self.global_search): p = ensure_path(venv) self.path_order = [(p / bin_dir).as_posix()] + self.path_order - self.paths[p] = PathEntry.create( - path=p, is_root=True, only_python=False - ) + self.paths[p] = PathEntry.create(path=p, is_root=True, only_python=False) if self.system: syspath = Path(sys.executable) syspath_bin = syspath.parent @@ -141,7 +143,7 @@ def _setup_pyenv(self): before_path + [p.path.as_posix() for p in root_paths] + after_path ) self.paths.update({p.path: p for p in root_paths}) - self._register_finder('pyenv', self.pyenv_finder) + self._register_finder("pyenv", self.pyenv_finder) def _setup_windows(self): from .windows import WindowsFinder @@ -151,13 +153,13 @@ def _setup_windows(self): path_addition = [p.path.as_posix() for p in root_paths] self.path_order = self.path_order[:] + path_addition self.paths.update({p.path: p for p in root_paths}) - self._register_finder('windows', self.windows_finder) + self._register_finder("windows", self.windows_finder) def get_path(self, path): path = ensure_path(path) _path = self.paths.get(path.as_posix()) if not _path and path.as_posix() in self.path_order: - _path = PathEntry.create( + _path = PathEntry.create( path=path.absolute(), is_root=True, only_python=self.only_python ) self.paths[path.as_posix()] = _path @@ -185,7 +187,9 @@ def which(self, executable): filtered = filter(None, (sub_which(self.get_path(k)) for k in self.path_order)) return next((f for f in filtered), None) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. Return all copies :param major: Major python version to search for. @@ -200,18 +204,28 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, """ sub_finder = operator.methodcaller( - "find_all_python_versions", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_all_python_versions", + major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) if os.name == "nt" and self.windows_finder: windows_finder_version = sub_finder(self.windows_finder) if windows_finder_version: return windows_finder_version paths = (self.get_path(k) for k in self.path_order) - path_filter = filter(None, unnest((sub_finder(p) for p in paths if p is not None))) + path_filter = filter( + None, unnest((sub_finder(p) for p in paths if p is not None)) + ) version_sort = operator.attrgetter("as_python.version_sort") return [c for c in sorted(path_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): """Search for a specific python version on the path. :param major: Major python version to search for. @@ -226,7 +240,13 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= """ sub_finder = operator.methodcaller( - "find_python_version", major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "find_python_version", + major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) if major and minor and patch: _tuple_pre = pre if pre is not None else False @@ -247,7 +267,7 @@ def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev= if ver.as_python.version_tuple[:5] in self.python_version_dict: self.python_version_dict[ver.as_python.version_tuple[:5]].append(ver) else: - self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver,] + self.python_version_dict[ver.as_python.version_tuple[:5]] = [ver] return ver @classmethod @@ -280,7 +300,13 @@ def create(cls, path=None, system=False, only_python=False, global_search=True): for p in _path_objects } ) - return cls(paths=path_entries, path_order=paths, only_python=only_python, system=system, global_search=global_search) + return cls( + paths=path_entries, + path_order=paths, + only_python=only_python, + system=system, + global_search=global_search, + ) @attr.s @@ -293,7 +319,7 @@ class PathEntry(BasePath): pythons = attr.ib() def __str__(self): - return fs_str('{0}'.format(self.path.as_posix())) + return fs_str("{0}".format(self.path.as_posix())) def _filter_children(self): if self.only_python: @@ -333,6 +359,7 @@ def as_python(self): if not self.py_version: try: from .python import PythonVersion + self.py_version = PythonVersion.from_path(self.path) except (ValueError, InvalidPythonVersion): self.py_version = None @@ -355,11 +382,7 @@ def create(cls, path, is_root=False, only_python=False, pythons=None): """ target = ensure_path(path) - creation_args = { - "path": target, - "is_root": is_root, - "only_python": only_python - } + creation_args = {"path": target, "is_root": is_root, "only_python": only_python} if pythons: creation_args["pythons"] = pythons _new = cls(**creation_args) diff --git a/pipenv/vendor/pythonfinder/models/pyenv.py b/pipenv/vendor/pythonfinder/models/pyenv.py index 13476b6b7b..6c8909369d 100644 --- a/pipenv/vendor/pythonfinder/models/pyenv.py +++ b/pipenv/vendor/pythonfinder/models/pyenv.py @@ -32,7 +32,9 @@ def get_versions(self): version.get("is_prerelease"), version.get("is_devrelease"), ) - versions[version_tuple] = VersionPath.create(path=p.resolve(), only_python=True) + versions[version_tuple] = VersionPath.create( + path=p.resolve(), only_python=True + ) return versions @pythons.default diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index fcb48723af..6176fad4df 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -44,13 +44,7 @@ def version_sort(self): release_sort = 1 elif self.is_devrelease: release_sort = 0 - return ( - self.major, - self.minor, - self.patch if self.patch else 0, - release_sort - ) - + return (self.major, self.minor, self.patch if self.patch else 0, release_sort) @property def version_tuple(self): @@ -68,9 +62,11 @@ def version_tuple(self): self.is_devrelease, ) - def matches(self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None): - if arch and arch.isdigit(): - arch = '{0}bit'.format(arch) + def matches( + self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None + ): + if arch and arch.isnumeric(): + arch = "{0}bit".format(arch) return ( (major is None or self.major == major) and (minor is None or self.minor == minor) @@ -195,9 +191,9 @@ def from_windows_launcher(cls, launcher_entry): @classmethod def create(cls, **kwargs): - if 'architecture' in kwargs: - if kwargs['architecture'].isdigit(): - kwargs['architecture'] = '{0}bit'.format(kwargs['architecture']) + if "architecture" in kwargs: + if kwargs["architecture"].isnumeric(): + kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) return cls(**kwargs) @@ -221,4 +217,6 @@ def merge(self, target): current_entries = {p.path for p in self.versions.get(version)} new_entries = {p.path for p in entries} new_entries -= current_entries - self.versions[version].append([e for e in entries if e.path in new_entries]) + self.versions[version].append( + [e for e in entries if e.path in new_entries] + ) diff --git a/pipenv/vendor/pythonfinder/models/windows.py b/pipenv/vendor/pythonfinder/models/windows.py index 40dd93ede9..f731432c91 100644 --- a/pipenv/vendor/pythonfinder/models/windows.py +++ b/pipenv/vendor/pythonfinder/models/windows.py @@ -17,9 +17,17 @@ class WindowsFinder(BaseFinder): versions = attr.ib() pythons = attr.ib() - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): version_matcher = operator.methodcaller( - "matches", major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + "matches", + major=major, + minor=minor, + patch=patch, + pre=pre, + dev=dev, + arch=arch, ) py_filter = filter( None, filter(lambda c: version_matcher(c), self.version_list) @@ -27,11 +35,17 @@ def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, version_sort = operator.attrgetter("version_sort") return [c.comes_from for c in sorted(py_filter, key=version_sort, reverse=True)] - def find_python_version(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): - return next(( - v for v in self.find_all_python_versions( - major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch - )), None + def find_python_version( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): + return next( + ( + v + for v in self.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) + ), + None, ) @versions.default diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index c6edbaf96e..d6f49bf745 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -35,7 +35,9 @@ def __init__(self, path=None, system=False, global_search=True): def system_path(self): if not self._system_path: self._system_path = SystemPath.create( - path=self.path_prepend, system=self.system, global_search=self.global_search + path=self.path_prepend, + system=self.system, + global_search=self.global_search, ) return self._system_path @@ -50,12 +52,21 @@ def windows_finder(self): def which(self, exe): return self.system_path.which(exe) - def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_python_version( + self, major, minor=None, patch=None, pre=None, dev=None, arch=None + ): from .models import PythonVersion - if isinstance(major, six.string_types) and pre is None and minor is None and dev is None and patch is None: - if arch is None and '-' in major: - major, arch = major.rsplit('-', 1) - if not arch.isdigit(): + + if ( + isinstance(major, six.string_types) + and pre is None + and minor is None + and dev is None + and patch is None + ): + if arch is None and "-" in major: + major, arch = major.rsplit("-", 1) + if not arch.isnumeric(): major = "{0}-{1}".format(major, arch) else: arch = "{0}bit".format(arch) @@ -76,19 +87,28 @@ def find_python_version(self, major, minor=None, patch=None, pre=None, dev=None, major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch ) - def find_all_python_versions(self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None): + def find_all_python_versions( + self, major=None, minor=None, patch=None, pre=None, dev=None, arch=None + ): version_sort = operator.attrgetter("as_python.version_sort") - python_version_dict = getattr(self.system_path, 'python_version_dict') + python_version_dict = getattr(self.system_path, "python_version_dict") if python_version_dict: - paths = filter(None, [path for version in python_version_dict.values() for path in version if path.as_python]) + paths = filter( + None, + [ + path + for version in python_version_dict.values() + for path in version + if path.as_python + ], + ) paths = sorted(paths, key=version_sort, reverse=True) return paths - versions = self.system_path.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) + versions = self.system_path.find_all_python_versions( + major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch + ) if not isinstance(versions, list): - versions = [versions,] - # if os.name == 'nt': - # windows_versions = self.windows_finder.find_all_python_versions(major=major, minor=minor, patch=patch, pre=pre, dev=dev, arch=arch) - # versions = list(windows_versions) + versions + versions = [versions] paths = sorted(versions, key=version_sort, reverse=True) path_map = {} for path in paths: diff --git a/pipenv/vendor/pythonfinder/utils.py b/pipenv/vendor/pythonfinder/utils.py index fdc543811d..1a9bfa2c1a 100644 --- a/pipenv/vendor/pythonfinder/utils.py +++ b/pipenv/vendor/pythonfinder/utils.py @@ -32,10 +32,7 @@ def _run(cmd): """ encoding = locale.getdefaultlocale()[1] or "utf-8" c = subprocess.Popen( - cmd, - env=os.environ.copy(), - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, + cmd, env=os.environ.copy(), stdout=subprocess.PIPE, stderr=subprocess.PIPE ) out, err = c.communicate() return out.decode(encoding).strip(), err.decode(encoding).strip() From 5ff8bdeed3b58915a6aca63b061a424783ffa715 Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Wed, 25 Jul 2018 17:23:28 -0400 Subject: [PATCH 63/69] Update vendor.txt Signed-off-by: Dan Ryan --- pipenv/vendor/vendor.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pipenv/vendor/vendor.txt b/pipenv/vendor/vendor.txt index c4f4412324..47253dd375 100644 --- a/pipenv/vendor/vendor.txt +++ b/pipenv/vendor/vendor.txt @@ -21,7 +21,7 @@ git+https://github.com/naiquevin/pipdeptree.git@ee5eaf86ed0f49ea97601475e048d81e pipreqs==0.4.9 docopt==0.6.2 yarg==0.1.9 -pythonfinder +pythonfinder==1.0.0 requests==2.19.1 chardet==3.0.4 idna==2.7 @@ -40,4 +40,3 @@ semver==2.8.0 shutilwhich==1.1.0 toml==0.9.4 cached-property==1.4.3 -pythonfinder==1.0.0 \ No newline at end of file From ed8e862d296293d544aada35284e6cf776477d04 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Thu, 26 Jul 2018 14:04:55 +0800 Subject: [PATCH 64/69] isnumeric -> isdigit --- pipenv/vendor/pythonfinder/models/python.py | 4 ++-- pipenv/vendor/pythonfinder/pythonfinder.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pipenv/vendor/pythonfinder/models/python.py b/pipenv/vendor/pythonfinder/models/python.py index 6176fad4df..f10ddb4e42 100644 --- a/pipenv/vendor/pythonfinder/models/python.py +++ b/pipenv/vendor/pythonfinder/models/python.py @@ -65,7 +65,7 @@ def version_tuple(self): def matches( self, major=None, minor=None, patch=None, pre=False, dev=False, arch=None ): - if arch and arch.isnumeric(): + if arch and arch.isdigit(): arch = "{0}bit".format(arch) return ( (major is None or self.major == major) @@ -192,7 +192,7 @@ def from_windows_launcher(cls, launcher_entry): @classmethod def create(cls, **kwargs): if "architecture" in kwargs: - if kwargs["architecture"].isnumeric(): + if kwargs["architecture"].isdigit(): kwargs["architecture"] = "{0}bit".format(kwargs["architecture"]) return cls(**kwargs) diff --git a/pipenv/vendor/pythonfinder/pythonfinder.py b/pipenv/vendor/pythonfinder/pythonfinder.py index d6f49bf745..035842e2c6 100644 --- a/pipenv/vendor/pythonfinder/pythonfinder.py +++ b/pipenv/vendor/pythonfinder/pythonfinder.py @@ -66,7 +66,7 @@ def find_python_version( ): if arch is None and "-" in major: major, arch = major.rsplit("-", 1) - if not arch.isnumeric(): + if not arch.isdigit(): major = "{0}-{1}".format(major, arch) else: arch = "{0}bit".format(arch) From 0170cdbf7717de0a045bcec0e877100434a42e80 Mon Sep 17 00:00:00 2001 From: Warix3 Date: Mon, 30 Jul 2018 01:28:42 +0200 Subject: [PATCH 65/69] PyCharm now fully supports pipenv As of the release version 2018.2 PyCharm has a full pipenv support. [Source](https://www.jetbrains.com/pycharm/whatsnew/#v2018-2-python "New features") --- docs/advanced.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/advanced.rst b/docs/advanced.rst index d3275368bb..ceaf64d5b2 100644 --- a/docs/advanced.rst +++ b/docs/advanced.rst @@ -242,11 +242,11 @@ different products which integrate with Pipenv projects: - `Emacs `_ (Editor Integration) - `Fish Shell `_ (Automatic ``$ pipenv shell``!) - `VS Code `_ (Editor Integration) +- `PyCharm `_ (Editor Integration) Works in progress: - `Sublime Text `_ (Editor Integration) -- `PyCharm `_ (Editor Integration) - Mysterious upcoming Google Cloud product (Cloud Hosting) From 553ed8258c7f217b4e5f61fa7182a239b2dc7fe9 Mon Sep 17 00:00:00 2001 From: Louie Lu Date: Mon, 23 Jul 2018 20:29:30 +0800 Subject: [PATCH 66/69] Add JSONDecodeError handling for get_lockfile_hash Now will return None when facing JSONDecodeError when loading the lockfile. do_init will remove the old lockfile and replace it. --- pipenv/core.py | 20 ++++++++++++++++++++ pipenv/project.py | 6 +++++- tests/integration/test_lock.py | 22 ++++++++++++++++++++++ 3 files changed, 47 insertions(+), 1 deletion(-) diff --git a/pipenv/core.py b/pipenv/core.py index 25bd0b7e45..d72d2e0fe3 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1257,6 +1257,26 @@ def do_init( ), err=True, ) + elif old_hash is None: + # Lockfile corrupted, remove it and replaced + click.echo( + crayons.red( + u"Pipfile.lock is corrupted, replaced with ({0})…".format( + new_hash[-6:] + ), + bold=True, + ), + err=True + ) + os.remove(project.lockfile_location) + do_lock( + system=system, + pre=pre, + keep_outdated=keep_outdated, + verbose=verbose, + write=True, + pypi_mirror=pypi_mirror, + ) else: click.echo( crayons.red( diff --git a/pipenv/project.py b/pipenv/project.py index b12f7c6451..275a6b41da 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -815,7 +815,11 @@ def get_lockfile_hash(self): if not os.path.exists(self.lockfile_location): return - lockfile = self.load_lockfile(expand_env_vars=False) + try: + lockfile = self.load_lockfile(expand_env_vars=False) + except json.decoder.JSONDecodeError: + # Lockfile corrupted + return if "_meta" in lockfile and hasattr(lockfile, "keys"): return lockfile["_meta"].get("hash", {}).get("sha256") # Lockfile exists but has no hash at all diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index 25195bfbec..a0cdfe155d 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -385,3 +385,25 @@ def test_lock_respecting_python_version(PipenvInstance, pypi): django_version = '==2.0.6' if py_version.startswith('3') else '==1.11.13' assert py_version == '2.7.14' assert p.lockfile['default']['django']['version'] == django_version + + +@pytest.mark.lock +@pytest.mark.install +def test_lockfile_corrupted(PipenvInstance): + with PipenvInstance() as p: + with open(p.lockfile_path, 'w') as f: + f.write('{corrupt}') + c = p.pipenv('install') + assert c.return_code == 0 + assert p.lockfile['_meta'] + + +@pytest.mark.lock +@pytest.mark.install +def test_lockfile_with_empty_dict(PipenvInstance): + with PipenvInstance() as p: + with open(p.lockfile_path, 'w') as f: + f.write('{}') + c = p.pipenv('install') + assert c.return_code == 0 + assert p.lockfile['_meta'] \ No newline at end of file From 79349dd8f1224d9235c5b9bf8d66bfcc6067e753 Mon Sep 17 00:00:00 2001 From: Louie Lu Date: Tue, 24 Jul 2018 09:45:12 +0800 Subject: [PATCH 67/69] Addressed uranusjr comments --- pipenv/core.py | 50 +++++++++++++++------------------- pipenv/project.py | 4 +-- tests/integration/test_lock.py | 6 ++-- 3 files changed, 28 insertions(+), 32 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index d72d2e0fe3..c1e7d0154e 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1257,40 +1257,34 @@ def do_init( ), err=True, ) - elif old_hash is None: - # Lockfile corrupted, remove it and replaced - click.echo( - crayons.red( - u"Pipfile.lock is corrupted, replaced with ({0})…".format( - new_hash[-6:] - ), - bold=True, - ), - err=True - ) - os.remove(project.lockfile_location) - do_lock( - system=system, - pre=pre, - keep_outdated=keep_outdated, - verbose=verbose, - write=True, - pypi_mirror=pypi_mirror, - ) else: - click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( - old_hash[-6:], new_hash[-6:] + if not old_hash: + # Lockfile corrupted, remove it and replaced + click.echo( + crayons.red( + u"Pipfile.lock is corrupted, replaced with ({0})…".format( + new_hash[-6:] + ), + bold=True, ), - bold=True, - ), - err=True, - ) + err=True + ) + os.remove(project.lockfile_location) + else: + click.echo( + crayons.red( + u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( + old_hash[-6:], new_hash[-6:] + ), + bold=True, + ), + err=True, + ) do_lock( system=system, pre=pre, keep_outdated=keep_outdated, + verbose=verbose, write=True, pypi_mirror=pypi_mirror, ) diff --git a/pipenv/project.py b/pipenv/project.py index 275a6b41da..c5f451e495 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -817,9 +817,9 @@ def get_lockfile_hash(self): try: lockfile = self.load_lockfile(expand_env_vars=False) - except json.decoder.JSONDecodeError: + except ValueError: # Lockfile corrupted - return + return "" if "_meta" in lockfile and hasattr(lockfile, "keys"): return lockfile["_meta"].get("hash", {}).get("sha256") # Lockfile exists but has no hash at all diff --git a/tests/integration/test_lock.py b/tests/integration/test_lock.py index a0cdfe155d..23d75f8e95 100644 --- a/tests/integration/test_lock.py +++ b/tests/integration/test_lock.py @@ -392,9 +392,10 @@ def test_lock_respecting_python_version(PipenvInstance, pypi): def test_lockfile_corrupted(PipenvInstance): with PipenvInstance() as p: with open(p.lockfile_path, 'w') as f: - f.write('{corrupt}') + f.write('{corrupted}') c = p.pipenv('install') assert c.return_code == 0 + assert 'Pipfile.lock is corrupted' in c.err assert p.lockfile['_meta'] @@ -406,4 +407,5 @@ def test_lockfile_with_empty_dict(PipenvInstance): f.write('{}') c = p.pipenv('install') assert c.return_code == 0 - assert p.lockfile['_meta'] \ No newline at end of file + assert 'Pipfile.lock is corrupted' in c.err + assert p.lockfile['_meta'] From bd131836cd95193a5d2fa0ec0873079f622594dc Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 12:26:07 +0800 Subject: [PATCH 68/69] Don't remove corrupt lock file --- pipenv/core.py | 28 +++++++--------------------- 1 file changed, 7 insertions(+), 21 deletions(-) diff --git a/pipenv/core.py b/pipenv/core.py index c1e7d0154e..2bb9140e45 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1258,28 +1258,14 @@ def do_init( err=True, ) else: - if not old_hash: - # Lockfile corrupted, remove it and replaced - click.echo( - crayons.red( - u"Pipfile.lock is corrupted, replaced with ({0})…".format( - new_hash[-6:] - ), - bold=True, - ), - err=True - ) - os.remove(project.lockfile_location) + if old_hash: + msg = u"Pipfile.lock ({1}) out of date, updating to ({0})…" else: - click.echo( - crayons.red( - u"Pipfile.lock ({0}) out of date, updating to ({1})…".format( - old_hash[-6:], new_hash[-6:] - ), - bold=True, - ), - err=True, - ) + msg = u"Pipfile.lock is corrupted, replaced with ({0})…" + click.echo(crayons.red( + msg.format(old_hash[-6:], new_hash[-6:]), + bold=True, + ), err=True) do_lock( system=system, pre=pre, From a67d0244a046a0a8efbd16d7ac37b611bd9145a1 Mon Sep 17 00:00:00 2001 From: Tzu-ping Chung Date: Mon, 30 Jul 2018 12:49:28 +0800 Subject: [PATCH 69/69] News --- news/2607.bugfix | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 news/2607.bugfix diff --git a/news/2607.bugfix b/news/2607.bugfix new file mode 100644 index 0000000000..101124a8da --- /dev/null +++ b/news/2607.bugfix @@ -0,0 +1,2 @@ +Catch JSON decoding error to prevent exception when the lock file is of +invalid format.