diff --git a/news/2384.feature b/news/2384.feature index 799c0c1d12..a16cb18c38 100644 --- a/news/2384.feature +++ b/news/2384.feature @@ -1 +1 @@ -Optimized hashing speed. +Pipenv will now generate hashes much more quickly by resolving them in a single pass during locking. diff --git a/news/2384.trivial b/news/2384.trivial index 21695a9c92..89abf16205 100644 --- a/news/2384.trivial +++ b/news/2384.trivial @@ -1 +1,2 @@ -Added pytz 2018.4 wheel for testing -- needed for dependency resolution. +Updated mocked pypi dependencies. +Removed ellipsis and em-dash characters from terminal output. diff --git a/pipenv/core.py b/pipenv/core.py index 1eca5c1f9a..804f469c52 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -833,7 +833,7 @@ def cleanup_procs(procs, concurrent): if failed_deps_list: click.echo( crayons.normal( - u'Installing initially–failed dependencies...', bold=True + u'Installing initially failed dependencies...', bold=True ) ) for dep, ignore_hash in progress.bar( @@ -1309,7 +1309,7 @@ def do_init( else: click.echo( crayons.red( - u'Pipfile.lock ({0}) out of date, updating to ({1})...¦'.format( + u'Pipfile.lock ({0}) out of date, updating to ({1})...'.format( old_hash[-6:], new_hash[-6:] ), bold=True, @@ -1334,7 +1334,7 @@ def do_init( sys.exit(1) else: click.echo( - crayons.normal(u'Pipfile.lock not found, creating...¦', bold=True), + crayons.normal(u'Pipfile.lock not found, creating...', bold=True), err=True, ) do_lock( @@ -1679,7 +1679,7 @@ def warn_in_virtualenv(): def ensure_lockfile(keep_outdated=False, pypi_mirror=None): - """Ensures that the lockfile is up–to–date.""" + """Ensures that the lockfile is up-to-date.""" if not keep_outdated: keep_outdated = project.settings.get('keep_outdated') # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored @@ -1689,7 +1689,7 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None): if new_hash != old_hash: click.echo( crayons.red( - u'Pipfile.lock ({0}) out of date, updating to ({1})...¦'.format( + u'Pipfile.lock ({0}) out of date, updating to ({1})...'.format( old_hash[-6:], new_hash[-6:] ), bold=True, @@ -1738,7 +1738,7 @@ def do_outdated(pypi_mirror=None): ) for package, new_version, old_version in outdated: click.echo( - 'Package {0!r} out–of–date: {1!r} installed, {2!r} available.'.format( + 'Package {0!r} out-of-date: {1!r} installed, {2!r} available.'.format( package, old_version, new_version ) ) @@ -1821,7 +1821,7 @@ def do_install( # Download requirements file click.echo( crayons.normal( - u'Remote requirements file provided! Downloading...¦', bold=True + u'Remote requirements file provided! Downloading...', bold=True ), err=True, ) @@ -2003,9 +2003,9 @@ def do_install( sys.exit(1) if converted.is_vcs and not converted.editable: click.echo( - '{0}: You installed a VCS dependency in non–editable mode. ' + '{0}: You installed a VCS dependency in non-editable mode. ' 'This will work fine, but sub-dependencies will not be resolved by {1}.' - '\n To enable this sub–dependency functionality, specify that this dependency is editable.' + '\n To enable this sub-dependency functionality, specify that this dependency is editable.' ''.format( crayons.red('Warning', bold=True), crayons.red('$ pipenv lock'), @@ -2441,7 +2441,7 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False): click.echo( u'{0}: {1}'.format( crayons.red('Warning', bold=True), - u'Unable to display currently–installed dependency graph information here. ' + u'Unable to display currently-installed dependency graph information here. ' u'Please run within a Pipenv project.', ), err=True, diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py index cd49dbf7d2..3fee3d89fc 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -25,6 +25,7 @@ from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier +from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable from pipenv.patched.notpip._vendor.pyparsing import ParseException from ..cache import CACHE_DIR @@ -392,19 +393,17 @@ def get_legacy_dependencies(self, ireq): specifierset = list(SpecifierSet(requires_python)) # for multiple specifiers, the correct way to represent that in # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` - first_spec, marker_str = specifierset[0]._spec - if len(specifierset) > 1: - marker_str = [marker_str,] - for spec in specifierset[1:]: - marker_str.append(str(spec)) - marker_str = ','.join(marker_str) - # join the leading specifier operator and the rest of the specifiers - marker_str = '{0}"{1}"'.format(first_spec, marker_str) - else: - marker_str = '=="{0}"'.format(requires_python.replace(' ', '')) + marker_key = Variable('python_version') + markers = [] + for spec in specifierset: + operator, val = spec._spec + operator = Op(operator) + val = Value(val) + markers.append(''.join([marker_key.serialize(), operator.serialize(), val.serialize()])) + marker_str = ' and '.join(markers) # The best way to add markers to a requirement is to make a separate requirement # with only markers on it, and then to transfer the object istelf - marker_to_add = Requirement('fakepkg; python_version{0}'.format(marker_str)).marker + marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker result.remove(ireq) ireq.req.marker = marker_to_add result.add(ireq) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 80cfaac7df..088aa06c2d 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644 # NOTE # We used to store the cache dir under ~/.pip-tools, which is not the diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py -index 1c4b943..4155869 100644 +index 1c4b943..07cd667 100644 --- a/pipenv/patched/piptools/repositories/pypi.py +++ b/pipenv/patched/piptools/repositories/pypi.py @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function, @@ -30,7 +30,7 @@ index 1c4b943..4155869 100644 from contextlib import contextmanager from shutil import rmtree -@@ -15,13 +16,23 @@ from .._compat import ( +@@ -15,13 +16,24 @@ from .._compat import ( Wheel, FAVORITE_HASH, TemporaryDirectory, @@ -44,6 +44,7 @@ index 1c4b943..4155869 100644 +from pip._vendor.packaging.requirements import InvalidRequirement, Requirement +from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version +from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier ++from pip._vendor.packaging.markers import Marker, Op, Value, Variable +from pip._vendor.pyparsing import ParseException + from ..cache import CACHE_DIR @@ -57,7 +58,7 @@ index 1c4b943..4155869 100644 from .base import BaseRepository -@@ -37,6 +48,40 @@ except ImportError: +@@ -37,6 +49,40 @@ except ImportError: from pip.wheel import WheelCache @@ -98,7 +99,7 @@ index 1c4b943..4155869 100644 class PyPIRepository(BaseRepository): DEFAULT_INDEX_URL = PyPI.simple_url -@@ -46,10 +91,11 @@ class PyPIRepository(BaseRepository): +@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository): config), but any other PyPI mirror can be used if index_urls is changed/configured on the Finder. """ @@ -112,7 +113,7 @@ index 1c4b943..4155869 100644 index_urls = [pip_options.index_url] + pip_options.extra_index_urls if pip_options.no_index: -@@ -74,11 +120,15 @@ class PyPIRepository(BaseRepository): +@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository): # of all secondary dependencies for the given requirement, so we # only have to go to disk once for each requirement self._dependencies_cache = {} @@ -130,7 +131,7 @@ index 1c4b943..4155869 100644 def freshen_build_caches(self): """ -@@ -114,10 +164,14 @@ class PyPIRepository(BaseRepository): +@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository): if ireq.editable: return ireq # return itself as the best match @@ -147,7 +148,7 @@ index 1c4b943..4155869 100644 # Reuses pip's internal candidate sort key to sort matching_candidates = [candidates_by_version[ver] for ver in matching_versions] -@@ -126,11 +180,71 @@ class PyPIRepository(BaseRepository): +@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository): best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key) # Turn the candidate into a pinned InstallRequirement @@ -222,7 +223,7 @@ index 1c4b943..4155869 100644 """ Given a pinned or an editable InstallRequirement, returns a set of dependencies (also InstallRequirements, but not necessarily pinned). -@@ -155,6 +269,20 @@ class PyPIRepository(BaseRepository): +@@ -155,6 +270,20 @@ class PyPIRepository(BaseRepository): os.makedirs(download_dir) if not os.path.isdir(self._wheel_download_dir): os.makedirs(self._wheel_download_dir) @@ -243,7 +244,7 @@ index 1c4b943..4155869 100644 try: # Pip < 9 and below -@@ -164,11 +292,14 @@ class PyPIRepository(BaseRepository): +@@ -164,11 +293,14 @@ class PyPIRepository(BaseRepository): download_dir=download_dir, wheel_download_dir=self._wheel_download_dir, session=self.session, @@ -260,7 +261,7 @@ index 1c4b943..4155869 100644 ) except TypeError: # Pip >= 10 (new resolver!) -@@ -188,17 +319,99 @@ class PyPIRepository(BaseRepository): +@@ -188,17 +320,97 @@ class PyPIRepository(BaseRepository): finder=self.finder, session=self.session, upgrade_strategy="to-satisfy-only", @@ -340,19 +341,17 @@ index 1c4b943..4155869 100644 + specifierset = list(SpecifierSet(requires_python)) + # for multiple specifiers, the correct way to represent that in + # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')` -+ first_spec, marker_str = specifierset[0]._spec -+ if len(specifierset) > 1: -+ marker_str = [marker_str,] -+ for spec in specifierset[1:]: -+ marker_str.append(str(spec)) -+ marker_str = ','.join(marker_str) -+ # join the leading specifier operator and the rest of the specifiers -+ marker_str = '{0}"{1}"'.format(first_spec, marker_str) -+ else: -+ marker_str = '=="{0}"'.format(requires_python.replace(' ', '')) ++ marker_key = Variable('python_version') ++ markers = [] ++ for spec in specifierset: ++ operator, val = spec._spec ++ operator = Op(operator) ++ val = Value(val) ++ markers.append(''.join([marker_key.serialize(), operator.serialize(), val.serialize()])) ++ marker_str = ' and '.join(markers) + # The best way to add markers to a requirement is to make a separate requirement + # with only markers on it, and then to transfer the object istelf -+ marker_to_add = Requirement('fakepkg; python_version{0}'.format(marker_str)).marker ++ marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker + result.remove(ireq) + ireq.req.marker = marker_to_add + result.add(ireq) @@ -363,7 +362,7 @@ index 1c4b943..4155869 100644 return set(self._dependencies_cache[ireq]) def get_hashes(self, ireq): -@@ -217,24 +430,22 @@ class PyPIRepository(BaseRepository): +@@ -217,24 +429,22 @@ class PyPIRepository(BaseRepository): # We need to get all of the candidates that match our current version # pin, these will represent all of the files that could possibly # satisfy this constraint. diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index 61ab41b2cd..6589b50936 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -191,20 +191,18 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir): @pytest.mark.needs_internet def test_get_vcs_refs(PipenvInstance, pip_src_dir): with PipenvInstance(chdir=True) as p: - c = p.pipenv('install -e git+https://github.com/hynek/structlog.git@16.1.0#egg=structlog') + c = p.pipenv('install -e git+https://github.com/benjaminp/six.git@1.9.0#egg=six') assert c.return_code == 0 - assert 'structlog' in p.pipfile['packages'] - assert 'structlog' in p.lockfile['default'] + assert 'six' in p.pipfile['packages'] assert 'six' in p.lockfile['default'] - assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492' + assert p.lockfile['default']['six']['ref'] == '5efb522b0647f7467248273ec1b893d06b984a59' pipfile = Path(p.pipfile_path) - new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0') + new_content = pipfile.read_bytes().replace(b'1.9.0', b'1.11.0') pipfile.write_bytes(new_content) c = p.pipenv('lock') assert c.return_code == 0 - assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034' - assert 'structlog' in p.pipfile['packages'] - assert 'structlog' in p.lockfile['default'] + assert p.lockfile['default']['six']['ref'] == '15e31431af97e5e64b80af0a3f598d382bcdd49a' + assert 'six' in p.pipfile['packages'] assert 'six' in p.lockfile['default']