Use the real marker parser -- confirmed working

Signed-off-by: Dan Ryan <dan@danryan.co>

Typo fix

Signed-off-by: Dan Ryan <dan@danryan.co>

Move away from structlog for testing

Signed-off-by: Dan Ryan <dan@danryan.co>

Remove em-dash and ellipsis characters

- Replace with dashes and dots

Signed-off-by: Dan Ryan <dan@danryan.co>

news/2384.feature

@@ -1 +1 @@
-Optimized hashing speed.
+Pipenv will now generate hashes much more quickly by resolving them in a single pass during locking.

news/2384.trivial

@@ -1 +1,2 @@
-Added pytz 2018.4 wheel for testing -- needed for dependency resolution.
+Updated mocked pypi dependencies.
+Removed ellipsis and em-dash characters from terminal output.

pipenv/core.py

@@ -833,7 +833,7 @@ def cleanup_procs(procs, concurrent):
  if failed_deps_list:
  click.echo(
  crayons.normal(
- u'Installing initially–failed dependencies...', bold=True
+ u'Installing initially failed dependencies...', bold=True
  )
  )
  for dep, ignore_hash in progress.bar(
@@ -1309,7 +1309,7 @@ def do_init(
  else:
  click.echo(
  crayons.red(
- u'Pipfile.lock ({0}) out of date, updating to ({1})...¦'.format(
+ u'Pipfile.lock ({0}) out of date, updating to ({1})...'.format(
  old_hash[-6:], new_hash[-6:]
  ),
  bold=True,
@@ -1334,7 +1334,7 @@ def do_init(
  sys.exit(1)
  else:
  click.echo(
- crayons.normal(u'Pipfile.lock not found, creating...¦', bold=True),
+ crayons.normal(u'Pipfile.lock not found, creating...', bold=True),
  err=True,
  )
  do_lock(
@@ -1679,7 +1679,7 @@ def warn_in_virtualenv():
 
 
 def ensure_lockfile(keep_outdated=False, pypi_mirror=None):
- """Ensures that the lockfile is up–to–date."""
+ """Ensures that the lockfile is up-to-date."""
  if not keep_outdated:
  keep_outdated = project.settings.get('keep_outdated')
  # Write out the lockfile if it doesn't exist, but not if the Pipfile is being ignored
@@ -1689,7 +1689,7 @@ def ensure_lockfile(keep_outdated=False, pypi_mirror=None):
  if new_hash != old_hash:
  click.echo(
  crayons.red(
- u'Pipfile.lock ({0}) out of date, updating to ({1})...¦'.format(
+ u'Pipfile.lock ({0}) out of date, updating to ({1})...'.format(
  old_hash[-6:], new_hash[-6:]
  ),
  bold=True,
@@ -1738,7 +1738,7 @@ def do_outdated(pypi_mirror=None):
  )
  for package, new_version, old_version in outdated:
  click.echo(
- 'Package {0!r} out–of–date: {1!r} installed, {2!r} available.'.format(
+ 'Package {0!r} out-of-date: {1!r} installed, {2!r} available.'.format(
  package, old_version, new_version
  )
  )
@@ -1821,7 +1821,7 @@ def do_install(
  # Download requirements file
  click.echo(
  crayons.normal(
- u'Remote requirements file provided! Downloading...¦', bold=True
+ u'Remote requirements file provided! Downloading...', bold=True
  ),
  err=True,
  )
@@ -2003,9 +2003,9 @@ def do_install(
  sys.exit(1)
  if converted.is_vcs and not converted.editable:
  click.echo(
- '{0}: You installed a VCS dependency in non–editable mode. '
+ '{0}: You installed a VCS dependency in non-editable mode. '
  'This will work fine, but sub-dependencies will not be resolved by {1}.'
- '\n To enable this sub–dependency functionality, specify that this dependency is editable.'
+ '\n To enable this sub-dependency functionality, specify that this dependency is editable.'
  ''.format(
  crayons.red('Warning', bold=True),
  crayons.red('$ pipenv lock'),
@@ -2436,7 +2436,7 @@ def do_graph(bare=False, json=False, json_tree=False, reverse=False):
  click.echo(
  u'{0}: {1}'.format(
  crayons.red('Warning', bold=True),
- u'Unable to display currently–installed dependency graph information here. '
+ u'Unable to display currently-installed dependency graph information here. '
  u'Please run within a Pipenv project.',
  ),
  err=True,

pipenv/patched/piptools/repositories/pypi.py

@@ -25,6 +25,7 @@
 from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
 from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
 from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
+from pipenv.patched.notpip._vendor.packaging.markers import Marker, Op, Value, Variable
 from pipenv.patched.notpip._vendor.pyparsing import ParseException
 
 from ..cache import CACHE_DIR
@@ -392,19 +393,17 @@ def get_legacy_dependencies(self, ireq):
  specifierset = list(SpecifierSet(requires_python))
  # for multiple specifiers, the correct way to represent that in
  # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
- first_spec, marker_str = specifierset[0]._spec
- if len(specifierset) > 1:
- marker_str = [marker_str,]
- for spec in specifierset[1:]:
- marker_str.append(str(spec))
- marker_str = ','.join(marker_str)
- # join the leading specifier operator and the rest of the specifiers
- marker_str = '{0}"{1}"'.format(first_spec, marker_str)
- else:
- marker_str = '=="{0}"'.format(requires_python.replace(' ', ''))
+ marker_key = Variable('python_version')
+ markers = []
+ for spec in specifierset:
+ operator, val = spec._spec
+ operator = Op(operator)
+ val = Value(val)
+ markers.append(''.join([marker_key.serialize(), operator.serialize(), val.serialize()]))
+ marker_str = ' and '.join(markers)
  # The best way to add markers to a requirement is to make a separate requirement
  # with only markers on it, and then to transfer the object istelf
- marker_to_add = Requirement('fakepkg; python_version{0}'.format(marker_str)).marker
+ marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker
  result.remove(ireq)
  ireq.req.marker = marker_to_add
  result.add(ireq)

tasks/vendoring/patches/patched/piptools.patch

@@ -19,7 +19,7 @@ index 4e6174c..75f9b49 100644
  # NOTE
  # We used to store the cache dir under ~/.pip-tools, which is not the
 diff --git a/pipenv/patched/piptools/repositories/pypi.py b/pipenv/patched/piptools/repositories/pypi.py
-index 1c4b943..4155869 100644
+index 1c4b943..07cd667 100644
 --- a/pipenv/patched/piptools/repositories/pypi.py
 +++ b/pipenv/patched/piptools/repositories/pypi.py
 @@ -4,6 +4,7 @@ from __future__ import (absolute_import, division, print_function,
@@ -30,7 +30,7 @@ index 1c4b943..4155869 100644
  from contextlib import contextmanager
  from shutil import rmtree
 
-@@ -15,13 +16,23 @@ from .._compat import (
+@@ -15,13 +16,24 @@ from .._compat import (
  Wheel,
  FAVORITE_HASH,
  TemporaryDirectory,
@@ -44,6 +44,7 @@ index 1c4b943..4155869 100644
 +from pip._vendor.packaging.requirements import InvalidRequirement, Requirement
 +from pip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
 +from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
++from pip._vendor.packaging.markers import Marker, Op, Value, Variable
 +from pip._vendor.pyparsing import ParseException
 +
  from ..cache import CACHE_DIR
@@ -57,7 +58,7 @@ index 1c4b943..4155869 100644
  from .base import BaseRepository
 
 
-@@ -37,6 +48,40 @@ except ImportError:
+@@ -37,6 +49,40 @@ except ImportError:
  from pip.wheel import WheelCache
 
 
@@ -98,7 +99,7 @@ index 1c4b943..4155869 100644
  class PyPIRepository(BaseRepository):
  DEFAULT_INDEX_URL = PyPI.simple_url
 
-@@ -46,10 +91,11 @@ class PyPIRepository(BaseRepository):
+@@ -46,10 +92,11 @@ class PyPIRepository(BaseRepository):
  config), but any other PyPI mirror can be used if index_urls is
  changed/configured on the Finder.
  """
@@ -112,7 +113,7 @@ index 1c4b943..4155869 100644
 
  index_urls = [pip_options.index_url] + pip_options.extra_index_urls
  if pip_options.no_index:
-@@ -74,11 +120,15 @@ class PyPIRepository(BaseRepository):
+@@ -74,11 +121,15 @@ class PyPIRepository(BaseRepository):
  # of all secondary dependencies for the given requirement, so we
  # only have to go to disk once for each requirement
  self._dependencies_cache = {}
@@ -130,7 +131,7 @@ index 1c4b943..4155869 100644
 
  def freshen_build_caches(self):
  """
-@@ -114,10 +164,14 @@ class PyPIRepository(BaseRepository):
+@@ -114,10 +165,14 @@ class PyPIRepository(BaseRepository):
  if ireq.editable:
  return ireq # return itself as the best match
 
@@ -147,7 +148,7 @@ index 1c4b943..4155869 100644
 
  # Reuses pip's internal candidate sort key to sort
  matching_candidates = [candidates_by_version[ver] for ver in matching_versions]
-@@ -126,11 +180,71 @@ class PyPIRepository(BaseRepository):
+@@ -126,11 +181,71 @@ class PyPIRepository(BaseRepository):
  best_candidate = max(matching_candidates, key=self.finder._candidate_sort_key)
 
  # Turn the candidate into a pinned InstallRequirement
@@ -222,7 +223,7 @@ index 1c4b943..4155869 100644
  """
  Given a pinned or an editable InstallRequirement, returns a set of
  dependencies (also InstallRequirements, but not necessarily pinned).
-@@ -155,6 +269,20 @@ class PyPIRepository(BaseRepository):
+@@ -155,6 +270,20 @@ class PyPIRepository(BaseRepository):
  os.makedirs(download_dir)
  if not os.path.isdir(self._wheel_download_dir):
  os.makedirs(self._wheel_download_dir)
@@ -243,7 +244,7 @@ index 1c4b943..4155869 100644
 
  try:
  # Pip < 9 and below
-@@ -164,11 +292,14 @@ class PyPIRepository(BaseRepository):
+@@ -164,11 +293,14 @@ class PyPIRepository(BaseRepository):
  download_dir=download_dir,
  wheel_download_dir=self._wheel_download_dir,
  session=self.session,
@@ -260,7 +261,7 @@ index 1c4b943..4155869 100644
  )
  except TypeError:
  # Pip >= 10 (new resolver!)
-@@ -188,17 +319,99 @@ class PyPIRepository(BaseRepository):
+@@ -188,17 +320,97 @@ class PyPIRepository(BaseRepository):
  finder=self.finder,
  session=self.session,
  upgrade_strategy="to-satisfy-only",
@@ -340,19 +341,17 @@ index 1c4b943..4155869 100644
 + specifierset = list(SpecifierSet(requires_python))
 + # for multiple specifiers, the correct way to represent that in
 + # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
-+ first_spec, marker_str = specifierset[0]._spec
-+ if len(specifierset) > 1:
-+ marker_str = [marker_str,]
-+ for spec in specifierset[1:]:
-+ marker_str.append(str(spec))
-+ marker_str = ','.join(marker_str)
-+ # join the leading specifier operator and the rest of the specifiers
-+ marker_str = '{0}"{1}"'.format(first_spec, marker_str)
-+ else:
-+ marker_str = '=="{0}"'.format(requires_python.replace(' ', ''))
++ marker_key = Variable('python_version')
++ markers = []
++ for spec in specifierset:
++ operator, val = spec._spec
++ operator = Op(operator)
++ val = Value(val)
++ markers.append(''.join([marker_key.serialize(), operator.serialize(), val.serialize()]))
++ marker_str = ' and '.join(markers)
 + # The best way to add markers to a requirement is to make a separate requirement
 + # with only markers on it, and then to transfer the object istelf
-+ marker_to_add = Requirement('fakepkg; python_version{0}'.format(marker_str)).marker
++ marker_to_add = Requirement('fakepkg; {0}'.format(marker_str)).marker
 + result.remove(ireq)
 + ireq.req.marker = marker_to_add
 + result.add(ireq)
@@ -363,7 +362,7 @@ index 1c4b943..4155869 100644
  return set(self._dependencies_cache[ireq])
 
  def get_hashes(self, ireq):
-@@ -217,24 +430,22 @@ class PyPIRepository(BaseRepository):
+@@ -217,24 +429,22 @@ class PyPIRepository(BaseRepository):
  # We need to get all of the candidates that match our current version
  # pin, these will represent all of the files that could possibly
  # satisfy this constraint.

tests/integration/test_install_uri.py

@@ -191,20 +191,18 @@ def test_install_local_vcs_not_in_lockfile(PipenvInstance, pip_src_dir):
 @pytest.mark.needs_internet
 def test_get_vcs_refs(PipenvInstance, pip_src_dir):
  with PipenvInstance(chdir=True) as p:
- c = p.pipenv('install -e git+https://github.com/hynek/structlog.git@16.1.0#egg=structlog')
+ c = p.pipenv('install -e git+https://github.com/benjaminp/six.git@1.9.0#egg=six')
  assert c.return_code == 0
- assert 'structlog' in p.pipfile['packages']
- assert 'structlog' in p.lockfile['default']
+ assert 'six' in p.pipfile['packages']
  assert 'six' in p.lockfile['default']
- assert p.lockfile['default']['structlog']['ref'] == 'a39f6906a268fb2f4c365042b31d0200468fb492'
+ assert p.lockfile['default']['six']['ref'] == '5efb522b0647f7467248273ec1b893d06b984a59'
  pipfile = Path(p.pipfile_path)
- new_content = pipfile.read_bytes().replace(b'16.1.0', b'18.1.0')
+ new_content = pipfile.read_bytes().replace(b'1.9.0', b'1.11.0')
  pipfile.write_bytes(new_content)
  c = p.pipenv('lock')
  assert c.return_code == 0
- assert p.lockfile['default']['structlog']['ref'] == 'a73fbd3a9c3cafb11f43168582083f839b883034'
- assert 'structlog' in p.pipfile['packages']
- assert 'structlog' in p.lockfile['default']
+ assert p.lockfile['default']['six']['ref'] == '15e31431af97e5e64b80af0a3f598d382bcdd49a'
+ assert 'six' in p.pipfile['packages']
  assert 'six' in p.lockfile['default']