Minor code cleanup

Signed-off-by: Dan Ryan <dan@danryan.co>

Add pytz and certifi updates

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix nondeterministic resolution bug

- Update dependencies
- Fix some issues with test logic
- Update piptools patch

Signed-off-by: Dan Ryan <dan@danryan.co>

Update more packages

Signed-off-by: Dan Ryan <dan@danryan.co>

Update tests and utils

Signed-off-by: Dan Ryan <dan@danryan.co>

Still need to tackle last few failures

- this will seriously help with resolution issues

Add alembic new version

Signed-off-by: Dan Ryan <dan@danryan.co>

news/2384.bugfix

@@ -1 +1 @@
-Dependencies with markers that don't match the current environment will now be skipped during ``pipenv lock``.
+Resolved a bug in our patched resolvers which could cause nondeterministic resolution failures in certain conditions.

news/2384.feature

@@ -0,0 +1 @@
+Optimized hashing speed.

news/2384.trivial

@@ -0,0 +1 @@
+Added pytz 2018.4 wheel for testing -- needed for dependency resolution.

pipenv/core.py

@@ -47,13 +47,11 @@
  is_star,
  rmtree,
  split_argument,
- extract_uri_from_vcs_dep,
  fs_str,
  clean_resolved_dep,
 )
 from ._compat import (
  TemporaryDirectory,
- vcs,
  Path
 )
 from .import pep508checker, progress
@@ -103,7 +101,7 @@
  ):
  INSTALL_LABEL = '🎅 '
  else:
- INSTALL_LABEL = '🐍 '
+ INSTALL_LABEL = '🝝 '
  INSTALL_LABEL2 = crayons.normal('☤ ', bold=True)
  STARTING_LABEL = ' '
 else:
@@ -1006,7 +1004,7 @@ def do_lock(
  pre=False,
  keep_outdated=False,
  write=True,
- pypi_mirror = None,
+ pypi_mirror=None,
 ):
  """Executes the freeze functionality."""
  from .utils import get_vcs_deps
@@ -1382,7 +1380,7 @@ def pip_install(
  selective_upgrade=False,
  requirements_dir=None,
  extra_indexes=None,
- pypi_mirror = None,
+ pypi_mirror=None,
 ):
  from notpip._internal import logger as piplogger
  from notpip._vendor.pyparsing import ParseException

pipenv/patched/piptools/repositories/local.py

@@ -56,7 +56,7 @@ def find_best_match(self, ireq, prereleases=None):
  if existing_pin and ireq_satisfied_by_existing_pin(ireq, existing_pin):
  project, version, _ = as_tuple(existing_pin)
  return make_install_requirement(
- project, version, ireq.extras, constraint=ireq.constraint
+ project, version, ireq.extras, constraint=ireq.constraint, markers=ireq.markers
  )
  else:
  return self.repository.find_best_match(ireq, prereleases)

pipenv/patched/piptools/repositories/pypi.py

@@ -24,14 +24,14 @@
 
 from pipenv.patched.notpip._vendor.packaging.requirements import InvalidRequirement, Requirement
 from pipenv.patched.notpip._vendor.packaging.version import Version, InvalidVersion, parse as parse_version
-from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
+from pipenv.patched.notpip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier, Specifier
 from pipenv.patched.notpip._vendor.pyparsing import ParseException
 
 from ..cache import CACHE_DIR
 from pipenv.environments import PIPENV_CACHE_DIR
 from ..exceptions import NoCandidateFound
 from ..utils import (fs_str, is_pinned_requirement, lookup_table, as_tuple, key_from_req,
- make_install_requirement, format_requirement, dedup)
+ make_install_requirement, format_requirement, dedup, clean_requires_python)
 
 from .base import BaseRepository
 
@@ -165,21 +165,7 @@ def find_best_match(self, ireq, prereleases=None):
  return ireq # return itself as the best match
 
  py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', str(sys.version_info[:3])))
- all_candidates = []
- for c in self.find_all_candidates(ireq.name):
- if c.requires_python:
- # Old specifications had people setting this to single digits
- # which is effectively the same as '>=digit,<digit+1'
- if c.requires_python.isdigit():
- c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
- try:
- specifier_set = SpecifierSet(c.requires_python)
- except InvalidSpecifier:
- pass
- else:
- if not specifier_set.contains(py_version):
- continue
- all_candidates.append(c)
+ all_candidates = clean_requires_python(self.find_all_candidates(ireq.name))
 
  candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version, unique=True)
  try:
@@ -284,6 +270,20 @@ def get_legacy_dependencies(self, ireq):
  os.makedirs(download_dir)
  if not os.path.isdir(self._wheel_download_dir):
  os.makedirs(self._wheel_download_dir)
+ # Collect setup_requires info from local eggs.
+ # Do this after we call the preparer on these reqs to make sure their
+ # egg info has been created
+ setup_requires = {}
+ dist = None
+ if ireq.editable:
+ try:
+ dist = ireq.get_dist()
+ if dist.has_metadata('requires.txt'):
+ setup_requires = self.finder.get_extras_links(
+ dist.get_metadata_lines('requires.txt')
+ )
+ except (TypeError, ValueError):
+ pass
 
  try:
  # Pip < 9 and below
@@ -320,7 +320,7 @@ def get_legacy_dependencies(self, ireq):
  finder=self.finder,
  session=self.session,
  upgrade_strategy="to-satisfy-only",
- force_reinstall=False,
+ force_reinstall=True,
  ignore_dependencies=False,
  ignore_requires_python=True,
  ignore_installed=True,
@@ -330,33 +330,37 @@ def get_legacy_dependencies(self, ireq):
  ignore_compatibility=False
  )
  self.resolver.resolve(reqset)
- result = reqset.requirements.values()
+ result = set(reqset.requirements.values())
 
- # Collect setup_requires info from local eggs.
- # Do this after we call the preparer on these reqs to make sure their
- # egg info has been created
- setup_requires = {}
- if ireq.editable:
+ # HACK: Sometimes the InstallRequirement doesn't properly get
+ # these values set on it during the resolution process. It's
+ # difficult to pin down what is going wrong. This fixes things.
+ if not getattr(ireq, 'version', None):
  try:
- dist = ireq.get_dist()
- if dist.has_metadata('requires.txt'):
- setup_requires = self.finder.get_extras_links(
- dist.get_metadata_lines('requires.txt')
- )
- # HACK: Sometimes the InstallRequirement doesn't properly get
- # these values set on it during the resolution process. It's
- # difficult to pin down what is going wrong. This fixes things.
- ireq.version = dist.version
- ireq.project_name = dist.project_name
- ireq.req = dist.as_requirement()
- except (TypeError, ValueError):
+ dist = ireq.get_dist() if not dist else None
+ ireq.version = ireq.get_dist().version
+ except (ValueError, OSError, TypeError) as e:
  pass
+ if not getattr(ireq, 'project_name', None):
+ try:
+ ireq.project_name = dist.project_name if dist else None
+ except (ValueError, TypeError) as e:
+ pass
+ if not getattr(ireq, 'req', None):
+ try:
+ ireq.req = dist.as_requirement() if dist else None
+ except (ValueError, TypeError) as e:
+ pass
+
  # Convert setup_requires dict into a somewhat usable form.
  if setup_requires:
  for section in setup_requires:
  python_version = section
  not_python = not (section.startswith('[') and ':' in section)
 
+ # This is for cleaning up :extras: formatted markers
+ # by adding them to the results of the resolver
+ # since any such extra would have been returned as a result anyway
  for value in setup_requires[section]:
  # This is a marker.
  if value.startswith('[') and ':' in value:
@@ -370,17 +374,45 @@ def get_legacy_dependencies(self, ireq):
  try:
  if not not_python:
  result = result + [InstallRequirement.from_line("{0}{1}".format(value, python_version).replace(':', ';'))]
- # Anything could go wrong here — can't be too careful.
+ # Anything could go wrong here -- can't be too careful.
  except Exception:
  pass
+
+ # this section properly creates 'python_version' markers for cross-python
+ # virtualenv creation and for multi-python compatibility.
  requires_python = reqset.requires_python if hasattr(reqset, 'requires_python') else self.resolver.requires_python
  if requires_python:
- marker = 'python_version=="{0}"'.format(requires_python.replace(' ', ''))
- new_req = InstallRequirement.from_line('{0}; {1}'.format(str(ireq.req), marker))
- result = [new_req]
+ marker_str = ''
+ # This corrects a logic error from the previous code which said that if 
+ # we Encountered any 'requires_python' attributes, basically only create a
+ # single result no matter how many we resolved. This should fix
+ # a majority of the remaining non-deterministic resolution issues.
+ if any(requires_python.startswith(op) for op in Specifier._operators.keys()):
+ # We are checking first if we have leading specifier operator 
+ # if not, we can assume we should be doing a == comparison
+ specifierset = list(SpecifierSet(requires_python))
+ # for multiple specifiers, the correct way to represent that in
+ # a specifierset is `Requirement('fakepkg; python_version<"3.0,>=2.6"')`
+ first_spec, marker_str = specifierset[0]._spec
+ if len(specifierset) > 1:
+ marker_str = [marker_str,]
+ for spec in specifierset[1:]:
+ marker_str.append(str(spec))
+ marker_str = ','.join(marker_str)
+ # join the leading specifier operator and the rest of the specifiers
+ marker_str = '{0}"{1}"'.format(first_spec, marker_str)
+ else:
+ marker_str = '=="{0}"'.format(requires_python.replace(' ', ''))
+ # The best way to add markers to a requirement is to make a separate requirement
+ # with only markers on it, and then to transfer the object istelf
+ marker_to_add = Requirement('fakepkg; python_version{0}'.format(marker_str)).marker
+ result.remove(ireq)
+ ireq.req.marker = marker_to_add
+ result.add(ireq)
 
  self._dependencies_cache[ireq] = result
  reqset.cleanup_files()
+
  return set(self._dependencies_cache[ireq])
 
  def get_hashes(self, ireq):
@@ -399,11 +431,16 @@ def get_hashes(self, ireq):
  # We need to get all of the candidates that match our current version
  # pin, these will represent all of the files that could possibly
  # satisfy this constraint.
- all_candidates = self.find_all_candidates(ireq.name)
- candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
- matching_versions = list(
- ireq.specifier.filter((candidate.version for candidate in all_candidates)))
- matching_candidates = candidates_by_version[matching_versions[0]]
+ ### Modification -- this is much more efficient....
+ ### modification again -- still more efficient
+ matching_candidates = (
+ c for c in clean_requires_python(self.find_all_candidates(ireq.name))
+ if c.version in ireq.specifier
+ )
+ # candidates_by_version = lookup_table(all_candidates, key=lambda c: c.version)
+ # matching_versions = list(
+ # ireq.specifier.filter((candidate.version for candidate in all_candidates)))
+ # matching_candidates = candidates_by_version[matching_versions[0]]
 
  return {
  self._hash_cache.get_hash(candidate.location)

pipenv/patched/piptools/utils.py

@@ -11,13 +11,30 @@
 from ._compat import InstallRequirement
 
 from first import first
-
+from pip._vendor.packaging.specifiers import SpecifierSet, InvalidSpecifier
 from .click import style
 
 
 UNSAFE_PACKAGES = {'setuptools', 'distribute', 'pip'}
 
 
+def clean_requires_python(candidates):
+ """Get a cleaned list of all the candidates with valid specifiers in the `requires_python` attributes."""
+ all_candidates = []
+ for c in candidates:
+ if c.requires_python:
+ # Old specifications had people setting this to single digits
+ # which is effectively the same as '>=digit,<digit+1'
+ if c.requires_python.isdigit():
+ c.requires_python = '>={0},<{1}'.format(c.requires_python, int(c.requires_python) + 1)
+ try:
+ specifier_set = SpecifierSet(c.requires_python)
+ except InvalidSpecifier:
+ pass
+ all_candidates.append(c)
+ return all_candidates
+
+
 def key_from_ireq(ireq):
  """Get a standardized key for an InstallRequirement."""
  if ireq.req is None and ireq.link is not None: