parent 8ae44bc

author Dan Ryan <dan@danryan.co> 1554074378 -0400
committer Dan Ryan <dan@danryan.co> 1558982736 -0400
gpgsig -----BEGIN PGP SIGNATURE-----

 iQIzBAABCgAdFiEEb6jpcpb+5zzDideCsyDQTvvOpJUFAlzsMFAACgkQsyDQTvvO
 pJWZmRAAtge5wdprlLnKdWUYK5USZb7Uk3zJHi9UIvnO7nKP6UA/L2D/5nxZitvx
 pZI7cGG+8sLp2yZNtQZdW6LNM1jmRXgzdMLYQh/5zo5gbj6KLOw7erh0FU3L3uM6
 wNBNM135Eqt7b+4a4C5TEK2UjwRxBHAsF+3ZzUy+UJQqgQEKxFVxW4gC4yxpfMtL
 jipE8ludwuOIM88ZJapmLpv2R6adQTxWZedTlmczdsy2/WKGHTCCpWs96PBbntdI
 pVBmoXfMhgZi+IuGR3iBYU0qS97vjJ8Te9tQZAaB9JGSqv3hHDWo1ht/rrG2RXzp
 3i0Cf2vG4035EUh56VYE9FCC9m6Vu3U9iIR34BZG9K5+lDP7pmJmjT+GymEgMP0N
 GoP3LYUO+dJjMjaUEMsC6QIi6DAots3uk4lxIw3wcA4Im/N/i5xafsRj1Eu3UdBL
 wBDKMz/FQjH+tD+mnvTlzaxD5vdhhCdBu1gK59rjNMlzg8hz6EF61QbHCaQHd4UI
 VOGIa8ThLlLI3addxzq/McceAc+OsLJ9hm06jkjvvoIuKrHyE3DybdbYQC7uEwyw
 2AvVuMDCPcciYQnkJhNTKmGvPcUDYD7cF91GKcUJKdPdyzDCeFEo5SUTKnLV4Cj2
 VD2sdVUk4jnmYfE4pCFvHKYooxQDMKQk/VCKl1c9QHL9/ijCx+o=
 =b+H1
 -----END PGP SIGNATURE-----

Ensure resolver doesn't compare editable specifiers

- Don't compare versions of editable dependencies when updating using
  `--keep-outdated` -- editable dependencies will now be updated to
  the latest version
- Ensure we don't drop markers from the lockfile when versions are not
  updated
- Fixes #3656
- Fixes #3659

Signed-off-by: Dan Ryan <dan@danryan.co>

Add future import for print function

Signed-off-by: Dan Ryan <dan@danryan.co>

Handle all possible markers in lockfiles

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix json import

Signed-off-by: Dan Ryan <dan@danryan.co>

point to correct reference for lockfile

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix marker merging errors

Signed-off-by: Dan Ryan <dan@danryan.co>

Prevent automatically setting `editable=True`

- Fixes #3647

Signed-off-by: Dan Ryan <dan@danryan.co>

Add new feature toggle for VCS dependency resolution

- Fixes #3577

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix syntax error

Signed-off-by: Dan Ryan <dan@danryan.co>

Use string for environment

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix class name resolution for py27

Signed-off-by: Dan Ryan <dan@danryan.co>

Write json files as unicode

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix resolution with env var

Signed-off-by: Dan Ryan <dan@danryan.co>

news/3577.feature.rst

@@ -0,0 +1 @@
+Added a new environment variable, ``PIPENV_RESOLVE_VCS``, to toggle dependency resolution off for non-editable VCS, file, and URL based dependencies.

news/3647.bugfix.rst

@@ -0,0 +1 @@
+Pipenv will no longer inadvertently set ``editable=True`` on all vcs dependencies.

news/3656.bugfix.rst

@@ -0,0 +1,2 @@
+The ``--keep-outdated`` argument to ``pipenv install`` and ``pipenv lock`` will now drop specifier constraints when encountering editable dependencies.
+- In addition, ``--keep-outdated`` will retain specifiers that would otherwise be dropped from any entries that have not been updated.

pipenv/core.py

@@ -1394,6 +1394,8 @@ def pip_install(
  src_dir = os.environ["PIP_SRC"]
  src = ["--src", os.environ["PIP_SRC"]]
  if not requirement.editable:
+ # Leave this off becauase old lockfiles don't have all deps included
+ # TODO: When can it be turned back on?
  no_deps = False
 
  if src_dir is not None:
@@ -1412,7 +1414,7 @@ def pip_install(
  prefix="pipenv-", suffix="-requirement.txt", dir=requirements_dir,
  delete=False
  )
- line = "-e" if requirement.editable else ""
+ line = "-e " if requirement.editable else ""
  if requirement.editable or requirement.name is not None:
  name = requirement.name
  if requirement.extras:
@@ -1640,7 +1642,6 @@ def system_which(command, mult=False):
  return result
 
 
-
 def format_help(help):
  """Formats the help string."""
  help = help.replace("Options:", str(crayons.normal("Options:", bold=True)))
@@ -1784,8 +1785,8 @@ def do_py(system=False):
  ),
  err=True,
  )
- return 
- 
+ return
+
  try:
  click.echo(which("python", allow_global=system))
  except AttributeError:

pipenv/environments.py

@@ -236,6 +236,15 @@ def _is_env_truthy(name):
 NOTE: This only affects the ``install`` and ``uninstall`` commands.
 """
 
+PIPENV_RESOLVE_VCS = _is_env_truthy(os.environ.get("PIPENV_RESOLVE_VCS", 'true'))
+"""Tells Pipenv whether to resolve all VCS dependencies in full.
+
+As of Pipenv 2018.11.26, only editable VCS dependencies were resolved in full.
+To retain this behavior and avoid handling any conflicts that arise from the new
+approach, you may set this to '0', 'off', or 'false'.
+"""
+
+
 PIPENV_PYUP_API_KEY = os.environ.get(
  "PIPENV_PYUP_API_KEY", "1ab8d58f-5122e025-83674263-bc1e79e0"
 )

pipenv/patched/piptools/utils.py

@@ -70,7 +70,7 @@ def clean_requires_python(candidates):
  all_candidates = []
  py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
  for c in candidates:
- if c.requires_python:
+ if getattr(c, "requires_python", None):
  # Old specifications had people setting this to single digits
  # which is effectively the same as '>=digit,<digit+1'
  if c.requires_python.isdigit():

pipenv/resolver.py

@@ -1,3 +1,6 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import absolute_import, print_function
 import json
 import logging
 import os
@@ -106,9 +109,14 @@ def __init__(self, name, entry_dict, project, resolver, reverse_deps=None, dev=F
  self.pipfile = project.parsed_pipfile.get(pipfile_section, {})
  self.lockfile = project.lockfile_content.get(section, {})
  self.pipfile_dict = self.pipfile.get(self.pipfile_name, {})
- self.lockfile_dict = self.lockfile.get(name, entry_dict)
+ if self.dev and self.name in project.lockfile_content.get("default", {}):
+ self.lockfile_dict = project.lockfile_content["default"][name]
+ else:
+ self.lockfile_dict = self.lockfile.get(name, entry_dict)
  self.resolver = resolver
  self.reverse_deps = reverse_deps
+ self._original_markers = None
+ self._markers = None
  self._entry = None
  self._lockfile_entry = None
  self._pipfile_entry = None
@@ -232,6 +240,10 @@ def get_cleaned_dict(self, keep_outdated=False):
  entry_extras.extend(list(self.lockfile_entry.extras))
  self._entry.req.extras = entry_extras
  self.entry_dict["extras"] = self.entry.extras
+ if self.original_markers and not self.markers:
+ original_markers = self.marker_to_str(self.original_markers)
+ self.markers = original_markers
+ self.entry_dict["markers"] = self.marker_to_str(original_markers)
  entry_hashes = set(self.entry.hashes)
  locked_hashes = set(self.lockfile_entry.hashes)
  if entry_hashes != locked_hashes and not self.is_updated:
@@ -248,6 +260,10 @@ def lockfile_entry(self):
  self._lockfile_entry = self.make_requirement(self.name, self.lockfile_dict)
  return self._lockfile_entry
 
+ @lockfile_entry.setter
+ def lockfile_entry(self, entry):
+ self._lockfile_entry = entry
+
  @property
  def pipfile_entry(self):
  if self._pipfile_entry is None:
@@ -359,6 +375,7 @@ def updated_version(self):
 
  @property
  def updated_specifier(self):
+ # type: () -> str
  return self.entry.specifiers
 
  @property
@@ -373,7 +390,7 @@ def original_version(self):
  return None
 
  def validate_specifiers(self):
- if self.is_in_pipfile:
+ if self.is_in_pipfile and not self.pipfile_entry.editable:
  return self.pipfile_entry.requirement.specifier.contains(self.updated_version)
  return True
 
@@ -550,8 +567,11 @@ def validate_constraints(self):
  constraint.check_if_exists(False)
  except Exception:
  from pipenv.exceptions import DependencyConflict
+ from pipenv.environments import is_verbose
+ if is_verbose():
+ print("Tried constraint: {0!r}".format(constraint), file=sys.stderr)
  msg = (
- "Cannot resolve conflicting version {0}{1} while {1}{2} is "
+ "Cannot resolve conflicting version {0}{1} while {2}{3} is "
  "locked.".format(
  self.name, self.updated_specifier, self.old_name, self.old_specifiers
  )
@@ -624,6 +644,7 @@ def clean_results(results, resolver, project, dev=False):
 
 def clean_outdated(results, resolver, project, dev=False):
  from pipenv.vendor.requirementslib.models.requirements import Requirement
+ from pipenv.environments import is_verbose
  if not project.lockfile_exists:
  return results
  lockfile = project.lockfile_content

pipenv/utils.py

@@ -31,7 +31,9 @@
 import parse
 
 from . import environments
-from .exceptions import PipenvUsageError, ResolutionFailure, RequirementError, PipenvCmdError
+from .exceptions import (
+ PipenvUsageError, RequirementError, PipenvCmdError, ResolutionFailure
+)
 from .pep508checker import lookup
 from .vendor.urllib3 import util as urllib3_util
 
@@ -398,7 +400,6 @@ def parse_line(
  ):
  # type: (...) -> Tuple[Requirement, Dict[str, str], Dict[str, str]]
  from .vendor.requirementslib.models.requirements import Requirement
- from .exceptions import ResolutionFailure
  if index_lookup is None:
  index_lookup = {}
  if markers_lookup is None:
@@ -444,6 +445,7 @@ def get_deps_from_req(cls, req, resolver=None):
  # type: (Requirement, Optional["Resolver"]) -> Tuple[Set[str], Dict[str, Dict[str, Union[str, bool, List[str]]]]]
  from .vendor.requirementslib.models.utils import _requirement_to_str_lowercase_name
  from .vendor.requirementslib.models.requirements import Requirement
+ from requirementslib.utils import is_installable_dir
  constraints = set() # type: Set[str]
  locked_deps = dict() # type: Dict[str, Dict[str, Union[str, bool, List[str]]]]
  if (req.is_file_or_url or req.is_vcs) and not req.is_wheel:
@@ -463,7 +465,16 @@ def get_deps_from_req(cls, req, resolver=None):
  setup_info = req.req.setup_info
  setup_info.get_info()
  locked_deps[pep423_name(name)] = entry
- requirements = [v for v in getattr(setup_info, "requires", {}).values()]
+ requirements = []
+ # Allow users to toggle resolution off for non-editable VCS packages
+ # but leave it on for local, installable folders on the filesystem
+ if environments.PIPENV_RESOLVE_VCS or (
+ req.editable or parsed_line.is_wheel or (
+ req.is_file_or_url and parsed_line.is_local and
+ is_installable_dir(parsed_line.path)
+ )
+ ):
+ requirements = [v for v in getattr(setup_info, "requires", {}).values()]
  for r in requirements:
  if getattr(r, "url", None) and not getattr(r, "editable", False):
  if r is not None:
@@ -1797,13 +1808,15 @@ def clean_resolved_dep(dep, is_top_level=False, pipfile_entry=None):
 
  # If a package is **PRESENT** in the pipfile but has no markers, make sure we
  # **NEVER** include markers in the lockfile
- if "markers" in dep:
+ if "markers" in dep and dep.get("markers", "").strip():
  # First, handle the case where there is no top level dependency in the pipfile
  if not is_top_level:
- try:
- lockfile["markers"] = translate_markers(dep)["markers"]
- except TypeError:
- pass
+ translated = translate_markers(dep).get("markers", "").strip()
+ if translated:
+ try:
+ lockfile["markers"] = translated
+ except TypeError:
+ pass
  # otherwise make sure we are prioritizing whatever the pipfile says about the markers
  # If the pipfile says nothing, then we should put nothing in the lockfile
  else:

tasks/vendoring/patches/patched/piptools.patch

@@ -608,7 +608,7 @@ index 9b4b4c2..8875543 100644
 + all_candidates = []
 + py_version = parse_version(os.environ.get('PIP_PYTHON_VERSION', '.'.join(map(str, sys.version_info[:3]))))
 + for c in candidates:
-+ if c.requires_python:
++ if getattr(c, "requires_python", None):
 + # Old specifications had people setting this to single digits
 + # which is effectively the same as '>=digit,<digit+1'
 + if c.requires_python.isdigit():

tests/integration/test_lock.py

@@ -1,10 +1,14 @@
+# -*- coding: utf-8 -*-
+
+import json
 import os
 import sys
 
 import pytest
 
 from flaky import flaky
 from vistir.compat import Path
+from vistir.misc import to_text
 from pipenv.utils import temp_environ
 
 
@@ -122,6 +126,21 @@ def test_keep_outdated_doesnt_upgrade_pipfile_pins(PipenvInstance, pypi):
  assert p.lockfile["default"]["urllib3"]["version"] == "==1.21.1"
 
 
+def test_keep_outdated_keeps_markers_not_removed(PipenvInstance, pypi):
+ with PipenvInstance(chdir=True, pypi=pypi) as p:
+ c = p.pipenv("install tablib")
+ assert c.ok
+ lockfile = Path(p.lockfile_path)
+ lockfile_content = lockfile.read_text()
+ lockfile_json = json.loads(lockfile_content)
+ assert "tablib" in lockfile_json["default"]
+ lockfile_json["default"]["tablib"]["markers"] = "python_version >= '2.7'"
+ lockfile.write_text(to_text(json.dumps(lockfile_json)))
+ c = p.pipenv("lock --keep-outdated")
+ assert c.ok
+ assert p.lockfile["default"]["tablib"].get("markers", "") == "python_version >= '2.7'"
+
+
 @pytest.mark.lock
 @pytest.mark.keep_outdated
 def test_keep_outdated_doesnt_update_satisfied_constraints(PipenvInstance, pypi):