Update requirementslib and fix VCS installation

Signed-off-by: Dan Ryan <dan@danryan.co>

Don't re-clone repos now that this works

Signed-off-by: Dan Ryan <dan@danryan.co>

Prune peeps directory from manifest

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix nonetype uris

Signed-off-by: Dan Ryan <dan@danryan.co>

Manually lock requirements?

Signed-off-by: Dan Ryan <dan@danryan.co>

Update requirementslib - leave context before updating sha

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix requirementslib vcs checkouts

Signed-off-by: Dan Ryan <dan@danryan.co>

fix tmpdir implementation

Signed-off-by: Dan Ryan <dan@danryan.co>

final fix for vcs uris

Signed-off-by: Dan Ryan <dan@danryan.co>

Allow for adding requirements objects directly to pipfile

Signed-off-by: Dan Ryan <dan@danryan.co>

Update piptools patch

Signed-off-by: Dan Ryan <dan@danryan.co>

MANIFEST.in

@@ -24,6 +24,8 @@ recursive-include docs/_templates *.html
 recursive-include docs/_static *.js *.css *.png
 recursive-exclude docs requirements*.txt
 
+
+prune peeps
 prune .buildkite
 prune .github
 prune .vsts-ci

pipenv/_compat.py

@@ -90,7 +90,7 @@ class TemporaryDirectory(object):
  in it are removed.
  """
 
- def __init__(self, suffix=None, prefix=None, dir=None):
+ def __init__(self, suffix="", prefix="", dir=None):
  if "RAM_DISK" in os.environ:
  import uuid
 

pipenv/core.py

@@ -1908,7 +1908,7 @@ def do_install(
  )
  # Add the package to the Pipfile.
  try:
- project.add_package_to_pipfile(pkg_requirement.as_line(), dev)
+ project.add_package_to_pipfile(pkg_requirement, dev)
  except ValueError as e:
  click.echo(
  "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e)

pipenv/project.py

@@ -770,13 +770,14 @@ def remove_package_from_pipfile(self, package_name, dev=False):
  del p[key][name]
  self.write_toml(p)
 
- def add_package_to_pipfile(self, package_name, dev=False):
+ def add_package_to_pipfile(self, package, dev=False):
  from .vendor.requirementslib import Requirement
 
  # Read and append Pipfile.
  p = self.parsed_pipfile
  # Don't re-capitalize file URLs or VCSs.
- package = Requirement.from_line(package_name.strip())
+ if not isinstance(package, Requirement):
+ package = Requirement.from_line(package.strip())
  _, converted = package.pipfile_entry
  key = "dev-packages" if dev else "packages"
  # Set empty group if it doesn't exist yet.

pipenv/utils.py

@@ -1146,6 +1146,7 @@ def get_vcs_deps(
  pypi_mirror=None,
 ):
  from ._compat import TemporaryDirectory, Path
+ import atexit
  from .vendor.requirementslib import Requirement
 
  section = "vcs_dev_packages" if dev else "vcs_packages"
@@ -1160,12 +1161,18 @@ def get_vcs_deps(
  os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src"))
  )
  src_dir.mkdir(mode=0o775, exist_ok=True)
+ else:
+ src_dir = TemporaryDirectory(prefix="pipenv-lock-dir")
+ atexit.register(src_dir.cleanup)
  for pkg_name, pkg_pipfile in packages.items():
  requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile)
  name = requirement.normalized_name
+ commit_hash = None
  if requirement.is_vcs:
- lockfile[name] = requirement.pipfile_entry[1]
- lockfile[name]['ref'] = requirement.req.repo.get_commit_hash()
+ with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo:
+ commit_hash = repo.get_commit_hash()
+ lockfile[name] = requirement.pipfile_entry[1]
+ lockfile[name]['ref'] = commit_hash
  reqs.append(requirement)
  return reqs, lockfile
 
@@ -1197,13 +1204,11 @@ def translate_markers(pipfile_entry):
  for m in pipfile_markers:
  entry = "{0}".format(pipfile_entry[m])
  if m != "markers":
- marker_set.add(str(Marker("{0}'{1}'".format(m, entry))))
+ marker_set.add(str(Marker("{0}{1}".format(m, entry))))
  new_pipfile.pop(m)
- marker_set.add(str(entry))
  if marker_set:
- new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s)
- if " and " in s else s
- for s in sorted(dedup(marker_set))])))
+ new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s
+ for s in sorted(dedup(marker_set))]))).replace('"', "'")
  return new_pipfile
 
 

pipenv/vendor/requirementslib/models/requirements.py

@@ -1,46 +1,50 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+import atexit
 import collections
 import hashlib
 import os
 
+from contextlib import contextmanager
+
 import attr
-import atexit
 
 from first import first
 from packaging.markers import Marker
+from packaging.requirements import Requirement as PackagingRequirement
 from packaging.specifiers import Specifier, SpecifierSet
 from packaging.utils import canonicalize_name
-from six.moves.urllib import parse as urllib_parse
-from six.moves.urllib.parse import unquote
-
 from pip_shims.shims import (
  InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url,
  url_to_path
 )
+from six.moves.urllib import parse as urllib_parse
+from six.moves.urllib.parse import unquote
 from vistir.compat import FileNotFoundError, Path, TemporaryDirectory
 from vistir.misc import dedup
-from vistir.path import get_converted_relative_path, is_valid_url, is_file_url, mkdir_p
+from vistir.path import (
+ create_tracked_tempdir, get_converted_relative_path, is_file_url,
+ is_valid_url, mkdir_p
+)
 
 from ..exceptions import RequirementError
-from ..utils import VCS_LIST, is_vcs, is_installable_file
+from ..utils import VCS_LIST, is_installable_file, is_vcs
 from .baserequirement import BaseRequirement
 from .dependencies import (
  AbstractDependency, find_all_matches, get_abstract_dependencies,
  get_dependencies, get_finder
 )
 from .markers import PipenvMarkers
 from .utils import (
- HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, filter_none,
- format_requirement, get_version, init_requirement,
+ HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string,
+ filter_none, format_requirement, get_version, init_requirement,
  is_pinned_requirement, make_install_requirement, optional_instance_of,
  parse_extras, specs_to_string, split_markers_from_line,
  split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path,
- validate_specifiers, validate_vcs, extras_to_string
+ validate_specifiers, validate_vcs
 )
 from .vcs import VCSRepository
-from packaging.requirements import Requirement as PackagingRequirement
 
 
 @attr.s
@@ -481,6 +485,9 @@ class VCSRequirement(FileRequirement):
  req = attr.ib()
 
  def __attrs_post_init__(self):
+ if not self.uri:
+ if self.path:
+ self.uri = path_to_url(self.path)
  split = urllib_parse.urlsplit(self.uri)
  scheme, rest = split[0], split[1:]
  vcs_type = ""
@@ -493,9 +500,10 @@ def __attrs_post_init__(self):
 
  @link.default
  def get_link(self):
+ uri = self.uri if self.uri else path_to_url(self.path)
  return build_vcs_link(
  self.vcs,
- add_ssh_scheme_to_git_uri(self.uri),
+ add_ssh_scheme_to_git_uri(uri),
  name=self.name,
  ref=self.ref,
  subdirectory=self.subdirectory,
@@ -564,21 +572,17 @@ def repo(self):
  def get_checkout_dir(self, src_dir=None):
  src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir
  checkout_dir = None
- if self.is_local and self.editable:
+ if self.is_local:
  path = self.path
  if not path:
  path = url_to_path(self.uri)
  if path and os.path.exists(path):
- checkout_dir = Path(self.path).absolute().as_posix()
+ checkout_dir = os.path.abspath(path)
  return checkout_dir
- return src_dir
+ return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name)
 
  def get_vcs_repo(self, src_dir=None):
  checkout_dir = self.get_checkout_dir(src_dir=src_dir)
- if not checkout_dir:
- _src_dir = TemporaryDirectory()
- atexit.register(_src_dir.cleanup)
- checkout_dir = Path(_src_dir.name).joinpath(self.name).absolute().as_posix()
  url = "{0}#egg={1}".format(self.vcs_uri, self.name)
  vcsrepo = VCSRepository(
  url=url,
@@ -587,25 +591,36 @@ def get_vcs_repo(self, src_dir=None):
  checkout_directory=checkout_dir,
  vcs_type=self.vcs
  )
- if not (self.is_local and self.editable):
+ if not self.is_local:
  vcsrepo.obtain()
  return vcsrepo
 
  def get_commit_hash(self):
+ hash_ = None
  hash_ = self.repo.get_commit_hash()
  return hash_
 
  def update_repo(self, src_dir=None, ref=None):
- ref = self.ref if not ref else ref
- if not (self.is_local and self.editable):
- self.repo.update()
+ if ref:
+ self.ref = ref
+ else:
+ if self.ref:
+ ref = self.ref
+ repo_hash = None
+ if not self.is_local and ref is not None:
  self.repo.checkout_ref(ref)
- hash_ = self.repo.get_commit_hash()
- return hash_
-
- def lock_vcs_ref(self):
+ repo_hash = self.repo.get_commit_hash()
+ return repo_hash
+
+ @contextmanager
+ def locked_vcs_repo(self, src_dir=None):
+ vcsrepo = self.get_vcs_repo(src_dir=src_dir)
+ if self.ref and not self.is_local:
+ vcsrepo.checkout_ref(self.ref)
  self.ref = self.get_commit_hash()
  self.req.revision = self.ref
+ yield vcsrepo
+ self._repo = vcsrepo
 
  @classmethod
  def from_pipfile(cls, name, pipfile):
@@ -627,12 +642,17 @@ def from_pipfile(cls, name, pipfile):
  "{0}+{1}".format(key, pipfile.get(key))
  ).split("+", 1)[1]
  url_keys = [pipfile.get(key), composed_uri]
- is_url = any(validity_fn(url_key) for url_key in url_keys for validity_fn in [is_valid_url, is_file_url])
- target_key = "uri" if is_url else "path"
- creation_args[target_key] = pipfile.get(key)
+ if any(is_valid_url(k) for k in url_keys) or any(is_file_url(k) for k in url_keys):
+ creation_args["uri"] = pipfile.get(key)
+ else:
+ creation_args["path"] = pipfile.get(key)
+ if os.path.isabs(pipfile.get(key)):
+ creation_args["uri"] = Path(pipfile.get(key)).absolute().as_uri()
  else:
  creation_args[key] = pipfile.get(key)
  creation_args["name"] = name
+ print("Creating req from pipfile: %s" % pipfile)
+ print("Using creation args: %s" % creation_args)
  return cls(**creation_args)
 
  @classmethod
@@ -669,7 +689,13 @@ def from_line(cls, line, editable=None, extras=None):
  @property
  def line_part(self):
  """requirements.txt compatible line part sans-extras"""
- if self.req:
+ if self.is_local:
+ base_link = self.link
+ if not self.link:
+ base_link = self.get_link()
+ final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}"
+ base = final_format.format(self.vcs_uri)
+ elif self.req:
  base = self.req.line
  if base and self.extras and not extras_to_string(self.extras) in base:
  if self.subdirectory:
@@ -694,7 +720,7 @@ def _choose_vcs_source(pipfile):
 
  @property
  def pipfile_part(self):
- pipfile_dict = attr.asdict(self, filter=lambda k, v: v is not None and k.name != '_repo').copy()
+ pipfile_dict = attr.asdict(self, filter=lambda k, v: bool(v) is True and k.name != '_repo').copy()
  if "vcs" in pipfile_dict:
  pipfile_dict = self._choose_vcs_source(pipfile_dict)
  name, _ = _strip_extras(pipfile_dict.pop("name"))
@@ -752,7 +778,10 @@ def extras_as_pip(self):
  def commit_hash(self):
  if not self.is_vcs:
  return None
- return self.req.get_commit_hash()
+ commit_hash = None
+ with self.req.locked_vcs_repo() as repo:
+ commit_hash = repo.get_commit_hash()
+ return commit_hash
 
  @specifiers.default
  def get_specifiers(self):

pipenv/vendor/requirementslib/models/vcs.py

@@ -23,34 +23,43 @@ def get_repo_instance(self):
  backend = VCS_SUPPORT._registry.get(self.vcs_type)
  return backend(url=self.url)
 
+ @property
+ def is_local(self):
+ url = self.url
+ if '+' in url:
+ url = url.split('+')[1]
+ return url.startswith("file")
+
  def obtain(self):
  if not os.path.exists(self.checkout_directory):
  self.repo_instance.obtain(self.checkout_directory)
  if self.ref:
- with vistir.contextmanagers.cd(self.checkout_directory):
- self.update(self.ref)
- self.commit_sha = self.get_commit_hash()
+ self.checkout_ref(self.ref)
+ self.commit_sha = self.get_commit_hash(self.ref)
  else:
  if not self.commit_sha:
  self.commit_sha = self.get_commit_hash()
 
  def checkout_ref(self, ref):
- target_rev = self.repo_instance.make_rev_options(ref)
  if not self.repo_instance.is_commit_id_equal(
  self.checkout_directory, self.get_commit_hash(ref)
  ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref):
- self.repo_instance.update(self.checkout_directory, self.url, target_rev)
- self.commit_hash = self.get_commit_hash()
+ if not self.is_local:
+  self.update(ref)
 
  def update(self, ref):
- target_rev = self.repo_instance.make_rev_options(ref)
+ target_ref = self.repo_instance.make_rev_options(ref)
+ sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev)
+ target_rev = target_ref.make_new(sha)
  if parse_version(pip_version) > parse_version("18.0"):
  self.repo_instance.update(self.checkout_directory, self.url, target_rev)
  else:
- self.repo_instance.update(self.checkout_directory, target_rev)
- self.commit_hash = self.get_commit_hash()
+ self.repo_instance.update(self.checkout_directory, target_ref)
+ self.commit_hash = self.get_commit_hash(ref)
 
  def get_commit_hash(self, ref=None):
  if ref:
- return self.repo_instance.get_revision(self.checkout_directory)
+ target_ref = self.repo_instance.make_rev_options(ref)
+ return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev)
+ # return self.repo_instance.get_revision(self.checkout_directory)
  return self.repo_instance.get_revision(self.checkout_directory)

tasks/vendoring/patches/patched/piptools.patch

@@ -717,7 +717,7 @@ index 7e8cdf3..0a0d27d 100644
 @@ -1,30 +1,42 @@
  # -*- coding=utf-8 -*-
  import importlib
-
+ 
 -def do_import(module_path, subimport=None, old_path=None):
 +
 +def do_import(module_path, subimport=None, old_path=None, vendored_name=None):
@@ -744,8 +744,8 @@ index 7e8cdf3..0a0d27d 100644
  if subimport:
  return getattr(_tmp, subimport, _tmp)
  return _tmp
--
-
+- 
+ 
 -InstallRequirement = do_import('req.req_install', 'InstallRequirement')
 -parse_requirements = do_import('req.req_file', 'parse_requirements')
 -RequirementSet = do_import('req.req_set', 'RequirementSet')

tests/integration/test_install_uri.py

@@ -97,16 +97,18 @@ def test_file_urls_work(PipenvInstance, pip_src_dir):
 @pytest.mark.files
 @pytest.mark.urls
 @pytest.mark.needs_internet
-def test_local_vcs_urls_work(PipenvInstance, pypi):
+def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir):
+ six_dir = tmpdir.join("six")
+ six_path = Path(six_dir.strpath)
  with PipenvInstance(pypi=pypi, chdir=True) as p:
- six_path = Path(p.path).joinpath("six").absolute()
  c = delegator.run(
- "git clone " "https://github.com/benjaminp/six.git {0}".format(six_path)
+ "git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath)
  )
  assert c.return_code == 0
 
  c = p.pipenv("install git+{0}#egg=six".format(six_path.as_uri()))
  assert c.return_code == 0
+ assert "six" in p.pipfile["packages"]
 
 
 @pytest.mark.e