Update requirementslib and fix VCS installation

Signed-off-by: Dan Ryan <dan@danryan.co>

Don't re-clone repos now that this works

Signed-off-by: Dan Ryan <dan@danryan.co>

Prune peeps directory from manifest

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix nonetype uris

Signed-off-by: Dan Ryan <dan@danryan.co>

Manually lock requirements?

Signed-off-by: Dan Ryan <dan@danryan.co>

Update requirementslib - leave context before updating sha

Signed-off-by: Dan Ryan <dan@danryan.co>

Fix requirementslib vcs checkouts

Signed-off-by: Dan Ryan <dan@danryan.co>

fix tmpdir implementation

Signed-off-by: Dan Ryan <dan@danryan.co>

final fix for vcs uris

Signed-off-by: Dan Ryan <dan@danryan.co>

Allow for adding requirements objects directly to pipfile

Signed-off-by: Dan Ryan <dan@danryan.co>

Update piptools patch

Signed-off-by: Dan Ryan <dan@danryan.co>

MANIFEST.in

@@ -24,6 +24,8 @@ recursive-include docs/_templates *.html
 recursive-include docs/_static *.js *.css *.png
 recursive-exclude docs requirements*.txt
 
+
+prune peeps
 prune .buildkite
 prune .github
 prune .vsts-ci

pipenv/_compat.py

@@ -90,7 +90,7 @@ class TemporaryDirectory(object):
     in it are removed.
     """
 
-    def __init__(self, suffix=None, prefix=None, dir=None):
+    def __init__(self, suffix="", prefix="", dir=None):
         if "RAM_DISK" in os.environ:
             import uuid
 

pipenv/core.py

@@ -1908,7 +1908,7 @@ def do_install(
             )
             # Add the package to the Pipfile.
             try:
-                project.add_package_to_pipfile(pkg_requirement.as_line(), dev)
+                project.add_package_to_pipfile(pkg_requirement, dev)
             except ValueError as e:
                 click.echo(
                     "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e)

pipenv/project.py

@@ -770,13 +770,14 @@ def remove_package_from_pipfile(self, package_name, dev=False):
             del p[key][name]
             self.write_toml(p)
 
-    def add_package_to_pipfile(self, package_name, dev=False):
+    def add_package_to_pipfile(self, package, dev=False):
         from .vendor.requirementslib import Requirement
 
         # Read and append Pipfile.
         p = self.parsed_pipfile
         # Don't re-capitalize file URLs or VCSs.
-        package = Requirement.from_line(package_name.strip())
+        if not isinstance(package, Requirement):
+            package = Requirement.from_line(package.strip())
         _, converted = package.pipfile_entry
         key = "dev-packages" if dev else "packages"
         # Set empty group if it doesn't exist yet.

pipenv/utils.py

@@ -1146,6 +1146,7 @@ def get_vcs_deps(
     pypi_mirror=None,
 ):
     from ._compat import TemporaryDirectory, Path
+    import atexit
     from .vendor.requirementslib import Requirement
 
     section = "vcs_dev_packages" if dev else "vcs_packages"
@@ -1160,12 +1161,18 @@ def get_vcs_deps(
             os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src"))
         )
         src_dir.mkdir(mode=0o775, exist_ok=True)
+    else:
+        src_dir = TemporaryDirectory(prefix="pipenv-lock-dir")
+        atexit.register(src_dir.cleanup)
     for pkg_name, pkg_pipfile in packages.items():
         requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile)
         name = requirement.normalized_name
+        commit_hash = None
         if requirement.is_vcs:
-            lockfile[name] = requirement.pipfile_entry[1]
-            lockfile[name]['ref'] = requirement.req.repo.get_commit_hash()
+            with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo:
+                commit_hash = repo.get_commit_hash()
+                lockfile[name] = requirement.pipfile_entry[1]
+                lockfile[name]['ref'] = commit_hash
         reqs.append(requirement)
     return reqs, lockfile
 
@@ -1197,13 +1204,11 @@ def translate_markers(pipfile_entry):
     for m in pipfile_markers:
         entry = "{0}".format(pipfile_entry[m])
         if m != "markers":
-            marker_set.add(str(Marker("{0}'{1}'".format(m, entry))))
+            marker_set.add(str(Marker("{0}{1}".format(m, entry))))
             new_pipfile.pop(m)
-            marker_set.add(str(entry))
     if marker_set:
-        new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s)
-                                                            if " and " in s else s
-                                                            for s in sorted(dedup(marker_set))])))
+        new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s
+                                        for s in sorted(dedup(marker_set))]))).replace('"', "'")
     return new_pipfile
 
 

pipenv/vendor/requirementslib/models/requirements.py

@@ -1,46 +1,50 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
 
+import atexit
 import collections
 import hashlib
 import os
 
+from contextlib import contextmanager
+
 import attr
-import atexit
 
 from first import first
 from packaging.markers import Marker
+from packaging.requirements import Requirement as PackagingRequirement
 from packaging.specifiers import Specifier, SpecifierSet
 from packaging.utils import canonicalize_name
-from six.moves.urllib import parse as urllib_parse
-from six.moves.urllib.parse import unquote
-
 from pip_shims.shims import (
     InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url,
     url_to_path
 )
+from six.moves.urllib import parse as urllib_parse
+from six.moves.urllib.parse import unquote
 from vistir.compat import FileNotFoundError, Path, TemporaryDirectory
 from vistir.misc import dedup
-from vistir.path import get_converted_relative_path, is_valid_url, is_file_url, mkdir_p
+from vistir.path import (
+    create_tracked_tempdir, get_converted_relative_path, is_file_url,
+    is_valid_url, mkdir_p
+)
 
 from ..exceptions import RequirementError
-from ..utils import VCS_LIST, is_vcs, is_installable_file
+from ..utils import VCS_LIST, is_installable_file, is_vcs
 from .baserequirement import BaseRequirement
 from .dependencies import (
     AbstractDependency, find_all_matches, get_abstract_dependencies,
     get_dependencies, get_finder
 )
 from .markers import PipenvMarkers
 from .utils import (
-    HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, filter_none,
-    format_requirement, get_version, init_requirement,
+    HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string,
+    filter_none, format_requirement, get_version, init_requirement,
     is_pinned_requirement, make_install_requirement, optional_instance_of,
     parse_extras, specs_to_string, split_markers_from_line,
     split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path,
-    validate_specifiers, validate_vcs, extras_to_string
+    validate_specifiers, validate_vcs
 )
 from .vcs import VCSRepository
-from packaging.requirements import Requirement as PackagingRequirement
 
 
 @attr.s
@@ -481,6 +485,9 @@ class VCSRequirement(FileRequirement):
     req = attr.ib()
 
     def __attrs_post_init__(self):
+        if not self.uri:
+            if self.path:
+                self.uri = path_to_url(self.path)
         split = urllib_parse.urlsplit(self.uri)
         scheme, rest = split[0], split[1:]
         vcs_type = ""
@@ -493,9 +500,10 @@ def __attrs_post_init__(self):
 
     @link.default
     def get_link(self):
+        uri = self.uri if self.uri else path_to_url(self.path)
         return build_vcs_link(
             self.vcs,
-            add_ssh_scheme_to_git_uri(self.uri),
+            add_ssh_scheme_to_git_uri(uri),
             name=self.name,
             ref=self.ref,
             subdirectory=self.subdirectory,
@@ -564,21 +572,17 @@ def repo(self):
     def get_checkout_dir(self, src_dir=None):
         src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir
         checkout_dir = None
-        if self.is_local and self.editable:
+        if self.is_local:
             path = self.path
             if not path:
                 path = url_to_path(self.uri)
             if path and os.path.exists(path):
-                checkout_dir = Path(self.path).absolute().as_posix()
+                checkout_dir = os.path.abspath(path)
                 return checkout_dir
-        return src_dir
+        return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name)
 
     def get_vcs_repo(self, src_dir=None):
         checkout_dir = self.get_checkout_dir(src_dir=src_dir)
-        if not checkout_dir:
-            _src_dir = TemporaryDirectory()
-            atexit.register(_src_dir.cleanup)
-            checkout_dir = Path(_src_dir.name).joinpath(self.name).absolute().as_posix()
         url = "{0}#egg={1}".format(self.vcs_uri, self.name)
         vcsrepo = VCSRepository(
             url=url,
@@ -587,25 +591,36 @@ def get_vcs_repo(self, src_dir=None):
             checkout_directory=checkout_dir,
             vcs_type=self.vcs
         )
-        if not (self.is_local and self.editable):
+        if not self.is_local:
             vcsrepo.obtain()
         return vcsrepo
 
     def get_commit_hash(self):
+        hash_ = None
         hash_ = self.repo.get_commit_hash()
         return hash_
 
     def update_repo(self, src_dir=None, ref=None):
-        ref = self.ref if not ref else ref
-        if not (self.is_local and self.editable):
-            self.repo.update()
+        if ref:
+            self.ref = ref
+        else:
+            if self.ref:
+                ref = self.ref
+        repo_hash = None
+        if not self.is_local and ref is not None:
             self.repo.checkout_ref(ref)
-        hash_ = self.repo.get_commit_hash()
-        return hash_
-
-    def lock_vcs_ref(self):
+        repo_hash = self.repo.get_commit_hash()
+        return repo_hash
+
+    @contextmanager
+    def locked_vcs_repo(self, src_dir=None):
+        vcsrepo = self.get_vcs_repo(src_dir=src_dir)
+        if self.ref and not self.is_local:
+            vcsrepo.checkout_ref(self.ref)
         self.ref = self.get_commit_hash()
         self.req.revision = self.ref
+        yield vcsrepo
+        self._repo = vcsrepo
 
     @classmethod
     def from_pipfile(cls, name, pipfile):
@@ -627,12 +642,17 @@ def from_pipfile(cls, name, pipfile):
                     "{0}+{1}".format(key, pipfile.get(key))
                 ).split("+", 1)[1]
                 url_keys = [pipfile.get(key), composed_uri]
-                is_url = any(validity_fn(url_key) for url_key in url_keys for validity_fn in [is_valid_url, is_file_url])
-                target_key = "uri" if is_url else "path"
-                creation_args[target_key] = pipfile.get(key)
+                if any(is_valid_url(k) for k in url_keys) or any(is_file_url(k) for k in url_keys):
+                    creation_args["uri"] = pipfile.get(key)
+                else:
+                    creation_args["path"] = pipfile.get(key)
+                    if os.path.isabs(pipfile.get(key)):
+                        creation_args["uri"] = Path(pipfile.get(key)).absolute().as_uri()
             else:
                 creation_args[key] = pipfile.get(key)
         creation_args["name"] = name
+        print("Creating req from pipfile: %s" % pipfile)
+        print("Using creation args: %s" % creation_args)
         return cls(**creation_args)
 
     @classmethod
@@ -669,7 +689,13 @@ def from_line(cls, line, editable=None, extras=None):
     @property
     def line_part(self):
         """requirements.txt compatible line part sans-extras"""
-        if self.req:
+        if self.is_local:
+            base_link = self.link
+            if not self.link:
+                base_link = self.get_link()
+            final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}"
+            base = final_format.format(self.vcs_uri)
+        elif self.req:
             base = self.req.line
         if base and self.extras and not extras_to_string(self.extras) in base:
             if self.subdirectory:
@@ -694,7 +720,7 @@ def _choose_vcs_source(pipfile):
 
     @property
     def pipfile_part(self):
-        pipfile_dict = attr.asdict(self, filter=lambda k, v: v is not None and k.name != '_repo').copy()
+        pipfile_dict = attr.asdict(self, filter=lambda k, v: bool(v) is True and k.name != '_repo').copy()
         if "vcs" in pipfile_dict:
             pipfile_dict = self._choose_vcs_source(pipfile_dict)
         name, _ = _strip_extras(pipfile_dict.pop("name"))
@@ -752,7 +778,10 @@ def extras_as_pip(self):
     def commit_hash(self):
         if not self.is_vcs:
             return None
-        return self.req.get_commit_hash()
+        commit_hash = None
+        with self.req.locked_vcs_repo() as repo:
+            commit_hash = repo.get_commit_hash()
+        return commit_hash
 
     @specifiers.default
     def get_specifiers(self):

pipenv/vendor/requirementslib/models/vcs.py

@@ -23,34 +23,43 @@ def get_repo_instance(self):
         backend = VCS_SUPPORT._registry.get(self.vcs_type)
         return backend(url=self.url)
 
+    @property
+    def is_local(self):
+        url = self.url
+        if '+' in url:
+            url = url.split('+')[1]
+        return url.startswith("file")
+
     def obtain(self):
         if not os.path.exists(self.checkout_directory):
             self.repo_instance.obtain(self.checkout_directory)
         if self.ref:
-            with vistir.contextmanagers.cd(self.checkout_directory):
-                self.update(self.ref)
-            self.commit_sha = self.get_commit_hash()
+            self.checkout_ref(self.ref)
+            self.commit_sha = self.get_commit_hash(self.ref)
         else:
             if not self.commit_sha:
                 self.commit_sha = self.get_commit_hash()
 
     def checkout_ref(self, ref):
-        target_rev = self.repo_instance.make_rev_options(ref)
         if not self.repo_instance.is_commit_id_equal(
             self.checkout_directory, self.get_commit_hash(ref)
         ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref):
-            self.repo_instance.update(self.checkout_directory, self.url, target_rev)
-        self.commit_hash = self.get_commit_hash()
+            if not self.is_local:
+                self.update(ref)
 
     def update(self, ref):
-        target_rev = self.repo_instance.make_rev_options(ref)
+        target_ref = self.repo_instance.make_rev_options(ref)
+        sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev)
+        target_rev = target_ref.make_new(sha)
         if parse_version(pip_version) > parse_version("18.0"):
             self.repo_instance.update(self.checkout_directory, self.url, target_rev)
         else:
-            self.repo_instance.update(self.checkout_directory, target_rev)
-        self.commit_hash = self.get_commit_hash()
+            self.repo_instance.update(self.checkout_directory, target_ref)
+        self.commit_hash = self.get_commit_hash(ref)
 
     def get_commit_hash(self, ref=None):
         if ref:
-            return self.repo_instance.get_revision(self.checkout_directory)
+            target_ref = self.repo_instance.make_rev_options(ref)
+            return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev)
+            # return self.repo_instance.get_revision(self.checkout_directory)
         return self.repo_instance.get_revision(self.checkout_directory)

tasks/vendoring/patches/patched/piptools.patch

@@ -717,7 +717,7 @@ index 7e8cdf3..0a0d27d 100644
 @@ -1,30 +1,42 @@
  # -*- coding=utf-8 -*-
  import importlib
-
+ 
 -def do_import(module_path, subimport=None, old_path=None):
 +
 +def do_import(module_path, subimport=None, old_path=None, vendored_name=None):
@@ -744,8 +744,8 @@ index 7e8cdf3..0a0d27d 100644
      if subimport:
          return getattr(_tmp, subimport, _tmp)
      return _tmp
--
-
+-    
+ 
 -InstallRequirement = do_import('req.req_install', 'InstallRequirement')
 -parse_requirements = do_import('req.req_file', 'parse_requirements')
 -RequirementSet = do_import('req.req_set', 'RequirementSet')

tests/integration/test_install_uri.py

@@ -97,16 +97,18 @@ def test_file_urls_work(PipenvInstance, pip_src_dir):
 @pytest.mark.files
 @pytest.mark.urls
 @pytest.mark.needs_internet
-def test_local_vcs_urls_work(PipenvInstance, pypi):
+def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir):
+    six_dir = tmpdir.join("six")
+    six_path = Path(six_dir.strpath)
     with PipenvInstance(pypi=pypi, chdir=True) as p:
-        six_path = Path(p.path).joinpath("six").absolute()
         c = delegator.run(
-            "git clone " "https://github.com/benjaminp/six.git {0}".format(six_path)
+            "git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath)
         )
         assert c.return_code == 0
 
         c = p.pipenv("install git+{0}#egg=six".format(six_path.as_uri()))
         assert c.return_code == 0
+        assert "six" in p.pipfile["packages"]
 
 
 @pytest.mark.e