From b33dfa65667ed3291608575344b20cdf9b2ff47d Mon Sep 17 00:00:00 2001 From: Dan Ryan Date: Mon, 3 Sep 2018 19:03:57 -0400 Subject: [PATCH] Update requirementslib and fix VCS installation Signed-off-by: Dan Ryan Don't re-clone repos now that this works Signed-off-by: Dan Ryan Prune peeps directory from manifest Signed-off-by: Dan Ryan Fix nonetype uris Signed-off-by: Dan Ryan Manually lock requirements? Signed-off-by: Dan Ryan Update requirementslib - leave context before updating sha Signed-off-by: Dan Ryan Fix requirementslib vcs checkouts Signed-off-by: Dan Ryan fix tmpdir implementation Signed-off-by: Dan Ryan final fix for vcs uris Signed-off-by: Dan Ryan Allow for adding requirements objects directly to pipfile Signed-off-by: Dan Ryan Update piptools patch Signed-off-by: Dan Ryan --- MANIFEST.in | 2 + pipenv/_compat.py | 2 +- pipenv/core.py | 2 +- pipenv/project.py | 5 +- pipenv/utils.py | 19 ++-- .../requirementslib/models/requirements.py | 93 ++++++++++++------- pipenv/vendor/requirementslib/models/vcs.py | 29 ++++-- .../vendoring/patches/patched/piptools.patch | 6 +- tests/integration/test_install_uri.py | 8 +- 9 files changed, 107 insertions(+), 59 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index f593cf781c..e5cc05f2b1 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -24,6 +24,8 @@ recursive-include docs/_templates *.html recursive-include docs/_static *.js *.css *.png recursive-exclude docs requirements*.txt + +prune peeps prune .buildkite prune .github prune .vsts-ci diff --git a/pipenv/_compat.py b/pipenv/_compat.py index b1861e4d21..558df3b846 100644 --- a/pipenv/_compat.py +++ b/pipenv/_compat.py @@ -90,7 +90,7 @@ class TemporaryDirectory(object): in it are removed. """ - def __init__(self, suffix=None, prefix=None, dir=None): + def __init__(self, suffix="", prefix="", dir=None): if "RAM_DISK" in os.environ: import uuid diff --git a/pipenv/core.py b/pipenv/core.py index 1f11ff7739..52c127b10a 100644 --- a/pipenv/core.py +++ b/pipenv/core.py @@ -1908,7 +1908,7 @@ def do_install( ) # Add the package to the Pipfile. try: - project.add_package_to_pipfile(pkg_requirement.as_line(), dev) + project.add_package_to_pipfile(pkg_requirement, dev) except ValueError as e: click.echo( "{0} {1}".format(crayons.red("ERROR (PACKAGE NOT INSTALLED):"), e) diff --git a/pipenv/project.py b/pipenv/project.py index ff1f5364a5..f9decbf641 100644 --- a/pipenv/project.py +++ b/pipenv/project.py @@ -770,13 +770,14 @@ def remove_package_from_pipfile(self, package_name, dev=False): del p[key][name] self.write_toml(p) - def add_package_to_pipfile(self, package_name, dev=False): + def add_package_to_pipfile(self, package, dev=False): from .vendor.requirementslib import Requirement # Read and append Pipfile. p = self.parsed_pipfile # Don't re-capitalize file URLs or VCSs. - package = Requirement.from_line(package_name.strip()) + if not isinstance(package, Requirement): + package = Requirement.from_line(package.strip()) _, converted = package.pipfile_entry key = "dev-packages" if dev else "packages" # Set empty group if it doesn't exist yet. diff --git a/pipenv/utils.py b/pipenv/utils.py index 9d02f92be7..b7295c2921 100644 --- a/pipenv/utils.py +++ b/pipenv/utils.py @@ -1146,6 +1146,7 @@ def get_vcs_deps( pypi_mirror=None, ): from ._compat import TemporaryDirectory, Path + import atexit from .vendor.requirementslib import Requirement section = "vcs_dev_packages" if dev else "vcs_packages" @@ -1160,12 +1161,18 @@ def get_vcs_deps( os.environ.get("PIP_SRC", os.path.join(project.virtualenv_location, "src")) ) src_dir.mkdir(mode=0o775, exist_ok=True) + else: + src_dir = TemporaryDirectory(prefix="pipenv-lock-dir") + atexit.register(src_dir.cleanup) for pkg_name, pkg_pipfile in packages.items(): requirement = Requirement.from_pipfile(pkg_name, pkg_pipfile) name = requirement.normalized_name + commit_hash = None if requirement.is_vcs: - lockfile[name] = requirement.pipfile_entry[1] - lockfile[name]['ref'] = requirement.req.repo.get_commit_hash() + with requirement.req.locked_vcs_repo(src_dir=src_dir) as repo: + commit_hash = repo.get_commit_hash() + lockfile[name] = requirement.pipfile_entry[1] + lockfile[name]['ref'] = commit_hash reqs.append(requirement) return reqs, lockfile @@ -1197,13 +1204,11 @@ def translate_markers(pipfile_entry): for m in pipfile_markers: entry = "{0}".format(pipfile_entry[m]) if m != "markers": - marker_set.add(str(Marker("{0}'{1}'".format(m, entry)))) + marker_set.add(str(Marker("{0}{1}".format(m, entry)))) new_pipfile.pop(m) - marker_set.add(str(entry)) if marker_set: - new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) - if " and " in s else s - for s in sorted(dedup(marker_set))]))) + new_pipfile["markers"] = str(Marker(" or ".join(["{0}".format(s) if " and " in s else s + for s in sorted(dedup(marker_set))]))).replace('"', "'") return new_pipfile diff --git a/pipenv/vendor/requirementslib/models/requirements.py b/pipenv/vendor/requirementslib/models/requirements.py index 0672d6db4e..471ebcea14 100644 --- a/pipenv/vendor/requirementslib/models/requirements.py +++ b/pipenv/vendor/requirementslib/models/requirements.py @@ -1,30 +1,35 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import +import atexit import collections import hashlib import os +from contextlib import contextmanager + import attr -import atexit from first import first from packaging.markers import Marker +from packaging.requirements import Requirement as PackagingRequirement from packaging.specifiers import Specifier, SpecifierSet from packaging.utils import canonicalize_name -from six.moves.urllib import parse as urllib_parse -from six.moves.urllib.parse import unquote - from pip_shims.shims import ( InstallRequirement, Link, Wheel, _strip_extras, parse_version, path_to_url, url_to_path ) +from six.moves.urllib import parse as urllib_parse +from six.moves.urllib.parse import unquote from vistir.compat import FileNotFoundError, Path, TemporaryDirectory from vistir.misc import dedup -from vistir.path import get_converted_relative_path, is_valid_url, is_file_url, mkdir_p +from vistir.path import ( + create_tracked_tempdir, get_converted_relative_path, is_file_url, + is_valid_url, mkdir_p +) from ..exceptions import RequirementError -from ..utils import VCS_LIST, is_vcs, is_installable_file +from ..utils import VCS_LIST, is_installable_file, is_vcs from .baserequirement import BaseRequirement from .dependencies import ( AbstractDependency, find_all_matches, get_abstract_dependencies, @@ -32,15 +37,14 @@ ) from .markers import PipenvMarkers from .utils import ( - HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, filter_none, - format_requirement, get_version, init_requirement, + HASH_STRING, add_ssh_scheme_to_git_uri, build_vcs_link, extras_to_string, + filter_none, format_requirement, get_version, init_requirement, is_pinned_requirement, make_install_requirement, optional_instance_of, parse_extras, specs_to_string, split_markers_from_line, split_vcs_method_from_uri, strip_ssh_from_git_uri, validate_path, - validate_specifiers, validate_vcs, extras_to_string + validate_specifiers, validate_vcs ) from .vcs import VCSRepository -from packaging.requirements import Requirement as PackagingRequirement @attr.s @@ -481,6 +485,9 @@ class VCSRequirement(FileRequirement): req = attr.ib() def __attrs_post_init__(self): + if not self.uri: + if self.path: + self.uri = path_to_url(self.path) split = urllib_parse.urlsplit(self.uri) scheme, rest = split[0], split[1:] vcs_type = "" @@ -493,9 +500,10 @@ def __attrs_post_init__(self): @link.default def get_link(self): + uri = self.uri if self.uri else path_to_url(self.path) return build_vcs_link( self.vcs, - add_ssh_scheme_to_git_uri(self.uri), + add_ssh_scheme_to_git_uri(uri), name=self.name, ref=self.ref, subdirectory=self.subdirectory, @@ -564,21 +572,17 @@ def repo(self): def get_checkout_dir(self, src_dir=None): src_dir = os.environ.get('PIP_SRC', None) if not src_dir else src_dir checkout_dir = None - if self.is_local and self.editable: + if self.is_local: path = self.path if not path: path = url_to_path(self.uri) if path and os.path.exists(path): - checkout_dir = Path(self.path).absolute().as_posix() + checkout_dir = os.path.abspath(path) return checkout_dir - return src_dir + return os.path.join(create_tracked_tempdir(prefix="requirementslib"), self.name) def get_vcs_repo(self, src_dir=None): checkout_dir = self.get_checkout_dir(src_dir=src_dir) - if not checkout_dir: - _src_dir = TemporaryDirectory() - atexit.register(_src_dir.cleanup) - checkout_dir = Path(_src_dir.name).joinpath(self.name).absolute().as_posix() url = "{0}#egg={1}".format(self.vcs_uri, self.name) vcsrepo = VCSRepository( url=url, @@ -587,25 +591,36 @@ def get_vcs_repo(self, src_dir=None): checkout_directory=checkout_dir, vcs_type=self.vcs ) - if not (self.is_local and self.editable): + if not self.is_local: vcsrepo.obtain() return vcsrepo def get_commit_hash(self): + hash_ = None hash_ = self.repo.get_commit_hash() return hash_ def update_repo(self, src_dir=None, ref=None): - ref = self.ref if not ref else ref - if not (self.is_local and self.editable): - self.repo.update() + if ref: + self.ref = ref + else: + if self.ref: + ref = self.ref + repo_hash = None + if not self.is_local and ref is not None: self.repo.checkout_ref(ref) - hash_ = self.repo.get_commit_hash() - return hash_ - - def lock_vcs_ref(self): + repo_hash = self.repo.get_commit_hash() + return repo_hash + + @contextmanager + def locked_vcs_repo(self, src_dir=None): + vcsrepo = self.get_vcs_repo(src_dir=src_dir) + if self.ref and not self.is_local: + vcsrepo.checkout_ref(self.ref) self.ref = self.get_commit_hash() self.req.revision = self.ref + yield vcsrepo + self._repo = vcsrepo @classmethod def from_pipfile(cls, name, pipfile): @@ -627,12 +642,17 @@ def from_pipfile(cls, name, pipfile): "{0}+{1}".format(key, pipfile.get(key)) ).split("+", 1)[1] url_keys = [pipfile.get(key), composed_uri] - is_url = any(validity_fn(url_key) for url_key in url_keys for validity_fn in [is_valid_url, is_file_url]) - target_key = "uri" if is_url else "path" - creation_args[target_key] = pipfile.get(key) + if any(is_valid_url(k) for k in url_keys) or any(is_file_url(k) for k in url_keys): + creation_args["uri"] = pipfile.get(key) + else: + creation_args["path"] = pipfile.get(key) + if os.path.isabs(pipfile.get(key)): + creation_args["uri"] = Path(pipfile.get(key)).absolute().as_uri() else: creation_args[key] = pipfile.get(key) creation_args["name"] = name + print("Creating req from pipfile: %s" % pipfile) + print("Using creation args: %s" % creation_args) return cls(**creation_args) @classmethod @@ -669,7 +689,13 @@ def from_line(cls, line, editable=None, extras=None): @property def line_part(self): """requirements.txt compatible line part sans-extras""" - if self.req: + if self.is_local: + base_link = self.link + if not self.link: + base_link = self.get_link() + final_format = "{{0}}#egg={0}".format(base_link.egg_fragment) if base_link.egg_fragment else "{0}" + base = final_format.format(self.vcs_uri) + elif self.req: base = self.req.line if base and self.extras and not extras_to_string(self.extras) in base: if self.subdirectory: @@ -694,7 +720,7 @@ def _choose_vcs_source(pipfile): @property def pipfile_part(self): - pipfile_dict = attr.asdict(self, filter=lambda k, v: v is not None and k.name != '_repo').copy() + pipfile_dict = attr.asdict(self, filter=lambda k, v: bool(v) is True and k.name != '_repo').copy() if "vcs" in pipfile_dict: pipfile_dict = self._choose_vcs_source(pipfile_dict) name, _ = _strip_extras(pipfile_dict.pop("name")) @@ -752,7 +778,10 @@ def extras_as_pip(self): def commit_hash(self): if not self.is_vcs: return None - return self.req.get_commit_hash() + commit_hash = None + with self.req.locked_vcs_repo() as repo: + commit_hash = repo.get_commit_hash() + return commit_hash @specifiers.default def get_specifiers(self): diff --git a/pipenv/vendor/requirementslib/models/vcs.py b/pipenv/vendor/requirementslib/models/vcs.py index 3bfb05d5fd..5d3ec08fc9 100644 --- a/pipenv/vendor/requirementslib/models/vcs.py +++ b/pipenv/vendor/requirementslib/models/vcs.py @@ -23,34 +23,43 @@ def get_repo_instance(self): backend = VCS_SUPPORT._registry.get(self.vcs_type) return backend(url=self.url) + @property + def is_local(self): + url = self.url + if '+' in url: + url = url.split('+')[1] + return url.startswith("file") + def obtain(self): if not os.path.exists(self.checkout_directory): self.repo_instance.obtain(self.checkout_directory) if self.ref: - with vistir.contextmanagers.cd(self.checkout_directory): - self.update(self.ref) - self.commit_sha = self.get_commit_hash() + self.checkout_ref(self.ref) + self.commit_sha = self.get_commit_hash(self.ref) else: if not self.commit_sha: self.commit_sha = self.get_commit_hash() def checkout_ref(self, ref): - target_rev = self.repo_instance.make_rev_options(ref) if not self.repo_instance.is_commit_id_equal( self.checkout_directory, self.get_commit_hash(ref) ) and not self.repo_instance.is_commit_id_equal(self.checkout_directory, ref): - self.repo_instance.update(self.checkout_directory, self.url, target_rev) - self.commit_hash = self.get_commit_hash() + if not self.is_local: + self.update(ref) def update(self, ref): - target_rev = self.repo_instance.make_rev_options(ref) + target_ref = self.repo_instance.make_rev_options(ref) + sha = self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) + target_rev = target_ref.make_new(sha) if parse_version(pip_version) > parse_version("18.0"): self.repo_instance.update(self.checkout_directory, self.url, target_rev) else: - self.repo_instance.update(self.checkout_directory, target_rev) - self.commit_hash = self.get_commit_hash() + self.repo_instance.update(self.checkout_directory, target_ref) + self.commit_hash = self.get_commit_hash(ref) def get_commit_hash(self, ref=None): if ref: - return self.repo_instance.get_revision(self.checkout_directory) + target_ref = self.repo_instance.make_rev_options(ref) + return self.repo_instance.get_revision_sha(self.checkout_directory, target_ref.arg_rev) + # return self.repo_instance.get_revision(self.checkout_directory) return self.repo_instance.get_revision(self.checkout_directory) diff --git a/tasks/vendoring/patches/patched/piptools.patch b/tasks/vendoring/patches/patched/piptools.patch index 6b937337ed..6dff468ac8 100644 --- a/tasks/vendoring/patches/patched/piptools.patch +++ b/tasks/vendoring/patches/patched/piptools.patch @@ -717,7 +717,7 @@ index 7e8cdf3..0a0d27d 100644 @@ -1,30 +1,42 @@ # -*- coding=utf-8 -*- import importlib - + -def do_import(module_path, subimport=None, old_path=None): + +def do_import(module_path, subimport=None, old_path=None, vendored_name=None): @@ -744,8 +744,8 @@ index 7e8cdf3..0a0d27d 100644 if subimport: return getattr(_tmp, subimport, _tmp) return _tmp -- - +- + -InstallRequirement = do_import('req.req_install', 'InstallRequirement') -parse_requirements = do_import('req.req_file', 'parse_requirements') -RequirementSet = do_import('req.req_set', 'RequirementSet') diff --git a/tests/integration/test_install_uri.py b/tests/integration/test_install_uri.py index c51db6ada2..c5915607c3 100644 --- a/tests/integration/test_install_uri.py +++ b/tests/integration/test_install_uri.py @@ -97,16 +97,18 @@ def test_file_urls_work(PipenvInstance, pip_src_dir): @pytest.mark.files @pytest.mark.urls @pytest.mark.needs_internet -def test_local_vcs_urls_work(PipenvInstance, pypi): +def test_local_vcs_urls_work(PipenvInstance, pypi, tmpdir): + six_dir = tmpdir.join("six") + six_path = Path(six_dir.strpath) with PipenvInstance(pypi=pypi, chdir=True) as p: - six_path = Path(p.path).joinpath("six").absolute() c = delegator.run( - "git clone " "https://github.com/benjaminp/six.git {0}".format(six_path) + "git clone https://github.com/benjaminp/six.git {0}".format(six_dir.strpath) ) assert c.return_code == 0 c = p.pipenv("install git+{0}#egg=six".format(six_path.as_uri())) assert c.return_code == 0 + assert "six" in p.pipfile["packages"] @pytest.mark.e