✨ Add VCS support

.github/workflows/main.yml

@@ -16,6 +16,8 @@ jobs:
         python-version: ['2.7', '3.5', '3.6', '3.7', '3.8', '3.9']
     steps:
     - uses: actions/checkout@v2
+      with:
+        fetch-depth: 0
 
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2

README.md

@@ -117,16 +117,16 @@ Exhaustive dependency trees without the need to install any packages (at most bu
 ```
 $ pipgrip --tree pipgrip
 
-pipgrip (0.6.7)
+pipgrip (0.7.0)
 ├── anytree (2.8.0)
-│   └── six>=1.9.0 (1.15.0)
-├── click>=7 (7.1.2)
-├── packaging>=17 (20.9)
-│   └── pyparsing>=2.0.2 (2.4.7)
-├── pip>=7.1.0 (21.0.1)
-├── pkginfo>=1.4.2 (1.7.0)
-├── setuptools>=38.3 (53.0.0)
-└── wheel (0.36.2)
+│   └── six>=1.9.0 (1.16.0)
+├── click>=7 (8.0.4)
+├── packaging>=17 (21.3)
+│   └── pyparsing!=3.0.5,>=2.0.2 (3.0.7)
+├── pip>=7.1.0 (22.0.3)
+├── pkginfo<1.8,>=1.4.2 (1.7.1)
+├── setuptools>=38.3 (60.9.3)
+└── wheel (0.37.1)
 ```
 
 For more details/further processing, combine `--tree` with `--json` for a detailed nested JSON dependency tree. See also `--tree-ascii` (no unicode tree markers), and `--tree-json` & `--tree-json-exact` (simplified JSON dependency trees).
@@ -223,10 +223,12 @@ keras==2.2.2 (2.2.2)
 
 - PubGrub doesn't support [version epochs](https://www.python.org/dev/peps/pep-0440/#version-epochs), the [main reason](https://github.com/pypa/pip/issues/8203#issuecomment-704931138) PyPA chose [resolvelib](https://github.com/sarugaku/resolvelib) over PubGrub for their new resolver.
 - Package names are canonicalised in wheel metadata, resulting in e.g. `path.py -> path-py` and `keras_preprocessing -> keras-preprocessing` in output.
-- [VCS Support](https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support) isn't implemented yet.
+- [VCS Support](https://pip.pypa.io/en/stable/reference/pip_install/#vcs-support): combining VCS requirements with `--editable`, as well as the [`@ -e svn+`](https://pip.pypa.io/en/stable/topics/vcs-support/#subversion) pattern aren't implemented yet.
+- Ommitting the `projectname @` prefix (the equivalent of e.g. `pip install git+https...`) is not supported neither for VCS requirements, nor for [PEP 440](https://www.python.org/dev/peps/pep-0440) direct references (the equivalent of e.g. `pip install https...`).
+- Parsing requirements files (`-r`) does not support: [custom file encodings](https://pip.pypa.io/en/stable/reference/requirements-file-format/#encoding), [line continuations](https://pip.pypa.io/en/stable/reference/requirements-file-format/#line-continuations), [global/per-requirement options](https://pip.pypa.io/en/stable/reference/requirements-file-format/#supported-options)
 - `--reversed-tree` isn't implemented yet.
 - Since `pip install -r` does not accept `.` as requirement, it is omitted from lockfiles, so `--install` or `--pipe` should be used when installing local projects (or using `--lock` and then passing it to pip using `--constraint`).
-- The equivalent of e.g. `pip install ../aiobotocore[boto3]` is not yet implemented. However, e.g. `pipgrip --install .[boto3]` is allowed.
+- Support for local paths, e.g. the equivalent of `pip install ../aiobotocore[boto3]` is not yet implemented. However, e.g. `pipgrip --install -e .[boto3]` is allowed.
 
 
 ## Development

src/pipgrip/cli.py

@@ -18,6 +18,7 @@
 from pipgrip.libs.mixology.failure import SolverFailure
 from pipgrip.libs.mixology.package import Package
 from pipgrip.libs.mixology.version_solver import VersionSolver
+from pipgrip.libs.semver import Version
 from pipgrip.package_source import PackageSource
 from pipgrip.pipper import install_packages, read_requirements
 
@@ -192,10 +193,21 @@ def render_json_tree_full(tree_root, max_depth, sort):
     return tree_dict_full
 
 
+def is_vcs_version(version):
+    return str(Version.parse(version).major) not in version
+
+
+def render_pin(package, version):
+    if package.startswith("."):
+        return package
+    sep = " @ " if is_vcs_version(version) else "=="
+    return sep.join((package, version))
+
+
 def render_lock(packages, include_dot=True, sort=False):
     fn = sorted if sort else list
     return fn(
-        "==".join(x) if not x[0].startswith(".") else x[0]
+        render_pin(x[0], x[1])
         for x in packages.items()
         if include_dot or not x[0].startswith(".")
     )
@@ -398,14 +410,6 @@ def main(
         if not install:
             raise click.ClickException("--user has no effect without --install")
 
-    for dep in dependencies:
-        if os.sep in dep:
-            raise click.ClickException(
-                "'{}' looks like a path, and is not supported yet by pipgrip".format(
-                    dep
-                )
-            )
-
     if no_cache_dir:
         cache_dir = tempfile.mkdtemp()
 

src/pipgrip/libs/mixology/package.py

@@ -7,11 +7,7 @@ class Package(object):
     """Represent a project's package."""
 
     def __init__(self, pip_string):  # type: (str) -> None
-        if pip_string == "_root_":
-            req = parse_req(".")
-            req.key = "_root_"
-        else:
-            req = parse_req(pip_string)
+        req = parse_req(pip_string)
         self._name = req.key
         self._req = req
 

src/pipgrip/libs/semver/__init__.py

@@ -165,4 +165,10 @@ def parse_single_constraint(constraint):  # type: (str) -> VersionConstraint
         else:
             return version
 
+    # VCS support
+    try:
+        return Version.parse(constraint)
+    except ValueError:
+        raise ValueError("Could not parse version constraint: {}".format(constraint))
+
     raise ValueError("Could not parse version constraint: {}".format(constraint))

src/pipgrip/libs/semver/version.py

@@ -1,4 +1,5 @@
 # flake8: noqa:A003
+import hashlib
 import re
 from typing import List, Optional, Union
 
@@ -195,13 +196,21 @@ def include_max(self):
 
     @classmethod
     def parse(cls, text):  # type: (str) -> Version
+        if not isinstance(text, ("".__class__, u"".__class__)):
+            raise ParseVersionError('Unable to parse "{}".'.format(text))
+
         try:
             match = COMPLETE_VERSION.match(text)
         except TypeError:
             match = None
 
         if match is None:
-            raise ParseVersionError('Unable to parse "{}".'.format(text))
+            # VCS support: use numerical hash
+            match = COMPLETE_VERSION.match(
+                str(
+                    int(hashlib.sha256(text.encode("utf-8")).hexdigest(), 16) % 10 ** 12
+                )
+            )
 
         text = text.rstrip(".")
 

src/pipgrip/package_source.py

@@ -122,7 +122,7 @@ def add(
         dependencies = []
         for dep in deps:
             req = parse_req(dep)
-            constraint = ",".join(["".join(tup) for tup in req.specs])
+            constraint = req.url or ",".join(["".join(tup) for tup in req.specs])
             dependencies.append(Dependency(req.key, constraint, req.__str__()))
 
         self._packages[name][extras][version] = dependencies
@@ -155,7 +155,7 @@ def root_dep(self, package):  # type: (str, str) -> None
         if is_unneeded_dep(package):
             return
         req = parse_req(package)
-        constraint = ",".join(["".join(tup) for tup in req.specs])
+        constraint = req.url or ",".join(["".join(tup) for tup in req.specs])
         self._root_dependencies.append(Dependency(req.key, constraint, req.__str__()))
         self.discover_and_add(req.__str__())
 

src/pipgrip/pipper.py

@@ -18,25 +18,22 @@
 
 
 def read_requirements(path):
+    re_comments = re.compile(r"(?:^|\s+)#")
     try:
         with io.open(path, mode="rt", encoding="utf-8") as fp:
-            return list(filter(None, (line.split("#")[0].strip() for line in fp)))
+            return list(
+                filter(None, (re_comments.split(line, 1)[0].strip() for line in fp))
+            )
     except IndexError:
         raise RuntimeError("{} is broken".format(path))
 
 
 def parse_req(requirement, extras=None):
-    from pipgrip.libs.mixology.package import Package
-
-    if isinstance(requirement, Package) and extras != requirement.req.extras:
-        raise RuntimeError(
-            "Conflict between package extras and extras kwarg. Please file an issue on GitHub."
-        )
-    if requirement.startswith("."):
-        req = pkg_resources.Requirement.parse(requirement.replace(".", "rubbish", 1))
+    if requirement == "_root_" or requirement == "." or requirement.startswith(".["):
+        req = pkg_resources.Requirement.parse("rubbish")
         if extras is not None:
             req.extras = extras
-        req.key = "."
+        req.key = "." if requirement.startswith(".[") else requirement
         full_str = req.__str__().replace(req.name, req.key)
         req.name = req.key
     else:
@@ -406,6 +403,7 @@ def discover_dependencies_and_versions(
 
     """
     req = parse_req(package)
+
     extras_requested = sorted(req.extras)
 
     logger.info("discovering %s", req)
@@ -414,10 +412,10 @@ def discover_dependencies_and_versions(
     )
     wheel_metadata = _extract_metadata(wheel_fname)
     wheel_requirements = _get_wheel_requirements(wheel_metadata, extras_requested)
-    wheel_version = wheel_metadata["version"]
+    wheel_version = req.url or wheel_metadata["version"]
     available_versions = (
         _get_available_versions(req.extras_name, index_url, extra_index_url, pre)
-        if req.key != "."
+        if req.key != "." and req.url is None
         else [wheel_version]
     )
     if wheel_version not in available_versions:

tests/test_cli.py

@@ -37,6 +37,9 @@ def mock_download_wheel(package, *args, **kwargs):
         "keras-applications==1.0.4": "./tests/assets/Keras_Applications-1.0.4-py2.py3-none-any.whl",
         "h5py": "./tests/assets/h5py-2.10.0-cp27-cp27m-macosx_10_6_intel.whl",
         "pip>=7.1.0": "./tests/assets/pip-20.0.2-py2.py3-none-any.whl",
+        "requests[socks]@ git+https://github.com/psf/requests": "./tests/assets/requests-2.22.0-py2.py3-none-any.whl",
+        "requests@ git+https://github.com/psf/requests": "./tests/assets/requests-2.22.0-py2.py3-none-any.whl",
+        "pysocks!=1.5.7,>=1.5.6": "./tests/assets/PySocks-1.7.1-py3-none-any.whl",
     }
     return wheelhouse[package]
 
@@ -64,6 +67,7 @@ def mock_get_available_versions(package, *args, **kwargs):
         "keras-applications": ["1.0.0", "1.0.1", "1.0.2", "1.0.4", "1.0.5", "1.0.6", "1.0.7", "1.0.8"],
         "h5py": ["2.10.0"],
         "pip": ["20.0.2"],
+        "pysocks": ["1.7.1"]
     }
     return versions[package]
 # fmt: on
@@ -248,6 +252,27 @@ def default_environment():
             ["keras_preprocessing", "requests; python_version < '3'"],
             ["keras-preprocessing==1.1.0", "six==1.13.0", "numpy==1.16.6"],
         ),
+        (
+            ["requests@git+https://github.com/psf/requests"],
+            [
+                "requests @ git+https://github.com/psf/requests",
+                "certifi==2019.11.28",
+                "chardet==3.0.4",
+                "idna==2.8",
+                "urllib3==1.25.7",
+            ],
+        ),
+        (
+            ["requests[socks]   @  git+https://github.com/psf/requests"],
+            [
+                "requests @ git+https://github.com/psf/requests",
+                "certifi==2019.11.28",
+                "chardet==3.0.4",
+                "idna==2.8",
+                "pysocks==1.7.1",
+                "urllib3==1.25.7",
+            ],
+        ),
     ],
     ids=(
         "pipgrip pipgrip",
@@ -263,6 +288,8 @@ def default_environment():
         "keras_preprocessing (underscore)",
         "-r",
         "environment marker",
+        "vcs",
+        "vcs with extras",
     ),
 )
 def test_solutions(arguments, expected, monkeypatch):

tests/tests_mixology/test_unsolvable.py

@@ -83,3 +83,16 @@ def test_no_valid_solution(source):
 So, because root depends on b (*), version solving failed."""
 
     check_solver_result(source, error=error, tries=2)
+
+
+def test_vcs_constraints(source):
+    source.root_dep("requests", "git+https://github.com/psf/requests")
+    source.root_dep("requests", "git+https://github.com/psf/requests.git")
+
+    source.add("requests", "git+https://github.com/psf/requests")
+    source.add("requests", "git+https://github.com/psf/requests.git")
+
+    error = """\
+Because root depends on both requests (git+https://github.com/psf/requests) and requests (git+https://github.com/psf/requests.git), version solving failed."""
+
+    check_solver_result(source, error=error)