feat: use packaging-based PEP 508 parser

python/deptry/dependency_getter/pdm.py

@@ -50,4 +50,4 @@ def _get_pdm_dev_dependencies(self) -> list[Dependency]:
         except KeyError:
             logging.debug("No section [tool.pdm.dev-dependencies] found in pyproject.toml")
 
-        return self._extract_pep_508_dependencies(dev_dependency_strings, self.package_module_name_map)
+        return self._extract_pep_508_dependencies(dev_dependency_strings)

python/deptry/dependency_getter/pep_621.py

@@ -2,16 +2,15 @@
 
 import itertools
 import logging
-import re
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-from deptry.dependency import Dependency
+from deptry.dependency import parse_pep_508_dependency
 from deptry.dependency_getter.base import DependenciesExtract, DependencyGetter
 from deptry.utils import load_pyproject_toml
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping, Sequence
+    from deptry.dependency import Dependency
 
 
 @dataclass
@@ -59,13 +58,13 @@ def get(self) -> DependenciesExtract:
     def _get_dependencies(self) -> list[Dependency]:
         pyproject_data = load_pyproject_toml(self.config)
         dependency_strings: list[str] = pyproject_data["project"]["dependencies"]
-        return self._extract_pep_508_dependencies(dependency_strings, self.package_module_name_map)
+        return self._extract_pep_508_dependencies(dependency_strings)
 
     def _get_optional_dependencies(self) -> dict[str, list[Dependency]]:
         pyproject_data = load_pyproject_toml(self.config)
 
         return {
-            group: self._extract_pep_508_dependencies(dependencies, self.package_module_name_map)
+            group: self._extract_pep_508_dependencies(dependencies)
             for group, dependencies in pyproject_data["project"].get("optional-dependencies", {}).items()
         }
 
@@ -98,31 +97,14 @@ def _split_development_dependencies_from_optional_dependencies(
         )
         return dev_dependencies, regular_dependencies
 
-    def _extract_pep_508_dependencies(
-        self, dependencies: list[str], package_module_name_map: Mapping[str, Sequence[str]]
-    ) -> list[Dependency]:
+    def _extract_pep_508_dependencies(self, dependencies: list[str]) -> list[Dependency]:
         """
         Given a list of dependency specifications (e.g. "django>2.1; os_name != 'nt'"), convert them to Dependency objects.
         """
-        extracted_dependencies = []
-
-        for spec in dependencies:
-            # An example of a spec is `"tomli>=1.1.0; python_version < \"3.11\""`
-            name = self._find_dependency_name_in(spec)
-            if name:
-                extracted_dependencies.append(
-                    Dependency(
-                        name,
-                        self.config,
-                        module_names=package_module_name_map.get(name),
-                    )
-                )
+        extracted_dependencies: list[Dependency] = []
 
-        return extracted_dependencies
+        for dependency in dependencies:
+            if extracted_dependency := parse_pep_508_dependency(dependency, self.config, self.package_module_name_map):
+                extracted_dependencies.append(extracted_dependency)
 
-    @staticmethod
-    def _find_dependency_name_in(spec: str) -> str | None:
-        match = re.search("[a-zA-Z0-9-_]+", spec)
-        if match:
-            return match.group(0)
-        return None
+        return extracted_dependencies

python/deptry/dependency_getter/requirements_files.py

@@ -6,11 +6,14 @@
 import re
 from dataclasses import dataclass
 from pathlib import Path
-from urllib.parse import urlparse
+from typing import TYPE_CHECKING
 
-from deptry.dependency import Dependency
+from deptry.dependency import parse_pep_508_dependency
 from deptry.dependency_getter.base import DependenciesExtract, DependencyGetter
 
+if TYPE_CHECKING:
+    from deptry.dependency import Dependency
+
 
 @dataclass
 class RequirementsTxtDependencyGetter(DependencyGetter):
@@ -66,63 +69,15 @@ def _extract_dependency_from_line(self, line: str, file_path: Path) -> Dependenc
         """
         Extract a dependency from a single line of a requirements.txt file.
         """
-        line = self._remove_comments_from(line)
-        line = self._remove_newlines_from(line)
-        name = self._find_dependency_name_in(line)
-        if name:
-            return Dependency(
-                name=name,
-                definition_file=file_path,
-                module_names=self.package_module_name_map.get(name),
-            )
-        else:
-            return None
-
-    def _find_dependency_name_in(self, line: str) -> str | None:
-        """
-        Find the dependency name of a dependency specified according to the pip-standards for requirement.txt
-        """
-        if self._line_is_url(line):
-            return self._extract_name_from_url(line)
-        else:
-            match = re.search("^[^-][a-zA-Z0-9-_]+", line)
-            if match:
-                return match.group(0)
-        return None
+        # Note that `packaging` does not strip comments on purpose (https://github.com/pypa/packaging/issues/807), so we
+        # need to remove the comments ourselves.
+        return parse_pep_508_dependency(self._remove_comments_from(line), file_path, self.package_module_name_map)
 
     @staticmethod
     def _remove_comments_from(line: str) -> str:
         """
-        Removes comments from a line. A comment is defined as any text
-        following a '#' that is either at the start of the line or preceded by a space.
+        Removes comments from a line. A comment is defined as any text following a '#' that is either at the start of
+        the line or preceded by a space.
         This ensures that fragments like '#egg=' in URLs are not mistakenly removed.
         """
         return re.sub(r"(?<!\S)#.*", "", line).strip()
-
-    @staticmethod
-    def _remove_newlines_from(line: str) -> str:
-        return line.replace("\n", "")
-
-    @staticmethod
-    def _line_is_url(line: str) -> bool:
-        return urlparse(line).scheme != ""
-
-    @staticmethod
-    def _extract_name_from_url(line: str) -> str | None:
-        # Try to find egg, for url like git+https://github.com/xxxxx/package@xxxxx#egg=package
-        match = re.search("egg=([a-zA-Z0-9-_]*)", line)
-        if match:
-            return match.group(1)
-
-        # for url like git+https://github.com/name/python-module.git@0d6dc38d58
-        match = re.search(r"\/((?:(?!\/).)*?)\.git", line)
-        if match:
-            return match.group(1)
-
-        # for url like https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
-        match = re.search(r"\/((?:(?!\/).)*?)\/archive\/", line)
-        if match:
-            return match.group(1)
-
-        logging.warning("Could not parse dependency name from url %s", line)
-        return None

tests/unit/dependency_getter/test_pdm.py

@@ -15,7 +15,7 @@ def test_dependency_getter(tmp_path: Path) -> None:
     "qux",
     "bar>=20.9",
     "optional-foo[option]>=0.12.11",
-    "conditional-bar>=1.1.0; python_version < 3.11",
+    "conditional-bar>=1.1.0; python_version < '3.11'",
     "fox-python",  # top level module is called "fox"
 ]
 """
@@ -57,12 +57,12 @@ def test_dev_dependency_getter(tmp_path: Path) -> None:
     "qux",
     "bar>=20.9",
     "optional-foo[option]>=0.12.11",
-    "conditional-bar>=1.1.0; python_version < 3.11",
+    "conditional-bar>=1.1.0; python_version < '3.11'",
 ]
 [tool.pdm.dev-dependencies]
 test = [
     "qux",
-    "bar; python_version < 3.11"
+    "bar; python_version < '3.11'"
     ]
 tox = [
     "foo-bar",

tests/unit/dependency_getter/test_pep_621.py

@@ -19,7 +19,7 @@ def test_dependency_getter(tmp_path: Path) -> None:
     "qux",
     "bar>=20.9",
     "optional-foo[option]>=0.12.11",
-    "conditional-bar>=1.1.0; python_version < 3.11",
+    "conditional-bar>=1.1.0; python_version < '3.11'",
     "fox-python",  # top level module is called "fox"
 ]
 

tests/unit/dependency_getter/test_requirements_txt.py

@@ -2,8 +2,6 @@
 
 from pathlib import Path
 
-import pytest
-
 from deptry.dependency_getter.requirements_files import RequirementsTxtDependencyGetter
 from tests.utils import run_within_dir
 
@@ -59,11 +57,15 @@ def test_parse_requirements_files(tmp_path: Path) -> None:
 
 
 def test_parse_requirements_files_urls(tmp_path: Path) -> None:
-    fake_requirements_files = """urllib3 @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
-https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
-git+https://github.com/baz/foo-bar.git@asd#egg=foo-bar
-git+https://github.com/baz/foo-bar.git@asd
-git+https://github.com/abc123/bar-foo@xyz789#egg=bar-fooo"""
+    fake_requirements_files = """git-archive @ git+https://github.com/foo/bar.git@1.2.3
+    remote-archive @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
+    another-remote-archive[with-extra] @ https://github.com/foo/bar/archive/refs/tags/v0.1.2.tar.gz
+    remote-archive-with-egg @ https://github.com/foo/bar/archive/refs/tags/v0.1.2.tar.gz#egg=foo-bar
+    local-sdist @ file:///local/sdist.tar.gz
+    local-wheel @ file:///local/wheel.whl
+    local-directory @ file:///local/directory
+    another-local-directory@file:///local/directory
+    """
 
     with run_within_dir(tmp_path):
         with Path("requirements.txt").open("w") as f:
@@ -72,14 +74,17 @@ def test_parse_requirements_files_urls(tmp_path: Path) -> None:
         dependencies_extract = RequirementsTxtDependencyGetter(Path("pyproject.toml")).get()
         dependencies = dependencies_extract.dependencies
 
-        assert len(dependencies) == 5
-        assert len(dependencies_extract.dev_dependencies) == 0
-
-        assert dependencies[0].name == "urllib3"
-        assert dependencies[1].name == "urllib3"
-        assert dependencies[2].name == "foo-bar"
-        assert dependencies[3].name == "foo-bar"
-        assert dependencies[4].name == "bar-fooo"
+        assert [dependency.name for dependency in dependencies] == [
+            "git-archive",
+            "remote-archive",
+            "another-remote-archive",
+            "remote-archive-with-egg",
+            "local-sdist",
+            "local-wheel",
+            "local-directory",
+            "another-local-directory",
+        ]
+        assert dependencies_extract.dev_dependencies == []
 
 
 def test_single(tmp_path: Path) -> None:
@@ -179,25 +184,3 @@ def test_dev_multiple_with_arguments(tmp_path: Path) -> None:
 
         assert dev_dependencies[0].name == "click"
         assert dev_dependencies[1].name == "bar"
-
-
-@pytest.mark.parametrize(
-    ("line", "expected"),
-    [
-        ("foo", False),
-        ("http", False),
-        ("https", False),
-        ("httpx", False),
-        ("git+http", False),
-        ("git+https", False),
-        ("http://", True),
-        ("https://", True),
-        ("git+http://", True),
-        ("git+https://", True),
-        ("file://", True),
-        ("file:///", True),
-        ("httpx://", True),
-    ],
-)
-def test__line_is_url(line: str, expected: bool) -> None:
-    assert RequirementsTxtDependencyGetter._line_is_url(line) is expected

tests/unit/test_dependency.py

@@ -2,13 +2,16 @@
 
 from importlib.metadata import PackageNotFoundError
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING
 from unittest.mock import patch
 
 import pytest
 
 from deptry.dependency import Dependency, parse_pep_508_dependency
 
+if TYPE_CHECKING:
+    from typing import Any
+
 
 def test_simple_dependency() -> None:
     dependency = Dependency("click", Path("pyproject.toml"))