feat: use packaging-based PEP 508 parser

deptry/dependency_getter/pdm.py

@@ -50,4 +50,4 @@ def _get_pdm_dev_dependencies(self) -> list[Dependency]:
         except KeyError:
             logging.debug("No section [tool.pdm.dev-dependencies] found in pyproject.toml")
 
-        return self._extract_pep_508_dependencies(dev_dependency_strings, self.package_module_name_map)
+        return self._extract_pep_508_dependencies(dev_dependency_strings)

deptry/dependency_getter/pep_621.py

@@ -1,16 +1,17 @@
 from __future__ import annotations
 
 import itertools
-import re
 from dataclasses import dataclass
 from typing import TYPE_CHECKING
 
-from deptry.dependency import Dependency
+from deptry.dependency import parse_pep_508_dependency
 from deptry.dependency_getter.base import DependenciesExtract, DependencyGetter
 from deptry.utils import load_pyproject_toml
 
 if TYPE_CHECKING:
-    from collections.abc import Mapping, Sequence
+    from deptry.dependency import Dependency
+
+    pass
 
 
 @dataclass
@@ -46,51 +47,24 @@ def get(self) -> DependenciesExtract:
     def _get_dependencies(self) -> list[Dependency]:
         pyproject_data = load_pyproject_toml(self.config)
         dependency_strings: list[str] = pyproject_data["project"]["dependencies"]
-        return self._extract_pep_508_dependencies(dependency_strings, self.package_module_name_map)
+        return self._extract_pep_508_dependencies(dependency_strings)
 
     def _get_optional_dependencies(self) -> dict[str, list[Dependency]]:
         pyproject_data = load_pyproject_toml(self.config)
 
         return {
-            group: self._extract_pep_508_dependencies(dependencies, self.package_module_name_map)
+            group: self._extract_pep_508_dependencies(dependencies)
             for group, dependencies in pyproject_data["project"].get("optional-dependencies", {}).items()
         }
 
-    def _extract_pep_508_dependencies(
-        self, dependencies: list[str], package_module_name_map: Mapping[str, Sequence[str]]
-    ) -> list[Dependency]:
+    def _extract_pep_508_dependencies(self, dependencies: list[str]) -> list[Dependency]:
         """
         Given a list of dependency specifications (e.g. "django>2.1; os_name != 'nt'"), convert them to Dependency objects.
         """
-        extracted_dependencies = []
-
-        for spec in dependencies:
-            # An example of a spec is `"tomli>=1.1.0; python_version < \"3.11\""`
-            name = self._find_dependency_name_in(spec)
-            if name:
-                extracted_dependencies.append(
-                    Dependency(
-                        name,
-                        self.config,
-                        conditional=self._is_conditional(spec),
-                        optional=self._is_optional(spec),
-                        module_names=package_module_name_map.get(name),
-                    )
-                )
-
-        return extracted_dependencies
+        extracted_dependencies: list[Dependency] = []
 
-    @staticmethod
-    def _is_optional(dependency_specification: str) -> bool:
-        return bool(re.findall(r"\[([a-zA-Z0-9-]+?)\]", dependency_specification))
+        for dependency in dependencies:
+            if extracted_dependency := parse_pep_508_dependency(dependency, self.config, self.package_module_name_map):
+                extracted_dependencies.append(extracted_dependency)
 
-    @staticmethod
-    def _is_conditional(dependency_specification: str) -> bool:
-        return ";" in dependency_specification
-
-    @staticmethod
-    def _find_dependency_name_in(spec: str) -> str | None:
-        match = re.search("[a-zA-Z0-9-_]+", spec)
-        if match:
-            return match.group(0)
-        return None
+        return extracted_dependencies

deptry/dependency_getter/requirements_txt.py

@@ -6,11 +6,14 @@
 import re
 from dataclasses import dataclass
 from pathlib import Path
-from urllib.parse import urlparse
+from typing import TYPE_CHECKING
 
-from deptry.dependency import Dependency
+from deptry.dependency import parse_pep_508_dependency
 from deptry.dependency_getter.base import DependenciesExtract, DependencyGetter
 
+if TYPE_CHECKING:
+    from deptry.dependency import Dependency
+
 
 @dataclass
 class RequirementsTxtDependencyGetter(DependencyGetter):
@@ -68,71 +71,8 @@ def _extract_dependency_from_line(self, line: str, file_path: Path) -> Dependenc
         """
         Extract a dependency from a single line of a requirements.txt file.
         """
-        line = self._remove_comments_from(line)
-        line = self._remove_newlines_from(line)
-        name = self._find_dependency_name_in(line)
-        if name:
-            line = line.replace(name, "")
-            optional = self._check_if_dependency_is_optional(line)
-            conditional = self._check_if_dependency_is_conditional(line)
-            return Dependency(
-                name=name,
-                definition_file=file_path,
-                optional=optional,
-                conditional=conditional,
-                module_names=self.package_module_name_map.get(name),
-            )
-        else:
-            return None
-
-    def _find_dependency_name_in(self, line: str) -> str | None:
-        """
-        Find the dependency name of a dependency specified according to the pip-standards for requirement.txt
-        """
-        if self._line_is_url(line):
-            return self._extract_name_from_url(line)
-        else:
-            match = re.search("^[^-][a-zA-Z0-9-_]+", line)
-            if match:
-                return match.group(0)
-        return None
+        return parse_pep_508_dependency(self._remove_comments_from(line), file_path, self.package_module_name_map)
 
     @staticmethod
     def _remove_comments_from(line: str) -> str:
         return re.sub(r"\s+#.*", "", line).strip()
-
-    @staticmethod
-    def _remove_newlines_from(line: str) -> str:
-        return line.replace("\n", "")
-
-    @staticmethod
-    def _check_if_dependency_is_optional(line: str) -> bool:
-        return bool(re.findall(r"\[([a-zA-Z0-9-]+?)\]", line))
-
-    @staticmethod
-    def _check_if_dependency_is_conditional(line: str) -> bool:
-        return ";" in line
-
-    @staticmethod
-    def _line_is_url(line: str) -> bool:
-        return urlparse(line).scheme != ""
-
-    @staticmethod
-    def _extract_name_from_url(line: str) -> str | None:
-        # Try to find egg, for url like git+https://github.com/xxxxx/package@xxxxx#egg=package
-        match = re.search("egg=([a-zA-Z0-9-_]*)", line)
-        if match:
-            return match.group(1)
-
-        # for url like git+https://github.com/name/python-module.git@0d6dc38d58
-        match = re.search(r"\/((?:(?!\/).)*?)\.git", line)
-        if match:
-            return match.group(1)
-
-        # for url like https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
-        match = re.search(r"\/((?:(?!\/).)*?)\/archive\/", line)
-        if match:
-            return match.group(1)
-
-        logging.warning("Could not parse dependency name from url %s", line)
-        return None

tests/unit/dependency_getter/test_pdm.py

@@ -15,7 +15,7 @@ def test_dependency_getter(tmp_path: Path) -> None:
     "qux",
     "bar>=20.9",
     "optional-foo[option]>=0.12.11",
-    "conditional-bar>=1.1.0; python_version < 3.11",
+    "conditional-bar>=1.1.0; python_version < '3.11'",
     "fox-python",  # top level module is called "fox"
 ]
 """
@@ -67,12 +67,12 @@ def test_dev_dependency_getter(tmp_path: Path) -> None:
     "qux",
     "bar>=20.9",
     "optional-foo[option]>=0.12.11",
-    "conditional-bar>=1.1.0; python_version < 3.11",
+    "conditional-bar>=1.1.0; python_version < '3.11'",
 ]
 [tool.pdm.dev-dependencies]
 test = [
     "qux",
-    "bar; python_version < 3.11"
+    "bar; python_version < '3.11'"
     ]
 tox = [
     "foo-bar",

tests/unit/dependency_getter/test_pep_621.py

@@ -14,7 +14,7 @@ def test_dependency_getter(tmp_path: Path) -> None:
     "qux",
     "bar>=20.9",
     "optional-foo[option]>=0.12.11",
-    "conditional-bar>=1.1.0; python_version < 3.11",
+    "conditional-bar>=1.1.0; python_version < '3.11'",
     "fox-python",  # top level module is called "fox"
 ]
 

tests/unit/dependency_getter/test_requirements_txt.py

@@ -66,11 +66,7 @@ def test_parse_requirements_txt(tmp_path: Path) -> None:
 
 
 def test_parse_requirements_txt_urls(tmp_path: Path) -> None:
-    fake_requirements_txt = """urllib3 @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
-https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip
-git+https://github.com/baz/foo-bar.git@asd#egg=foo-bar
-git+https://github.com/baz/foo-bar.git@asd
-git+https://github.com/abc123/bar-foo@xyz789#egg=bar-fooo"""
+    fake_requirements_txt = """urllib3 @ https://github.com/urllib3/urllib3/archive/refs/tags/1.26.8.zip"""
 
     with run_within_dir(tmp_path):
         with Path("requirements.txt").open("w") as f:
@@ -79,14 +75,10 @@ def test_parse_requirements_txt_urls(tmp_path: Path) -> None:
         dependencies_extract = RequirementsTxtDependencyGetter(Path("pyproject.toml")).get()
         dependencies = dependencies_extract.dependencies
 
-        assert len(dependencies) == 5
+        assert len(dependencies) == 1
         assert len(dependencies_extract.dev_dependencies) == 0
 
         assert dependencies[0].name == "urllib3"
-        assert dependencies[1].name == "urllib3"
-        assert dependencies[2].name == "foo-bar"
-        assert dependencies[3].name == "foo-bar"
-        assert dependencies[4].name == "bar-fooo"
 
 
 def test_single(tmp_path: Path) -> None:
@@ -188,25 +180,3 @@ def test_dev_multiple_with_arguments(tmp_path: Path) -> None:
 
         assert dev_dependencies[0].name == "click"
         assert dev_dependencies[1].name == "bar"
-
-
-@pytest.mark.parametrize(
-    ("line", "expected"),
-    [
-        ("foo", False),
-        ("http", False),
-        ("https", False),
-        ("httpx", False),
-        ("git+http", False),
-        ("git+https", False),
-        ("http://", True),
-        ("https://", True),
-        ("git+http://", True),
-        ("git+https://", True),
-        ("file://", True),
-        ("file:///", True),
-        ("httpx://", True),
-    ],
-)
-def test__line_is_url(line: str, expected: bool) -> None:
-    assert RequirementsTxtDependencyGetter._line_is_url(line) is expected

tests/unit/test_dependency.py

@@ -2,13 +2,16 @@
 
 from importlib.metadata import PackageNotFoundError
 from pathlib import Path
-from typing import Any
+from typing import TYPE_CHECKING
 from unittest.mock import patch
 
 import pytest
 
 from deptry.dependency import Dependency, parse_pep_508_dependency
 
+if TYPE_CHECKING:
+    from typing import Any
+
 
 def test_simple_dependency() -> None:
     dependency = Dependency("click", Path("pyproject.toml"))