A way to pre-configure a map of package names to module names

deptry/cli.py

@@ -1,14 +1,19 @@
 from __future__ import annotations
 
 import logging
+from collections import defaultdict
 from pathlib import Path
+from typing import TYPE_CHECKING
 
 import click
 
 from deptry.compat import metadata
 from deptry.config import read_configuration_from_pyproject_toml
 from deptry.core import Core
 
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
 DEFAULT_EXCLUDE = ("venv", r"\.venv", r"\.direnv", "tests", r"\.git", r"setup\.py")
 
 
@@ -37,6 +42,48 @@ def convert(
 COMMA_SEPARATED_TUPLE = CommaSeparatedTupleParamType()
 
 
+class CommaSeparatedMappingParamType(click.ParamType):
+    """
+    This class is used to uniformly handle configuration parameters that can be either passed as a comma-separated pair
+    string, or as a Mapping of strings to tuples of strings. Items in a pair string are separated by an equal sign,
+    where multiple values are separated by a pipe: key1=value1,key2=value2|value3.
+    For example, the value for a parameter can be a comma-separated pair string when passed as a command line argument,
+    or as a mapping of string to tuples of strings when passed through pyproject.toml.
+    """
+
+    name = "mapping"
+
+    def convert(
+        self,
+        # In the mapping value below, although a str is a Sequence[str] itself,
+        # they are treated differently from other sequences of str.
+        value: str | Mapping[str, Sequence[str] | str],
+        param: click.Parameter | None,
+        ctx: click.Context | None,
+    ) -> dict[str, tuple[str, ...]]:
+        converted: dict[str, tuple[str, ...]]
+        if isinstance(value, str):
+            map_: defaultdict[str, list[str]] = defaultdict(list)
+            for item in value.split(","):
+                pair = tuple(item.split("=", 1))
+                if len(pair) != 2:
+                    error_msg = (
+                        f"package name and module names pairs should be concatenated with an equal sign (=): {item}"
+                    )
+                    raise ValueError(error_msg)
+                package_name = pair[0]
+                module_names = pair[1].split("|")
+                map_[package_name].extend(module_names)
+            converted = {k: tuple(v) for k, v in map_.items()}
+        else:
+            converted = {k: (v,) if isinstance(v, str) else tuple(v) for k, v in value.items()}
+
+        return converted
+
+
+COMMA_SEPARATED_MAPPING = CommaSeparatedMappingParamType()
+
+
 def configure_logger(_ctx: click.Context, _param: click.Parameter, value: bool) -> None:
     log_level = logging.DEBUG if value else logging.INFO
     logging.basicConfig(level=log_level, handlers=[logging.StreamHandler()], format="%(message)s")
@@ -201,6 +248,14 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b
     help="""If specified, a summary of the dependency issues found will be written to the output location specified. e.g. `deptry . -o deptry.json`""",
     show_default=True,
 )
+@click.option(
+    "--package-module-name-map",
+    "-pmnm",
+    type=COMMA_SEPARATED_MAPPING,
+    help="""Manually defined module names belonging to packages. For example; `deptry . --package-module-name-map package_1=module_a,package_2=module_b|module_c`.""",
+    default={},
+    show_default=False,
+)
 def deptry(
     root: Path,
     config: Path,
@@ -219,6 +274,7 @@ def deptry(
     requirements_txt_dev: tuple[str, ...],
     known_first_party: tuple[str, ...],
     json_output: str,
+    package_module_name_map: Mapping[str, Sequence[str]],
 ) -> None:
     """Find dependency issues in your Python project.
 
@@ -246,4 +302,5 @@ def deptry(
         requirements_txt_dev=requirements_txt_dev,
         known_first_party=known_first_party,
         json_output=json_output,
+        package_module_name_map=package_module_name_map,
     ).run()

deptry/core.py

@@ -24,6 +24,7 @@
 from deptry.stdlibs import STDLIBS_PYTHON
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
     from pathlib import Path
 
     from deptry.dependency import Dependency
@@ -51,6 +52,7 @@ class Core:
     requirements_txt_dev: tuple[str, ...]
     known_first_party: tuple[str, ...]
     json_output: str
+    package_module_name_map: Mapping[str, Sequence[str]]
 
     def run(self) -> None:
         self._log_config()
@@ -105,13 +107,15 @@ def _find_issues(self, imported_modules: list[Module], dependencies: list[Depend
 
     def _get_dependencies(self, dependency_management_format: DependencyManagementFormat) -> DependenciesExtract:
         if dependency_management_format is DependencyManagementFormat.POETRY:
-            return PoetryDependencyGetter(self.config).get()
+            return PoetryDependencyGetter(self.config, self.package_module_name_map).get()
         if dependency_management_format is DependencyManagementFormat.PDM:
-            return PDMDependencyGetter(self.config).get()
+            return PDMDependencyGetter(self.config, self.package_module_name_map).get()
         if dependency_management_format is DependencyManagementFormat.PEP_621:
-            return PEP621DependencyGetter(self.config).get()
+            return PEP621DependencyGetter(self.config, self.package_module_name_map).get()
         if dependency_management_format is DependencyManagementFormat.REQUIREMENTS_TXT:
-            return RequirementsTxtDependencyGetter(self.config, self.requirements_txt, self.requirements_txt_dev).get()
+            return RequirementsTxtDependencyGetter(
+                self.config, self.package_module_name_map, self.requirements_txt, self.requirements_txt_dev
+            ).get()
         raise IncorrectDependencyFormatError
 
     def _get_local_modules(self) -> set[str]:

deptry/dependency.py

@@ -2,9 +2,13 @@
 
 import logging
 import re
+from typing import TYPE_CHECKING
 
 from deptry.compat import PackageNotFoundError, metadata
 
+if TYPE_CHECKING:
+    from collections.abc import Sequence
+
 
 class Dependency:
     """
@@ -14,7 +18,9 @@ class Dependency:
     By default, we also add the dependency's name with '-' replaced by '_' to the top-level modules.
     """
 
-    def __init__(self, name: str, conditional: bool = False, optional: bool = False) -> None:
+    def __init__(
+        self, name: str, conditional: bool = False, optional: bool = False, module_names: Sequence[str] | None = None
+    ) -> None:
         """
         Args:
             name: Name of the dependency, as shown in pyproject.toml
@@ -24,18 +30,20 @@ def __init__(self, name: str, conditional: bool = False, optional: bool = False)
         self.is_conditional = conditional
         self.is_optional = optional
         self.found = self.find_metadata(name)
-        self.top_levels = self._get_top_levels(name)
+        self.top_levels = self._get_top_levels(name, module_names)
 
-    def _get_top_levels(self, name: str) -> set[str]:
-        top_levels = []
+    def _get_top_levels(self, name: str, module_names: Sequence[str] | None) -> set[str]:
+        top_levels: set[str] = set()
 
-        if self.found:
-            top_levels += self._get_top_level_module_names_from_top_level_txt()
+        if module_names is not None:
+            top_levels.update(module_names)
+        elif self.found:
+            top_levels.update(self._get_top_level_module_names_from_top_level_txt())
             if not top_levels:
-                top_levels += self._get_top_level_module_names_from_record_file()
+                top_levels.update(self._get_top_level_module_names_from_record_file())
 
-        top_levels.append(name.replace("-", "_").lower())
-        return set(top_levels)
+        top_levels.add(name.replace("-", "_").lower())
+        return top_levels
 
     def __repr__(self) -> str:
         return f"Dependency '{self.name}'"

deptry/dependency_getter/base.py

@@ -2,10 +2,11 @@
 
 import logging
 from abc import ABC, abstractmethod
-from dataclasses import dataclass
+from dataclasses import dataclass, field
 from typing import TYPE_CHECKING
 
 if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
     from pathlib import Path
 
     from deptry.dependency import Dependency
@@ -26,6 +27,7 @@ class DependencyGetter(ABC):
     """
 
     config: Path
+    package_module_name_map: Mapping[str, Sequence[str]] = field(default_factory=dict)
 
     @abstractmethod
     def get(self) -> DependenciesExtract:

deptry/dependency_getter/pdm.py

@@ -50,4 +50,4 @@ def _get_pdm_dev_dependencies(self) -> list[Dependency]:
         except KeyError:
             logging.debug("No section [tool.pdm.dev-dependencies] found in pyproject.toml")
 
-        return self._extract_pep_508_dependencies(dev_dependency_strings)
+        return self._extract_pep_508_dependencies(dev_dependency_strings, self.package_module_name_map)

deptry/dependency_getter/pep_621.py

@@ -3,11 +3,15 @@
 import itertools
 import re
 from dataclasses import dataclass
+from typing import TYPE_CHECKING
 
 from deptry.dependency import Dependency
 from deptry.dependency_getter.base import DependenciesExtract, DependencyGetter
 from deptry.utils import load_pyproject_toml
 
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
 
 @dataclass
 class PEP621DependencyGetter(DependencyGetter):
@@ -42,18 +46,20 @@ def get(self) -> DependenciesExtract:
     def _get_dependencies(self) -> list[Dependency]:
         pyproject_data = load_pyproject_toml(self.config)
         dependency_strings: list[str] = pyproject_data["project"]["dependencies"]
-        return self._extract_pep_508_dependencies(dependency_strings)
+        return self._extract_pep_508_dependencies(dependency_strings, self.package_module_name_map)
 
     def _get_optional_dependencies(self) -> dict[str, list[Dependency]]:
         pyproject_data = load_pyproject_toml(self.config)
 
         return {
-            group: self._extract_pep_508_dependencies(dependencies)
+            group: self._extract_pep_508_dependencies(dependencies, self.package_module_name_map)
             for group, dependencies in pyproject_data["project"].get("optional-dependencies", {}).items()
         }
 
     @classmethod
-    def _extract_pep_508_dependencies(cls, dependencies: list[str]) -> list[Dependency]:
+    def _extract_pep_508_dependencies(
+        cls, dependencies: list[str], package_module_name_map: Mapping[str, Sequence[str]]
+    ) -> list[Dependency]:
         """
         Given a list of dependency specifications (e.g. "django>2.1; os_name != 'nt'"), convert them to Dependency objects.
         """
@@ -64,7 +70,12 @@ def _extract_pep_508_dependencies(cls, dependencies: list[str]) -> list[Dependen
             name = cls._find_dependency_name_in(spec)
             if name:
                 extracted_dependencies.append(
-                    Dependency(name, conditional=cls._is_conditional(spec), optional=cls._is_optional(spec))
+                    Dependency(
+                        name,
+                        conditional=cls._is_conditional(spec),
+                        optional=cls._is_optional(spec),
+                        module_names=package_module_name_map.get(name),
+                    )
                 )
 
         return extracted_dependencies

deptry/dependency_getter/poetry.py

@@ -2,12 +2,15 @@
 
 import contextlib
 from dataclasses import dataclass
-from typing import Any
+from typing import TYPE_CHECKING, Any
 
 from deptry.dependency import Dependency
 from deptry.dependency_getter.base import DependenciesExtract, DependencyGetter
 from deptry.utils import load_pyproject_toml
 
+if TYPE_CHECKING:
+    from collections.abc import Mapping, Sequence
+
 
 @dataclass
 class PoetryDependencyGetter(DependencyGetter):
@@ -25,7 +28,7 @@ def get(self) -> DependenciesExtract:
     def _get_poetry_dependencies(self) -> list[Dependency]:
         pyproject_data = load_pyproject_toml(self.config)
         dependencies: dict[str, Any] = pyproject_data["tool"]["poetry"]["dependencies"]
-        return self._get_dependencies(dependencies)
+        return self._get_dependencies(dependencies, self.package_module_name_map)
 
     def _get_poetry_dev_dependencies(self) -> list[Dependency]:
         """
@@ -45,17 +48,23 @@ def _get_poetry_dev_dependencies(self) -> list[Dependency]:
         with contextlib.suppress(KeyError):
             dependencies = {**pyproject_data["tool"]["poetry"]["group"]["dev"]["dependencies"], **dependencies}
 
-        return self._get_dependencies(dependencies)
+        return self._get_dependencies(dependencies, self.package_module_name_map)
 
     @classmethod
-    def _get_dependencies(cls, poetry_dependencies: dict[str, Any]) -> list[Dependency]:
+    def _get_dependencies(
+        cls, poetry_dependencies: dict[str, Any], package_module_name_map: Mapping[str, Sequence[str]]
+    ) -> list[Dependency]:
         dependencies = []
         for dep, spec in poetry_dependencies.items():
             # dep is the dependency name, spec is the version specification, e.g. "^0.2.2" or {"*", optional = true}
             if dep != "python":
                 optional = cls._is_optional(spec)
                 conditional = cls._is_conditional(spec)
-                dependencies.append(Dependency(dep, conditional=conditional, optional=optional))
+                dependencies.append(
+                    Dependency(
+                        dep, conditional=conditional, optional=optional, module_names=package_module_name_map.get(dep)
+                    )
+                )
 
         return dependencies
 

deptry/dependency_getter/requirements_txt.py

@@ -72,7 +72,12 @@ def _extract_dependency_from_line(self, line: str) -> Dependency | None:
             line = line.replace(name, "")
             optional = self._check_if_dependency_is_optional(line)
             conditional = self._check_if_dependency_is_conditional(line)
-            return Dependency(name=name, optional=optional, conditional=conditional)
+            return Dependency(
+                name=name,
+                optional=optional,
+                conditional=conditional,
+                module_names=self.package_module_name_map.get(name),
+            )
         else:
             return None
 

docs/usage.md

@@ -396,3 +396,51 @@ json_output = "deptry_report.txt"
 ```shell
 deptry . --json-output deptry_report.txt
 ```
+
+#### Manually map Package Names to Top Level Module Names
+
+Deptry will automatically detect top level modules names that belong to a
+module in two ways.
+The first is by inspecting the installed packages. The second is by translating
+the package name to a module name (`foo-bar` translates to `foo_bar`).
+
+This however is not always sufficient. A situation may occur where a package is
+not installed because it is optional and unused in the current installation.
+Then when the package name doesn't directly translate to the top level module
+name, or there are more top level modules names, Deptry may report both
+obsolete packages, and missing packages. A concrete example is deptry reporting obsolete (optional) dependency
+`foo-python`, and missing package `foo`, while package `foo-python` would
+install top level module `foo`, if it were installed.
+
+A solution is to pre-define a mapping between the pacakge name and the top
+level module name(s).
+
+* Type `dict[str, list[str] | str]`
+* Default: `{}`
+* `pyproject.toml` option name: `package_module_name_map`
+* CLI option name: `--package-module-name-map` (short: `-pmnm`)
+* `pyproject.toml` examples:
+```toml
+[tool.deptry.package_module_name_map]
+foo-python = "foo"
+```
+Or for multiple top level module names:
+```toml
+[tool.deptry.package_module_name_map]
+foo-python = [
+    "foo",
+    "bar",
+]
+```
+* CLI examples:
+```shell
+deptry . --package-module-name-map 'foo-python=foo'
+```
+Multiple module names are joined by a pipe (`|`):
+```shell
+deptry . --package-module-name-map 'foo-python=foo|bar'
+```
+Multiple package name to module name mappings are joined by a comma (`,`):
+```shell
+deptry . --package-module-name-map 'foo-python=foo,bar-python=bar'
+```