From fdc4ef165dc0c063e137c5ffe92603ee139923e7 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Mon, 17 Jun 2024 07:38:06 +0200 Subject: [PATCH 1/4] start with dep100 --- python/deptry/violations/base.py | 10 ++- .../violations/dep001_missing/finder.py | 3 +- .../deptry/violations/dep002_unused/finder.py | 3 +- .../violations/dep003_transitive/finder.py | 3 +- .../violations/dep004_misplaced_dev/finder.py | 3 +- .../dep100_unused_ignores/__init__.py | 0 .../dep100_unused_ignores/finder.py | 90 +++++++++++++++++++ .../dep100_unused_ignores/violation.py | 19 ++++ .../violations/dep001_missing/test_finder.py | 2 +- .../violations/dep002_unused/test_finder.py | 2 +- .../dep003_transitive/test_finder.py | 4 +- 11 files changed, 130 insertions(+), 9 deletions(-) create mode 100644 python/deptry/violations/dep100_unused_ignores/__init__.py create mode 100644 python/deptry/violations/dep100_unused_ignores/finder.py create mode 100644 python/deptry/violations/dep100_unused_ignores/violation.py diff --git a/python/deptry/violations/base.py b/python/deptry/violations/base.py index f513e9a5..f1c5a43d 100644 --- a/python/deptry/violations/base.py +++ b/python/deptry/violations/base.py @@ -21,15 +21,19 @@ class ViolationsFinder(ABC): imported_modules_with_locations: A list of ModuleLocations objects representing the modules imported by the project and their locations. dependencies: A list of Dependency objects representing the project's dependencies. - ignored_modules: A tuple of module names to ignore when scanning for issues. Defaults to an + modules_to_ignore: A tuple of module names to ignore when scanning for issues. Defaults to an empty tuple. + used_ignores: a list to keep track of which values in 'modules_to_ignore' were actually used. This list should + be updated during the `find()` method, so that the set difference of `modules_to_ignore` and `used_ignores` + results in a set of modules in `modules_to_ignore` that no longer need to be ignored. """ violation: ClassVar[type[Violation]] imported_modules_with_locations: list[ModuleLocations] dependencies: list[Dependency] - ignored_modules: tuple[str, ...] = () + modules_to_ignore: tuple[str, ...] = () + used_ignores: list[str] = [] @abstractmethod def find(self) -> list[Violation]: @@ -50,12 +54,14 @@ class Violation(ABC): issue: An attribute representing the module or dependency where the violation occurred. location: An attribute representing the location in the code where the violation occurred. + ignored: Boolean flag indicating if a violation should be ignored """ error_code: ClassVar[str] = "" error_template: ClassVar[str] = "" issue: Dependency | Module location: Location + ignored: bool = False @abstractmethod def get_error_message(self) -> str: diff --git a/python/deptry/violations/dep001_missing/finder.py b/python/deptry/violations/dep001_missing/finder.py index 45f18547..f9275bdc 100644 --- a/python/deptry/violations/dep001_missing/finder.py +++ b/python/deptry/violations/dep001_missing/finder.py @@ -44,8 +44,9 @@ def _is_missing(self, module: Module) -> bool: ]): return False - if module.name in self.ignored_modules: + if module.name in self.modules_to_ignore: logging.debug("Identified module '%s' as a missing dependency, but ignoring.", module.name) + self.used_ignores.append(module.name) return False logging.debug("No package found to import module '%s' from. Marked as a missing dependency.", module.name) diff --git a/python/deptry/violations/dep002_unused/finder.py b/python/deptry/violations/dep002_unused/finder.py index f364e39a..825c0636 100644 --- a/python/deptry/violations/dep002_unused/finder.py +++ b/python/deptry/violations/dep002_unused/finder.py @@ -45,8 +45,9 @@ def _is_unused(self, dependency: Dependency) -> bool: if self._dependency_found_in_imported_modules(dependency) or self._any_of_the_top_levels_imported(dependency): return False - if dependency.name in self.ignored_modules: + if dependency.name in self.modules_to_ignore: logging.debug("Dependency '%s' found to be unused, but ignoring.", dependency.name) + self.used_ignores.append(dependency.name) return False logging.debug("Dependency '%s' does not seem to be used.", dependency.name) diff --git a/python/deptry/violations/dep003_transitive/finder.py b/python/deptry/violations/dep003_transitive/finder.py index dcca3a4c..e20d3eb1 100644 --- a/python/deptry/violations/dep003_transitive/finder.py +++ b/python/deptry/violations/dep003_transitive/finder.py @@ -52,7 +52,8 @@ def _is_transitive(self, module: Module) -> bool: ]): return False - if module.name in self.ignored_modules: + if module.name in self.modules_to_ignore: + self.used_ignores.append(module.name) logging.debug("Dependency '%s' found to be a transitive dependency, but ignoring.", module.package) return False diff --git a/python/deptry/violations/dep004_misplaced_dev/finder.py b/python/deptry/violations/dep004_misplaced_dev/finder.py index 6fe506ad..415b830f 100644 --- a/python/deptry/violations/dep004_misplaced_dev/finder.py +++ b/python/deptry/violations/dep004_misplaced_dev/finder.py @@ -50,7 +50,8 @@ def _is_development_dependency(self, module: Module, corresponding_package_name: if not module.is_provided_by_dev_dependency or module.is_provided_by_dependency: return False - if module.name in self.ignored_modules: + if module.name in self.modules_to_ignore: + self.used_ignores.append(module.name) logging.debug( "Dependency '%s' found to be a misplaced development dependency, but ignoring.", corresponding_package_name, diff --git a/python/deptry/violations/dep100_unused_ignores/__init__.py b/python/deptry/violations/dep100_unused_ignores/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/python/deptry/violations/dep100_unused_ignores/finder.py b/python/deptry/violations/dep100_unused_ignores/finder.py new file mode 100644 index 00000000..c9a1f111 --- /dev/null +++ b/python/deptry/violations/dep100_unused_ignores/finder.py @@ -0,0 +1,90 @@ +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import TYPE_CHECKING + +from deptry.violations.dep004_misplaced_dev.violation import DEP004MisplacedDevDependencyViolation + +if TYPE_CHECKING: + from deptry.module import Module + from deptry.violations import Violation + + violation: ClassVar[type[Violation]] + imported_modules_with_locations: list[ModuleLocations] + dependencies: list[Dependency] + modules_to_ignore: tuple[str, ...] = () + used_ignores: list[str] = [] + + @abstractmethod + def find(self) -> list[Violation]: + """Find issues about dependencies.""" + raise NotImplementedError() + + +@dataclass +class DEP100UnusedIgnoresFinder: + """ + Given a list of imported modules and a list of project dependencies, determine which development dependencies + should actually be regular dependencies. + + This is the case for any development dependency encountered, since files solely used for development purposes should be excluded from scanning. + """ + + violation: ClassVar[type[Violation]] = DEP004MisplacedDevDependencyViolation + + def find(self) -> list[Violation]: + """ + In this function, we use 'corresponding_package_name' instead of module.package, since it can happen that a + development dependency is not installed, but it's still found to be used in the codebase, due to simple name + matching. In that case, it's added under module.dev_top_levels. _get_package_name is added for these edge-cases. + """ + logging.debug("\nScanning for incorrect development dependencies...") + misplaced_dev_dependencies: list[Violation] = [] + + for module_with_locations in self.imported_modules_with_locations: + module = module_with_locations.module + + logging.debug("Scanning module %s...", module.name) + corresponding_package_name = self._get_package_name(module) + + if corresponding_package_name and self._is_development_dependency(module, corresponding_package_name): + for location in module_with_locations.locations: + misplaced_dev_dependencies.append(self.violation(module, location)) + + return misplaced_dev_dependencies + + def _is_development_dependency(self, module: Module, corresponding_package_name: str) -> bool: + # Module can be provided both by a regular and by a development dependency. + # Only continue if module is ONLY provided by a dev dependency. + if not module.is_provided_by_dev_dependency or module.is_provided_by_dependency: + return False + + if module.name in self.modules_to_ignore: + self.used_ignores.append(module.name) + logging.debug( + "Dependency '%s' found to be a misplaced development dependency, but ignoring.", + corresponding_package_name, + ) + return False + + logging.debug("Dependency '%s' marked as a misplaced development dependency.", corresponding_package_name) + return True + + def _get_package_name(self, module: Module) -> str | None: + if module.package: + return module.package + if module.dev_top_levels: + if len(module.dev_top_levels) > 1: + logging.debug( + "Module %s is found in the top-level module names of multiple development dependencies. Skipping.", + module.name, + ) + elif len(module.dev_top_levels) == 0: + logging.debug( + "Module %s has no metadata and it is not found in any top-level module names. Skipping.", + module.name, + ) + else: + return module.dev_top_levels[0] + return None diff --git a/python/deptry/violations/dep100_unused_ignores/violation.py b/python/deptry/violations/dep100_unused_ignores/violation.py new file mode 100644 index 00000000..d326a881 --- /dev/null +++ b/python/deptry/violations/dep100_unused_ignores/violation.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +from dataclasses import dataclass +from typing import TYPE_CHECKING, ClassVar + +from deptry.violations.base import Violation + +if TYPE_CHECKING: + from deptry.module import Module + + +@dataclass +class DEP004MisplacedDevDependencyViolation(Violation): + error_code: ClassVar[str] = "DEP100" + error_template: ClassVar[str] = "'{name}' imported but declared as a dev dependency" + issue: Module + + def get_error_message(self) -> str: + return self.error_template.format(name=self.issue.name) diff --git a/tests/unit/violations/dep001_missing/test_finder.py b/tests/unit/violations/dep001_missing/test_finder.py index 0c970091..41788490 100644 --- a/tests/unit/violations/dep001_missing/test_finder.py +++ b/tests/unit/violations/dep001_missing/test_finder.py @@ -41,7 +41,7 @@ def test_simple_with_ignore() -> None: ) ] - assert DEP001MissingDependenciesFinder(modules_locations, dependencies, ignored_modules=("foobar",)).find() == [] + assert DEP001MissingDependenciesFinder(modules_locations, dependencies, modules_to_ignore=("foobar",)).find() == [] def test_no_error() -> None: diff --git a/tests/unit/violations/dep002_unused/test_finder.py b/tests/unit/violations/dep002_unused/test_finder.py index 14b3c4aa..8319081c 100644 --- a/tests/unit/violations/dep002_unused/test_finder.py +++ b/tests/unit/violations/dep002_unused/test_finder.py @@ -30,7 +30,7 @@ def test_simple_with_ignore() -> None: ) ] - assert DEP002UnusedDependenciesFinder(modules_locations, dependencies, ignored_modules=("click",)).find() == [] + assert DEP002UnusedDependenciesFinder(modules_locations, dependencies, modules_to_ignore=("click",)).find() == [] def test_top_level() -> None: diff --git a/tests/unit/violations/dep003_transitive/test_finder.py b/tests/unit/violations/dep003_transitive/test_finder.py index 31090fb0..9a163a8d 100644 --- a/tests/unit/violations/dep003_transitive/test_finder.py +++ b/tests/unit/violations/dep003_transitive/test_finder.py @@ -36,4 +36,6 @@ def test_simple_with_ignore() -> None: ) ] - assert DEP003TransitiveDependenciesFinder(modules_locations, dependencies, ignored_modules=("foobar",)).find() == [] + assert ( + DEP003TransitiveDependenciesFinder(modules_locations, dependencies, modules_to_ignore=("foobar",)).find() == [] + ) From 86025de9c79dd7f5c9b0e85dde2e553c7d22f032 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Mon, 17 Jun 2024 07:41:55 +0200 Subject: [PATCH 2/4] start with dep100 --- .../dep100_unused_ignores/finder.py | 67 +------------------ .../dep100_unused_ignores/violation.py | 2 +- 2 files changed, 4 insertions(+), 65 deletions(-) diff --git a/python/deptry/violations/dep100_unused_ignores/finder.py b/python/deptry/violations/dep100_unused_ignores/finder.py index c9a1f111..95539642 100644 --- a/python/deptry/violations/dep100_unused_ignores/finder.py +++ b/python/deptry/violations/dep100_unused_ignores/finder.py @@ -4,7 +4,7 @@ from dataclasses import dataclass from typing import TYPE_CHECKING -from deptry.violations.dep004_misplaced_dev.violation import DEP004MisplacedDevDependencyViolation +from deptry.violations.dep004_misplaced_dev.violation import DEP if TYPE_CHECKING: from deptry.module import Module @@ -25,66 +25,5 @@ def find(self) -> list[Violation]: @dataclass class DEP100UnusedIgnoresFinder: """ - Given a list of imported modules and a list of project dependencies, determine which development dependencies - should actually be regular dependencies. - - This is the case for any development dependency encountered, since files solely used for development purposes should be excluded from scanning. - """ - - violation: ClassVar[type[Violation]] = DEP004MisplacedDevDependencyViolation - - def find(self) -> list[Violation]: - """ - In this function, we use 'corresponding_package_name' instead of module.package, since it can happen that a - development dependency is not installed, but it's still found to be used in the codebase, due to simple name - matching. In that case, it's added under module.dev_top_levels. _get_package_name is added for these edge-cases. - """ - logging.debug("\nScanning for incorrect development dependencies...") - misplaced_dev_dependencies: list[Violation] = [] - - for module_with_locations in self.imported_modules_with_locations: - module = module_with_locations.module - - logging.debug("Scanning module %s...", module.name) - corresponding_package_name = self._get_package_name(module) - - if corresponding_package_name and self._is_development_dependency(module, corresponding_package_name): - for location in module_with_locations.locations: - misplaced_dev_dependencies.append(self.violation(module, location)) - - return misplaced_dev_dependencies - - def _is_development_dependency(self, module: Module, corresponding_package_name: str) -> bool: - # Module can be provided both by a regular and by a development dependency. - # Only continue if module is ONLY provided by a dev dependency. - if not module.is_provided_by_dev_dependency or module.is_provided_by_dependency: - return False - - if module.name in self.modules_to_ignore: - self.used_ignores.append(module.name) - logging.debug( - "Dependency '%s' found to be a misplaced development dependency, but ignoring.", - corresponding_package_name, - ) - return False - - logging.debug("Dependency '%s' marked as a misplaced development dependency.", corresponding_package_name) - return True - - def _get_package_name(self, module: Module) -> str | None: - if module.package: - return module.package - if module.dev_top_levels: - if len(module.dev_top_levels) > 1: - logging.debug( - "Module %s is found in the top-level module names of multiple development dependencies. Skipping.", - module.name, - ) - elif len(module.dev_top_levels) == 0: - logging.debug( - "Module %s has no metadata and it is not found in any top-level module names. Skipping.", - module.name, - ) - else: - return module.dev_top_levels[0] - return None + TODO + """ \ No newline at end of file diff --git a/python/deptry/violations/dep100_unused_ignores/violation.py b/python/deptry/violations/dep100_unused_ignores/violation.py index d326a881..330320b3 100644 --- a/python/deptry/violations/dep100_unused_ignores/violation.py +++ b/python/deptry/violations/dep100_unused_ignores/violation.py @@ -12,7 +12,7 @@ @dataclass class DEP004MisplacedDevDependencyViolation(Violation): error_code: ClassVar[str] = "DEP100" - error_template: ClassVar[str] = "'{name}' imported but declared as a dev dependency" + error_template: ClassVar[str] = "'{name}' TODO" issue: Module def get_error_message(self) -> str: From 345d5cee283f51c6321bedd93ed135ec9b3a823a Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Mon, 17 Jun 2024 07:43:21 +0200 Subject: [PATCH 3/4] start with dep100 --- python/deptry/violations/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/deptry/violations/base.py b/python/deptry/violations/base.py index f1c5a43d..87e715fd 100644 --- a/python/deptry/violations/base.py +++ b/python/deptry/violations/base.py @@ -61,7 +61,6 @@ class Violation(ABC): error_template: ClassVar[str] = "" issue: Dependency | Module location: Location - ignored: bool = False @abstractmethod def get_error_message(self) -> str: From fb7a73d42aa18dd981572c0852a354cd35a71b9c Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Mon, 17 Jun 2024 07:44:13 +0200 Subject: [PATCH 4/4] start with dep100 --- python/deptry/violations/base.py | 1 - 1 file changed, 1 deletion(-) diff --git a/python/deptry/violations/base.py b/python/deptry/violations/base.py index 87e715fd..9227731c 100644 --- a/python/deptry/violations/base.py +++ b/python/deptry/violations/base.py @@ -54,7 +54,6 @@ class Violation(ABC): issue: An attribute representing the module or dependency where the violation occurred. location: An attribute representing the location in the code where the violation occurred. - ignored: Boolean flag indicating if a violation should be ignored """ error_code: ClassVar[str] = ""