From 57d0989d3922c4e9abc9878f490f51de24952e1b Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Tue, 9 May 2023 20:52:43 +0200 Subject: [PATCH 1/4] feat: add ability to pass multiple source directories --- deptry/cli.py | 15 ++++-- deptry/core.py | 8 +-- deptry/python_file_finder.py | 21 ++++---- .../another_directory/__init__.py | 0 .../another_directory/foo.py | 2 + .../pyproject.toml | 14 +++++ .../src/__init__.py | 0 .../src/foo.py | 3 ++ .../src/foobar.py | 2 + .../worker/__init__.py | 0 .../worker/foo.py | 3 ++ .../worker/foobaz.py | 2 + .../test_cli_multiple_source_directories.py | 36 +++++++++++++ tests/unit/test_core.py | 2 +- tests/unit/test_python_file_finder.py | 51 +++++++++++++++++-- 15 files changed, 137 insertions(+), 22 deletions(-) create mode 100644 tests/data/project_with_multiple_source_directories/another_directory/__init__.py create mode 100644 tests/data/project_with_multiple_source_directories/another_directory/foo.py create mode 100644 tests/data/project_with_multiple_source_directories/pyproject.toml create mode 100644 tests/data/project_with_multiple_source_directories/src/__init__.py create mode 100644 tests/data/project_with_multiple_source_directories/src/foo.py create mode 100644 tests/data/project_with_multiple_source_directories/src/foobar.py create mode 100644 tests/data/project_with_multiple_source_directories/worker/__init__.py create mode 100644 tests/data/project_with_multiple_source_directories/worker/foo.py create mode 100644 tests/data/project_with_multiple_source_directories/worker/foobaz.py create mode 100644 tests/functional/cli/test_cli_multiple_source_directories.py diff --git a/deptry/cli.py b/deptry/cli.py index 0b69d1ec..f8dd35eb 100644 --- a/deptry/cli.py +++ b/deptry/cli.py @@ -104,7 +104,7 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b @click.command() -@click.argument("root", type=click.Path(exists=True, path_type=Path)) +@click.argument("root", type=click.Path(exists=True, path_type=Path), nargs=-1, required=True) @click.option( "--verbose", "-v", @@ -268,7 +268,7 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b show_default=False, ) def deptry( - root: Path, + root: tuple[Path, ...], config: Path, no_ansi: bool, ignore_obsolete: tuple[str, ...], @@ -290,8 +290,15 @@ def deptry( ) -> None: """Find dependency issues in your Python project. - [ROOT] is the path to the root directory of the project to be scanned. - All other arguments should be specified relative to [ROOT]. + ROOT is the path to the root directory of the project to be scanned. For instance, to invoke deptry in the current + directory: + + deptry . + + If your project has multiple source directories, multiple ROOT can be specified. For instance, to invoke deptry in + both 'src' and 'worker' directories: + + deptry src worker """ diff --git a/deptry/core.py b/deptry/core.py index dada2c94..52da1979 100644 --- a/deptry/core.py +++ b/deptry/core.py @@ -33,7 +33,7 @@ @dataclass class Core: - root: Path + root: tuple[Path, ...] config: Path no_ansi: bool ignore_obsolete: tuple[str, ...] @@ -148,13 +148,15 @@ def _get_dependencies(self, dependency_management_format: DependencyManagementFo def _get_local_modules(self) -> set[str]: """ - Get all local Python modules from the root directory and `known_first_party` list. + Get all local Python modules from the source directories and `known_first_party` list. A module is considered a local Python module if it matches at least one of those conditions: - it is a directory that contains at least one Python file - it is a Python file that is not named `__init__.py` (since it is a special case) - it is set in the `known_first_party` list """ - guessed_local_modules = {path.stem for path in self.root.iterdir() if self._is_local_module(path)} + guessed_local_modules = { + path.stem for source in self.root for path in source.iterdir() if self._is_local_module(path) + } return guessed_local_modules | set(self.known_first_party) diff --git a/deptry/python_file_finder.py b/deptry/python_file_finder.py index 48832d3e..7508a3fd 100644 --- a/deptry/python_file_finder.py +++ b/deptry/python_file_finder.py @@ -26,7 +26,7 @@ class PythonFileFinder: using_default_exclude: bool ignore_notebooks: bool = False - def get_all_python_files_in(self, directory: Path) -> list[Path]: + def get_all_python_files_in(self, directories: tuple[Path, ...]) -> list[Path]: logging.debug("Collecting Python files to scan...") source_files = [] @@ -36,17 +36,18 @@ def get_all_python_files_in(self, directory: Path) -> list[Path]: gitignore_spec = self._generate_gitignore_pathspec(Path(".")) - for root_str, dirs, files in os.walk(directory, topdown=True): - root = Path(root_str) + for directory in directories: + for root_str, dirs, files in os.walk(directory, topdown=True): + root = Path(root_str) - if self._is_directory_ignored(root, ignore_regex): - dirs[:] = [] - continue + if self._is_directory_ignored(root, ignore_regex): + dirs[:] = [] + continue - for file_str in files: - file = root / file_str - if not self._is_file_ignored(file, file_lookup_suffixes, ignore_regex, gitignore_spec): - source_files.append(file) + for file_str in files: + file = root / file_str + if not self._is_file_ignored(file, file_lookup_suffixes, ignore_regex, gitignore_spec): + source_files.append(file) logging.debug("Python files to scan for imports:\n%s\n", "\n".join([str(file) for file in source_files])) diff --git a/tests/data/project_with_multiple_source_directories/another_directory/__init__.py b/tests/data/project_with_multiple_source_directories/another_directory/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/project_with_multiple_source_directories/another_directory/foo.py b/tests/data/project_with_multiple_source_directories/another_directory/foo.py new file mode 100644 index 00000000..4a228026 --- /dev/null +++ b/tests/data/project_with_multiple_source_directories/another_directory/foo.py @@ -0,0 +1,2 @@ +import a_non_existing_dependency +import toml diff --git a/tests/data/project_with_multiple_source_directories/pyproject.toml b/tests/data/project_with_multiple_source_directories/pyproject.toml new file mode 100644 index 00000000..8f3c4584 --- /dev/null +++ b/tests/data/project_with_multiple_source_directories/pyproject.toml @@ -0,0 +1,14 @@ +[project] +# PEP 621 project metadata +# See https://www.python.org/dev/peps/pep-0621/ +name = "foo" +version = "1.2.3" +requires-python = ">=3.7" +dependencies = ["toml"] + +[build-system] +requires = ["setuptools>=61.0.0"] +build-backend = "setuptools.build_meta" + +[tool.deptry] +ignore_obsolete = ["pkginfo"] diff --git a/tests/data/project_with_multiple_source_directories/src/__init__.py b/tests/data/project_with_multiple_source_directories/src/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/project_with_multiple_source_directories/src/foo.py b/tests/data/project_with_multiple_source_directories/src/foo.py new file mode 100644 index 00000000..780145d4 --- /dev/null +++ b/tests/data/project_with_multiple_source_directories/src/foo.py @@ -0,0 +1,3 @@ +import httpx + +from foobar import a_local_method diff --git a/tests/data/project_with_multiple_source_directories/src/foobar.py b/tests/data/project_with_multiple_source_directories/src/foobar.py new file mode 100644 index 00000000..7df15676 --- /dev/null +++ b/tests/data/project_with_multiple_source_directories/src/foobar.py @@ -0,0 +1,2 @@ +def a_local_method(): + ... diff --git a/tests/data/project_with_multiple_source_directories/worker/__init__.py b/tests/data/project_with_multiple_source_directories/worker/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/project_with_multiple_source_directories/worker/foo.py b/tests/data/project_with_multiple_source_directories/worker/foo.py new file mode 100644 index 00000000..a655cc32 --- /dev/null +++ b/tests/data/project_with_multiple_source_directories/worker/foo.py @@ -0,0 +1,3 @@ +import celery + +from foobaz import a_local_method diff --git a/tests/data/project_with_multiple_source_directories/worker/foobaz.py b/tests/data/project_with_multiple_source_directories/worker/foobaz.py new file mode 100644 index 00000000..7df15676 --- /dev/null +++ b/tests/data/project_with_multiple_source_directories/worker/foobaz.py @@ -0,0 +1,2 @@ +def a_local_method(): + ... diff --git a/tests/functional/cli/test_cli_multiple_source_directories.py b/tests/functional/cli/test_cli_multiple_source_directories.py new file mode 100644 index 00000000..ba4ddf02 --- /dev/null +++ b/tests/functional/cli/test_cli_multiple_source_directories.py @@ -0,0 +1,36 @@ +from __future__ import annotations + +from pathlib import Path +from typing import TYPE_CHECKING + +from click.testing import CliRunner + +from deptry.cli import deptry +from tests.utils import get_issues_report, run_within_dir + +if TYPE_CHECKING: + from tests.functional.types import ToolSpecificProjectBuilder + + +def test_cli_with_multiple_source_directories(pip_project_builder: ToolSpecificProjectBuilder) -> None: + with run_within_dir(pip_project_builder("project_with_multiple_source_directories")): + result = CliRunner().invoke(deptry, "src worker -o report.json") + + assert result.exit_code == 1 + assert get_issues_report() == [ + { + "error": {"code": "DEP002", "message": "'toml' defined as a dependency but not used in the codebase"}, + "module": "toml", + "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None}, + }, + { + "error": {"code": "DEP001", "message": "'httpx' imported but missing from the dependency definitions"}, + "module": "httpx", + "location": {"file": str(Path("src/foo.py")), "line": 1, "column": 0}, + }, + { + "error": {"code": "DEP001", "message": "'celery' imported but missing from the dependency definitions"}, + "module": "celery", + "location": {"file": str(Path("worker/foo.py")), "line": 1, "column": 0}, + }, + ] diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 81f8169b..49b4ae07 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -79,7 +79,7 @@ def test__get_local_modules( assert ( Core( - root=tmp_path / root_suffix, + root=(tmp_path / root_suffix,), config=Path("pyproject.toml"), no_ansi=False, ignore_obsolete=(), diff --git a/tests/unit/test_python_file_finder.py b/tests/unit/test_python_file_finder.py index 2c96ba64..a749a536 100644 --- a/tests/unit/test_python_file_finder.py +++ b/tests/unit/test_python_file_finder.py @@ -24,7 +24,7 @@ def test_simple(tmp_path: Path) -> None: files = PythonFileFinder( exclude=(".venv",), extend_exclude=("other_dir",), using_default_exclude=False - ).get_all_python_files_in(Path(".")) + ).get_all_python_files_in((Path("."),)) assert sorted(files) == [ Path("dir/subdir/file1.py"), @@ -50,7 +50,7 @@ def test_only_matches_start(tmp_path: Path) -> None: files = PythonFileFinder( exclude=("subdir",), extend_exclude=(), using_default_exclude=False - ).get_all_python_files_in(Path(".")) + ).get_all_python_files_in((Path("."),)) assert sorted(files) == [ Path("dir/subdir/file1.py"), @@ -78,7 +78,7 @@ def test_matches_ipynb(ignore_notebooks: bool, expected: list[Path], tmp_path: P files = PythonFileFinder( exclude=(), extend_exclude=(), using_default_exclude=False, ignore_notebooks=ignore_notebooks - ).get_all_python_files_in(Path(".")) + ).get_all_python_files_in((Path("."),)) assert sorted(files) == expected @@ -128,7 +128,50 @@ def test_regex_argument(exclude: tuple[str], expected: list[Path], tmp_path: Pat files = PythonFileFinder( exclude=exclude, extend_exclude=(), using_default_exclude=False - ).get_all_python_files_in(Path(".")) + ).get_all_python_files_in((Path("."),)) + + assert sorted(files) == expected + + +@pytest.mark.parametrize( + ("exclude", "expected"), + [ + ( + (".*file1",), + [ + Path("dir/subdir/file2.py"), + Path("dir/subdir/file3.py"), + Path("other_dir/subdir/file2.py"), + ], + ), + ( + (".*file1|.*file2",), + [ + Path("dir/subdir/file3.py"), + ], + ), + ( + (".*/subdir/",), + [], + ), + ], +) +def test_multiple_source_directories(exclude: tuple[str], expected: list[Path], tmp_path: Path) -> None: + with run_within_dir(tmp_path): + create_files( + [ + Path("dir/subdir/file1.py"), + Path("dir/subdir/file2.py"), + Path("dir/subdir/file3.py"), + Path("other_dir/subdir/file1.py"), + Path("other_dir/subdir/file2.py"), + Path("another_dir/subdir/file1.py"), + ] + ) + + files = PythonFileFinder( + exclude=exclude, extend_exclude=(), using_default_exclude=False + ).get_all_python_files_in((Path("dir"), Path("other_dir"))) assert sorted(files) == expected From 419c4a8b1a7b79c16423366ce650f506aa52d641 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Tue, 9 May 2023 21:19:45 +0200 Subject: [PATCH 2/4] feat(python_file_finder): remove duplicate from files --- deptry/python_file_finder.py | 10 ++++++---- tests/unit/test_python_file_finder.py | 11 +++++++++++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/deptry/python_file_finder.py b/deptry/python_file_finder.py index 7508a3fd..e162838f 100644 --- a/deptry/python_file_finder.py +++ b/deptry/python_file_finder.py @@ -29,7 +29,7 @@ class PythonFileFinder: def get_all_python_files_in(self, directories: tuple[Path, ...]) -> list[Path]: logging.debug("Collecting Python files to scan...") - source_files = [] + source_files = set() ignore_regex = re.compile("|".join(self.exclude + self.extend_exclude)) file_lookup_suffixes = {".py"} if self.ignore_notebooks else {".py", ".ipynb"} @@ -47,11 +47,13 @@ def get_all_python_files_in(self, directories: tuple[Path, ...]) -> list[Path]: for file_str in files: file = root / file_str if not self._is_file_ignored(file, file_lookup_suffixes, ignore_regex, gitignore_spec): - source_files.append(file) + source_files.add(file) - logging.debug("Python files to scan for imports:\n%s\n", "\n".join([str(file) for file in source_files])) + source_files_list = list(source_files) - return source_files + logging.debug("Python files to scan for imports:\n%s\n", "\n".join([str(file) for file in source_files_list])) + + return source_files_list def _is_directory_ignored(self, directory: Path, ignore_regex: Pattern[str]) -> bool: return bool((self.exclude + self.extend_exclude) and ignore_regex.match(str(directory))) diff --git a/tests/unit/test_python_file_finder.py b/tests/unit/test_python_file_finder.py index a749a536..884a37c6 100644 --- a/tests/unit/test_python_file_finder.py +++ b/tests/unit/test_python_file_finder.py @@ -176,6 +176,17 @@ def test_multiple_source_directories(exclude: tuple[str], expected: list[Path], assert sorted(files) == expected +def test_duplicates_are_removed(tmp_path: Path) -> None: + with run_within_dir(tmp_path): + create_files([Path("dir/subdir/file1.py")]) + + files = PythonFileFinder(exclude=(), extend_exclude=(), using_default_exclude=False).get_all_python_files_in( + (Path("."), Path(".")) + ) + + assert sorted(files) == [Path("dir/subdir/file1.py")] + + def test__generate_gitignore_pathspec_with_non_default_exclude(tmp_path: Path) -> None: gitignore_pathspec = PythonFileFinder( exclude=(), extend_exclude=(), using_default_exclude=False From cc4757434020d58146fc17f60f247b41d97e1b6f Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Tue, 9 May 2023 21:35:29 +0200 Subject: [PATCH 3/4] docs(usage): add multiple source directories example --- docs/usage.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 9011c1bb..55d19a11 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -8,7 +8,13 @@ _deptry_ can be run with: deptry . ``` -where `.` is the path to the root directory of the project to be scanned. All other arguments should be specified relative to this directory. +where `.` is the path to the root directory of the project to be scanned. + +If your project have multiple source directories, multiple root directories can be provided: + +```shell +deptry a_directory another_directory +``` If you want to configure _deptry_ using `pyproject.toml`, or if your dependencies are stored in `pyproject.toml`, but it is located in another location than the one _deptry_ is run from, you can specify the location to it by using `--config ` argument. From ce46162fada49948b02916c404ed2bfc85468355 Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Wed, 10 May 2023 11:39:45 +0200 Subject: [PATCH 4/4] docs(usage): s/have/s/ Co-authored-by: Florian Maas --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 55d19a11..edabfb12 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -10,7 +10,7 @@ deptry . where `.` is the path to the root directory of the project to be scanned. -If your project have multiple source directories, multiple root directories can be provided: +If your project has multiple source directories, multiple root directories can be provided: ```shell deptry a_directory another_directory