diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 8caebe58..231447e8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -46,6 +46,9 @@ jobs: - name: Check typing run: pdm run mypy + - name: Run Rust tests + run: cargo test + - name: Run unit tests run: pdm run pytest tests/unit --cov --cov-config=pyproject.toml --cov-report=xml @@ -72,6 +75,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Run Rust tests + run: cargo test + - name: Run unit tests run: pdm run pytest tests/unit --cov --cov-config=pyproject.toml --cov-report=xml @@ -92,6 +98,9 @@ jobs: with: python-version: ${{ matrix.python-version }} + - name: Run Rust tests + run: cargo test + - name: Run unit tests run: pdm run pytest tests/unit --cov --cov-config=pyproject.toml --cov-report=xml diff --git a/.gitignore b/.gitignore index 9345bd42..b5dbdd00 100644 --- a/.gitignore +++ b/.gitignore @@ -137,9 +137,6 @@ dmypy.json # Cython debug symbols cython_debug/ -# PyCharm -# JetBrains specific template is maintained in a separate JetBrains.gitignore that can -# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore -# and can be added to the global gitignore or merged into this file. For a more nuclear -# option (not recommended) you can uncomment the following to ignore the entire idea folder. -#.idea/ +# Tests +## `.cache` is ignored, but in this directory we want to have this directory in git as it is used for tests. +!/tests/data/file_finder/.cache diff --git a/Cargo.lock b/Cargo.lock index cb3a9646..7bf395ff 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -119,6 +119,7 @@ dependencies = [ "pyo3-log", "rayon", "regex", + "rstest", "ruff_python_ast", "ruff_python_parser", "ruff_source_file", @@ -141,6 +142,101 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "futures" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "645c6916888f6cb6350d2550b80fb63e734897a8498abe35cfb732b6487804b0" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-channel" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eac8f7d7865dcb88bd4373ab671c8cf4508703796caa2b1985a9ca867b3fcb78" +dependencies = [ + "futures-core", + "futures-sink", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-executor" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a576fc72ae164fca6b9db127eaa9a9dda0d61316034f33a0a0d4eda41f02b01d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a44623e20b9681a318efdd71c299b6b222ed6f231972bfe2f224ebad6311f0c1" + +[[package]] +name = "futures-macro" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87750cf4b7a4c0625b1529e4c543c2182106e4dedc60a2a6455e00d212c489ac" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9fb8e00e87438d937621c1c6269e53f536c14d3fbd6a042bb24879e57d474fb5" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-timer" +version = "3.0.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f288b0a4f20f9a56b5d1da57e2227c661b7b16168e2f72365f57b63326e29b24" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-channel", + "futures-core", + "futures-io", + "futures-macro", + "futures-sink", + "futures-task", + "memchr", + "pin-project-lite", + "pin-utils", + "slab", +] + [[package]] name = "getopts" version = "0.2.21" @@ -161,6 +257,12 @@ dependencies = [ "wasi", ] +[[package]] +name = "glob" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d2fabcfbdc87f4758337ca535fb41a6d701b65693ce38287d856d1674551ec9b" + [[package]] name = "globset" version = "0.4.14" @@ -345,6 +447,18 @@ dependencies = [ "siphasher", ] +[[package]] +name = "pin-project-lite" +version = "0.2.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bda66fc9667c18cb2758a2ac84d1167245054bcf85d5d1aaa6923f45801bdd02" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + [[package]] name = "portable-atomic" version = "1.6.0" @@ -537,6 +651,41 @@ version = "0.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "adad44e29e4c806119491a7f06f03de4d1af22c3a680dd47f1e6e179439d1f56" +[[package]] +name = "relative-path" +version = "1.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e898588f33fdd5b9420719948f9f2a32c922a246964576f71ba7f24f80610fbc" + +[[package]] +name = "rstest" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d5316d2a1479eeef1ea21e7f9ddc67c191d497abc8fc3ba2467857abbb68330" +dependencies = [ + "futures", + "futures-timer", + "rstest_macros", + "rustc_version", +] + +[[package]] +name = "rstest_macros" +version = "0.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04a9df72cc1f67020b0d63ad9bfe4a323e459ea7eb68e03bd9824db49f9a4c25" +dependencies = [ + "cfg-if", + "glob", + "proc-macro2", + "quote", + "regex", + "relative-path", + "rustc_version", + "syn", + "unicode-ident", +] + [[package]] name = "ruff_python_ast" version = "0.0.0" @@ -607,6 +756,15 @@ version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2" +[[package]] +name = "rustc_version" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bfa0f585226d2e68097d4f95d113b15b83a82e819ab25717ec0590d9584ef366" +dependencies = [ + "semver", +] + [[package]] name = "ryu" version = "1.0.17" @@ -628,6 +786,12 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" + [[package]] name = "serde" version = "1.0.197" @@ -665,6 +829,15 @@ version = "0.3.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "38b58827f4464d87d377d175e90bf58eb00fd8716ff0a62f80356b5e61555d0d" +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.2" diff --git a/Cargo.toml b/Cargo.toml index 6a3211e9..0da9ba29 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,6 +24,9 @@ ruff_source_file = { git = "https://github.com/astral-sh/ruff", tag = "v0.3.4" } ruff_text_size = { git = "https://github.com/astral-sh/ruff", tag = "v0.3.4" } serde_json = "1.0.115" +[dev-dependencies] +rstest = "0.19.0" + [profile.release] lto = true codegen-units = 1 diff --git a/python/deptry/core.py b/python/deptry/core.py index 265d4fae..328db3ca 100644 --- a/python/deptry/core.py +++ b/python/deptry/core.py @@ -3,20 +3,20 @@ import logging import sys from dataclasses import dataclass +from pathlib import Path from typing import TYPE_CHECKING from deptry.dependency_getter.builder import DependencyGetterBuilder from deptry.exceptions import UnsupportedPythonVersionError from deptry.imports.extract import get_imported_modules_from_list_of_files from deptry.module import ModuleBuilder, ModuleLocations -from deptry.python_file_finder import get_all_python_files_in from deptry.reporters import JSONReporter, TextReporter +from deptry.rust import find_python_files from deptry.stdlibs import STDLIBS_PYTHON from deptry.violations.finder import find_violations if TYPE_CHECKING: from collections.abc import Mapping - from pathlib import Path from deptry.dependency_getter.base import DependenciesExtract from deptry.violations import Violation @@ -93,9 +93,12 @@ def run(self) -> None: def _find_python_files(self) -> list[Path]: logging.debug("Collecting Python files to scan...") - python_files = get_all_python_files_in( - self.root, self.exclude, self.extend_exclude, self.using_default_exclude, self.ignore_notebooks - ) + python_files = [ + Path(f) + for f in find_python_files( + self.root, self.exclude, self.extend_exclude, self.using_default_exclude, self.ignore_notebooks + ) + ] logging.debug( "Python files to scan for imports:\n%s\n", "\n".join(str(python_file) for python_file in python_files) diff --git a/python/deptry/python_file_finder.py b/python/deptry/python_file_finder.py deleted file mode 100644 index 2594c1d6..00000000 --- a/python/deptry/python_file_finder.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -from deptry.rust import find_python_files - - -def get_all_python_files_in( - directories: tuple[Path, ...], - exclude: tuple[str, ...], - extend_exclude: tuple[str, ...], - using_default_exclude: bool, - ignore_notebooks: bool = False, -) -> list[Path]: - return [ - Path(f) - for f in find_python_files(directories, exclude, extend_exclude, using_default_exclude, ignore_notebooks) - ] diff --git a/src/python_file_finder.rs b/src/python_file_finder.rs index e34b3ba8..06475ddb 100644 --- a/src/python_file_finder.rs +++ b/src/python_file_finder.rs @@ -91,3 +91,223 @@ fn entry_satisfies_predicate(entry: &DirEntry, regex: Option<&Regex>) -> bool { .unwrap() .is_match(path_str.strip_prefix("./").unwrap_or(&path_str).as_ref()) } + +#[cfg(test)] +mod tests { + use super::*; + use rstest::rstest; + use std::env; + use std::path::Path; + + fn get_test_data_directory() -> &'static Path { + Path::new(concat!( + env!("CARGO_MANIFEST_DIR"), + "/tests/data/file_finder" + )) + } + + fn py_object_to_sorted_vec_string(py: Python, py_object: PyObject) -> Vec { + let mut list: Vec = py_object.extract(py).unwrap(); + list.sort(); + list + } + + #[test] + fn test_simple() { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + env::set_current_dir(get_test_data_directory()).unwrap(); + + let result = find_python_files( + py, + vec![PathBuf::from("./")], + vec![".venv"], + vec!["other_dir"], + false, + false, + ) + .unwrap(); + + assert_eq!( + py_object_to_sorted_vec_string(py, result), + vec![ + ".cache/file1.py", + ".cache/file2.py", + "another_dir/subdir/file1.py", + "dir/subdir/file1.ipynb", + "dir/subdir/file1.py", + "dir/subdir/file2.py", + "dir/subdir/file3.py", + "subdir/file1.py", + ], + ); + }) + } + + #[test] + fn test_only_matches_start() { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + env::set_current_dir(get_test_data_directory()).unwrap(); + + let result = find_python_files( + py, + vec![PathBuf::from("./")], + vec!["foo"], + vec!["subdir"], + false, + false, + ) + .unwrap(); + + assert_eq!( + py_object_to_sorted_vec_string(py, result), + vec![ + ".cache/file1.py", + ".cache/file2.py", + "another_dir/subdir/file1.py", + "dir/subdir/file1.ipynb", + "dir/subdir/file1.py", + "dir/subdir/file2.py", + "dir/subdir/file3.py", + "other_dir/subdir/file1.py", + ], + ); + }) + } + + #[test] + fn test_ignores_notebooks() { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + env::set_current_dir(get_test_data_directory()).unwrap(); + + let result = + find_python_files(py, vec![PathBuf::from("./")], vec![], vec![], false, true) + .unwrap(); + + assert_eq!( + py_object_to_sorted_vec_string(py, result), + vec![ + ".cache/file1.py", + ".cache/file2.py", + "another_dir/subdir/file1.py", + "dir/subdir/file1.py", + "dir/subdir/file2.py", + "dir/subdir/file3.py", + "other_dir/subdir/file1.py", + "subdir/file1.py", + ], + ); + }) + } + + #[rstest] + #[case( + vec![".*file1"], + vec![ + ".cache/file2.py", + "dir/subdir/file2.py", + "dir/subdir/file3.py", + ], + )] + #[case( + vec![".cache|other.*subdir"], + vec![ + "another_dir/subdir/file1.py", + "dir/subdir/file1.ipynb", + "dir/subdir/file1.py", + "dir/subdir/file2.py", + "dir/subdir/file3.py", + "subdir/file1.py", + ], + )] + #[case( + vec![".*/subdir/"], + vec![ + ".cache/file1.py", + ".cache/file2.py", + "subdir/file1.py", + ], + )] + fn test_regex_argument(#[case] exclude: Vec<&str>, #[case] expected: Vec<&str>) { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + env::set_current_dir(get_test_data_directory()).unwrap(); + + let result = + find_python_files(py, vec![PathBuf::from("./")], exclude, vec![], false, false) + .unwrap(); + + assert_eq!(py_object_to_sorted_vec_string(py, result), expected); + }) + } + + #[rstest] + #[case( + vec![".*file1"], + vec![ + "dir/subdir/file2.py", + "dir/subdir/file3.py", + ], + )] + #[case( + vec![".*file1|.*file2"], + vec!["dir/subdir/file3.py"], + )] + #[case( + vec![".*/subdir/"], + vec![], + )] + fn test_multiple_source_directories(#[case] exclude: Vec<&str>, #[case] expected: Vec<&str>) { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + env::set_current_dir(get_test_data_directory()).unwrap(); + + let result = find_python_files( + py, + vec![PathBuf::from("./dir"), PathBuf::from("./other_dir")], + exclude, + vec![], + false, + false, + ) + .unwrap(); + + assert_eq!(py_object_to_sorted_vec_string(py, result), expected); + }) + } + + #[test] + fn test_duplicates_are_removed() { + pyo3::prepare_freethreaded_python(); + Python::with_gil(|py| { + env::set_current_dir(get_test_data_directory()).unwrap(); + + let result = find_python_files( + py, + vec![PathBuf::from("./"), PathBuf::from("./")], + vec![], + vec![], + false, + false, + ) + .unwrap(); + + assert_eq!( + py_object_to_sorted_vec_string(py, result), + vec![ + ".cache/file1.py", + ".cache/file2.py", + "another_dir/subdir/file1.py", + "dir/subdir/file1.ipynb", + "dir/subdir/file1.py", + "dir/subdir/file2.py", + "dir/subdir/file3.py", + "other_dir/subdir/file1.py", + "subdir/file1.py", + ], + ); + }) + } +} diff --git a/tests/data/file_finder/.cache/file1.py b/tests/data/file_finder/.cache/file1.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/.cache/file2.py b/tests/data/file_finder/.cache/file2.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/another_dir/subdir/file1.py b/tests/data/file_finder/another_dir/subdir/file1.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/dir/subdir/file1.ipynb b/tests/data/file_finder/dir/subdir/file1.ipynb new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/dir/subdir/file1.py b/tests/data/file_finder/dir/subdir/file1.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/dir/subdir/file2.py b/tests/data/file_finder/dir/subdir/file2.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/dir/subdir/file3.py b/tests/data/file_finder/dir/subdir/file3.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/other_dir/subdir/file1.py b/tests/data/file_finder/other_dir/subdir/file1.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/data/file_finder/subdir/file1.py b/tests/data/file_finder/subdir/file1.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/unit/test_python_file_finder.py b/tests/unit/test_python_file_finder.py deleted file mode 100644 index d777c035..00000000 --- a/tests/unit/test_python_file_finder.py +++ /dev/null @@ -1,176 +0,0 @@ -from __future__ import annotations - -from pathlib import Path - -import pytest - -from deptry.python_file_finder import get_all_python_files_in -from tests.utils import create_files, run_within_dir - - -def test_simple(tmp_path: Path) -> None: - with run_within_dir(tmp_path): - create_files([ - Path("dir/subdir/file1.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - Path("other_dir/subdir/file1.py"), - Path("other_dir/subdir/file2.py"), - ]) - - files = get_all_python_files_in( - (Path(),), - exclude=(".venv",), - extend_exclude=("other_dir",), - using_default_exclude=False, - ) - - assert sorted(files) == [ - Path("dir/subdir/file1.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - ] - - -def test_only_matches_start(tmp_path: Path) -> None: - """ - Test that adding 'subdir' as exclude argument does not also exclude dir/subdir. - """ - with run_within_dir(tmp_path): - create_files([ - Path("dir/subdir/file1.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - Path("subdir/file1.py"), - Path("subdir/file2.py"), - ]) - - files = get_all_python_files_in( - (Path(),), exclude=("foo",), extend_exclude=("subdir",), using_default_exclude=False - ) - - assert sorted(files) == [ - Path("dir/subdir/file1.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - ] - - -@pytest.mark.parametrize( - ("ignore_notebooks", "expected"), - [ - ( - False, - [Path("dir/subdir/file1.ipynb")], - ), - ( - True, - [], - ), - ], -) -def test_matches_ipynb(ignore_notebooks: bool, expected: list[Path], tmp_path: Path) -> None: - with run_within_dir(tmp_path): - create_files([Path("dir/subdir/file1.ipynb")]) - - files = get_all_python_files_in( - (Path(),), exclude=(), extend_exclude=(), using_default_exclude=False, ignore_notebooks=ignore_notebooks - ) - - assert sorted(files) == expected - - -@pytest.mark.parametrize( - ("exclude", "expected"), - [ - ( - (".*file1",), - [ - Path(".cache/file2.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - Path("other_dir/subdir/file2.py"), - ], - ), - ( - (".cache|other.*subdir",), - [ - Path("dir/subdir/file1.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - ], - ), - ( - (".*/subdir/",), - [ - Path(".cache/file1.py"), - Path(".cache/file2.py"), - ], - ), - ], -) -def test_regex_argument(exclude: tuple[str], expected: list[Path], tmp_path: Path) -> None: - with run_within_dir(tmp_path): - create_files([ - Path(".cache/file1.py"), - Path(".cache/file2.py"), - Path("dir/subdir/file1.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - Path("other_dir/subdir/file1.py"), - Path("other_dir/subdir/file2.py"), - ]) - - files = get_all_python_files_in((Path(),), exclude=exclude, extend_exclude=(), using_default_exclude=False) - - assert sorted(files) == expected - - -@pytest.mark.parametrize( - ("exclude", "expected"), - [ - ( - (".*file1",), - [ - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - Path("other_dir/subdir/file2.py"), - ], - ), - ( - (".*file1|.*file2",), - [ - Path("dir/subdir/file3.py"), - ], - ), - ( - (".*/subdir/",), - [], - ), - ], -) -def test_multiple_source_directories(exclude: tuple[str], expected: list[Path], tmp_path: Path) -> None: - with run_within_dir(tmp_path): - create_files([ - Path("dir/subdir/file1.py"), - Path("dir/subdir/file2.py"), - Path("dir/subdir/file3.py"), - Path("other_dir/subdir/file1.py"), - Path("other_dir/subdir/file2.py"), - Path("another_dir/subdir/file1.py"), - ]) - - files = get_all_python_files_in( - (Path("dir"), Path("other_dir")), exclude=exclude, extend_exclude=(), using_default_exclude=False - ) - - assert sorted(files) == expected - - -def test_duplicates_are_removed(tmp_path: Path) -> None: - with run_within_dir(tmp_path): - create_files([Path("dir/subdir/file1.py")]) - - files = get_all_python_files_in((Path(), Path()), exclude=(), extend_exclude=(), using_default_exclude=False) - - assert sorted(files) == [Path("dir/subdir/file1.py")]