Merge branch 'main' into chore/bump-ruff-ast-parser-0.3.4

Cargo.toml

@@ -11,7 +11,9 @@ crate-type = ["cdylib"]
 [dependencies]
 chardetng = "0.1.17"
 encoding_rs = "0.8.33"
+ignore = "0.4.22"
 log = "0.4.21"
+path-slash = "0.2.1"
 pyo3 = { version = "0.20.3", features = ["abi3-py38"] }
 pyo3-log = "0.9.0"
 rayon = "1.9.0"

pyproject.toml

@@ -25,7 +25,6 @@ classifiers = [
 ]
 dependencies = [
  "click>=8.0.0,<9",
- "pathspec>=0.9.0",
  "colorama>=0.4.6; sys_platform == 'win32'",
  "tomli>=2.0.1; python_version < '3.11'"
 ]

python/deptry/core.py

@@ -14,7 +14,7 @@
 from deptry.exceptions import IncorrectDependencyFormatError, UnsupportedPythonVersionError
 from deptry.imports.extract import get_imported_modules_from_list_of_files
 from deptry.module import ModuleBuilder, ModuleLocations
-from deptry.python_file_finder import PythonFileFinder
+from deptry.python_file_finder import get_all_python_files_in
 from deptry.reporters import JSONReporter, TextReporter
 from deptry.stdlibs import STDLIBS_PYTHON
 from deptry.violations import (
@@ -65,10 +65,7 @@ def run(self) -> None:
 
  self._log_dependencies(dependencies_extract)
 
- all_python_files = PythonFileFinder(
- self.exclude, self.extend_exclude, self.using_default_exclude, self.ignore_notebooks
- ).get_all_python_files_in(self.root)
-
+ python_files = self._find_python_files()
  local_modules = self._get_local_modules()
  stdlib_modules = self._get_stdlib_modules()
 
@@ -83,7 +80,7 @@ def run(self) -> None:
  ).build(),
  locations,
  )
- for module, locations in get_imported_modules_from_list_of_files(all_python_files).items()
+ for module, locations in get_imported_modules_from_list_of_files(python_files).items()
  ]
  imported_modules_with_locations = [
  module_with_locations
@@ -99,6 +96,19 @@ def run(self) -> None:
 
  self._exit(violations)
 
+ def _find_python_files(self) -> list[Path]:
+ logging.debug("Collecting Python files to scan...")
+
+ python_files = get_all_python_files_in(
+ self.root, self.exclude, self.extend_exclude, self.using_default_exclude, self.ignore_notebooks
+ )
+
+ logging.debug(
+ "Python files to scan for imports:\n%s\n", "\n".join(str(python_file) for python_file in python_files)
+ )
+
+ return python_files
+
  def _find_violations(
  self, imported_modules_with_locations: list[ModuleLocations], dependencies: list[Dependency]
  ) -> list[Violation]:

python/deptry/python_file_finder.py

@@ -1,79 +1,18 @@
 from __future__ import annotations
 
-import logging
-import os
-import re
-from dataclasses import dataclass
 from pathlib import Path
-from typing import Pattern
 
-from pathspec import PathSpec
+from deptry.rust import find_python_files
 
 
-@dataclass
-class PythonFileFinder:
- """
- Get a list of all .py and .ipynb files recursively within a directory.
- Args:
- exclude: A list of regex patterns of paths to ignore.
- extend_exclude: An additional list of regex patterns of paths to ignore.
- using_default_exclude: Whether the exclude list was explicitly set, or the default was used.
- ignore_notebooks: If ignore_notebooks is set to True, .ipynb files are ignored and only .py files are returned.
- """
-
- exclude: tuple[str, ...]
- extend_exclude: tuple[str, ...]
- using_default_exclude: bool
- ignore_notebooks: bool = False
-
- def get_all_python_files_in(self, directories: tuple[Path, ...]) -> list[Path]:
- logging.debug("Collecting Python files to scan...")
-
- source_files = set()
-
- ignore_regex = re.compile("|".join(self.exclude + self.extend_exclude))
- file_lookup_suffixes = {".py"} if self.ignore_notebooks else {".py", ".ipynb"}
-
- gitignore_spec = self._generate_gitignore_pathspec(Path())
-
- for directory in directories:
- for root_str, dirs, files in os.walk(directory, topdown=True):
- root = Path(root_str)
-
- if self._is_directory_ignored(root, ignore_regex):
- dirs[:] = []
- continue
-
- for file_str in files:
- file = root / file_str
- if not self._is_file_ignored(file, file_lookup_suffixes, ignore_regex, gitignore_spec):
- source_files.add(file)
-
- source_files_list = list(source_files)
-
- logging.debug("Python files to scan for imports:\n%s\n", "\n".join([str(file) for file in source_files_list]))
-
- return source_files_list
-
- def _is_directory_ignored(self, directory: Path, ignore_regex: Pattern[str]) -> bool:
- return bool((self.exclude + self.extend_exclude) and ignore_regex.match(str(directory)))
-
- def _is_file_ignored(
- self, file: Path, file_lookup_suffixes: set[str], ignore_regex: Pattern[str], gitignore_spec: PathSpec | None
- ) -> bool:
- return bool(
- file.suffix not in file_lookup_suffixes
- or ((self.exclude + self.extend_exclude) and ignore_regex.match(file.as_posix()))
- or (gitignore_spec and gitignore_spec.match_file(file))
- )
-
- def _generate_gitignore_pathspec(self, directory: Path) -> PathSpec | None:
- # If `exclude` is explicitly set, `.gitignore` is not taken into account.
- if not self.using_default_exclude:
- return None
-
- try:
- with (directory / ".gitignore").open() as gitignore:
- return PathSpec.from_lines("gitwildmatch", gitignore)
- except FileNotFoundError:
- return None
+def get_all_python_files_in(
+ directories: tuple[Path, ...],
+ exclude: tuple[str, ...],
+ extend_exclude: tuple[str, ...],
+ using_default_exclude: bool,
+ ignore_notebooks: bool = False,
+) -> list[Path]:
+ return [
+ Path(f)
+ for f in find_python_files(directories, exclude, extend_exclude, using_default_exclude, ignore_notebooks)
+ ]

python/deptry/rust.pyi

@@ -1,7 +1,16 @@
+from pathlib import Path
+
 from .rust import Location as RustLocation
 
 def get_imports_from_py_files(file_paths: list[str]) -> dict[str, list[RustLocation]]: ...
 def get_imports_from_ipynb_files(file_paths: list[str]) -> dict[str, list[RustLocation]]: ...
+def find_python_files(
+ directories: tuple[Path, ...],
+ exclude: tuple[str, ...],
+ extend_exclude: tuple[str, ...],
+ using_default_exclude: bool,
+ ignore_notebooks: bool = False,
+) -> list[str]: ...
 
 class Location:
  file: str

rust-toolchain.toml

@@ -1,2 +1,2 @@
 [toolchain]
-channel = "1.75"
+channel = "1.77"

src/file_utils.rs

@@ -20,8 +20,7 @@ pub fn read_file(file_path: &str) -> PyResult<String> {
  Ok(content) => Ok(content),
  Err(e) => match e.kind() {
  ErrorKind::NotFound => Err(PyFileNotFoundError::new_err(format!(
- "File not found: '{}'",
- file_path
+ "File not found: '{file_path}'",
  ))),
  ErrorKind::InvalidData => {
  let file = File::open(path).unwrap();
@@ -32,7 +31,7 @@ pub fn read_file(file_path: &str) -> PyResult<String> {
  .unwrap_or_else(|| guess_encoding(&buffer));
  read_with_encoding(&buffer, encoding)
  }
- _ => Err(PyIOError::new_err(format!("An error occurred: '{}'", e))),
+ _ => Err(PyIOError::new_err(format!("An error occurred: '{e}'"))),
  },
  }
 }

src/imports/ipynb.rs

@@ -63,7 +63,7 @@ fn _extract_code_from_notebook_cells(cells: &[serde_json::Value]) -> String {
  let code_lines: Vec<String> = cells
  .iter()
  .filter(|cell| cell["cell_type"] == "code")
- .flat_map(|cell| cell["source"].as_array())
+ .filter_map(|cell| cell["source"].as_array())
  .flatten()
  .filter_map(|line| line.as_str())
  .map(str::to_owned)

src/imports/shared.rs

@@ -29,7 +29,7 @@ pub fn get_ast_from_file_content(file_content: &str) -> PyResult<Mod> {
 }
 
 /// Iterates through an AST to identify and collect import statements, and returns them together with their
-/// respective TextRange for each occurrence.
+/// respective `TextRange` for each occurrence.
 pub fn extract_imports_from_ast(ast: Mod) -> HashMap<String, Vec<TextRange>> {
  let mut visitor = ImportVisitor::new();
 
@@ -62,7 +62,7 @@ pub fn convert_imports_with_textranges_to_location_objects(
  .column
  .get();
  Location {
- file: file_path.to_string(),
+ file: file_path.to_owned(),
  line: Some(start_line),
  column: Some(start_col),
  }
@@ -73,7 +73,7 @@ pub fn convert_imports_with_textranges_to_location_objects(
  imports_with_locations
 }
 
-/// Transforms a Rust HashMap containing import data into a Python dictionary suitable for Python-side consumption.
+/// Transforms a Rust `HashMap` containing import data into a Python dictionary suitable for Python-side consumption.
 pub fn convert_to_python_dict(
  py: Python<'_>,
  imports_with_locations: FileToImportsMap,

src/lib.rs

@@ -5,6 +5,7 @@ use pyo3::prelude::*;
 mod file_utils;
 mod imports;
 mod location;
+mod python_file_finder;
 mod visitor;
 
 use location::Location;
@@ -18,6 +19,7 @@ fn rust(_py: Python, m: &PyModule) -> PyResult<()> {
  imports::ipynb::get_imports_from_ipynb_files,
  m
  )?)?;
+ m.add_function(wrap_pyfunction!(python_file_finder::find_python_files, m)?)?;
  m.add_class::<Location>()?;
  Ok(())
 }