diff --git a/deptry/python_file_finder.py b/deptry/python_file_finder.py
index 7b1549fc..48832d3e 100644
--- a/deptry/python_file_finder.py
+++ b/deptry/python_file_finder.py
@@ -34,7 +34,7 @@ def get_all_python_files_in(self, directory: Path) -> list[Path]:
         ignore_regex = re.compile("|".join(self.exclude + self.extend_exclude))
         file_lookup_suffixes = {".py"} if self.ignore_notebooks else {".py", ".ipynb"}
 
-        gitignore_spec = self._generate_gitignore_pathspec(directory)
+        gitignore_spec = self._generate_gitignore_pathspec(Path("."))
 
         for root_str, dirs, files in os.walk(directory, topdown=True):
             root = Path(root_str)
diff --git a/tests/data/project_with_src_directory/.gitignore b/tests/data/project_with_src_directory/.gitignore
new file mode 100644
index 00000000..891399da
--- /dev/null
+++ b/tests/data/project_with_src_directory/.gitignore
@@ -0,0 +1 @@
+src/this_file_is_gitignored.py
diff --git a/tests/data/project_with_src_directory/src/this_file_is_gitignored.py b/tests/data/project_with_src_directory/src/this_file_is_gitignored.py
new file mode 100644
index 00000000..517a14f3
--- /dev/null
+++ b/tests/data/project_with_src_directory/src/this_file_is_gitignored.py
@@ -0,0 +1 @@
+import a_non_existing_module