Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refine how we detect namespace packages #12169

Merged
merged 18 commits into from
Apr 9, 2024
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
Refactor code and improve docs
nicoddemus committed Apr 7, 2024
commit fddb7db189264e2d64fbf960e105e3293f9b8300
53 changes: 31 additions & 22 deletions src/_pytest/pathlib.py
Original file line number Diff line number Diff line change
@@ -37,8 +37,6 @@
import uuid
import warnings

from typing_extensions import TypeGuard

from _pytest.compat import assert_never
from _pytest.outcomes import skip
from _pytest.warning_types import PytestWarning
@@ -637,6 +635,7 @@ def _import_module_using_spec(
spec = importlib.util.spec_from_file_location(module_name, str(module_path))

if spec_matches_module_path(spec, module_path):
assert spec is not None
mod = importlib.util.module_from_spec(spec)
sys.modules[module_name] = mod
spec.loader.exec_module(mod) # type: ignore[union-attr]
@@ -649,7 +648,8 @@ def _import_module_using_spec(

def spec_matches_module_path(
module_spec: Optional[ModuleSpec], module_path: Path
) -> TypeGuard[ModuleSpec]:
) -> bool:
"""Return true if the given ModuleSpec can be used to import the given module path."""
if module_spec is not None and module_spec.origin is not None:
if Path(module_spec.origin) == module_path:
return True
@@ -775,7 +775,7 @@ def resolve_pkg_root_and_module_name(
Passing the full path to `models.py` will yield Path("src") and "app.core.models".
If consider_namespace_packages is True, then we additionally check upwards in the hierarchy
until we find a directory that is reachable from sys.path, which marks it as a namespace package:
for namespace packages:
https://packaging.python.org/en/latest/guides/packaging-namespace-packages
@@ -788,39 +788,48 @@ def resolve_pkg_root_and_module_name(
if consider_namespace_packages:
start = path.parent if pkg_path is None else pkg_path.parent
for candidate in (start, *start.parents):
if _is_importable(candidate, path):
if is_importable(candidate, path):
# Point the pkg_root to the root of the namespace package.
pkg_root = candidate.parent
pkg_root = candidate
break

if pkg_root is not None:
names = list(path.with_suffix("").relative_to(pkg_root).parts)
if names[-1] == "__init__":
names.pop()
module_name = ".".join(names)
module_name = compute_module_name(pkg_root, path)
return pkg_root, module_name

raise CouldNotResolvePathError(f"Could not resolve for {path}")


def _is_importable(root: Path, path: Path) -> bool:
def is_importable(root: Path, module_path: Path) -> bool:
"""
Return if the given module path could be imported normally by Python, akin to the user
nicoddemus marked this conversation as resolved.
Show resolved Hide resolved
entering the REPL and importing the corresponding module name directly.
"""
module_name = compute_module_name(root, module_path)
try:
# Note this is different from what we do in ``import_path``, where we also search sys.meta_path.
nicoddemus marked this conversation as resolved.
Show resolved Hide resolved
# Searching sys.meta_path will eventually find a spec which can load the file even if the interpreter would
# not find this module normally in the REPL, which is exactly what we want to be able to do in
# ``import_path``, but not here.
spec = importlib.util.find_spec(module_name)
except (ImportError, ValueError, ImportWarning):
return False
else:
return spec_matches_module_path(spec, module_path)


def compute_module_name(root: Path, path: Path) -> str:
"""Compute a module name based on a path and a root anchor."""
try:
path_without_suffix = path.with_suffix("")
except ValueError:
# Empty paths (such as Path.cwd()) might break meta_path hooks (like our own assertion rewriter).
return False
return ""

names = list(path_without_suffix.relative_to(root.parent).parts)
if names[-1] == "__init__":
names = list(path_without_suffix.relative_to(root).parts)
if names and names[-1] == "__init__":
names.pop()
module_name = ".".join(names)

try:
spec = importlib.util.find_spec(module_name)
except (ImportError, ValueError, ImportWarning):
return False
else:
return spec_matches_module_path(spec, path)
return ".".join(names)


class CouldNotResolvePathError(Exception):
34 changes: 27 additions & 7 deletions testing/test_pathlib.py
Original file line number Diff line number Diff line change
@@ -18,9 +18,9 @@
import unittest.mock

from _pytest.monkeypatch import MonkeyPatch
from _pytest.pathlib import _is_importable
from _pytest.pathlib import bestrelpath
from _pytest.pathlib import commonpath
from _pytest.pathlib import compute_module_name
from _pytest.pathlib import CouldNotResolvePathError
from _pytest.pathlib import ensure_deletable
from _pytest.pathlib import fnmatch_ex
@@ -30,6 +30,7 @@
from _pytest.pathlib import ImportMode
from _pytest.pathlib import ImportPathMismatchError
from _pytest.pathlib import insert_missing_modules
from _pytest.pathlib import is_importable
from _pytest.pathlib import maybe_delete_a_numbered_dir
from _pytest.pathlib import module_name_from_path
from _pytest.pathlib import resolve_package_path
@@ -1321,13 +1322,25 @@ def test_is_importable_bad_arguments(self, pytester: Pytester) -> None:
pytester.syspathinsert()
path = pytester.path / "bar.x"
path.mkdir()
assert _is_importable(path.parent, path) is False
assert is_importable(path.parent, path) is False

path = pytester.path / ".bar.x"
path.mkdir()
assert _is_importable(path.parent, path) is False
assert is_importable(path.parent, path) is False

assert _is_importable(Path(), Path()) is False
def test_compute_module_name(self, tmp_path: Path) -> None:
assert compute_module_name(tmp_path, tmp_path) == ""
assert compute_module_name(Path(), Path()) == ""

assert compute_module_name(tmp_path, tmp_path / "mod.py") == "mod"
assert compute_module_name(tmp_path, tmp_path / "src/app/bar") == "src.app.bar"
assert (
compute_module_name(tmp_path, tmp_path / "src/app/bar.py") == "src.app.bar"
)
assert (
compute_module_name(tmp_path, tmp_path / "src/app/bar/__init__.py")
== "src.app.bar"
)

@pytest.mark.parametrize("insert", [True, False])
def test_full_ns_packages_without_init_files(
@@ -1362,14 +1375,21 @@ def test_full_ns_packages_without_init_files(


def validate_namespace_package(
pytester: Pytester, paths: Sequence[Path], imports: Sequence[str]
pytester: Pytester, paths: Sequence[Path], modules: Sequence[str]
) -> RunResult:
"""Validate a Python namespace package by importing modules from it in a Python subprocess"""
"""
Validate that a Python namespace package is set up correctly.
In a sub interpreter, add 'paths' to sys.path and attempt to import the given modules.
In this module many tests configure a set of files as a namespace package, this function
is used as sanity check that our files are configured correctly from the point of view of Python.
"""
lines = [
"import sys",
# Configure sys.path.
*[f"sys.path.append(r{str(x)!r})" for x in paths],
# Imports.
*[f"import {x}" for x in imports],
*[f"import {x}" for x in modules],
]
return pytester.runpython_c("\n".join(lines))