Check functional test directory for correct structure

pylint/testutils/functional/find_functional_tests.py

@@ -3,29 +3,81 @@
 
 import os
 from pathlib import Path
-from typing import List, Union
+from typing import List, Set, Union
 
 from pylint.testutils.functional.test_file import FunctionalTestFile
 
-# 'Wet finger' number of files that are reasonable to display by an IDE
-# 'Wet finger' as in 'in my settings there are precisely this many'.
 REASONABLY_DISPLAYABLE_VERTICALLY = 48
+"""'Wet finger' number of files that are reasonable to display by an IDE."""
+SHOULD_BE_IN_THE_SAME_DIRECTORY = 5
+"""'Wet finger' as in 'in my settings there are precisely this many'."""
+
+IGNORED_PARENT_DIRS = {
+    "deprecated_relative_import",
+    "ext",
+    "regression",
+    "regression_02",
+}
+"""Direct parent directories that should be ignored."""
+
+IGNORED_PARENT_PARENT_DIRS = {
+    "docparams",
+    "deprecated_relative_import",
+    "ext",
+}
+"""Parents of direct parent directories that should be ignored."""
 
 
 def get_functional_test_files_from_directory(
     input_dir: Union[Path, str]
 ) -> List[FunctionalTestFile]:
     """Get all functional tests in the input_dir."""
     suite = []
+
+    _check_functional_tests_structure(Path(input_dir))
+
     for dirpath, _, filenames in os.walk(input_dir):
         if dirpath.endswith("__pycache__"):
             continue
-
-        assert (
-            len(filenames) <= REASONABLY_DISPLAYABLE_VERTICALLY
-        ), f"{dirpath} contains too many functional tests files."
-
         for filename in filenames:
             if filename != "__init__.py" and filename.endswith(".py"):
                 suite.append(FunctionalTestFile(dirpath, filename))
     return suite
+
+
+def _check_functional_tests_structure(directory: Path) -> None:
+    """Check if test directories follow correct file/folder structure."""
+    # Ignore underscored directories
+    if Path(directory).stem.startswith("_"):
+        return
+
+    files: Set[Path] = set()
+    dirs: Set[Path] = set()
+
+    # Collect all subdirectories and files in directory
+    for file_or_dir in directory.iterdir():
+        if file_or_dir.is_file():
+            if file_or_dir.suffix == ".py" and not file_or_dir.stem.startswith("_"):
+                files.add(file_or_dir)
+        elif file_or_dir.is_dir():
+            dirs.add(file_or_dir)
+            _check_functional_tests_structure(file_or_dir)
+
+    assert len(files) <= REASONABLY_DISPLAYABLE_VERTICALLY, (
+        f"{directory} contains too many functional tests files "
+        + f"({len(files)} > {REASONABLY_DISPLAYABLE_VERTICALLY})."
+    )
+
+    for file in files:
+        possible_dir = file.parent / file.stem.split("_")[0]
+        assert not possible_dir.exists(), f"{file} should go in {possible_dir}."
+
+        # Exclude some directories as they follow a different structure
+        if (
+            not len(file.parent.stem) == 1  # First letter subdirectories
+            and file.parent.stem not in IGNORED_PARENT_DIRS
+            and file.parent.parent.stem not in IGNORED_PARENT_PARENT_DIRS
+        ):
+            assert file.stem.startswith(
+                file.parent.stem
+            ), f"{file} should not go in {file.parent}"

tests/test_functional_directories.py

@@ -0,0 +1,12 @@
+"""Test that the directory structure of the functional tests is correct."""
+from pathlib import Path
+
+from pylint.testutils.functional.find_functional_tests import (
+    get_functional_test_files_from_directory,
+)
+
+
+def test_directories() -> None:
+    """Test that the directory structure of the functional tests is correct."""
+    functional_dir = Path(__file__).parent / "functional"
+    get_functional_test_files_from_directory(functional_dir)

tests/testutils/test_functional_testutils.py

@@ -8,7 +8,10 @@
 import pytest
 
 from pylint import testutils
-from pylint.testutils.functional import FunctionalTestFile
+from pylint.testutils.functional import (
+    FunctionalTestFile,
+    get_functional_test_files_from_directory,
+)
 
 HERE = Path(__file__).parent
 DATA_DIRECTORY = HERE / "data"
@@ -19,3 +22,9 @@ def test_parsing_of_pylintrc_init_hook() -> None:
     with pytest.raises(RuntimeError):
         test_file = FunctionalTestFile(str(DATA_DIRECTORY), "init_hook.py")
         testutils.LintModuleTest(test_file)
+
+
+def test_get_functional_test_files_from_directory() -> None:
+    """Test that we correctly check the functional test directory structures."""
+    with pytest.raises(AssertionError, match="using_dir.py should not go in"):
+        get_functional_test_files_from_directory(DATA_DIRECTORY)