Tests autodiscovery replacing hardcoded file list (#13143)

This PR hopefully resolves #8650.

The main change is the `find_test_files ` helper function that looks for files matching the pattern in the unit tests directory. It has been used in the following test files for autodiscovery:
-  `testcheck.py`
-  `testdeps.py`
- ` testfinegrained.py`
- `testparse.py`
- `testsemanal.py`

I've also updated the readme.

mypy/test/helpers.py

@@ -1,11 +1,12 @@
 import os
+import pathlib
 import re
 import sys
 import time
 import shutil
 import contextlib
 
-from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator, Union, Pattern
+from typing import List, Iterable, Dict, Tuple, Callable, Any, Iterator, Union, Pattern, Optional
 
 from mypy import defaults
 import mypy.api as api
@@ -494,3 +495,11 @@ def normalize_file_output(content: List[str], current_abs_path: str) -> List[str
     result = [re.sub(r'\b' + re.escape(base_version) + r'\b', '$VERSION', x) for x in result]
     result = [timestamp_regex.sub('$TIMESTAMP', x) for x in result]
     return result
+
+
+def find_test_files(pattern: str, exclude: Optional[List[str]] = None) -> List[str]:
+    return [
+        path.name
+        for path in (pathlib.Path("./test-data/unit").rglob(pattern))
+        if path.name not in (exclude or [])
+    ]

mypy/test/testcheck.py

@@ -17,6 +17,7 @@
     assert_string_arrays_equal, normalize_error_messages, assert_module_equivalence,
     update_testcase_output, parse_options,
     assert_target_equivalence, check_test_output_files, perform_file_operations,
+    find_test_files,
 )
 from mypy.errors import CompileError
 from mypy.semanal_main import core_modules
@@ -29,96 +30,20 @@
 import pytest
 
 # List of files that contain test case descriptions.
-typecheck_files = [
-    'check-basic.test',
-    'check-union-or-syntax.test',
-    'check-callable.test',
-    'check-classes.test',
-    'check-statements.test',
-    'check-generics.test',
-    'check-dynamic-typing.test',
-    'check-inference.test',
-    'check-inference-context.test',
-    'check-kwargs.test',
-    'check-overloading.test',
-    'check-type-checks.test',
-    'check-abstract.test',
-    'check-multiple-inheritance.test',
-    'check-super.test',
-    'check-modules.test',
-    'check-modules-fast.test',
-    'check-typevar-values.test',
-    'check-unsupported.test',
-    'check-unreachable-code.test',
-    'check-unions.test',
-    'check-isinstance.test',
-    'check-lists.test',
-    'check-namedtuple.test',
-    'check-narrowing.test',
-    'check-typeddict.test',
-    'check-type-aliases.test',
-    'check-ignore.test',
-    'check-type-promotion.test',
-    'check-semanal-error.test',
-    'check-flags.test',
-    'check-incremental.test',
-    'check-serialize.test',
-    'check-bound.test',
-    'check-optional.test',
-    'check-fastparse.test',
-    'check-warnings.test',
-    'check-async-await.test',
-    'check-newtype.test',
-    'check-class-namedtuple.test',
-    'check-selftype.test',
-    'check-python2.test',
-    'check-columns.test',
-    'check-functions.test',
-    'check-tuples.test',
-    'check-expressions.test',
-    'check-generic-subtyping.test',
-    'check-varargs.test',
-    'check-newsyntax.test',
-    'check-protocols.test',
-    'check-underscores.test',
-    'check-classvar.test',
-    'check-enum.test',
-    'check-incomplete-fixture.test',
-    'check-custom-plugin.test',
-    'check-default-plugin.test',
-    'check-attr.test',
-    'check-ctypes.test',
-    'check-dataclasses.test',
-    'check-final.test',
-    'check-redefine.test',
-    'check-literal.test',
-    'check-newsemanal.test',
-    'check-inline-config.test',
-    'check-reports.test',
-    'check-errorcodes.test',
-    'check-annotated.test',
-    'check-parameter-specification.test',
-    'check-typevar-tuple.test',
-    'check-generic-alias.test',
-    'check-typeguard.test',
-    'check-functools.test',
-    'check-singledispatch.test',
-    'check-slots.test',
-    'check-formatting.test',
-    'check-native-int.test',
-]
+# Includes all check-* files with the .test extension in the test-data/unit directory
+typecheck_files = find_test_files(pattern="check-*.test")
 
 # Tests that use Python 3.8-only AST features (like expression-scoped ignores):
-if sys.version_info >= (3, 8):
-    typecheck_files.append('check-python38.test')
-if sys.version_info >= (3, 9):
-    typecheck_files.append('check-python39.test')
-if sys.version_info >= (3, 10):
-    typecheck_files.append('check-python310.test')
+if sys.version_info < (3, 8):
+    typecheck_files.remove('check-python38.test')
+if sys.version_info < (3, 9):
+    typecheck_files.remove('check-python39.test')
+if sys.version_info < (3, 10):
+    typecheck_files.remove('check-python310.test')
 
 # Special tests for platforms with case-insensitive filesystems.
-if sys.platform in ('darwin', 'win32'):
-    typecheck_files.extend(['check-modules-case.test'])
+if sys.platform not in ('darwin', 'win32'):
+    typecheck_files.remove('check-modules-case.test')
 
 
 class TypeCheckSuite(DataSuite):

mypy/test/testdeps.py

@@ -14,7 +14,7 @@
 from mypy.server.deps import get_dependencies
 from mypy.test.config import test_temp_dir
 from mypy.test.data import DataDrivenTestCase, DataSuite
-from mypy.test.helpers import assert_string_arrays_equal, parse_options
+from mypy.test.helpers import assert_string_arrays_equal, parse_options, find_test_files
 from mypy.types import Type
 from mypy.typestate import TypeState
 
@@ -23,14 +23,7 @@
 
 
 class GetDependenciesSuite(DataSuite):
-    files = [
-        'deps.test',
-        'deps-types.test',
-        'deps-generics.test',
-        'deps-expressions.test',
-        'deps-statements.test',
-        'deps-classes.test',
-    ]
+    files = find_test_files(pattern="deps*.test")
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         src = '\n'.join(testcase.input)

mypy/test/testfinegrained.py

@@ -27,7 +27,7 @@
 )
 from mypy.test.helpers import (
     assert_string_arrays_equal, parse_options, assert_module_equivalence,
-    assert_target_equivalence, perform_file_operations,
+    assert_target_equivalence, perform_file_operations, find_test_files,
 )
 from mypy.server.mergecheck import check_consistency
 from mypy.dmypy_util import DEFAULT_STATUS_FILE
@@ -42,15 +42,9 @@
 
 
 class FineGrainedSuite(DataSuite):
-    files = [
-        'fine-grained.test',
-        'fine-grained-cycles.test',
-        'fine-grained-blockers.test',
-        'fine-grained-modules.test',
-        'fine-grained-follow-imports.test',
-        'fine-grained-suggest.test',
-        'fine-grained-attr.test',
-    ]
+    files = find_test_files(
+        pattern="fine-grained*.test", exclude=["fine-grained-cache-incremental.test"]
+    )
 
     # Whether to use the fine-grained cache in the testing. This is overridden
     # by a trivial subclass to produce a suite that uses the cache.

mypy/test/testparse.py

@@ -5,7 +5,7 @@
 from pytest import skip
 
 from mypy import defaults
-from mypy.test.helpers import assert_string_arrays_equal, parse_options
+from mypy.test.helpers import assert_string_arrays_equal, parse_options, find_test_files
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.parse import parse
 from mypy.errors import CompileError
@@ -15,11 +15,10 @@
 class ParserSuite(DataSuite):
     required_out_section = True
     base_path = '.'
-    files = ['parse.test',
-             'parse-python2.test']
+    files = find_test_files(pattern="parse*.test", exclude=["parse-errors.test"])
 
-    if sys.version_info >= (3, 10):
-        files.append('parse-python310.test')
+    if sys.version_info < (3, 10):
+        files.remove('parse-python310.test')
 
     def run_case(self, testcase: DataDrivenTestCase) -> None:
         test_parser(testcase)

mypy/test/testsemanal.py

@@ -9,7 +9,8 @@
 from mypy.modulefinder import BuildSource
 from mypy.defaults import PYTHON3_VERSION
 from mypy.test.helpers import (
-    assert_string_arrays_equal, normalize_error_messages, testfile_pyversion, parse_options
+    assert_string_arrays_equal, normalize_error_messages, testfile_pyversion, parse_options,
+    find_test_files,
 )
 from mypy.test.data import DataDrivenTestCase, DataSuite
 from mypy.test.config import test_temp_dir
@@ -21,26 +22,19 @@
 # Semantic analyzer test cases: dump parse tree
 
 # Semantic analysis test case description files.
-semanal_files = [
-    'semanal-basic.test',
-    'semanal-expressions.test',
-    'semanal-classes.test',
-    'semanal-types.test',
-    'semanal-typealiases.test',
-    'semanal-modules.test',
-    'semanal-statements.test',
-    'semanal-abstractclasses.test',
-    'semanal-namedtuple.test',
-    'semanal-typeddict.test',
-    'semenal-literal.test',
-    'semanal-classvar.test',
-    'semanal-python2.test',
-    'semanal-lambda.test',
-]
-
-
-if sys.version_info >= (3, 10):
-    semanal_files.append('semanal-python310.test')
+semanal_files = find_test_files(
+    pattern="semanal-*.test",
+    exclude=[
+        "semanal-errors-python310.test",
+        "semanal-errors.test",
+        "semanal-typeinfo.test",
+        "semanal-symtable.test",
+    ],
+)
+
+
+if sys.version_info < (3, 10):
+    semanal_files.remove('semanal-python310.test')
 
 
 def get_semanal_options(program_text: str, testcase: DataDrivenTestCase) -> Options:

test-data/unit/README.md

@@ -7,8 +7,7 @@ Quick Start
 
 To add a simple unit test for a new feature you developed, open or create a
 `test-data/unit/check-*.test` file with a name that roughly relates to the
-feature you added. If you added a new `check-*.test` file, add it to the list
-of files in `mypy/test/testcheck.py`.
+feature you added. If you added a new `check-*.test` file, it will be autodiscovered during unittests run.
 
 Add the test in this format anywhere in the file: