pythongh-82054: Executing tests in parallel by sharding.

Lib/test/libregrtest/runtest.py

@@ -143,6 +143,14 @@ def __str__(self) -> str:
 # set of tests that we don't want to be executed when using regrtest
 NOTTESTS = set()
 
+# If these test directories are encountered recurse into them and treat each
+# test_ .py or dir as a separate test module. This can increase parallelism.
+# Beware this can't generally be done for any directory with sub-tests as the
+# __init__.py may do things which alter what tests are to be run.
+SPLITTESTDIRS = {
+ "test_asyncio",
+ "test_compiler",
+}
 
 # Storage of uncollectable objects
 FOUND_GARBAGE = []
@@ -158,18 +166,26 @@ def findtestdir(path=None):
  return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
 
 
-def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS):
- """Return a list of all applicable test modules."""
- testdir = findtestdir(testdir)
- names = os.listdir(testdir)
- tests = []
- others = set(stdtests) | nottests
- for name in names:
- mod, ext = os.path.splitext(name)
- if mod[:5] == "test_" and ext in (".py", "") and mod not in others:
- tests.append(mod)
- return stdtests + sorted(tests)
 
+ def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, splittestdirs=SPLITTESTDIRS, base_mod=""):
+ """Return a list of all applicable test modules."""
+ testdir = findtestdir(testdir)
+ names = os.listdir(testdir)
+ tests = []
+ others = set(stdtests) | nottests
+ for name in names:
+ mod, ext = os.path.splitext(name)
+ if mod[:5] == "test_" and mod not in others:
+ if mod in splittestdirs:
+ subdir = os.path.join(testdir, mod)
+ if len(base_mod):
+ mod = f"{base_mod}.{mod}"
+ else:
+ mod = f"test.{mod}"
+ tests.extend(findtests(subdir, [], nottests, splittestdirs, mod))
+ elif ext in (".py", ""):
+ tests.append(f"{base_mod}.{mod}" if len(base_mod) else mod)
+ return stdtests + sorted(tests)
 
 def get_abs_module(ns: Namespace, test_name: str) -> str:
  if test_name.startswith('test.') or ns.testdir: