Skip to content

Commit 852b6a2

Browse files
committed
pythongh-108388: regrtest runs slowest tests first
The Python test suite (regrtest) now runs the 20 slowest tests first and then other tests, to better use all available CPUs when running tests in parallel.
1 parent 7a6cc3e commit 852b6a2

File tree

3 files changed

+66
-37
lines changed

3 files changed

+66
-37
lines changed

Lib/test/libregrtest/main.py

+52-9
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@
1212
from test.libregrtest.cmdline import _parse_args
1313
from test.libregrtest.runtest import (
1414
findtests, runtest, get_abs_module, is_failed,
15-
STDTESTS, NOTTESTS, PROGRESS_MIN_TIME,
15+
PROGRESS_MIN_TIME,
1616
Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted,
1717
ChildError, DidNotRun)
1818
from test.libregrtest.setup import setup_tests
@@ -42,6 +42,32 @@
4242
EXITCODE_ENV_CHANGED = 3
4343
EXITCODE_NO_TESTS_RAN = 4
4444

45+
# Coarse heuristic: tests taking at least 1 minute on a modern
46+
# developer laptop. The list should have less than 20 tests.
47+
SLOWEST_TESTS = frozenset((
48+
# more or less sorted from the slowest to the fastest
49+
"test_concurrent_futures",
50+
"test_multiprocessing_spawn",
51+
"test_multiprocessing_forkserver",
52+
"test_multiprocessing_fork",
53+
"test_multiprocessing_main_handling",
54+
"test_pickle",
55+
"test_compileall",
56+
"test_cppext",
57+
"test_venv",
58+
"test_gdb",
59+
"test_tools",
60+
"test_peg_generator",
61+
"test_perf_profiler",
62+
"test_buffer",
63+
"test_subprocess",
64+
"test_signal",
65+
"test_tarfile",
66+
"test_regrtest",
67+
"test_socket",
68+
"test_io",
69+
))
70+
4571

4672
class Regrtest:
4773
"""Execute a test suite.
@@ -246,21 +272,18 @@ def find_tests(self, tests):
246272
# add default PGO tests if no tests are specified
247273
setup_pgo_tests(self.ns)
248274

249-
stdtests = STDTESTS[:]
250-
nottests = NOTTESTS.copy()
275+
exclude_tests = set()
251276
if self.ns.exclude:
252277
for arg in self.ns.args:
253-
if arg in stdtests:
254-
stdtests.remove(arg)
255-
nottests.add(arg)
278+
exclude_tests.add(arg)
256279
self.ns.args = []
257280

258281
# if testdir is set, then we are not running the python tests suite, so
259282
# don't add default tests to be executed or skipped (pass empty values)
260283
if self.ns.testdir:
261-
alltests = findtests(self.ns.testdir, list(), set())
284+
alltests = findtests(self.ns.testdir)
262285
else:
263-
alltests = findtests(self.ns.testdir, stdtests, nottests)
286+
alltests = findtests(self.ns.testdir, exclude=exclude_tests)
264287

265288
if not self.ns.fromfile:
266289
self.selected = self.tests or self.ns.args or alltests
@@ -282,11 +305,31 @@ def find_tests(self, tests):
282305
print("Couldn't find starting test (%s), using all tests"
283306
% self.ns.start, file=sys.stderr)
284307

308+
self.group_randomize_tests()
309+
310+
def group_randomize_tests(self):
285311
if self.ns.randomize:
286312
if self.ns.random_seed is None:
287313
self.ns.random_seed = random.randrange(10000000)
288314
random.seed(self.ns.random_seed)
289-
random.shuffle(self.selected)
315+
316+
# group slow tests
317+
slow = []
318+
other = []
319+
for name in self.selected:
320+
if name in SLOWEST_TESTS:
321+
slow.append(name)
322+
else:
323+
other.append(name)
324+
325+
if self.ns.randomize:
326+
if slow:
327+
random.shuffle(slow)
328+
if other:
329+
random.shuffle(other)
330+
331+
# gh-108388: Run the slowest first, and then other tests
332+
self.selected = slow + other
290333

291334
def list_tests(self):
292335
for name in self.selected:

Lib/test/libregrtest/runtest.py

+11-28
Original file line numberDiff line numberDiff line change
@@ -125,24 +125,6 @@ def __str__(self) -> str:
125125
# the test is running in background
126126
PROGRESS_MIN_TIME = 30.0 # seconds
127127

128-
# small set of tests to determine if we have a basically functioning interpreter
129-
# (i.e. if any of these fail, then anything else is likely to follow)
130-
STDTESTS = [
131-
'test_grammar',
132-
'test_opcodes',
133-
'test_dict',
134-
'test_builtin',
135-
'test_exceptions',
136-
'test_types',
137-
'test_unittest',
138-
'test_doctest',
139-
'test_doctest2',
140-
'test_support'
141-
]
142-
143-
# set of tests that we don't want to be executed when using regrtest
144-
NOTTESTS = set()
145-
146128
#If these test directories are encountered recurse into them and treat each
147129
# test_ .py or dir as a separate test module. This can increase parallelism.
148130
# Beware this can't generally be done for any directory with sub-tests as the
@@ -166,22 +148,23 @@ def findtestdir(path=None):
166148
return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
167149

168150

169-
def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""):
151+
def findtests(testdir=None, *, exclude=(), split_test_dirs=SPLITTESTDIRS, base_mod=""):
170152
"""Return a list of all applicable test modules."""
171153
testdir = findtestdir(testdir)
172154
names = os.listdir(testdir)
173155
tests = []
174-
others = set(stdtests) | nottests
175156
for name in names:
176157
mod, ext = os.path.splitext(name)
177-
if mod[:5] == "test_" and mod not in others:
178-
if mod in split_test_dirs:
179-
subdir = os.path.join(testdir, mod)
180-
mod = f"{base_mod or 'test'}.{mod}"
181-
tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod))
182-
elif ext in (".py", ""):
183-
tests.append(f"{base_mod}.{mod}" if base_mod else mod)
184-
return stdtests + sorted(tests)
158+
if not mod.startswith("test_") or mod in exclude:
159+
continue
160+
161+
if mod in split_test_dirs:
162+
subdir = os.path.join(testdir, mod)
163+
mod = f"{base_mod or 'test'}.{mod}"
164+
tests.extend(findtests(subdir, exclude=exclude, split_test_dirs=split_test_dirs, base_mod=mod))
165+
elif ext in (".py", ""):
166+
tests.append(f"{base_mod}.{mod}" if base_mod else mod)
167+
return sorted(tests)
185168

186169

187170
def get_abs_module(ns: Namespace, test_name: str) -> str:
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
The Python test suite (regrtest) now runs the 20 slowest tests first and
2+
then other tests, to better use all available CPUs when running tests in
3+
parallel. Patch Victor Stinner.

0 commit comments

Comments
 (0)