diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py
index 361189199d3820..d6e5031c771d07 100644
--- a/Lib/test/libregrtest/main.py
+++ b/Lib/test/libregrtest/main.py
@@ -12,7 +12,7 @@
from test.libregrtest.cmdline import _parse_args
from test.libregrtest.runtest import (
findtests, runtest, get_abs_module, is_failed,
- STDTESTS, NOTTESTS, PROGRESS_MIN_TIME,
+ PROGRESS_MIN_TIME,
Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted,
ChildError, DidNotRun)
from test.libregrtest.setup import setup_tests
@@ -42,6 +42,32 @@
EXITCODE_ENV_CHANGED = 3
EXITCODE_NO_TESTS_RAN = 4
+# Coarse heuristic: tests taking at least 1 minute on a modern
+# developer laptop. The list should have less than 20 tests.
+SLOWEST_TESTS = frozenset((
+ # more or less sorted from the slowest to the fastest
+ "test_concurrent_futures",
+ "test_multiprocessing_spawn",
+ "test_multiprocessing_forkserver",
+ "test_multiprocessing_fork",
+ "test_multiprocessing_main_handling",
+ "test_pickle",
+ "test_compileall",
+ "test_cppext",
+ "test_venv",
+ "test_gdb",
+ "test_tools",
+ "test_peg_generator",
+ "test_perf_profiler",
+ "test_buffer",
+ "test_subprocess",
+ "test_signal",
+ "test_tarfile",
+ "test_regrtest",
+ "test_socket",
+ "test_io",
+))
+
class Regrtest:
"""Execute a test suite.
@@ -246,21 +272,18 @@ def find_tests(self, tests):
# add default PGO tests if no tests are specified
setup_pgo_tests(self.ns)
- stdtests = STDTESTS[:]
- nottests = NOTTESTS.copy()
+ exclude_tests = set()
if self.ns.exclude:
for arg in self.ns.args:
- if arg in stdtests:
- stdtests.remove(arg)
- nottests.add(arg)
+ exclude_tests.add(arg)
self.ns.args = []
# if testdir is set, then we are not running the python tests suite, so
# don't add default tests to be executed or skipped (pass empty values)
if self.ns.testdir:
- alltests = findtests(self.ns.testdir, list(), set())
+ alltests = findtests(self.ns.testdir)
else:
- alltests = findtests(self.ns.testdir, stdtests, nottests)
+ alltests = findtests(self.ns.testdir, exclude=exclude_tests)
if not self.ns.fromfile:
self.selected = self.tests or self.ns.args or alltests
@@ -282,11 +305,31 @@ def find_tests(self, tests):
print("Couldn't find starting test (%s), using all tests"
% self.ns.start, file=sys.stderr)
+ self.group_randomize_tests()
+
+ def group_randomize_tests(self):
if self.ns.randomize:
if self.ns.random_seed is None:
self.ns.random_seed = random.randrange(10000000)
random.seed(self.ns.random_seed)
- random.shuffle(self.selected)
+
+ # group slow tests
+ slow = []
+ other = []
+ for name in self.selected:
+ if name in SLOWEST_TESTS:
+ slow.append(name)
+ else:
+ other.append(name)
+
+ if self.ns.randomize:
+ if slow:
+ random.shuffle(slow)
+ if other:
+ random.shuffle(other)
+
+ # gh-108388: Run the slowest first, and then other tests
+ self.selected = slow + other
def list_tests(self):
for name in self.selected:
@@ -420,8 +463,8 @@ def display_result(self):
if self.ns.print_slow:
self.test_times.sort(reverse=True)
print()
- print("10 slowest tests:")
- for test_time, test in self.test_times[:10]:
+ print("20 slowest tests:")
+ for test_time, test in self.test_times[:20]:
print("- %s: %s" % (test, format_duration(test_time)))
if self.bad:
diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py
index 61595277ed6d5a..9cc1ab7a9c389e 100644
--- a/Lib/test/libregrtest/runtest.py
+++ b/Lib/test/libregrtest/runtest.py
@@ -125,24 +125,6 @@ def __str__(self) -> str:
# the test is running in background
PROGRESS_MIN_TIME = 30.0 # seconds
-# small set of tests to determine if we have a basically functioning interpreter
-# (i.e. if any of these fail, then anything else is likely to follow)
-STDTESTS = [
- 'test_grammar',
- 'test_opcodes',
- 'test_dict',
- 'test_builtin',
- 'test_exceptions',
- 'test_types',
- 'test_unittest',
- 'test_doctest',
- 'test_doctest2',
- 'test_support'
-]
-
-# set of tests that we don't want to be executed when using regrtest
-NOTTESTS = set()
-
#If these test directories are encountered recurse into them and treat each
# test_ .py or dir as a separate test module. This can increase parallelism.
# Beware this can't generally be done for any directory with sub-tests as the
@@ -166,22 +148,23 @@ def findtestdir(path=None):
return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
-def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""):
+def findtests(testdir=None, *, exclude=(), split_test_dirs=SPLITTESTDIRS, base_mod=""):
"""Return a list of all applicable test modules."""
testdir = findtestdir(testdir)
names = os.listdir(testdir)
tests = []
- others = set(stdtests) | nottests
for name in names:
mod, ext = os.path.splitext(name)
- if mod[:5] == "test_" and mod not in others:
- if mod in split_test_dirs:
- subdir = os.path.join(testdir, mod)
- mod = f"{base_mod or 'test'}.{mod}"
- tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod))
- elif ext in (".py", ""):
- tests.append(f"{base_mod}.{mod}" if base_mod else mod)
- return stdtests + sorted(tests)
+ if not mod.startswith("test_") or mod in exclude:
+ continue
+
+ if mod in split_test_dirs:
+ subdir = os.path.join(testdir, mod)
+ mod = f"{base_mod or 'test'}.{mod}"
+ tests.extend(findtests(subdir, exclude=exclude, split_test_dirs=split_test_dirs, base_mod=mod))
+ elif ext in (".py", ""):
+ tests.append(f"{base_mod}.{mod}" if base_mod else mod)
+ return sorted(tests)
def get_abs_module(ns: Namespace, test_name: str) -> str:
diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py
index c23542e254c99f..87281eb5e03c7f 100644
--- a/Lib/test/test_peg_generator/__init__.py
+++ b/Lib/test/test_peg_generator/__init__.py
@@ -4,8 +4,10 @@
from test.support import load_package_tests
-# Creating a virtual environment and building C extensions is slow
-support.requires('cpu')
+if support.check_sanitizer(address=True, memory=True):
+ # gh-90791: Skip the test because it is too slow when Python is built
+ # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
+ raise unittest.SkipTest("test too slow on ASAN/MSAN build")
# Load all tests in package
diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py
index c4395c7c0ad0c9..dde5d84e9edc6b 100644
--- a/Lib/test/test_tools/__init__.py
+++ b/Lib/test/test_tools/__init__.py
@@ -7,6 +7,12 @@
from test.support import import_helper
+if support.check_sanitizer(address=True, memory=True):
+ # gh-90791: Skip the test because it is too slow when Python is built
+ # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
+ raise unittest.SkipTest("test too slow on ASAN/MSAN build")
+
+
if not support.has_subprocess_support:
raise unittest.SkipTest("test module requires subprocess")
diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py
index 3e9a48b5bc6a89..2ba36ca208f967 100644
--- a/Lib/test/test_tools/test_freeze.py
+++ b/Lib/test/test_tools/test_freeze.py
@@ -18,9 +18,6 @@
class TestFreeze(unittest.TestCase):
def test_freeze_simple_script(self):
- # Building Python is slow
- support.requires('cpu')
-
script = textwrap.dedent("""
import sys
print('running...')
diff --git a/Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst b/Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst
new file mode 100644
index 00000000000000..11980539ba360b
--- /dev/null
+++ b/Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst
@@ -0,0 +1,3 @@
+The Python test suite (regrtest) now runs the 20 slowest tests first and
+then other tests, to better use all available CPUs when running tests in
+parallel. Patch Victor Stinner.