gh-108388: regrtest runs slowest tests first

Lib/test/libregrtest/main.py

@@ -12,7 +12,7 @@
 from test.libregrtest.cmdline import _parse_args
 from test.libregrtest.runtest import (
     findtests, runtest, get_abs_module, is_failed,
-    STDTESTS, NOTTESTS, PROGRESS_MIN_TIME,
+    PROGRESS_MIN_TIME,
     Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted,
     ChildError, DidNotRun)
 from test.libregrtest.setup import setup_tests
@@ -42,6 +42,32 @@
 EXITCODE_ENV_CHANGED = 3
 EXITCODE_NO_TESTS_RAN = 4
 
+# Coarse heuristic: tests taking at least 1 minute on a modern
+# developer laptop. The list should have less than 20 tests.
+SLOWEST_TESTS = frozenset((
+    # more or less sorted from the slowest to the fastest
+    "test_concurrent_futures",
+    "test_multiprocessing_spawn",
+    "test_multiprocessing_forkserver",
+    "test_multiprocessing_fork",
+    "test_multiprocessing_main_handling",
+    "test_pickle",
+    "test_compileall",
+    "test_cppext",
+    "test_venv",
+    "test_gdb",
+    "test_tools",
+    "test_peg_generator",
+    "test_perf_profiler",
+    "test_buffer",
+    "test_subprocess",
+    "test_signal",
+    "test_tarfile",
+    "test_regrtest",
+    "test_socket",
+    "test_io",
+))
+
 
 class Regrtest:
     """Execute a test suite.
@@ -246,21 +272,18 @@ def find_tests(self, tests):
             # add default PGO tests if no tests are specified
             setup_pgo_tests(self.ns)
 
-        stdtests = STDTESTS[:]
-        nottests = NOTTESTS.copy()
+        exclude_tests = set()
         if self.ns.exclude:
             for arg in self.ns.args:
-                if arg in stdtests:
-                    stdtests.remove(arg)
-                nottests.add(arg)
+                exclude_tests.add(arg)
             self.ns.args = []
 
         # if testdir is set, then we are not running the python tests suite, so
         # don't add default tests to be executed or skipped (pass empty values)
         if self.ns.testdir:
-            alltests = findtests(self.ns.testdir, list(), set())
+            alltests = findtests(self.ns.testdir)
         else:
-            alltests = findtests(self.ns.testdir, stdtests, nottests)
+            alltests = findtests(self.ns.testdir, exclude=exclude_tests)
 
         if not self.ns.fromfile:
             self.selected = self.tests or self.ns.args or alltests
@@ -282,11 +305,31 @@ def find_tests(self, tests):
                 print("Couldn't find starting test (%s), using all tests"
                       % self.ns.start, file=sys.stderr)
 
+        self.group_randomize_tests()
+
+    def group_randomize_tests(self):
         if self.ns.randomize:
             if self.ns.random_seed is None:
                 self.ns.random_seed = random.randrange(10000000)
             random.seed(self.ns.random_seed)
-            random.shuffle(self.selected)
+
+        # group slow tests
+        slow = []
+        other = []
+        for name in self.selected:
+            if name in SLOWEST_TESTS:
+                slow.append(name)
+            else:
+                other.append(name)
+
+        if self.ns.randomize:
+            if slow:
+                random.shuffle(slow)
+            if other:
+                random.shuffle(other)
+
+        # gh-108388: Run the slowest first, and then other tests
+        self.selected = slow + other
 
     def list_tests(self):
         for name in self.selected:
@@ -420,8 +463,8 @@ def display_result(self):
         if self.ns.print_slow:
             self.test_times.sort(reverse=True)
             print()
-            print("10 slowest tests:")
-            for test_time, test in self.test_times[:10]:
+            print("20 slowest tests:")
+            for test_time, test in self.test_times[:20]:
                 print("- %s: %s" % (test, format_duration(test_time)))
 
         if self.bad:

Lib/test/libregrtest/runtest.py

@@ -125,24 +125,6 @@ def __str__(self) -> str:
 # the test is running in background
 PROGRESS_MIN_TIME = 30.0   # seconds
 
-# small set of tests to determine if we have a basically functioning interpreter
-# (i.e. if any of these fail, then anything else is likely to follow)
-STDTESTS = [
-    'test_grammar',
-    'test_opcodes',
-    'test_dict',
-    'test_builtin',
-    'test_exceptions',
-    'test_types',
-    'test_unittest',
-    'test_doctest',
-    'test_doctest2',
-    'test_support'
-]
-
-# set of tests that we don't want to be executed when using regrtest
-NOTTESTS = set()
-
 #If these test directories are encountered recurse into them and treat each
 # test_ .py or dir as a separate test module. This can increase parallelism.
 # Beware this can't generally be done for any directory with sub-tests as the
@@ -166,22 +148,23 @@ def findtestdir(path=None):
     return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir
 
 
-def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""):
+def findtests(testdir=None, *, exclude=(), split_test_dirs=SPLITTESTDIRS, base_mod=""):
     """Return a list of all applicable test modules."""
     testdir = findtestdir(testdir)
     names = os.listdir(testdir)
     tests = []
-    others = set(stdtests) | nottests
     for name in names:
         mod, ext = os.path.splitext(name)
-        if mod[:5] == "test_" and mod not in others:
-            if mod in split_test_dirs:
-                subdir = os.path.join(testdir, mod)
-                mod = f"{base_mod or 'test'}.{mod}"
-                tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod))
-            elif ext in (".py", ""):
-                tests.append(f"{base_mod}.{mod}" if base_mod else mod)
-    return stdtests + sorted(tests)
+        if not mod.startswith("test_") or mod in exclude:
+            continue
+
+        if mod in split_test_dirs:
+            subdir = os.path.join(testdir, mod)
+            mod = f"{base_mod or 'test'}.{mod}"
+            tests.extend(findtests(subdir, exclude=exclude, split_test_dirs=split_test_dirs, base_mod=mod))
+        elif ext in (".py", ""):
+            tests.append(f"{base_mod}.{mod}" if base_mod else mod)
+    return sorted(tests)
 
 
 def get_abs_module(ns: Namespace, test_name: str) -> str:

Lib/test/test_peg_generator/__init__.py

@@ -4,8 +4,10 @@
 from test.support import load_package_tests
 
 
-# Creating a virtual environment and building C extensions is slow
-support.requires('cpu')
+if support.check_sanitizer(address=True, memory=True):
+    # gh-90791: Skip the test because it is too slow when Python is built
+    # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
+    raise unittest.SkipTest("test too slow on ASAN/MSAN build")
 
 
 # Load all tests in package

Lib/test/test_tools/__init__.py

@@ -7,6 +7,12 @@
 from test.support import import_helper
 
 
+if support.check_sanitizer(address=True, memory=True):
+    # gh-90791: Skip the test because it is too slow when Python is built
+    # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions.
+    raise unittest.SkipTest("test too slow on ASAN/MSAN build")
+
+
 if not support.has_subprocess_support:
     raise unittest.SkipTest("test module requires subprocess")
 

Lib/test/test_tools/test_freeze.py

@@ -18,9 +18,6 @@
 class TestFreeze(unittest.TestCase):
 
     def test_freeze_simple_script(self):
-        # Building Python is slow
-        support.requires('cpu')
-
         script = textwrap.dedent("""
             import sys
             print('running...')

Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst

@@ -0,0 +1,3 @@
+The Python test suite (regrtest) now runs the 20 slowest tests first and
+then other tests, to better use all available CPUs when running tests in
+parallel. Patch Victor Stinner.