From 852b6a2f0c0d1ab70eb618f6e2cae7dc9aaa49ff Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 02:23:44 +0200 Subject: [PATCH 1/3] gh-108388: regrtest runs slowest tests first The Python test suite (regrtest) now runs the 20 slowest tests first and then other tests, to better use all available CPUs when running tests in parallel. --- Lib/test/libregrtest/main.py | 61 ++++++++++++++++--- Lib/test/libregrtest/runtest.py | 39 ++++-------- ...-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst | 3 + 3 files changed, 66 insertions(+), 37 deletions(-) create mode 100644 Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 361189199d3820..8305816523f59f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -12,7 +12,7 @@ from test.libregrtest.cmdline import _parse_args from test.libregrtest.runtest import ( findtests, runtest, get_abs_module, is_failed, - STDTESTS, NOTTESTS, PROGRESS_MIN_TIME, + PROGRESS_MIN_TIME, Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted, ChildError, DidNotRun) from test.libregrtest.setup import setup_tests @@ -42,6 +42,32 @@ EXITCODE_ENV_CHANGED = 3 EXITCODE_NO_TESTS_RAN = 4 +# Coarse heuristic: tests taking at least 1 minute on a modern +# developer laptop. The list should have less than 20 tests. +SLOWEST_TESTS = frozenset(( + # more or less sorted from the slowest to the fastest + "test_concurrent_futures", + "test_multiprocessing_spawn", + "test_multiprocessing_forkserver", + "test_multiprocessing_fork", + "test_multiprocessing_main_handling", + "test_pickle", + "test_compileall", + "test_cppext", + "test_venv", + "test_gdb", + "test_tools", + "test_peg_generator", + "test_perf_profiler", + "test_buffer", + "test_subprocess", + "test_signal", + "test_tarfile", + "test_regrtest", + "test_socket", + "test_io", +)) + class Regrtest: """Execute a test suite. @@ -246,21 +272,18 @@ def find_tests(self, tests): # add default PGO tests if no tests are specified setup_pgo_tests(self.ns) - stdtests = STDTESTS[:] - nottests = NOTTESTS.copy() + exclude_tests = set() if self.ns.exclude: for arg in self.ns.args: - if arg in stdtests: - stdtests.remove(arg) - nottests.add(arg) + exclude_tests.add(arg) self.ns.args = [] # if testdir is set, then we are not running the python tests suite, so # don't add default tests to be executed or skipped (pass empty values) if self.ns.testdir: - alltests = findtests(self.ns.testdir, list(), set()) + alltests = findtests(self.ns.testdir) else: - alltests = findtests(self.ns.testdir, stdtests, nottests) + alltests = findtests(self.ns.testdir, exclude=exclude_tests) if not self.ns.fromfile: self.selected = self.tests or self.ns.args or alltests @@ -282,11 +305,31 @@ def find_tests(self, tests): print("Couldn't find starting test (%s), using all tests" % self.ns.start, file=sys.stderr) + self.group_randomize_tests() + + def group_randomize_tests(self): if self.ns.randomize: if self.ns.random_seed is None: self.ns.random_seed = random.randrange(10000000) random.seed(self.ns.random_seed) - random.shuffle(self.selected) + + # group slow tests + slow = [] + other = [] + for name in self.selected: + if name in SLOWEST_TESTS: + slow.append(name) + else: + other.append(name) + + if self.ns.randomize: + if slow: + random.shuffle(slow) + if other: + random.shuffle(other) + + # gh-108388: Run the slowest first, and then other tests + self.selected = slow + other def list_tests(self): for name in self.selected: diff --git a/Lib/test/libregrtest/runtest.py b/Lib/test/libregrtest/runtest.py index 61595277ed6d5a..9cc1ab7a9c389e 100644 --- a/Lib/test/libregrtest/runtest.py +++ b/Lib/test/libregrtest/runtest.py @@ -125,24 +125,6 @@ def __str__(self) -> str: # the test is running in background PROGRESS_MIN_TIME = 30.0 # seconds -# small set of tests to determine if we have a basically functioning interpreter -# (i.e. if any of these fail, then anything else is likely to follow) -STDTESTS = [ - 'test_grammar', - 'test_opcodes', - 'test_dict', - 'test_builtin', - 'test_exceptions', - 'test_types', - 'test_unittest', - 'test_doctest', - 'test_doctest2', - 'test_support' -] - -# set of tests that we don't want to be executed when using regrtest -NOTTESTS = set() - #If these test directories are encountered recurse into them and treat each # test_ .py or dir as a separate test module. This can increase parallelism. # Beware this can't generally be done for any directory with sub-tests as the @@ -166,22 +148,23 @@ def findtestdir(path=None): return path or os.path.dirname(os.path.dirname(__file__)) or os.curdir -def findtests(testdir=None, stdtests=STDTESTS, nottests=NOTTESTS, *, split_test_dirs=SPLITTESTDIRS, base_mod=""): +def findtests(testdir=None, *, exclude=(), split_test_dirs=SPLITTESTDIRS, base_mod=""): """Return a list of all applicable test modules.""" testdir = findtestdir(testdir) names = os.listdir(testdir) tests = [] - others = set(stdtests) | nottests for name in names: mod, ext = os.path.splitext(name) - if mod[:5] == "test_" and mod not in others: - if mod in split_test_dirs: - subdir = os.path.join(testdir, mod) - mod = f"{base_mod or 'test'}.{mod}" - tests.extend(findtests(subdir, [], nottests, split_test_dirs=split_test_dirs, base_mod=mod)) - elif ext in (".py", ""): - tests.append(f"{base_mod}.{mod}" if base_mod else mod) - return stdtests + sorted(tests) + if not mod.startswith("test_") or mod in exclude: + continue + + if mod in split_test_dirs: + subdir = os.path.join(testdir, mod) + mod = f"{base_mod or 'test'}.{mod}" + tests.extend(findtests(subdir, exclude=exclude, split_test_dirs=split_test_dirs, base_mod=mod)) + elif ext in (".py", ""): + tests.append(f"{base_mod}.{mod}" if base_mod else mod) + return sorted(tests) def get_abs_module(ns: Namespace, test_name: str) -> str: diff --git a/Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst b/Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst new file mode 100644 index 00000000000000..11980539ba360b --- /dev/null +++ b/Misc/NEWS.d/next/Tests/2023-08-24-02-36-15.gh-issue-108388.Z08JFZ.rst @@ -0,0 +1,3 @@ +The Python test suite (regrtest) now runs the 20 slowest tests first and +then other tests, to better use all available CPUs when running tests in +parallel. Patch Victor Stinner. From 9d5c2033cf5cbb0a9a29658bd19806e071a33eb6 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 02:38:52 +0200 Subject: [PATCH 2/3] Revert "test_peg_generator and test_freeze require cpu (#108386)" This reverts commit 7a6cc3eb66805e6515d5db5b79e58e2f76b550c8. --- Lib/test/test_peg_generator/__init__.py | 6 ++++-- Lib/test/test_tools/__init__.py | 6 ++++++ Lib/test/test_tools/test_freeze.py | 3 --- 3 files changed, 10 insertions(+), 5 deletions(-) diff --git a/Lib/test/test_peg_generator/__init__.py b/Lib/test/test_peg_generator/__init__.py index c23542e254c99f..87281eb5e03c7f 100644 --- a/Lib/test/test_peg_generator/__init__.py +++ b/Lib/test/test_peg_generator/__init__.py @@ -4,8 +4,10 @@ from test.support import load_package_tests -# Creating a virtual environment and building C extensions is slow -support.requires('cpu') +if support.check_sanitizer(address=True, memory=True): + # gh-90791: Skip the test because it is too slow when Python is built + # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. + raise unittest.SkipTest("test too slow on ASAN/MSAN build") # Load all tests in package diff --git a/Lib/test/test_tools/__init__.py b/Lib/test/test_tools/__init__.py index c4395c7c0ad0c9..dde5d84e9edc6b 100644 --- a/Lib/test/test_tools/__init__.py +++ b/Lib/test/test_tools/__init__.py @@ -7,6 +7,12 @@ from test.support import import_helper +if support.check_sanitizer(address=True, memory=True): + # gh-90791: Skip the test because it is too slow when Python is built + # with ASAN/MSAN: between 5 and 20 minutes on GitHub Actions. + raise unittest.SkipTest("test too slow on ASAN/MSAN build") + + if not support.has_subprocess_support: raise unittest.SkipTest("test module requires subprocess") diff --git a/Lib/test/test_tools/test_freeze.py b/Lib/test/test_tools/test_freeze.py index 3e9a48b5bc6a89..2ba36ca208f967 100644 --- a/Lib/test/test_tools/test_freeze.py +++ b/Lib/test/test_tools/test_freeze.py @@ -18,9 +18,6 @@ class TestFreeze(unittest.TestCase): def test_freeze_simple_script(self): - # Building Python is slow - support.requires('cpu') - script = textwrap.dedent(""" import sys print('running...') From 3345a176b25bd23b29b571068e8d839d91aecbd5 Mon Sep 17 00:00:00 2001 From: Victor Stinner Date: Thu, 24 Aug 2023 02:42:29 +0200 Subject: [PATCH 3/3] DEBUG: Log the 20 slowest tests --- Lib/test/libregrtest/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 8305816523f59f..d6e5031c771d07 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -463,8 +463,8 @@ def display_result(self): if self.ns.print_slow: self.test_times.sort(reverse=True) print() - print("10 slowest tests:") - for test_time, test in self.test_times[:10]: + print("20 slowest tests:") + for test_time, test in self.test_times[:20]: print("- %s: %s" % (test, format_duration(test_time))) if self.bad: