[3.12] gh-108822: regrtest computes statistics (#108793)

Lib/test/libregrtest/main.py

@@ -11,15 +11,14 @@
 import unittest
 from test.libregrtest.cmdline import _parse_args
 from test.libregrtest.runtest import (
-    findtests, split_test_packages, runtest, get_abs_module, is_failed,
-    PROGRESS_MIN_TIME,
-    Passed, Failed, EnvChanged, Skipped, ResourceDenied, Interrupted,
-    ChildError, DidNotRun)
+    findtests, split_test_packages, runtest, get_abs_module,
+    PROGRESS_MIN_TIME, State)
 from test.libregrtest.setup import setup_tests
 from test.libregrtest.pgo import setup_pgo_tests
 from test.libregrtest.utils import (removepy, count, format_duration,
                                     printlist, get_build_info)
 from test import support
+from test.support import TestStats
 from test.support import os_helper
 from test.support import threading_helper
 
@@ -78,13 +77,14 @@ def __init__(self):
         self.good = []
         self.bad = []
         self.skipped = []
-        self.resource_denieds = []
+        self.resource_denied = []
         self.environment_changed = []
         self.run_no_tests = []
         self.need_rerun = []
         self.rerun = []
         self.first_result = None
         self.interrupted = False
+        self.stats_dict: dict[str, TestStats] = {}
 
         # used by --slow
         self.test_times = []
@@ -93,7 +93,7 @@ def __init__(self):
         self.tracer = None
 
         # used to display the progress bar "[ 3/100]"
-        self.start_time = time.monotonic()
+        self.start_time = time.perf_counter()
         self.test_count = ''
         self.test_count_width = 1
 
@@ -111,36 +111,41 @@ def __init__(self):
 
     def get_executed(self):
         return (set(self.good) | set(self.bad) | set(self.skipped)
-                | set(self.resource_denieds) | set(self.environment_changed)
+                | set(self.resource_denied) | set(self.environment_changed)
                 | set(self.run_no_tests))
 
     def accumulate_result(self, result, rerun=False):
-        test_name = result.name
-
-        if not isinstance(result, (ChildError, Interrupted)) and not rerun:
-            self.test_times.append((result.duration_sec, test_name))
-
-        if isinstance(result, Passed):
-            self.good.append(test_name)
-        elif isinstance(result, ResourceDenied):
-            self.skipped.append(test_name)
-            self.resource_denieds.append(test_name)
-        elif isinstance(result, Skipped):
-            self.skipped.append(test_name)
-        elif isinstance(result, EnvChanged):
-            self.environment_changed.append(test_name)
-        elif isinstance(result, Failed):
-            if not rerun:
-                self.bad.append(test_name)
-                self.need_rerun.append(result)
-        elif isinstance(result, DidNotRun):
-            self.run_no_tests.append(test_name)
-        elif isinstance(result, Interrupted):
-            self.interrupted = True
-        else:
-            raise ValueError("invalid test result: %r" % result)
+        test_name = result.test_name
+
+        if result.has_meaningful_duration() and not rerun:
+            self.test_times.append((result.duration, test_name))
 
-        if rerun and not isinstance(result, (Failed, Interrupted)):
+        match result.state:
+            case State.PASSED:
+                self.good.append(test_name)
+            case State.ENV_CHANGED:
+                self.environment_changed.append(test_name)
+            case State.SKIPPED:
+                self.skipped.append(test_name)
+            case State.RESOURCE_DENIED:
+                self.skipped.append(test_name)
+                self.resource_denied.append(test_name)
+            case State.INTERRUPTED:
+                self.interrupted = True
+            case State.DID_NOT_RUN:
+                self.run_no_tests.append(test_name)
+            case _:
+                if result.is_failed(self.ns.fail_env_changed):
+                    if not rerun:
+                        self.bad.append(test_name)
+                        self.need_rerun.append(result)
+                else:
+                    raise ValueError(f"invalid test state: {state!r}")
+
+        if result.stats is not None:
+            self.stats_dict[result.test_name] = result.stats
+
+        if rerun and not(result.is_failed(False) or result.state == State.INTERRUPTED):
             self.bad.remove(test_name)
 
         xml_data = result.xml_data
@@ -162,7 +167,7 @@ def log(self, line=''):
             line = f"load avg: {load_avg:.2f} {line}"
 
         # add the timestamp prefix:  "0:01:05 "
-        test_time = time.monotonic() - self.start_time
+        test_time = time.perf_counter() - self.start_time
 
         mins, secs = divmod(int(test_time), 60)
         hours, mins = divmod(mins, 60)
@@ -337,7 +342,7 @@ def rerun_failed_tests(self):
         rerun_list = list(self.need_rerun)
         self.need_rerun.clear()
         for result in rerun_list:
-            test_name = result.name
+            test_name = result.test_name
             self.rerun.append(test_name)
 
             errors = result.errors or []
@@ -364,7 +369,7 @@ def rerun_failed_tests(self):
 
             self.accumulate_result(result, rerun=True)
 
-            if isinstance(result, Interrupted):
+            if result.state == State.INTERRUPTED:
                 break
 
         if self.bad:
@@ -461,7 +466,7 @@ def run_tests_sequential(self):
 
         previous_test = None
         for test_index, test_name in enumerate(self.tests, 1):
-            start_time = time.monotonic()
+            start_time = time.perf_counter()
 
             text = test_name
             if previous_test:
@@ -480,14 +485,14 @@ def run_tests_sequential(self):
                 result = runtest(self.ns, test_name)
                 self.accumulate_result(result)
 
-            if isinstance(result, Interrupted):
+            if result.state == State.INTERRUPTED:
                 break
 
             previous_test = str(result)
-            test_time = time.monotonic() - start_time
+            test_time = time.perf_counter() - start_time
             if test_time >= PROGRESS_MIN_TIME:
                 previous_test = "%s in %s" % (previous_test, format_duration(test_time))
-            elif isinstance(result, Passed):
+            elif result.state == State.PASSED:
                 # be quiet: say nothing if the test passed shortly
                 previous_test = None
 
@@ -496,7 +501,7 @@ def run_tests_sequential(self):
                 if module not in save_modules and module.startswith("test."):
                     support.unload(module)
 
-            if self.ns.failfast and is_failed(result, self.ns):
+            if self.ns.failfast and result.is_failed(self.ns.fail_env_changed):
                 break
 
         if previous_test:
@@ -631,13 +636,48 @@ def finalize(self):
                             coverdir=self.ns.coverdir)
 
         print()
-        duration = time.monotonic() - self.start_time
-        print("Total duration: %s" % format_duration(duration))
-        print("Tests result: %s" % self.get_tests_result())
+        self.display_summary()
 
         if self.ns.runleaks:
             os.system("leaks %d" % os.getpid())
 
+    def display_summary(self):
+        duration = time.perf_counter() - self.start_time
+
+        # Total duration
+        print("Total duration: %s" % format_duration(duration))
+
+        # Total tests
+        total = TestStats()
+        for stats in self.stats_dict.values():
+            total.accumulate(stats)
+        stats = [f'run={total.tests_run:,}']
+        if total.failures:
+            stats.append(f'failures={total.failures:,}')
+        if total.skipped:
+            stats.append(f'skipped={total.skipped:,}')
+        print(f"Total tests: {' '.join(stats)}")
+
+        # Total test files
+        report = [f'success={len(self.good)}']
+        if self.bad:
+            report.append(f'failed={len(self.bad)}')
+        if self.environment_changed:
+            report.append(f'env_changed={len(self.environment_changed)}')
+        if self.skipped:
+            report.append(f'skipped={len(self.skipped)}')
+        if self.resource_denied:
+            report.append(f'resource_denied={len(self.resource_denied)}')
+        if self.rerun:
+            report.append(f'rerun={len(self.rerun)}')
+        if self.run_no_tests:
+            report.append(f'run_no_tests={len(self.run_no_tests)}')
+        print(f"Total test files: {' '.join(report)}")
+
+        # Result
+        result = self.get_tests_result()
+        print(f"Result: {result}")
+
     def save_xml_result(self):
         if not self.ns.xmlpath and not self.testsuite_xml:
             return

Lib/test/libregrtest/refleak.py

@@ -83,11 +83,12 @@ def get_pooled_int(value):
         print(("1234567890"*(repcount//10 + 1))[:repcount], file=sys.stderr,
               flush=True)
 
+    results = None
     dash_R_cleanup(fs, ps, pic, zdc, abcs)
     support.gc_collect()
 
     for i in rep_range:
-        test_func()
+        results = test_func()
 
         dash_R_cleanup(fs, ps, pic, zdc, abcs)
         support.gc_collect()
@@ -151,7 +152,7 @@ def check_fd_deltas(deltas):
                 print(msg, file=refrep)
                 refrep.flush()
             failed = True
-    return failed
+    return (failed, results)
 
 
 def dash_R_cleanup(fs, ps, pic, zdc, abcs):

Lib/test/libregrtest/runtest_mp.py

@@ -1,3 +1,4 @@
+import dataclasses
 import faulthandler
 import json
 import os.path
@@ -13,12 +14,13 @@
 
 from test import support
 from test.support import os_helper
+from test.support import TestStats
 
 from test.libregrtest.cmdline import Namespace
 from test.libregrtest.main import Regrtest
 from test.libregrtest.runtest import (
-    runtest, is_failed, TestResult, Interrupted, Timeout, ChildError,
-    PROGRESS_MIN_TIME, Passed, EnvChanged)
+    runtest, TestResult, State,
+    PROGRESS_MIN_TIME)
 from test.libregrtest.setup import setup_tests
 from test.libregrtest.utils import format_duration, print_warning
 
@@ -43,9 +45,9 @@
 
 
 def must_stop(result: TestResult, ns: Namespace) -> bool:
-    if isinstance(result, Interrupted):
+    if result.state == State.INTERRUPTED:
         return True
-    if ns.failfast and is_failed(result, ns):
+    if ns.failfast and result.is_failed(ns.fail_env_changed):
         return True
     return False
 
@@ -130,8 +132,8 @@ def stop(self):
 class MultiprocessResult(NamedTuple):
     result: TestResult
     # bpo-45410: stderr is written into stdout to keep messages order
-    stdout: str
-    error_msg: str
+    worker_stdout: str | None = None
+    err_msg: str | None = None
 
 
 ExcStr = str
@@ -209,15 +211,12 @@ def stop(self) -> None:
     def mp_result_error(
         self,
         test_result: TestResult,
-        stdout: str = '',
+        stdout: str | None = None,
         err_msg=None
     ) -> MultiprocessResult:
-        test_result.duration_sec = time.monotonic() - self.start_time
         return MultiprocessResult(test_result, stdout, err_msg)
 
     def _run_process(self, test_name: str, tmp_dir: str, stdout_fh: TextIO) -> int:
-        self.start_time = time.monotonic()
-
         self.current_test_name = test_name
         try:
             popen = run_test_in_subprocess(test_name, self.ns, tmp_dir, stdout_fh)
@@ -306,38 +305,41 @@ def _runtest(self, test_name: str) -> MultiprocessResult:
                 # gh-101634: Catch UnicodeDecodeError if stdout cannot be
                 # decoded from encoding
                 err_msg = f"Cannot read process stdout: {exc}"
-                return self.mp_result_error(ChildError(test_name), '', err_msg)
+                result = TestResult(test_name, state=State.MULTIPROCESSING_ERROR)
+                return self.mp_result_error(result, err_msg=err_msg)
 
         if retcode is None:
-            return self.mp_result_error(Timeout(test_name), stdout)
+            result = TestResult(test_name, state=State.TIMEOUT)
+            return self.mp_result_error(result, stdout)
 
         err_msg = None
         if retcode != 0:
             err_msg = "Exit code %s" % retcode
         else:
-            stdout, _, result = stdout.rpartition("\n")
+            stdout, _, worker_json = stdout.rpartition("\n")
             stdout = stdout.rstrip()
-            if not result:
+            if not worker_json:
                 err_msg = "Failed to parse worker stdout"
             else:
                 try:
                     # deserialize run_tests_worker() output
-                    result = json.loads(result, object_hook=decode_test_result)
+                    result = json.loads(worker_json,
+                                        object_hook=decode_test_result)
                 except Exception as exc:
                     err_msg = "Failed to parse worker JSON: %s" % exc
 
-        if err_msg is not None:
-            return self.mp_result_error(ChildError(test_name), stdout, err_msg)
+        if err_msg:
+            result = TestResult(test_name, state=State.MULTIPROCESSING_ERROR)
+            return self.mp_result_error(result, stdout, err_msg)
 
         if tmp_files:
             msg = (f'\n\n'
                    f'Warning -- {test_name} leaked temporary files '
                    f'({len(tmp_files)}): {", ".join(sorted(tmp_files))}')
             stdout += msg
-            if isinstance(result, Passed):
-                result = EnvChanged.from_passed(result)
+            result.set_env_changed()
 
-        return MultiprocessResult(result, stdout, err_msg)
+        return MultiprocessResult(result, stdout)
 
     def run(self) -> None:
         while not self._stopped:
@@ -347,7 +349,9 @@ def run(self) -> None:
                 except StopIteration:
                     break
 
+                self.start_time = time.monotonic()
                 mp_result = self._runtest(test_name)
+                mp_result.result.duration = time.monotonic() - self.start_time
                 self.output.put((False, mp_result))
 
                 if must_stop(mp_result.result, self.ns):
@@ -473,11 +477,11 @@ def display_result(self, mp_result: MultiprocessResult) -> None:
         result = mp_result.result
 
         text = str(result)
-        if mp_result.error_msg is not None:
-            # CHILD_ERROR
-            text += ' (%s)' % mp_result.error_msg
-        elif (result.duration_sec >= PROGRESS_MIN_TIME and not self.ns.pgo):
-            text += ' (%s)' % format_duration(result.duration_sec)
+        if mp_result.err_msg:
+            # MULTIPROCESSING_ERROR
+            text += ' (%s)' % mp_result.err_msg
+        elif (result.duration >= PROGRESS_MIN_TIME and not self.ns.pgo):
+            text += ' (%s)' % format_duration(result.duration)
         running = get_running(self.workers)
         if running and not self.ns.pgo:
             text += ' -- running: %s' % ', '.join(running)
@@ -489,7 +493,7 @@ def _process_result(self, item: QueueOutput) -> bool:
             # Thread got an exception
             format_exc = item[1]
             print_warning(f"regrtest worker thread failed: {format_exc}")
-            result = ChildError("<regrtest worker>")
+            result = TestResult("<regrtest worker>", state=State.MULTIPROCESSING_ERROR)
             self.regrtest.accumulate_result(result)
             return True
 
@@ -498,8 +502,8 @@ def _process_result(self, item: QueueOutput) -> bool:
         self.regrtest.accumulate_result(mp_result.result)
         self.display_result(mp_result)
 
-        if mp_result.stdout:
-            print(mp_result.stdout, flush=True)
+        if mp_result.worker_stdout:
+            print(mp_result.worker_stdout, flush=True)
 
         if must_stop(mp_result.result, self.ns):
             return True
@@ -541,32 +545,20 @@ class EncodeTestResult(json.JSONEncoder):
 
     def default(self, o: Any) -> dict[str, Any]:
         if isinstance(o, TestResult):
-            result = vars(o)
+            result = dataclasses.asdict(o)
             result["__test_result__"] = o.__class__.__name__
             return result
 
         return super().default(o)
 
 
-def decode_test_result(d: dict[str, Any]) -> TestResult | dict[str, Any]:
+def decode_test_result(d: dict[str, Any]) -> TestResult | TestStats | dict[str, Any]:
     """Decode a TestResult (sub)class object from a JSON dict."""
 
     if "__test_result__" not in d:
         return d
 
-    cls_name = d.pop("__test_result__")
-    for cls in get_all_test_result_classes():
-        if cls.__name__ == cls_name:
-            return cls(**d)
-
-
-def get_all_test_result_classes() -> set[type[TestResult]]:
-    prev_count = 0
-    classes = {TestResult}
-    while len(classes) > prev_count:
-        prev_count = len(classes)
-        to_add = []
-        for cls in classes:
-            to_add.extend(cls.__subclasses__())
-        classes.update(to_add)
-    return classes
+    d.pop('__test_result__')
+    if d['stats'] is not None:
+        d['stats'] = TestStats(**d['stats'])
+    return TestResult(**d)

Lib/test/libregrtest/save_env.py

@@ -23,7 +23,7 @@ class SkipTestEnvironment(Exception):
 class saved_test_environment:
     """Save bits of the test environment and restore them at block exit.
 
-        with saved_test_environment(testname, verbose, quiet):
+        with saved_test_environment(test_name, verbose, quiet):
             #stuff
 
     Unless quiet is True, a warning is printed to stderr if any of
@@ -34,8 +34,8 @@ class saved_test_environment:
     items is also printed.
     """
 
-    def __init__(self, testname, verbose=0, quiet=False, *, pgo=False):
-        self.testname = testname
+    def __init__(self, test_name, verbose=0, quiet=False, *, pgo=False):
+        self.test_name = test_name
         self.verbose = verbose
         self.quiet = quiet
         self.pgo = pgo
@@ -323,7 +323,7 @@ def __exit__(self, exc_type, exc_val, exc_tb):
                 restore(original)
                 if not self.quiet and not self.pgo:
                     print_warning(
-                        f"{name} was modified by {self.testname}\n"
+                        f"{name} was modified by {self.test_name}\n"
                         f"  Before: {original}\n"
                         f"  After:  {current} ")
         return False

Lib/test/support/__init__.py

@@ -4,6 +4,7 @@
     raise ImportError('support must be imported from the test package')
 
 import contextlib
+import dataclasses
 import functools
 import getpass
 import opcode
@@ -118,17 +119,20 @@ class Error(Exception):
 
 class TestFailed(Error):
     """Test failed."""
+    def __init__(self, msg, *args, stats=None):
+        self.msg = msg
+        self.stats = stats
+        super().__init__(msg, *args)
+
+    def __str__(self):
+        return self.msg
 
 class TestFailedWithDetails(TestFailed):
     """Test failed."""
-    def __init__(self, msg, errors, failures):
-        self.msg = msg
+    def __init__(self, msg, errors, failures, stats):
         self.errors = errors
         self.failures = failures
-        super().__init__(msg, errors, failures)
-
-    def __str__(self):
-        return self.msg
+        super().__init__(msg, errors, failures, stats=stats)
 
 class TestDidNotRun(Error):
     """Test did not run any subtests."""
@@ -1108,6 +1112,29 @@ def _filter_suite(suite, pred):
                 newtests.append(test)
     suite._tests = newtests
 
+@dataclasses.dataclass(slots=True)
+class TestStats:
+    tests_run: int = 0
+    failures: int = 0
+    skipped: int = 0
+
+    @staticmethod
+    def from_unittest(result):
+        return TestStats(result.testsRun,
+                         len(result.failures),
+                         len(result.skipped))
+
+    @staticmethod
+    def from_doctest(results):
+        return TestStats(results.attempted,
+                         results.failed)
+
+    def accumulate(self, stats):
+        self.tests_run += stats.tests_run
+        self.failures += stats.failures
+        self.skipped += stats.skipped
+
+
 def _run_suite(suite):
     """Run tests from a unittest.TestSuite-derived class."""
     runner = get_test_runner(sys.stdout,
@@ -1122,6 +1149,7 @@ def _run_suite(suite):
     if not result.testsRun and not result.skipped and not result.errors:
         raise TestDidNotRun
     if not result.wasSuccessful():
+        stats = TestStats.from_unittest(result)
         if len(result.errors) == 1 and not result.failures:
             err = result.errors[0][1]
         elif len(result.failures) == 1 and not result.errors:
@@ -1131,7 +1159,8 @@ def _run_suite(suite):
             if not verbose: err += "; run in verbose mode for details"
         errors = [(str(tc), exc_str) for tc, exc_str in result.errors]
         failures = [(str(tc), exc_str) for tc, exc_str in result.failures]
-        raise TestFailedWithDetails(err, errors, failures)
+        raise TestFailedWithDetails(err, errors, failures, stats=stats)
+    return result
 
 
 # By default, don't filter tests
@@ -1240,7 +1269,7 @@ def run_unittest(*classes):
         else:
             suite.addTest(loader.loadTestsFromTestCase(cls))
     _filter_suite(suite, match_test)
-    _run_suite(suite)
+    return _run_suite(suite)
 
 #=======================================================================
 # Check for the presence of docstrings.
@@ -1280,13 +1309,18 @@ def run_doctest(module, verbosity=None, optionflags=0):
     else:
         verbosity = None
 
-    f, t = doctest.testmod(module, verbose=verbosity, optionflags=optionflags)
-    if f:
-        raise TestFailed("%d of %d doctests failed" % (f, t))
+    results = doctest.testmod(module,
+                             verbose=verbosity,
+                             optionflags=optionflags)
+    if results.failed:
+        stats = TestStats.from_doctest(results)
+        raise TestFailed(f"{results.failed} of {results.attempted} "
+                         f"doctests failed",
+                         stats=stats)
     if verbose:
         print('doctest (%s) ... %d tests with zero failures' %
-              (module.__name__, t))
-    return f, t
+              (module.__name__, results.attempted))
+    return results
 
 
 #=======================================================================

Lib/test/test_netrc.py

@@ -309,7 +309,7 @@ def test_security(self):
                              ('anonymous', '', 'pass'))
 
 def test_main():
-    run_unittest(NetrcTestCase)
+    return run_unittest(NetrcTestCase)
 
 if __name__ == "__main__":
     test_main()

Lib/test/test_pep646_syntax.py

@@ -320,7 +320,7 @@
 def test_main(verbose=False):
     from test import support
     from test import test_pep646_syntax
-    support.run_doctest(test_pep646_syntax, verbose)
+    return support.run_doctest(test_pep646_syntax, verbose)
 
 if __name__ == "__main__":
     test_main(verbose=True)

Lib/test/test_xml_etree.py

@@ -4250,7 +4250,7 @@ def test_main(module=None):
         old_factories = None
 
     try:
-        support.run_unittest(*test_classes)
+        return support.run_unittest(*test_classes)
     finally:
         from xml.etree import ElementPath
         # Restore mapping and path cache