From 852175f017fd246a68fb5f9dc7e56beb979a4658 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 19:28:58 +0000 Subject: [PATCH 1/3] gh-118846: Fix free-threading test failures when run sequentially The free-threaded build currently immortalizes some objects once the first thread is started. This can lead to test failures depending on the order in which tests are run. This PR addresses those failures by suppressing immortalization or skipping the affected tests. --- Lib/test/seq_tests.py | 1 + Lib/test/test_capi/test_misc.py | 5 ++++- Lib/test/test_descr.py | 1 + Lib/test/test_gc.py | 12 ++++++++++-- Lib/test/test_inspect/test_inspect.py | 3 ++- Lib/test/test_module/__init__.py | 3 +++ Lib/test/test_trace.py | 4 +++- Lib/test/test_zoneinfo/test_zoneinfo.py | 3 ++- 8 files changed, 26 insertions(+), 6 deletions(-) diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index a41970d8f3f55a..719c9434a16820 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -426,6 +426,7 @@ def test_pickle(self): self.assertEqual(lst2, lst) self.assertNotEqual(id(lst2), id(lst)) + @support.suppress_immortalization() def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index 020e8493e57c0c..ed42d7b64302f9 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -26,7 +26,8 @@ from test.support import threading_helper from test.support import warnings_helper from test.support import requires_limited_api -from test.support import requires_gil_enabled, expected_failure_if_gil_disabled +from test.support import suppress_immortalization +from test.support import expected_failure_if_gil_disabled from test.support import Py_GIL_DISABLED from test.support.script_helper import assert_python_failure, assert_python_ok, run_python_until_end try: @@ -481,6 +482,7 @@ def test_heap_ctype_doc_and_text_signature(self): def test_null_type_doc(self): self.assertEqual(_testcapi.NullTpDocType.__doc__, None) + @suppress_immortalization() def test_subclass_of_heap_gc_ctype_with_tpdealloc_decrefs_once(self): class HeapGcCTypeSubclass(_testcapi.HeapGcCType): def __init__(self): @@ -498,6 +500,7 @@ def __init__(self): del subclass_instance self.assertEqual(type_refcnt - 1, sys.getrefcount(HeapGcCTypeSubclass)) + @suppress_immortalization() def test_subclass_of_heap_gc_ctype_with_del_modifying_dunder_class_only_decrefs_once(self): class A(_testcapi.HeapGcCType): def __init__(self): diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index 18144c8cbb2f0a..c3f292467a6738 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -5014,6 +5014,7 @@ def __new__(cls): cls.lst = [2**i for i in range(10000)] X.descr + @support.suppress_immortalization() def test_remove_subclass(self): # bpo-46417: when the last subclass of a type is deleted, # remove_subclass() clears the internal dictionary of subclasses: diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 52681dc18cfb86..49386dc94e80ec 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -3,7 +3,8 @@ from test import support from test.support import (verbose, refcount_test, cpython_only, requires_subprocess, - requires_gil_enabled) + requires_gil_enabled, suppress_immortalization, + Py_GIL_DISABLED) from test.support.import_helper import import_module from test.support.os_helper import temp_dir, TESTFN, unlink from test.support.script_helper import assert_python_ok, make_script @@ -109,6 +110,7 @@ def test_tuple(self): del l self.assertEqual(gc.collect(), 2) + @suppress_immortalization() def test_class(self): class A: pass @@ -117,6 +119,7 @@ class A: del A self.assertNotEqual(gc.collect(), 0) + @suppress_immortalization() def test_newstyleclass(self): class A(object): pass @@ -133,6 +136,7 @@ class A: del a self.assertNotEqual(gc.collect(), 0) + @suppress_immortalization() def test_newinstance(self): class A(object): pass @@ -219,6 +223,7 @@ class B(object): self.fail("didn't find obj in garbage (finalizer)") gc.garbage.remove(obj) + @suppress_immortalization() def test_function(self): # Tricky: f -> d -> f, code should call d.clear() after the exec to # break the cycle. @@ -561,6 +566,7 @@ def test_get_referents(self): self.assertEqual(gc.get_referents(1, 'a', 4j), []) + @suppress_immortalization() def test_is_tracked(self): # Atomic built-in types are not tracked, user-defined objects and # mutable containers are. @@ -598,7 +604,9 @@ class UserFloatSlots(float): class UserIntSlots(int): __slots__ = () - self.assertTrue(gc.is_tracked(gc)) + if not Py_GIL_DISABLED: + # gh-117783: modules may be immortalized in free-threaded build + self.assertTrue(gc.is_tracked(gc)) self.assertTrue(gc.is_tracked(UserClass)) self.assertTrue(gc.is_tracked(UserClass())) self.assertTrue(gc.is_tracked(UserInt())) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 82e466e978624f..5bed2093cb105b 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -34,7 +34,7 @@ except ImportError: ThreadPoolExecutor = None -from test.support import cpython_only, import_helper +from test.support import cpython_only, import_helper, suppress_immortalization from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ from test.support.import_helper import DirsOnSysPath, ready_to_import from test.support.os_helper import TESTFN, temp_cwd @@ -2430,6 +2430,7 @@ def __getattribute__(self, attr): self.assertFalse(test.called) + @suppress_immortalization() def test_cache_does_not_cause_classes_to_persist(self): # regression test for gh-118013: # check that the internal _shadowed_dict cache does not cause diff --git a/Lib/test/test_module/__init__.py b/Lib/test/test_module/__init__.py index 98d1cbe824df12..952ba43f72504d 100644 --- a/Lib/test/test_module/__init__.py +++ b/Lib/test/test_module/__init__.py @@ -4,6 +4,7 @@ import weakref from test.support import gc_collect from test.support import import_helper +from test.support import suppress_immortalization from test.support.script_helper import assert_python_ok import sys @@ -103,6 +104,7 @@ def f(): gc_collect() self.assertEqual(f().__dict__["bar"], 4) + @suppress_immortalization() def test_clear_dict_in_ref_cycle(self): destroyed = [] m = ModuleType("foo") @@ -118,6 +120,7 @@ def __del__(self): gc_collect() self.assertEqual(destroyed, [1]) + @suppress_immortalization() def test_weakref(self): m = ModuleType("foo") wr = weakref.ref(m) diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index 93966ee31d0a01..7ff3fe4091dfa4 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,7 +1,7 @@ import os from pickle import dump import sys -from test.support import captured_stdout, requires_resource +from test.support import captured_stdout, requires_resource, requires_gil_enabled from test.support.os_helper import (TESTFN, rmtree, unlink) from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap @@ -301,6 +301,7 @@ def test_loop_caller_importing(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_inst_method_calling(self): obj = TracedClass(20) self.tracer.runfunc(obj.inst_method_calling, 1) @@ -334,6 +335,7 @@ def setUp(self): @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) diff --git a/Lib/test/test_zoneinfo/test_zoneinfo.py b/Lib/test/test_zoneinfo/test_zoneinfo.py index 8414721555731e..8bcd6d2e9951b9 100644 --- a/Lib/test/test_zoneinfo/test_zoneinfo.py +++ b/Lib/test/test_zoneinfo/test_zoneinfo.py @@ -17,7 +17,7 @@ from datetime import date, datetime, time, timedelta, timezone from functools import cached_property -from test.support import MISSING_C_DOCSTRINGS +from test.support import MISSING_C_DOCSTRINGS, requires_gil_enabled from test.test_zoneinfo import _support as test_support from test.test_zoneinfo._support import OS_ENV_LOCK, TZPATH_TEST_LOCK, ZoneInfoTestBase from test.support.import_helper import import_module, CleanImport @@ -1931,6 +1931,7 @@ def test_cache_location(self): self.assertFalse(hasattr(c_zoneinfo.ZoneInfo, "_weak_cache")) self.assertTrue(hasattr(py_zoneinfo.ZoneInfo, "_weak_cache")) + @requires_gil_enabled("gh-117783: types may be immortalized") def test_gc_tracked(self): import gc From f469a2e881ef9e3489fcec928f0c5aa9519d5b90 Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 22:21:23 +0000 Subject: [PATCH 2/3] Fix test_pydoc failure --- Lib/test/test_inspect/test_inspect.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py index 5bed2093cb105b..8bd13033490b81 100644 --- a/Lib/test/test_inspect/test_inspect.py +++ b/Lib/test/test_inspect/test_inspect.py @@ -768,6 +768,7 @@ def test_getfile_builtin_function_or_method(self): inspect.getfile(list.append) self.assertIn('expected, got', str(e_append.exception)) + @suppress_immortalization() def test_getfile_class_without_module(self): class CM(type): @property From 0c0c9b4312e1e2668f1294cfca955355ca4788cb Mon Sep 17 00:00:00 2001 From: Sam Gross Date: Thu, 9 May 2024 23:09:41 +0000 Subject: [PATCH 3/3] Fix test_gc when run sequentially --- Lib/test/test_gc.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/Lib/test/test_gc.py b/Lib/test/test_gc.py index 49386dc94e80ec..906f9884d6792f 100644 --- a/Lib/test/test_gc.py +++ b/Lib/test/test_gc.py @@ -1355,6 +1355,10 @@ def callback(ignored): junk = [] i = 0 detector = GC_Detector() + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: @@ -1423,6 +1427,10 @@ def __del__(self): detector = GC_Detector() junk = [] i = 0 + if Py_GIL_DISABLED: + # The free-threaded build doesn't have multiple generations, so + # just trigger a GC manually. + gc.collect() while not detector.gc_happened: i += 1 if i > 10000: