From a5bd3f1fa3a58431b66a2bb9514675e7e5f10e8b Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Mon, 11 Nov 2024 20:06:43 +0800 Subject: [PATCH 01/17] remove `_PyOptimizer_NewCounter` --- Include/internal/pycore_optimizer.h | 1 - Lib/test/test_monitoring.py | 29 ----------------------------- Modules/_testinternalcapi.c | 7 ------- Python/optimizer.c | 12 ------------ 4 files changed, 49 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index bc7cfcde613d65..9aa537497918cb 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -119,7 +119,6 @@ PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); // Export for '_testinternalcapi' shared extension. PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); -PyAPI_FUNC(PyObject *) _PyOptimizer_NewCounter(void); PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 43e3e56639db62..ba88a4c6d451d4 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2085,35 +2085,6 @@ def callback(code, instruction_offset): sys.monitoring.set_events(0, 0) -class TestOptimizer(MonitoringTestBase, unittest.TestCase): - - def setUp(self): - _testinternalcapi = import_module("_testinternalcapi") - if hasattr(_testinternalcapi, "get_optimizer"): - self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(opt) - super(TestOptimizer, self).setUp() - - def tearDown(self): - super(TestOptimizer, self).tearDown() - import _testinternalcapi - if hasattr(_testinternalcapi, "get_optimizer"): - _testinternalcapi.set_optimizer(self.old_opt) - - def test_for_loop(self): - def test_func(x): - i = 0 - while i < x: - i += 1 - - code = test_func.__code__ - sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) - self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) - test_func(1000) - sys.monitoring.set_local_events(TEST_TOOL, code, 0) - self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) - class TestTier2Optimizer(CheckEvents): def test_monitoring_already_opimized_loop(self): diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 150d34d168f5e4..38a8e31f1d105b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -989,12 +989,6 @@ get_co_framesize(PyObject *self, PyObject *arg) #ifdef _Py_TIER2 -static PyObject * -new_counter_optimizer(PyObject *self, PyObject *arg) -{ - return _PyOptimizer_NewCounter(); -} - static PyObject * new_uop_optimizer(PyObject *self, PyObject *arg) { @@ -2101,7 +2095,6 @@ static PyMethodDef module_functions[] = { #ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, - {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, diff --git a/Python/optimizer.c b/Python/optimizer.c index 52b3f0a84afedf..3815c45c0f8952 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1410,18 +1410,6 @@ PyTypeObject _PyCounterOptimizer_Type = { .tp_dealloc = (destructor)PyObject_Free, }; -PyObject * -_PyOptimizer_NewCounter(void) -{ - _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type); - if (opt == NULL) { - return NULL; - } - opt->base.optimize = counter_optimize; - opt->count = 0; - return (PyObject *)opt; -} - /***************************************** * Executor management From a77915c59ea5ec5971e8e244b6af8e5f32cca283 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 15 Nov 2024 11:00:54 +0800 Subject: [PATCH 02/17] remove uop optimize tests --- Lib/test/test_capi/test_opt.py | 6 ++++-- Modules/_testinternalcapi.c | 7 ------- 2 files changed, 4 insertions(+), 9 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 12542d8b7fa62e..5feaffe06bf5e0 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -141,7 +141,8 @@ def get_opnames(ex): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and + hasattr(_testinternalcapi, "new_uop_optimizer"), "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): @@ -589,7 +590,8 @@ def testfunc(n): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and + hasattr(_testinternalcapi, "new_uop_optimizer"), "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38a8e31f1d105b..2e57fbee2fd4fd 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -989,12 +989,6 @@ get_co_framesize(PyObject *self, PyObject *arg) #ifdef _Py_TIER2 -static PyObject * -new_uop_optimizer(PyObject *self, PyObject *arg) -{ - return _PyOptimizer_NewUOpOptimizer(); -} - static PyObject * set_optimizer(PyObject *self, PyObject *opt) { @@ -2095,7 +2089,6 @@ static PyMethodDef module_functions[] = { #ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, - {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, #endif From 089f2b91e8711dc66ebe6e6541dd6dce85782748 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 15 Nov 2024 11:33:52 +0800 Subject: [PATCH 03/17] remove `_PyOptimizer_NewUOpOptimizer` --- Include/internal/pycore_optimizer.h | 2 +- Python/optimizer.c | 9 ++------- Python/pylifecycle.c | 3 ++- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 9aa537497918cb..0c238824182c51 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -119,7 +119,7 @@ PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); // Export for '_testinternalcapi' shared extension. PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); -PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); +PyAPI_FUNC(void) _Py_SetUOpOptimize(_PyOptimizerObject* opt); #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 diff --git a/Python/optimizer.c b/Python/optimizer.c index 3815c45c0f8952..61f46b3c7632e7 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1321,15 +1321,10 @@ PyTypeObject _PyUOpOptimizer_Type = { .tp_dealloc = uop_opt_dealloc, }; -PyObject * -_PyOptimizer_NewUOpOptimizer(void) +void +_Py_SetUOpOptimize(_PyOptimizerObject *opt) { - _PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type); - if (opt == NULL) { - return NULL; - } opt->optimize = uop_optimize; - return (PyObject *)opt; } static void diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 9a8a92aaceb0be..2e7d40070e9d82 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1351,10 +1351,11 @@ init_interp_main(PyThreadState *tstate) } else #endif { - PyObject *opt = _PyOptimizer_NewUOpOptimizer(); + PyObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } + _Py_SetUOpOptimize((_PyOptimizerObject *)opt); if (_Py_SetTier2Optimizer((_PyOptimizerObject *)opt)) { return _PyStatus_ERR("can't install optimizer"); } From 16681b0758e3aae5e9c77b18f3a823062099535a Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 15 Nov 2024 18:23:47 +0800 Subject: [PATCH 04/17] revert commits --- Include/internal/pycore_optimizer.h | 3 +- Lib/test/test_capi/test_opt.py | 88 +++++++++++++++++++++++++++-- Lib/test/test_monitoring.py | 29 ++++++++++ Modules/_testinternalcapi.c | 14 +++++ Python/optimizer.c | 21 ++++++- Python/pylifecycle.c | 3 +- 6 files changed, 149 insertions(+), 9 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 0c238824182c51..bc7cfcde613d65 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -119,7 +119,8 @@ PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); // Export for '_testinternalcapi' shared extension. PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); -PyAPI_FUNC(void) _Py_SetUOpOptimize(_PyOptimizerObject* opt); +PyAPI_FUNC(PyObject *) _PyOptimizer_NewCounter(void); +PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 #define _Py_MAX_ALLOWED_GLOBALS_MODIFICATIONS 6 diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 5feaffe06bf5e0..1e4dead2240436 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -141,8 +141,89 @@ def get_opnames(ex): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and - hasattr(_testinternalcapi, "new_uop_optimizer"), +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") +class TestExecutorInvalidation(unittest.TestCase): + + def setUp(self): + self.old = _testinternalcapi.get_optimizer() + self.opt = _testinternalcapi.new_counter_optimizer() + _testinternalcapi.set_optimizer(self.opt) + + def tearDown(self): + _testinternalcapi.set_optimizer(self.old) + + def test_invalidate_object(self): + # Generate a new set of functions at each call + ns = {} + func_src = "\n".join( + f""" + def f{n}(): + for _ in range(1000): + pass + """ for n in range(5) + ) + exec(textwrap.dedent(func_src), ns, ns) + funcs = [ ns[f'f{n}'] for n in range(5)] + objects = [object() for _ in range(5)] + + for f in funcs: + f() + executors = [get_first_executor(f) for f in funcs] + # Set things up so each executor depends on the objects + # with an equal or lower index. + for i, exe in enumerate(executors): + self.assertTrue(exe.is_valid()) + for obj in objects[:i+1]: + _testinternalcapi.add_executor_dependency(exe, obj) + self.assertTrue(exe.is_valid()) + # Assert that the correct executors are invalidated + # and check that nothing crashes when we invalidate + # an executor multiple times. + for i in (4,3,2,1,0): + _testinternalcapi.invalidate_executors(objects[i]) + for exe in executors[i:]: + self.assertFalse(exe.is_valid()) + for exe in executors[:i]: + self.assertTrue(exe.is_valid()) + + def test_uop_optimizer_invalidation(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def f(): + for i in range(1000): + pass + """), ns, ns) + f = ns['f'] + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + _testinternalcapi.invalidate_executors(f.__code__) + self.assertFalse(exe.is_valid()) + + def test_sys__clear_internal_caches(self): + def f(): + for _ in range(1000): + pass + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + sys._clear_internal_caches() + self.assertFalse(exe.is_valid()) + exe = get_first_executor(f) + self.assertIsNone(exe) + + +@requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): @@ -590,8 +671,7 @@ def testfunc(n): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and - hasattr(_testinternalcapi, "new_uop_optimizer"), +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), "Requires optimizer infrastructure") @unittest.skipIf(os.getenv("PYTHON_UOPS_OPTIMIZE") == "0", "Needs uop optimizer to run.") class TestUopsOptimization(unittest.TestCase): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index ba88a4c6d451d4..43e3e56639db62 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2085,6 +2085,35 @@ def callback(code, instruction_offset): sys.monitoring.set_events(0, 0) +class TestOptimizer(MonitoringTestBase, unittest.TestCase): + + def setUp(self): + _testinternalcapi = import_module("_testinternalcapi") + if hasattr(_testinternalcapi, "get_optimizer"): + self.old_opt = _testinternalcapi.get_optimizer() + opt = _testinternalcapi.new_counter_optimizer() + _testinternalcapi.set_optimizer(opt) + super(TestOptimizer, self).setUp() + + def tearDown(self): + super(TestOptimizer, self).tearDown() + import _testinternalcapi + if hasattr(_testinternalcapi, "get_optimizer"): + _testinternalcapi.set_optimizer(self.old_opt) + + def test_for_loop(self): + def test_func(x): + i = 0 + while i < x: + i += 1 + + code = test_func.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + test_func(1000) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + class TestTier2Optimizer(CheckEvents): def test_monitoring_already_opimized_loop(self): diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 2e57fbee2fd4fd..150d34d168f5e4 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -989,6 +989,18 @@ get_co_framesize(PyObject *self, PyObject *arg) #ifdef _Py_TIER2 +static PyObject * +new_counter_optimizer(PyObject *self, PyObject *arg) +{ + return _PyOptimizer_NewCounter(); +} + +static PyObject * +new_uop_optimizer(PyObject *self, PyObject *arg) +{ + return _PyOptimizer_NewUOpOptimizer(); +} + static PyObject * set_optimizer(PyObject *self, PyObject *opt) { @@ -2089,6 +2101,8 @@ static PyMethodDef module_functions[] = { #ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, + {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, + {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, #endif diff --git a/Python/optimizer.c b/Python/optimizer.c index 61f46b3c7632e7..52b3f0a84afedf 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1321,10 +1321,15 @@ PyTypeObject _PyUOpOptimizer_Type = { .tp_dealloc = uop_opt_dealloc, }; -void -_Py_SetUOpOptimize(_PyOptimizerObject *opt) +PyObject * +_PyOptimizer_NewUOpOptimizer(void) { + _PyOptimizerObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type); + if (opt == NULL) { + return NULL; + } opt->optimize = uop_optimize; + return (PyObject *)opt; } static void @@ -1405,6 +1410,18 @@ PyTypeObject _PyCounterOptimizer_Type = { .tp_dealloc = (destructor)PyObject_Free, }; +PyObject * +_PyOptimizer_NewCounter(void) +{ + _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type); + if (opt == NULL) { + return NULL; + } + opt->base.optimize = counter_optimize; + opt->count = 0; + return (PyObject *)opt; +} + /***************************************** * Executor management diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c index 2e7d40070e9d82..9a8a92aaceb0be 100644 --- a/Python/pylifecycle.c +++ b/Python/pylifecycle.c @@ -1351,11 +1351,10 @@ init_interp_main(PyThreadState *tstate) } else #endif { - PyObject *opt = PyObject_New(_PyOptimizerObject, &_PyUOpOptimizer_Type); + PyObject *opt = _PyOptimizer_NewUOpOptimizer(); if (opt == NULL) { return _PyStatus_ERR("can't initialize optimizer"); } - _Py_SetUOpOptimize((_PyOptimizerObject *)opt); if (_Py_SetTier2Optimizer((_PyOptimizerObject *)opt)) { return _PyStatus_ERR("can't install optimizer"); } From 68dd2b1722275115f6aed147324c478b5ed711f5 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Fri, 15 Nov 2024 21:10:21 +0800 Subject: [PATCH 05/17] add export explanation --- Include/internal/pycore_optimizer.h | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index bc7cfcde613d65..684c636f5b6557 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -113,6 +113,8 @@ void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorDetach(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); + +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); // For testing @@ -276,6 +278,7 @@ extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr, int chain_depth); static inline int is_terminator(const _PyUOpInstruction *uop) From f3542b6a3542ace3b9915b0161af84e29d87b51b Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Sat, 23 Nov 2024 23:49:16 +0800 Subject: [PATCH 06/17] remove `_PyCounterOptimizer_Type` and `_PyOptimizer_NewCounter` --- Include/internal/pycore_optimizer.h | 2 - Lib/test/test_capi/test_opt.py | 88 ++-------------------------- Lib/test/test_monitoring.py | 4 +- Modules/_testinternalcapi.c | 7 --- Objects/object.c | 1 - Python/optimizer.c | 22 ------- Tools/c-analyzer/cpython/ignored.tsv | 1 - 7 files changed, 7 insertions(+), 118 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 684c636f5b6557..cce19786b44dc6 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -121,7 +121,6 @@ PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); // Export for '_testinternalcapi' shared extension. PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); -PyAPI_FUNC(PyObject *) _PyOptimizer_NewCounter(void); PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); #define _Py_MAX_ALLOWED_BUILTINS_MODIFICATIONS 3 @@ -153,7 +152,6 @@ int _Py_uop_analyze_and_optimize(struct _PyInterpreterFrame *frame, _PyBloomFilter *dependencies); extern PyTypeObject _PyCounterExecutor_Type; -extern PyTypeObject _PyCounterOptimizer_Type; extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 1e4dead2240436..f03991215f6426 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -37,7 +37,8 @@ def clear_executors(func): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and + hasattr(_testinternalcapi, "new_counter_optimizer"), "Requires optimizer infrastructure") class TestOptimizerAPI(unittest.TestCase): @@ -141,89 +142,8 @@ def get_opnames(ex): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), - "Requires optimizer infrastructure") -class TestExecutorInvalidation(unittest.TestCase): - - def setUp(self): - self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(self.opt) - - def tearDown(self): - _testinternalcapi.set_optimizer(self.old) - - def test_invalidate_object(self): - # Generate a new set of functions at each call - ns = {} - func_src = "\n".join( - f""" - def f{n}(): - for _ in range(1000): - pass - """ for n in range(5) - ) - exec(textwrap.dedent(func_src), ns, ns) - funcs = [ ns[f'f{n}'] for n in range(5)] - objects = [object() for _ in range(5)] - - for f in funcs: - f() - executors = [get_first_executor(f) for f in funcs] - # Set things up so each executor depends on the objects - # with an equal or lower index. - for i, exe in enumerate(executors): - self.assertTrue(exe.is_valid()) - for obj in objects[:i+1]: - _testinternalcapi.add_executor_dependency(exe, obj) - self.assertTrue(exe.is_valid()) - # Assert that the correct executors are invalidated - # and check that nothing crashes when we invalidate - # an executor multiple times. - for i in (4,3,2,1,0): - _testinternalcapi.invalidate_executors(objects[i]) - for exe in executors[i:]: - self.assertFalse(exe.is_valid()) - for exe in executors[:i]: - self.assertTrue(exe.is_valid()) - - def test_uop_optimizer_invalidation(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(""" - def f(): - for i in range(1000): - pass - """), ns, ns) - f = ns['f'] - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - f() - exe = get_first_executor(f) - self.assertIsNotNone(exe) - self.assertTrue(exe.is_valid()) - _testinternalcapi.invalidate_executors(f.__code__) - self.assertFalse(exe.is_valid()) - - def test_sys__clear_internal_caches(self): - def f(): - for _ in range(1000): - pass - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - f() - exe = get_first_executor(f) - self.assertIsNotNone(exe) - self.assertTrue(exe.is_valid()) - sys._clear_internal_caches() - self.assertFalse(exe.is_valid()) - exe = get_first_executor(f) - self.assertIsNone(exe) - - -@requires_specialization -@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and + hasattr(_testinternalcapi, "new_counter_optimizer"), "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 43e3e56639db62..f7a4a7cdb851b8 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -15,6 +15,7 @@ from test.support.import_helper import import_module _testcapi = test.support.import_helper.import_module("_testcapi") +_testinternalcapi = test.support.import_helper.import_module("_testinternalcapi") PAIR = (0,1) @@ -2085,10 +2086,11 @@ def callback(code, instruction_offset): sys.monitoring.set_events(0, 0) +@unittest.skipUnless(hasattr(_testinternalcapi, "new_counter_optimizer"), + "Requires counter optimizer") class TestOptimizer(MonitoringTestBase, unittest.TestCase): def setUp(self): - _testinternalcapi = import_module("_testinternalcapi") if hasattr(_testinternalcapi, "get_optimizer"): self.old_opt = _testinternalcapi.get_optimizer() opt = _testinternalcapi.new_counter_optimizer() diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 150d34d168f5e4..38a8e31f1d105b 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -989,12 +989,6 @@ get_co_framesize(PyObject *self, PyObject *arg) #ifdef _Py_TIER2 -static PyObject * -new_counter_optimizer(PyObject *self, PyObject *arg) -{ - return _PyOptimizer_NewCounter(); -} - static PyObject * new_uop_optimizer(PyObject *self, PyObject *arg) { @@ -2101,7 +2095,6 @@ static PyMethodDef module_functions[] = { #ifdef _Py_TIER2 {"get_optimizer", get_optimizer, METH_NOARGS, NULL}, {"set_optimizer", set_optimizer, METH_O, NULL}, - {"new_counter_optimizer", new_counter_optimizer, METH_NOARGS, NULL}, {"new_uop_optimizer", new_uop_optimizer, METH_NOARGS, NULL}, {"add_executor_dependency", add_executor_dependency, METH_VARARGS, NULL}, {"invalidate_executors", invalidate_executors, METH_O, NULL}, diff --git a/Objects/object.c b/Objects/object.c index 9befd92e3231c8..cf0d4b46bd9004 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2381,7 +2381,6 @@ static PyTypeObject* static_types[] = { &_PyCoroWrapper_Type, #ifdef _Py_TIER2 &_PyCounterExecutor_Type, - &_PyCounterOptimizer_Type, &_PyDefaultOptimizer_Type, #endif &_Py_GenericAliasIterType, diff --git a/Python/optimizer.c b/Python/optimizer.c index 52b3f0a84afedf..b762dd4a6b6258 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1400,28 +1400,6 @@ static PyMethodDef counter_optimizer_methods[] = { { NULL, NULL }, }; -PyTypeObject _PyCounterOptimizer_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "Counter optimizer", - .tp_basicsize = sizeof(_PyCounterOptimizerObject), - .tp_itemsize = 0, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION, - .tp_methods = counter_optimizer_methods, - .tp_dealloc = (destructor)PyObject_Free, -}; - -PyObject * -_PyOptimizer_NewCounter(void) -{ - _PyCounterOptimizerObject *opt = (_PyCounterOptimizerObject *)_PyObject_New(&_PyCounterOptimizer_Type); - if (opt == NULL) { - return NULL; - } - opt->base.optimize = counter_optimize; - opt->count = 0; - return (PyObject *)opt; -} - /***************************************** * Executor management diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index c8c30a7985aa2e..09bd16acaf1e8b 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -385,7 +385,6 @@ Python/sysmodule.c - _PySys_ImplName - Python/sysmodule.c - whatstrings - Python/optimizer.c - _PyDefaultOptimizer_Type - Python/optimizer.c - _PyCounterExecutor_Type - -Python/optimizer.c - _PyCounterOptimizer_Type - Python/optimizer.c - _PyUOpExecutor_Type - Python/optimizer.c - _PyUOpOptimizer_Type - Python/optimizer.c - _PyOptimizer_Default - From d763008466dc8938e0798658113338bc8006d15c Mon Sep 17 00:00:00 2001 From: "blurb-it[bot]" <43283697+blurb-it[bot]@users.noreply.github.com> Date: Wed, 1 Jan 2025 03:25:39 +0000 Subject: [PATCH 07/17] =?UTF-8?q?=F0=9F=93=9C=F0=9F=A4=96=20Added=20by=20b?= =?UTF-8?q?lurb=5Fit.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst | 1 + 1 file changed, 1 insertion(+) create mode 100644 Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst diff --git a/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst b/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst new file mode 100644 index 00000000000000..d441c95d34d6c9 --- /dev/null +++ b/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst @@ -0,0 +1 @@ +Remove the internal opmization API ``_PyOptimizer_NewCounter``. From a7f9df3129e06117ddc74969493307f959c7740d Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Sun, 12 Jan 2025 21:55:58 +0800 Subject: [PATCH 08/17] remove `_INTERNAL_INCREMENT_OPT_COUNTER` --- Include/internal/pycore_uop_ids.h | 177 ++++++++++++------------- Include/internal/pycore_uop_metadata.h | 4 - Python/bytecodes.c | 7 - Python/executor_cases.c.h | 10 -- Python/optimizer.c | 37 ------ Python/optimizer_cases.c.h | 6 - 6 files changed, 88 insertions(+), 153 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3841363b411eed..3d4c5d3d3805db 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -162,43 +162,42 @@ extern "C" { #define _INSTRUMENTED_POP_JUMP_IF_NONE INSTRUMENTED_POP_JUMP_IF_NONE #define _INSTRUMENTED_POP_JUMP_IF_NOT_NONE INSTRUMENTED_POP_JUMP_IF_NOT_NONE #define _INSTRUMENTED_POP_JUMP_IF_TRUE INSTRUMENTED_POP_JUMP_IF_TRUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 397 -#define _IS_NONE 398 +#define _IS_NONE 397 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 399 -#define _ITER_CHECK_RANGE 400 -#define _ITER_CHECK_TUPLE 401 -#define _ITER_JUMP_LIST 402 -#define _ITER_JUMP_RANGE 403 -#define _ITER_JUMP_TUPLE 404 -#define _ITER_NEXT_LIST 405 -#define _ITER_NEXT_RANGE 406 -#define _ITER_NEXT_TUPLE 407 -#define _JUMP_TO_TOP 408 +#define _ITER_CHECK_LIST 398 +#define _ITER_CHECK_RANGE 399 +#define _ITER_CHECK_TUPLE 400 +#define _ITER_JUMP_LIST 401 +#define _ITER_JUMP_RANGE 402 +#define _ITER_JUMP_TUPLE 403 +#define _ITER_NEXT_LIST 404 +#define _ITER_NEXT_RANGE 405 +#define _ITER_NEXT_TUPLE 406 +#define _JUMP_TO_TOP 407 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND -#define _LOAD_ATTR 409 -#define _LOAD_ATTR_CLASS 410 -#define _LOAD_ATTR_CLASS_0 411 -#define _LOAD_ATTR_CLASS_1 412 +#define _LOAD_ATTR 408 +#define _LOAD_ATTR_CLASS 409 +#define _LOAD_ATTR_CLASS_0 410 +#define _LOAD_ATTR_CLASS_1 411 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 413 -#define _LOAD_ATTR_INSTANCE_VALUE_0 414 -#define _LOAD_ATTR_INSTANCE_VALUE_1 415 -#define _LOAD_ATTR_METHOD_LAZY_DICT 416 -#define _LOAD_ATTR_METHOD_NO_DICT 417 -#define _LOAD_ATTR_METHOD_WITH_VALUES 418 -#define _LOAD_ATTR_MODULE 419 -#define _LOAD_ATTR_MODULE_FROM_KEYS 420 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 421 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 422 -#define _LOAD_ATTR_PROPERTY_FRAME 423 -#define _LOAD_ATTR_SLOT 424 -#define _LOAD_ATTR_SLOT_0 425 -#define _LOAD_ATTR_SLOT_1 426 -#define _LOAD_ATTR_WITH_HINT 427 +#define _LOAD_ATTR_INSTANCE_VALUE 412 +#define _LOAD_ATTR_INSTANCE_VALUE_0 413 +#define _LOAD_ATTR_INSTANCE_VALUE_1 414 +#define _LOAD_ATTR_METHOD_LAZY_DICT 415 +#define _LOAD_ATTR_METHOD_NO_DICT 416 +#define _LOAD_ATTR_METHOD_WITH_VALUES 417 +#define _LOAD_ATTR_MODULE 418 +#define _LOAD_ATTR_MODULE_FROM_KEYS 419 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 420 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 421 +#define _LOAD_ATTR_PROPERTY_FRAME 422 +#define _LOAD_ATTR_SLOT 423 +#define _LOAD_ATTR_SLOT_0 424 +#define _LOAD_ATTR_SLOT_1 425 +#define _LOAD_ATTR_WITH_HINT 426 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS -#define _LOAD_BYTECODE 428 +#define _LOAD_BYTECODE 427 #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL @@ -208,98 +207,98 @@ extern "C" { #define _LOAD_CONST_INLINE_WITH_NULL 432 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 433 -#define _LOAD_FAST_0 434 -#define _LOAD_FAST_1 435 -#define _LOAD_FAST_2 436 -#define _LOAD_FAST_3 437 -#define _LOAD_FAST_4 438 -#define _LOAD_FAST_5 439 -#define _LOAD_FAST_6 440 -#define _LOAD_FAST_7 441 +#define _LOAD_FAST 432 +#define _LOAD_FAST_0 433 +#define _LOAD_FAST_1 434 +#define _LOAD_FAST_2 435 +#define _LOAD_FAST_3 436 +#define _LOAD_FAST_4 437 +#define _LOAD_FAST_5 438 +#define _LOAD_FAST_6 439 +#define _LOAD_FAST_7 440 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 442 -#define _LOAD_GLOBAL_BUILTINS 443 -#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 444 -#define _LOAD_GLOBAL_MODULE 445 -#define _LOAD_GLOBAL_MODULE_FROM_KEYS 446 +#define _LOAD_GLOBAL 441 +#define _LOAD_GLOBAL_BUILTINS 442 +#define _LOAD_GLOBAL_BUILTINS_FROM_KEYS 443 +#define _LOAD_GLOBAL_MODULE 444 +#define _LOAD_GLOBAL_MODULE_FROM_KEYS 445 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME -#define _LOAD_SMALL_INT 447 -#define _LOAD_SMALL_INT_0 448 -#define _LOAD_SMALL_INT_1 449 -#define _LOAD_SMALL_INT_2 450 -#define _LOAD_SMALL_INT_3 451 +#define _LOAD_SMALL_INT 446 +#define _LOAD_SMALL_INT_0 447 +#define _LOAD_SMALL_INT_1 448 +#define _LOAD_SMALL_INT_2 449 +#define _LOAD_SMALL_INT_3 450 #define _LOAD_SPECIAL LOAD_SPECIAL #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR #define _LOAD_SUPER_ATTR_METHOD LOAD_SUPER_ATTR_METHOD -#define _MAKE_CALLARGS_A_TUPLE 452 +#define _MAKE_CALLARGS_A_TUPLE 451 #define _MAKE_CELL MAKE_CELL #define _MAKE_FUNCTION MAKE_FUNCTION -#define _MAKE_WARM 453 +#define _MAKE_WARM 452 #define _MAP_ADD MAP_ADD #define _MATCH_CLASS MATCH_CLASS #define _MATCH_KEYS MATCH_KEYS #define _MATCH_MAPPING MATCH_MAPPING #define _MATCH_SEQUENCE MATCH_SEQUENCE -#define _MAYBE_EXPAND_METHOD 454 -#define _MAYBE_EXPAND_METHOD_KW 455 -#define _MONITOR_CALL 456 -#define _MONITOR_JUMP_BACKWARD 457 -#define _MONITOR_RESUME 458 +#define _MAYBE_EXPAND_METHOD 453 +#define _MAYBE_EXPAND_METHOD_KW 454 +#define _MONITOR_CALL 455 +#define _MONITOR_JUMP_BACKWARD 456 +#define _MONITOR_RESUME 457 #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_JUMP_IF_FALSE 459 -#define _POP_JUMP_IF_TRUE 460 +#define _POP_JUMP_IF_FALSE 458 +#define _POP_JUMP_IF_TRUE 459 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 461 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 460 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 462 +#define _PUSH_FRAME 461 #define _PUSH_NULL PUSH_NULL -#define _PY_FRAME_GENERAL 463 -#define _PY_FRAME_KW 464 -#define _QUICKEN_RESUME 465 -#define _REPLACE_WITH_TRUE 466 +#define _PY_FRAME_GENERAL 462 +#define _PY_FRAME_KW 463 +#define _QUICKEN_RESUME 464 +#define _REPLACE_WITH_TRUE 465 #define _RESUME_CHECK RESUME_CHECK #define _RETURN_GENERATOR RETURN_GENERATOR #define _RETURN_VALUE RETURN_VALUE -#define _SAVE_RETURN_OFFSET 467 -#define _SEND 468 -#define _SEND_GEN_FRAME 469 +#define _SAVE_RETURN_OFFSET 466 +#define _SEND 467 +#define _SEND_GEN_FRAME 468 #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 470 -#define _STORE_ATTR 471 -#define _STORE_ATTR_INSTANCE_VALUE 472 -#define _STORE_ATTR_SLOT 473 -#define _STORE_ATTR_WITH_HINT 474 +#define _START_EXECUTOR 469 +#define _STORE_ATTR 470 +#define _STORE_ATTR_INSTANCE_VALUE 471 +#define _STORE_ATTR_SLOT 472 +#define _STORE_ATTR_WITH_HINT 473 #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 475 -#define _STORE_FAST_0 476 -#define _STORE_FAST_1 477 -#define _STORE_FAST_2 478 -#define _STORE_FAST_3 479 -#define _STORE_FAST_4 480 -#define _STORE_FAST_5 481 -#define _STORE_FAST_6 482 -#define _STORE_FAST_7 483 +#define _STORE_FAST 474 +#define _STORE_FAST_0 475 +#define _STORE_FAST_1 476 +#define _STORE_FAST_2 477 +#define _STORE_FAST_3 478 +#define _STORE_FAST_4 479 +#define _STORE_FAST_5 480 +#define _STORE_FAST_6 481 +#define _STORE_FAST_7 482 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME -#define _STORE_SLICE 484 -#define _STORE_SUBSCR 485 +#define _STORE_SLICE 483 +#define _STORE_SUBSCR 484 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TIER2_RESUME_CHECK 486 -#define _TO_BOOL 487 +#define _TIER2_RESUME_CHECK 485 +#define _TO_BOOL 486 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -309,13 +308,13 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 488 +#define _UNPACK_SEQUENCE 487 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START #define _YIELD_VALUE YIELD_VALUE -#define MAX_UOP_ID 488 +#define MAX_UOP_ID 487 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 721fa94da19615..a334ffc6d3a78d 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -286,7 +286,6 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_LOAD_GLOBAL_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_GLOBAL_BUILTINS] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_LOAD_ATTR_MODULE] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, - [_INTERNAL_INCREMENT_OPT_COUNTER] = 0, [_DYNAMIC_EXIT] = HAS_ESCAPES_FLAG, [_START_EXECUTOR] = HAS_ESCAPES_FLAG, [_MAKE_WARM] = 0, @@ -441,7 +440,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_2] = "_INIT_CALL_PY_EXACT_ARGS_2", [_INIT_CALL_PY_EXACT_ARGS_3] = "_INIT_CALL_PY_EXACT_ARGS_3", [_INIT_CALL_PY_EXACT_ARGS_4] = "_INIT_CALL_PY_EXACT_ARGS_4", - [_INTERNAL_INCREMENT_OPT_COUNTER] = "_INTERNAL_INCREMENT_OPT_COUNTER", [_IS_NONE] = "_IS_NONE", [_IS_OP] = "_IS_OP", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", @@ -1119,8 +1117,6 @@ int _PyUop_num_popped(int opcode, int oparg) return 0; case _LOAD_ATTR_MODULE: return 1; - case _INTERNAL_INCREMENT_OPT_COUNTER: - return 1; case _DYNAMIC_EXIT: return 0; case _START_EXECUTOR: diff --git a/Python/bytecodes.c b/Python/bytecodes.c index ef7086c3c020b0..48c10e93bb0ee5 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -5065,13 +5065,6 @@ dummy_func( DECREF_INPUTS(); } - /* Internal -- for testing executors */ - op(_INTERNAL_INCREMENT_OPT_COUNTER, (opt --)) { - _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); - exe->count++; - DEAD(opt); - } - tier2 op(_DYNAMIC_EXIT, (exit_p/4 --)) { tstate->previous_executor = (PyObject *)current_executor; _PyExitData *exit = (_PyExitData *)exit_p; diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0d3b9b269ab247..c15bf7047969f4 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -6069,16 +6069,6 @@ break; } - case _INTERNAL_INCREMENT_OPT_COUNTER: { - _PyStackRef opt; - opt = stack_pointer[-1]; - _PyCounterOptimizerObject *exe = (_PyCounterOptimizerObject *)PyStackRef_AsPyObjectBorrow(opt); - exe->count++; - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _DYNAMIC_EXIT: { PyObject *exit_p = (PyObject *)CURRENT_OPERAND0(); tstate->previous_executor = (PyObject *)current_executor; diff --git a/Python/optimizer.c b/Python/optimizer.c index b762dd4a6b6258..d6e2b96aa35a2e 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1352,43 +1352,6 @@ PyTypeObject _PyCounterExecutor_Type = { .tp_clear = (inquiry)executor_clear, }; -static int -counter_optimize( - _PyOptimizerObject* self, - _PyInterpreterFrame *frame, - _Py_CODEUNIT *instr, - _PyExecutorObject **exec_ptr, - int Py_UNUSED(curr_stackentries), - bool Py_UNUSED(progress_needed) -) -{ - PyCodeObject *code = _PyFrame_GetCode(frame); - int oparg = instr->op.arg; - while (instr->op.code == EXTENDED_ARG) { - instr++; - oparg = (oparg << 8) | instr->op.arg; - } - if (instr->op.code != JUMP_BACKWARD) { - /* Counter optimizer can only handle backward edges */ - return 0; - } - _Py_CODEUNIT *target = instr + 1 + _PyOpcode_Caches[JUMP_BACKWARD] - oparg; - _PyUOpInstruction buffer[4] = { - { .opcode = _START_EXECUTOR, .jump_target = 3, .format=UOP_FORMAT_JUMP }, - { .opcode = _LOAD_CONST_INLINE, .operand0 = (uintptr_t)self }, - { .opcode = _INTERNAL_INCREMENT_OPT_COUNTER }, - { .opcode = _EXIT_TRACE, .target = (uint32_t)(target - _PyCode_CODE(code)), .format=UOP_FORMAT_TARGET } - }; - _PyExecutorObject *executor = make_executor_from_uops(buffer, 4, &EMPTY_FILTER); - if (executor == NULL) { - return -1; - } - Py_INCREF(self); - Py_SET_TYPE(executor, &_PyCounterExecutor_Type); - *exec_ptr = executor; - return 1; -} - static PyObject * counter_get_counter(PyObject *self, PyObject *args) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index a64dd465467122..0e649e977f16f2 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -2620,12 +2620,6 @@ break; } - case _INTERNAL_INCREMENT_OPT_COUNTER: { - stack_pointer += -1; - assert(WITHIN_STACK_BOUNDS()); - break; - } - case _DYNAMIC_EXIT: { break; } From 44aa7abee8211dc77a3e0f6826514c2495fd53af Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Sun, 12 Jan 2025 22:06:38 +0800 Subject: [PATCH 09/17] remove `_PyCounterOptimizerObject` --- Include/internal/pycore_optimizer.h | 5 - Lib/test/test_capi/test_opt.py | 168 ---------------------------- Python/optimizer.c | 12 -- 3 files changed, 185 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index cce19786b44dc6..224d5488ee8677 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -98,11 +98,6 @@ struct _PyOptimizerObject { }; /** Test support **/ -typedef struct { - _PyOptimizerObject base; - int64_t count; -} _PyCounterOptimizerObject; - _PyOptimizerObject *_Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject* optimizer); diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index f03991215f6426..37c937f8762de5 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -35,91 +35,6 @@ def clear_executors(func): func.__code__ = func.__code__.replace() -@requires_specialization -@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and - hasattr(_testinternalcapi, "new_counter_optimizer"), - "Requires optimizer infrastructure") -class TestOptimizerAPI(unittest.TestCase): - - def test_new_counter_optimizer_dealloc(self): - # See gh-108727 - def f(): - _testinternalcapi.new_counter_optimizer() - - f() - - def test_get_set_optimizer(self): - old = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.new_counter_optimizer() - try: - _testinternalcapi.set_optimizer(opt) - self.assertEqual(_testinternalcapi.get_optimizer(), opt) - _testinternalcapi.set_optimizer(None) - self.assertEqual(_testinternalcapi.get_optimizer(), None) - finally: - _testinternalcapi.set_optimizer(old) - - - def test_counter_optimizer(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(f""" - def loop(): - for _ in range({TIER2_THRESHOLD + 1000}): - pass - """), ns, ns) - loop = ns['loop'] - - for repeat in range(5): - opt = _testinternalcapi.new_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - with clear_executors(loop): - loop() - self.assertEqual(opt.get_count(), 1001) - - def test_long_loop(self): - "Check that we aren't confused by EXTENDED_ARG" - - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(f""" - def nop(): - pass - - def long_loop(): - for _ in range({TIER2_THRESHOLD + 20}): - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - nop(); nop(); nop(); nop(); nop(); nop(); nop(); nop(); - """), ns, ns) - long_loop = ns['long_loop'] - - opt = _testinternalcapi.new_counter_optimizer() - with temporary_optimizer(opt): - self.assertEqual(opt.get_count(), 0) - long_loop() - self.assertEqual(opt.get_count(), 21) # Need iterations to warm up - - def test_code_restore_for_ENTER_EXECUTOR(self): - def testfunc(x): - i = 0 - while i < x: - i += 1 - - opt = _testinternalcapi.new_counter_optimizer() - with temporary_optimizer(opt): - testfunc(1000) - code, replace_code = testfunc.__code__, testfunc.__code__.replace() - self.assertEqual(code, replace_code) - self.assertEqual(hash(code), hash(replace_code)) - - def get_first_executor(func): code = func.__code__ co_code = code.co_code @@ -140,89 +55,6 @@ def get_opnames(ex): return list(iter_opnames(ex)) -@requires_specialization -@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer") and - hasattr(_testinternalcapi, "new_counter_optimizer"), - "Requires optimizer infrastructure") -class TestExecutorInvalidation(unittest.TestCase): - - def setUp(self): - self.old = _testinternalcapi.get_optimizer() - self.opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(self.opt) - - def tearDown(self): - _testinternalcapi.set_optimizer(self.old) - - def test_invalidate_object(self): - # Generate a new set of functions at each call - ns = {} - func_src = "\n".join( - f""" - def f{n}(): - for _ in range({TIER2_THRESHOLD}): - pass - """ for n in range(5) - ) - exec(textwrap.dedent(func_src), ns, ns) - funcs = [ ns[f'f{n}'] for n in range(5)] - objects = [object() for _ in range(5)] - - for f in funcs: - f() - executors = [get_first_executor(f) for f in funcs] - # Set things up so each executor depends on the objects - # with an equal or lower index. - for i, exe in enumerate(executors): - self.assertTrue(exe.is_valid()) - for obj in objects[:i+1]: - _testinternalcapi.add_executor_dependency(exe, obj) - self.assertTrue(exe.is_valid()) - # Assert that the correct executors are invalidated - # and check that nothing crashes when we invalidate - # an executor multiple times. - for i in (4,3,2,1,0): - _testinternalcapi.invalidate_executors(objects[i]) - for exe in executors[i:]: - self.assertFalse(exe.is_valid()) - for exe in executors[:i]: - self.assertTrue(exe.is_valid()) - - def test_uop_optimizer_invalidation(self): - # Generate a new function at each call - ns = {} - exec(textwrap.dedent(f""" - def f(): - for i in range({TIER2_THRESHOLD}): - pass - """), ns, ns) - f = ns['f'] - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - f() - exe = get_first_executor(f) - self.assertIsNotNone(exe) - self.assertTrue(exe.is_valid()) - _testinternalcapi.invalidate_executors(f.__code__) - self.assertFalse(exe.is_valid()) - - def test_sys__clear_internal_caches(self): - def f(): - for _ in range(TIER2_THRESHOLD): - pass - opt = _testinternalcapi.new_uop_optimizer() - with temporary_optimizer(opt): - f() - exe = get_first_executor(f) - self.assertIsNotNone(exe) - self.assertTrue(exe.is_valid()) - sys._clear_internal_caches() - self.assertFalse(exe.is_valid()) - exe = get_first_executor(f) - self.assertIsNone(exe) - - @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), diff --git a/Python/optimizer.c b/Python/optimizer.c index d6e2b96aa35a2e..1c057db0afc938 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1352,18 +1352,6 @@ PyTypeObject _PyCounterExecutor_Type = { .tp_clear = (inquiry)executor_clear, }; -static PyObject * -counter_get_counter(PyObject *self, PyObject *args) -{ - return PyLong_FromLongLong(((_PyCounterOptimizerObject *)self)->count); -} - -static PyMethodDef counter_optimizer_methods[] = { - { "get_count", counter_get_counter, METH_NOARGS, NULL }, - { NULL, NULL }, -}; - - /***************************************** * Executor management ****************************************/ From 0e96ecbddc3d4b0ea1479d05ddf40df654c3417b Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Sun, 12 Jan 2025 23:44:27 +0800 Subject: [PATCH 10/17] remove misc. stuff --- Include/internal/pycore_optimizer.h | 1 - Lib/test/test_monitoring.py | 30 ---------------------------- Modules/_testinternalcapi.c | 6 ------ Objects/object.c | 1 - Python/optimizer.c | 19 ------------------ Tools/c-analyzer/cpython/ignored.tsv | 1 - 6 files changed, 58 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 224d5488ee8677..29fa7ad101a13d 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -146,7 +146,6 @@ int _Py_uop_analyze_and_optimize(struct _PyInterpreterFrame *frame, _PyUOpInstruction *trace, int trace_len, int curr_stackentries, _PyBloomFilter *dependencies); -extern PyTypeObject _PyCounterExecutor_Type; extern PyTypeObject _PyDefaultOptimizer_Type; extern PyTypeObject _PyUOpExecutor_Type; extern PyTypeObject _PyUOpOptimizer_Type; diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index f7a4a7cdb851b8..9294d642a98f74 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2086,36 +2086,6 @@ def callback(code, instruction_offset): sys.monitoring.set_events(0, 0) -@unittest.skipUnless(hasattr(_testinternalcapi, "new_counter_optimizer"), - "Requires counter optimizer") -class TestOptimizer(MonitoringTestBase, unittest.TestCase): - - def setUp(self): - if hasattr(_testinternalcapi, "get_optimizer"): - self.old_opt = _testinternalcapi.get_optimizer() - opt = _testinternalcapi.new_counter_optimizer() - _testinternalcapi.set_optimizer(opt) - super(TestOptimizer, self).setUp() - - def tearDown(self): - super(TestOptimizer, self).tearDown() - import _testinternalcapi - if hasattr(_testinternalcapi, "get_optimizer"): - _testinternalcapi.set_optimizer(self.old_opt) - - def test_for_loop(self): - def test_func(x): - i = 0 - while i < x: - i += 1 - - code = test_func.__code__ - sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) - self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) - test_func(1000) - sys.monitoring.set_local_events(TEST_TOOL, code, 0) - self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) - class TestTier2Optimizer(CheckEvents): def test_monitoring_already_opimized_loop(self): diff --git a/Modules/_testinternalcapi.c b/Modules/_testinternalcapi.c index 38a8e31f1d105b..98aea5b596e920 100644 --- a/Modules/_testinternalcapi.c +++ b/Modules/_testinternalcapi.c @@ -1028,12 +1028,6 @@ add_executor_dependency(PyObject *self, PyObject *args) if (!PyArg_ParseTuple(args, "OO", &exec, &obj)) { return NULL; } - /* No way to tell in general if exec is an executor, so we only accept - * counting_executor */ - if (strcmp(Py_TYPE(exec)->tp_name, "counting_executor")) { - PyErr_SetString(PyExc_TypeError, "argument must be a counting_executor"); - return NULL; - } _Py_Executor_DependsOn((_PyExecutorObject *)exec, obj); Py_RETURN_NONE; } diff --git a/Objects/object.c b/Objects/object.c index cf0d4b46bd9004..4e900d8e79d91a 100644 --- a/Objects/object.c +++ b/Objects/object.c @@ -2380,7 +2380,6 @@ static PyTypeObject* static_types[] = { &_PyContextTokenMissing_Type, &_PyCoroWrapper_Type, #ifdef _Py_TIER2 - &_PyCounterExecutor_Type, &_PyDefaultOptimizer_Type, #endif &_Py_GenericAliasIterType, diff --git a/Python/optimizer.c b/Python/optimizer.c index 1c057db0afc938..6a7960658200f3 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -1332,25 +1332,6 @@ _PyOptimizer_NewUOpOptimizer(void) return (PyObject *)opt; } -static void -counter_dealloc(_PyExecutorObject *self) { - /* The optimizer is the operand of the second uop. */ - PyObject *opt = (PyObject *)self->trace[1].operand0; - Py_DECREF(opt); - uop_dealloc(self); -} - -PyTypeObject _PyCounterExecutor_Type = { - PyVarObject_HEAD_INIT(&PyType_Type, 0) - .tp_name = "counting_executor", - .tp_basicsize = offsetof(_PyExecutorObject, exits), - .tp_itemsize = 1, - .tp_flags = Py_TPFLAGS_DEFAULT | Py_TPFLAGS_DISALLOW_INSTANTIATION | Py_TPFLAGS_HAVE_GC, - .tp_dealloc = (destructor)counter_dealloc, - .tp_methods = executor_methods, - .tp_traverse = executor_traverse, - .tp_clear = (inquiry)executor_clear, -}; /***************************************** * Executor management diff --git a/Tools/c-analyzer/cpython/ignored.tsv b/Tools/c-analyzer/cpython/ignored.tsv index 09bd16acaf1e8b..39112a600e4103 100644 --- a/Tools/c-analyzer/cpython/ignored.tsv +++ b/Tools/c-analyzer/cpython/ignored.tsv @@ -384,7 +384,6 @@ Python/sysmodule.c - _PySys_ImplCacheTag - Python/sysmodule.c - _PySys_ImplName - Python/sysmodule.c - whatstrings - Python/optimizer.c - _PyDefaultOptimizer_Type - -Python/optimizer.c - _PyCounterExecutor_Type - Python/optimizer.c - _PyUOpExecutor_Type - Python/optimizer.c - _PyUOpOptimizer_Type - Python/optimizer.c - _PyOptimizer_Default - From ce188a45498e600d05818a219af8abf87951d12d Mon Sep 17 00:00:00 2001 From: Xuanteng Huang <44627253+xuantengh@users.noreply.github.com> Date: Sun, 12 Jan 2025 23:45:51 +0800 Subject: [PATCH 11/17] Update Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst Co-authored-by: Brandt Bucher --- .../next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst b/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst index d441c95d34d6c9..8362ee3a2b1760 100644 --- a/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst +++ b/Misc/NEWS.d/next/C_API/2025-01-01-03-25-38.gh-issue-126599.MRCYlH.rst @@ -1 +1 @@ -Remove the internal opmization API ``_PyOptimizer_NewCounter``. +Remove some internal test APIs for the experimental JIT compiler. From 949352ec46c801e9f9ce776dfaa7ccdd887e6054 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 14 Jan 2025 09:12:31 +0800 Subject: [PATCH 12/17] remove unnecessary comments --- Include/internal/pycore_optimizer.h | 3 -- Lib/test/test_capi/test_opt.py | 74 +++++++++++++++++++++++++++++ Lib/test/test_monitoring.py | 15 ++++++ 3 files changed, 89 insertions(+), 3 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 29fa7ad101a13d..2f4794ed5fb2cc 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -109,11 +109,9 @@ void _Py_ExecutorDetach(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); -// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); // For testing -// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); @@ -270,7 +268,6 @@ extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); -// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(int) _PyOptimizer_Optimize(struct _PyInterpreterFrame *frame, _Py_CODEUNIT *start, _PyStackRef *stack_pointer, _PyExecutorObject **exec_ptr, int chain_depth); static inline int is_terminator(const _PyUOpInstruction *uop) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 37c937f8762de5..7c9da5eec85ed5 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -55,6 +55,80 @@ def get_opnames(ex): return list(iter_opnames(ex)) +@requires_specialization +@unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +class TestExecutorInvalidation(unittest.TestCase): + + def test_invalidate_object(self): + # Generate a new set of functions at each call + ns = {} + func_src = "\n".join( + f""" + def f{n}(): + for _ in range(1000): + pass + """ for n in range(5) + ) + exec(textwrap.dedent(func_src), ns, ns) + funcs = [ ns[f'f{n}'] for n in range(5)] + objects = [object() for _ in range(5)] + + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + for f in funcs: + f() + executors = [get_first_executor(f) for f in funcs] + # Set things up so each executor depends on the objects + # with an equal or lower index. + for i, exe in enumerate(executors): + self.assertTrue(exe.is_valid()) + for obj in objects[:i+1]: + _testinternalcapi.add_executor_dependency(exe, obj) + self.assertTrue(exe.is_valid()) + # Assert that the correct executors are invalidated + # and check that nothing crashes when we invalidate + # an executor multiple times. + for i in (4,3,2,1,0): + _testinternalcapi.invalidate_executors(objects[i]) + for exe in executors[i:]: + self.assertFalse(exe.is_valid()) + for exe in executors[:i]: + self.assertTrue(exe.is_valid()) + + def test_uop_optimizer_invalidation(self): + # Generate a new function at each call + ns = {} + exec(textwrap.dedent(""" + def f(): + for i in range(1000): + pass + """), ns, ns) + f = ns['f'] + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + _testinternalcapi.invalidate_executors(f.__code__) + self.assertFalse(exe.is_valid()) + + def test_sys__clear_internal_caches(self): + def f(): + for _ in range(1000): + pass + opt = _testinternalcapi.new_uop_optimizer() + with temporary_optimizer(opt): + f() + exe = get_first_executor(f) + self.assertIsNotNone(exe) + self.assertTrue(exe.is_valid()) + sys._clear_internal_caches() + self.assertFalse(exe.is_valid()) + exe = get_first_executor(f) + self.assertIsNone(exe) + + @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") @unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index 9294d642a98f74..f2f22fd9ead9d1 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -2086,6 +2086,21 @@ def callback(code, instruction_offset): sys.monitoring.set_events(0, 0) +class TestOptimizer(MonitoringTestBase, unittest.TestCase): + + def test_for_loop(self): + def test_func(x): + i = 0 + while i < x: + i += 1 + + code = test_func.__code__ + sys.monitoring.set_local_events(TEST_TOOL, code, E.PY_START) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), E.PY_START) + test_func(1000) + sys.monitoring.set_local_events(TEST_TOOL, code, 0) + self.assertEqual(sys.monitoring.get_local_events(TEST_TOOL, code), 0) + class TestTier2Optimizer(CheckEvents): def test_monitoring_already_opimized_loop(self): From 46f0fa62f85a6844d5020007055dd05647991c03 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 14 Jan 2025 09:20:22 +0800 Subject: [PATCH 13/17] re-gen cases --- Include/internal/pycore_uop_ids.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 3d4c5d3d3805db..a250bf855bb104 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -201,10 +201,10 @@ extern "C" { #define _LOAD_COMMON_CONSTANT LOAD_COMMON_CONSTANT #define _LOAD_CONST LOAD_CONST #define _LOAD_CONST_IMMORTAL LOAD_CONST_IMMORTAL -#define _LOAD_CONST_INLINE 429 -#define _LOAD_CONST_INLINE_BORROW 430 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 431 -#define _LOAD_CONST_INLINE_WITH_NULL 432 +#define _LOAD_CONST_INLINE 428 +#define _LOAD_CONST_INLINE_BORROW 429 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 430 +#define _LOAD_CONST_INLINE_WITH_NULL 431 #define _LOAD_CONST_MORTAL LOAD_CONST_MORTAL #define _LOAD_DEREF LOAD_DEREF #define _LOAD_FAST 432 From f0ffafd95048bff0d1a920169c0ef546a90a1134 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 14 Jan 2025 09:28:41 +0800 Subject: [PATCH 14/17] fix capi test failures --- Lib/test/test_capi/test_opt.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 7c9da5eec85ed5..b1bf88a70542ab 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -65,7 +65,7 @@ def test_invalidate_object(self): func_src = "\n".join( f""" def f{n}(): - for _ in range(1000): + for _ in range({TIER2_THRESHOLD}): pass """ for n in range(5) ) @@ -98,9 +98,9 @@ def f{n}(): def test_uop_optimizer_invalidation(self): # Generate a new function at each call ns = {} - exec(textwrap.dedent(""" + exec(textwrap.dedent(f""" def f(): - for i in range(1000): + for i in range({TIER2_THRESHOLD}): pass """), ns, ns) f = ns['f'] @@ -115,7 +115,7 @@ def f(): def test_sys__clear_internal_caches(self): def f(): - for _ in range(1000): + for _ in range(TIER2_THRESHOLD): pass opt = _testinternalcapi.new_uop_optimizer() with temporary_optimizer(opt): From 85580ca8ed00d4099b112b8e719c795436c0a032 Mon Sep 17 00:00:00 2001 From: Xuanteng Huang Date: Tue, 14 Jan 2025 09:41:39 +0800 Subject: [PATCH 15/17] add decorator for required API --- Lib/test/test_capi/test_opt.py | 1 + 1 file changed, 1 insertion(+) diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index b1bf88a70542ab..89f2fb44e5a4b0 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -57,6 +57,7 @@ def get_opnames(ex): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") +@unittest.skipUnless(hasattr(_testinternalcapi, "new_uop_optimizer"), "required infrastructure") class TestExecutorInvalidation(unittest.TestCase): def test_invalidate_object(self): From c3f4c6bf5b6f5ddab8eae5e17b8b8058129795f7 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 16 Jan 2025 15:14:08 -0800 Subject: [PATCH 16/17] Clean up the diff --- Include/internal/pycore_optimizer.h | 2 +- Lib/test/test_capi/test_opt.py | 3 ++- Lib/test/test_monitoring.py | 2 -- 3 files changed, 3 insertions(+), 4 deletions(-) diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 2f4794ed5fb2cc..a02b9ab4291bfc 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -108,10 +108,10 @@ void _Py_ExecutorInit(_PyExecutorObject *, const _PyBloomFilter *); void _Py_ExecutorDetach(_PyExecutorObject *); void _Py_BloomFilter_Init(_PyBloomFilter *); void _Py_BloomFilter_Add(_PyBloomFilter *bloom, void *obj); - PyAPI_FUNC(void) _Py_Executor_DependsOn(_PyExecutorObject *executor, void *obj); // For testing +// Export for '_testinternalcapi' shared extension. PyAPI_FUNC(_PyOptimizerObject *) _Py_GetOptimizer(void); PyAPI_FUNC(int) _Py_SetTier2Optimizer(_PyOptimizerObject* optimizer); PyAPI_FUNC(PyObject *) _PyOptimizer_NewUOpOptimizer(void); diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 89f2fb44e5a4b0..a74b8fdd3923b7 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -57,7 +57,8 @@ def get_opnames(ex): @requires_specialization @unittest.skipIf(Py_GIL_DISABLED, "optimizer not yet supported in free-threaded builds") -@unittest.skipUnless(hasattr(_testinternalcapi, "new_uop_optimizer"), "required infrastructure") +@unittest.skipUnless(hasattr(_testinternalcapi, "get_optimizer"), + "Requires optimizer infrastructure") class TestExecutorInvalidation(unittest.TestCase): def test_invalidate_object(self): diff --git a/Lib/test/test_monitoring.py b/Lib/test/test_monitoring.py index f2f22fd9ead9d1..dd2542619a0be0 100644 --- a/Lib/test/test_monitoring.py +++ b/Lib/test/test_monitoring.py @@ -12,10 +12,8 @@ import test.support from test.support import requires_specialization_ft, script_helper -from test.support.import_helper import import_module _testcapi = test.support.import_helper.import_module("_testcapi") -_testinternalcapi = test.support.import_helper.import_module("_testinternalcapi") PAIR = (0,1) From fcb9d07234a4fbdc2095d1cf9f2a7db9b61dbd47 Mon Sep 17 00:00:00 2001 From: Brandt Bucher Date: Thu, 16 Jan 2025 15:14:35 -0800 Subject: [PATCH 17/17] Fix some unused code warnings --- Python/optimizer.c | 9 --------- 1 file changed, 9 deletions(-) diff --git a/Python/optimizer.c b/Python/optimizer.c index 6a7960658200f3..9beb47246eb3d6 100644 --- a/Python/optimizer.c +++ b/Python/optimizer.c @@ -132,8 +132,6 @@ _Py_GetOptimizer(void) static _PyExecutorObject * make_executor_from_uops(_PyUOpInstruction *buffer, int length, const _PyBloomFilter *dependencies); -static const _PyBloomFilter EMPTY_FILTER = { 0 }; - _PyOptimizerObject * _Py_SetOptimizer(PyInterpreterState *interp, _PyOptimizerObject *optimizer) { @@ -251,13 +249,6 @@ get_oparg(PyObject *self, PyObject *Py_UNUSED(ignored)) return PyLong_FromUnsignedLong(((_PyExecutorObject *)self)->vm_data.oparg); } -static PyMethodDef executor_methods[] = { - { "is_valid", is_valid, METH_NOARGS, NULL }, - { "get_opcode", get_opcode, METH_NOARGS, NULL }, - { "get_oparg", get_oparg, METH_NOARGS, NULL }, - { NULL, NULL }, -}; - ///////////////////// Experimental UOp Optimizer ///////////////////// static int executor_clear(_PyExecutorObject *executor);