gh-91053: Add an optional callback that is invoked whenever a function is modified

Doc/c-api/function.rst

@@ -118,3 +118,59 @@ There are a few functions specific to Python functions.
  must be a dictionary or ``Py_None``.
 
  Raises :exc:`SystemError` and returns ``-1`` on failure.
+
+
+.. c:function:: int PyFunction_AddWatcher(PyFunction_WatchCallback callback)
+
+ Register *callback* as a function watcher for the current interpreter.
+ Return an ID which may be passed to :c:func:`PyFunction_ClearWatcher`.
+ In case of error (e.g. no more watcher IDs available),
+ return ``-1`` and set an exception.
+
+ .. versionadded:: 3.12
+
+
+.. c:function:: int PyFunction_ClearWatcher(int watcher_id)
+
+ Clear watcher identified by *watcher_id* previously returned from
+ :c:func:`PyFunction_AddWatcher` for the current interpreter.
+ Return ``0`` on success, or ``-1`` and set an exception on error
+ (e.g. if the given *watcher_id* was never registered.)
+
+ .. versionadded:: 3.12
+
+
+.. c:type:: PyFunction_WatchEvent
+
+ Enumeration of possible function watcher events:
+ - ``PyFunction_EVENT_CREATE``
+ - ``PyFunction_EVENT_DESTROY``
+ - ``PyFunction_EVENT_MODIFY_CODE``
+ - ``PyFunction_EVENT_MODIFY_DEFAULTS``
+ - ``PyFunction_EVENT_MODIFY_KWDEFAULTS``
+
+ .. versionadded:: 3.12
+
+
+.. c:type:: int (*PyFunction_WatchCallback)(PyFunction_WatchEvent event, PyFunctionObject *func, PyObject *new_value)
+
+ Type of a function watcher callback function.
+
+ If *event* is ``PyFunction_EVENT_CREATE`` or ``PyFunction_EVENT_DESTROY``
+ then *new_value* will be ``NULL``. Otherwise, *new_value* will hold a
+ :term:`borrowed reference` to the new value that is about to be stored in
+ *func* for the attribute that is being modified.
+
+ The callback may inspect but must not modify *func*; doing so could have
+ unpredictable effects, including infinite recursion.
+
+ If *event* is ``PyFunction_EVENT_CREATE``, then the callback is invoked
+ after `func` has been fully initialized. Otherwise, the callback is invoked
+ before the modification to *func* takes place, so the prior state of *func*
+ can be inspected.
+
+ If the callback returns with an exception set, it must return ``-1``; this
+ exception will be printed as an unraisable exception using
+ :c:func:`PyErr_WriteUnraisable`. Otherwise it should return ``0``.
+
+ .. versionadded:: 3.12

Include/cpython/funcobject.h

@@ -131,6 +131,55 @@ PyAPI_DATA(PyTypeObject) PyStaticMethod_Type;
 PyAPI_FUNC(PyObject *) PyClassMethod_New(PyObject *);
 PyAPI_FUNC(PyObject *) PyStaticMethod_New(PyObject *);
 
+#define FOREACH_FUNC_EVENT(V) \
+ V(CREATE) \
+ V(DESTROY) \
+ V(MODIFY_CODE) \
+ V(MODIFY_DEFAULTS) \
+ V(MODIFY_KWDEFAULTS)
+
+typedef enum {
+ #define DEF_EVENT(EVENT) PyFunction_EVENT_##EVENT,
+ FOREACH_FUNC_EVENT(DEF_EVENT)
+ #undef DEF_EVENT
+} PyFunction_WatchEvent;
+
+/*
+ * A callback that is invoked for different events in a function's lifecycle.
+ *
+ * The callback is invoked with a borrowed reference to func, after it is
+ * created and before it is modified or destroyed. The callback should not
+ * modify func.
+ *
+ * When a function's code object, defaults, or kwdefaults are modified the
+ * callback will be invoked with the respective event and new_value will
+ * contain a borrowed reference to the new value that is about to be stored in
+ * the function. Otherwise the third argument is NULL.
+ *
+ * If the callback returns with an exception set, it must return -1. Otherwise
+ * it should return 0.
+ */
+typedef int (*PyFunction_WatchCallback)(
+ PyFunction_WatchEvent event,
+ PyFunctionObject *func,
+ PyObject *new_value);
+
+/*
+ * Register a per-interpreter callback that will be invoked for function lifecycle
+ * events.
+ *
+ * Returns a handle that may be passed to PyFunction_ClearWatcher on success,
+ * or -1 and sets an error if no more handles are available.
+ */
+PyAPI_FUNC(int) PyFunction_AddWatcher(PyFunction_WatchCallback callback);
+
+/*
+ * Clear the watcher associated with the watcher_id handle.
+ *
+ * Returns 0 on success or -1 if no watcher exists for the supplied id.
+ */
+PyAPI_FUNC(int) PyFunction_ClearWatcher(int watcher_id);
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_function.h

@@ -8,6 +8,8 @@ extern "C" {
 # error "this header requires Py_BUILD_CORE define"
 #endif
 
+#define FUNC_MAX_WATCHERS 8
+
 extern PyFunctionObject* _PyFunction_FromConstructor(PyFrameConstructor *constr);
 
 extern uint32_t _PyFunction_GetVersionForCurrentState(PyFunctionObject *func);

Include/internal/pycore_interp.h

@@ -17,6 +17,7 @@ extern "C" {
 #include "pycore_dict.h" // struct _Py_dict_state
 #include "pycore_exceptions.h" // struct _Py_exc_state
 #include "pycore_floatobject.h" // struct _Py_float_state
+#include "pycore_function.h" // FUNC_MAX_WATCHERS
 #include "pycore_genobject.h" // struct _Py_async_gen_state
 #include "pycore_gc.h" // struct _gc_runtime_state
 #include "pycore_list.h" // struct _Py_list_state
@@ -147,6 +148,7 @@ struct _is {
  _PyFrameEvalFunction eval_frame;
 
  PyDict_WatchCallback dict_watchers[DICT_MAX_WATCHERS];
+ PyFunction_WatchCallback func_watchers[FUNC_MAX_WATCHERS];
 
  Py_ssize_t co_extra_user_count;
  freefunc co_extra_freefuncs[MAX_CO_EXTRA_USERS];

Lib/test/test_capi.py

@@ -37,6 +37,19 @@
 # Skip this test if the _testcapi module isn't available.
 _testcapi = import_helper.import_module('_testcapi')
 
+from _testcapi import (
+ PYFUNC_EVENT_CREATE,
+ PYFUNC_EVENT_DESTROY,
+ PYFUNC_EVENT_MODIFY_CODE,
+ PYFUNC_EVENT_MODIFY_DEFAULTS,
+ PYFUNC_EVENT_MODIFY_KWDEFAULTS,
+ _add_func_watcher,
+ _allocate_too_many_func_watchers,
+ _clear_func_watcher,
+ _set_func_defaults_via_capi,
+ _set_func_kwdefaults_via_capi,
+)
+
 import _testinternalcapi
 
 
@@ -1556,5 +1569,98 @@ def test_clear_unassigned_watcher_id(self):
  self.clear_watcher(1)
 
 
+class FuncEventsTest(unittest.TestCase):
+ @contextmanager
+ def add_watcher(self, func):
+ wid = _add_func_watcher(func)
+ try:
+ yield
+ finally:
+ _clear_func_watcher(wid)
+
+ def test_func_events_dispatched(self):
+ events = []
+ def watcher(*args):
+ events.append(args)
+
+ with self.add_watcher(watcher):
+ def myfunc():
+ pass
+ self.assertIn((PYFUNC_EVENT_CREATE, myfunc, None), events)
+ myfunc_id = id(myfunc)
+
+ new_code = self.test_func_events_dispatched.__code__
+ myfunc.__code__ = new_code
+ self.assertIn((PYFUNC_EVENT_MODIFY_CODE, myfunc, new_code), events)
+
+ new_defaults = (123,)
+ myfunc.__defaults__ = new_defaults
+ self.assertIn((PYFUNC_EVENT_MODIFY_DEFAULTS, myfunc, new_defaults), events)
+
+ new_defaults = (456,)
+ _set_func_defaults_via_capi(myfunc, new_defaults)
+ self.assertIn((PYFUNC_EVENT_MODIFY_DEFAULTS, myfunc, new_defaults), events)
+
+ new_kwdefaults = {"self": 123}
+ myfunc.__kwdefaults__ = new_kwdefaults
+ self.assertIn((PYFUNC_EVENT_MODIFY_KWDEFAULTS, myfunc, new_kwdefaults), events)
+
+ new_kwdefaults = {"self": 456}
+ _set_func_kwdefaults_via_capi(myfunc, new_kwdefaults)
+ self.assertIn((PYFUNC_EVENT_MODIFY_KWDEFAULTS, myfunc, new_kwdefaults), events)
+
+ # Clear events reference to func
+ events = []
+ del myfunc
+ self.assertIn((PYFUNC_EVENT_DESTROY, myfunc_id, None), events)
+
+ def test_multiple_watchers(self):
+ events0 = []
+ def first_watcher(*args):
+ events0.append(args)
+
+ events1 = []
+ def second_watcher(*args):
+ events1.append(args)
+
+ with self.add_watcher(first_watcher):
+ with self.add_watcher(second_watcher):
+ def myfunc():
+ pass
+
+ event = (PYFUNC_EVENT_CREATE, myfunc, None)
+ self.assertIn(event, events0)
+ self.assertIn(event, events1)
+
+ def test_watcher_raises_error(self):
+ class MyError(Exception):
+ pass
+
+ def watcher(*args):
+ raise MyError("testing 123")
+
+ with self.add_watcher(watcher):
+ with catch_unraisable_exception() as cm:
+ def myfunc():
+ pass
+
+ self.assertIs(cm.unraisable.object, myfunc)
+ self.assertIsInstance(cm.unraisable.exc_value, MyError)
+
+ def test_clear_out_of_range_watcher_id(self):
+ with self.assertRaisesRegex(ValueError, r"invalid func watcher ID -1"):
+ _clear_func_watcher(-1)
+ with self.assertRaisesRegex(ValueError, r"invalid func watcher ID 8"):
+ _clear_func_watcher(8) # FUNC_MAX_WATCHERS = 8
+
+ def test_clear_unassigned_watcher_id(self):
+ with self.assertRaisesRegex(ValueError, r"no func watcher set for ID 1"):
+ _clear_func_watcher(1)
+
+ def test_allocate_too_many_watchers(self):
+ with self.assertRaisesRegex(RuntimeError, r"no more func watcher IDs"):
+ _allocate_too_many_func_watchers()
+
+
 if __name__ == "__main__":
  unittest.main()

Misc/NEWS.d/next/Core and Builtins/2022-10-05-11-44-52.gh-issue-91053.f5Bo3p.rst

@@ -0,0 +1,4 @@
+Optimizing interpreters and JIT compilers may need to invalidate internal
+metadata when functions are modified. This change adds the ability to
+provide a callback that will be invoked each time a function is created,
+modified, or destroyed.

Modules/Setup.stdlib.in

@@ -169,7 +169,7 @@
 @MODULE__XXTESTFUZZ_TRUE@_xxtestfuzz _xxtestfuzz/_xxtestfuzz.c _xxtestfuzz/fuzzer.c
 @MODULE__TESTBUFFER_TRUE@_testbuffer _testbuffer.c
 @MODULE__TESTINTERNALCAPI_TRUE@_testinternalcapi _testinternalcapi.c
-@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c
+@MODULE__TESTCAPI_TRUE@_testcapi _testcapimodule.c _testcapi/vectorcall.c _testcapi/vectorcall_limited.c _testcapi/heaptype.c _testcapi/unicode.c _testcapi/func_events.c
 
 # Some testing modules MUST be built as shared libraries.
 *shared*