bpo-47045: Remove f_state field (GH-31963)

* Remove the f_state field from _PyInterpreterFrame

*  Make ownership of the frame explicit, replacing the is_generator field with an owner field.

Author: markshannon

Include/cpython/genobject.h

@@ -27,7 +27,7 @@ extern "C" {
     char prefix##_closed;                                                   \
     char prefix##_running_async;                                            \
     /* The frame */                                                         \
-    char prefix##_frame_valid;                                              \
+    int8_t prefix##_frame_state;                                            \
     PyObject *prefix##_iframe[1];
 
 typedef struct {

Include/cpython/pystate.h

@@ -103,6 +103,7 @@ struct _ts {
        This is to prevent the actual trace/profile code from being recorded in
        the trace/profile. */
     int tracing;
+    int tracing_what; /* The event currently being traced, if any. */
 
     /* Pointer to current _PyCFrame in the C stack frame of the currently,
      * or most recently, executing _PyEval_EvalFrameDefault. */

Include/internal/pycore_frame.h

@@ -5,6 +5,7 @@ extern "C" {
 #endif
 
 #include <stdbool.h>
+#include <stddef.h>
 
 struct _frame {
     PyObject_HEAD
@@ -14,7 +15,6 @@ struct _frame {
     int f_lineno;               /* Current line number. Only valid if non-zero */
     char f_trace_lines;         /* Emit per-line trace events? */
     char f_trace_opcodes;       /* Emit per-opcode trace events? */
-    char f_owns_frame;          /* This frame owns the frame */
     /* The frame data, if this frame object owns the frame */
     PyObject *_f_frame_data[1];
 };
@@ -24,20 +24,19 @@ extern PyFrameObject* _PyFrame_New_NoTrack(PyCodeObject *code);
 
 /* other API */
 
-/* These values are chosen so that the inline functions below all
- * compare f_state to zero.
- */
-enum _framestate {
+typedef enum _framestate {
     FRAME_CREATED = -2,
     FRAME_SUSPENDED = -1,
     FRAME_EXECUTING = 0,
-    FRAME_RETURNED = 1,
-    FRAME_UNWINDING = 2,
-    FRAME_RAISED = 3,
+    FRAME_COMPLETED = 1,
     FRAME_CLEARED = 4
-};
+} PyFrameState;
 
-typedef signed char PyFrameState;
+enum _frameowner {
+    FRAME_OWNED_BY_THREAD = 0,
+    FRAME_OWNED_BY_GENERATOR = 1,
+    FRAME_OWNED_BY_FRAME_OBJECT = 2
+};
 
 /*
     frame->f_lasti refers to the index of the last instruction,
@@ -54,24 +53,11 @@ typedef struct _PyInterpreterFrame {
     struct _PyInterpreterFrame *previous;
     int f_lasti;       /* Last instruction if called */
     int stacktop;     /* Offset of TOS from localsplus  */
-    PyFrameState f_state;  /* What state the frame is in */
     bool is_entry;  // Whether this is the "root" frame for the current _PyCFrame.
-    bool is_generator;
+    char owner;
     PyObject *localsplus[1];
 } _PyInterpreterFrame;
 
-static inline int _PyFrame_IsRunnable(_PyInterpreterFrame *f) {
-    return f->f_state < FRAME_EXECUTING;
-}
-
-static inline int _PyFrame_IsExecuting(_PyInterpreterFrame *f) {
-    return f->f_state == FRAME_EXECUTING;
-}
-
-static inline int _PyFrameHasCompleted(_PyInterpreterFrame *f) {
-    return f->f_state > FRAME_EXECUTING;
-}
-
 static inline PyObject **_PyFrame_Stackbase(_PyInterpreterFrame *f) {
     return f->localsplus + f->f_code->co_nlocalsplus;
 }
@@ -111,9 +97,8 @@ _PyFrame_InitializeSpecials(
     frame->stacktop = nlocalsplus;
     frame->frame_obj = NULL;
     frame->f_lasti = -1;
-    frame->f_state = FRAME_CREATED;
     frame->is_entry = false;
-    frame->is_generator = false;
+    frame->owner = FRAME_OWNED_BY_THREAD;
 }
 
 /* Gets the pointer to the locals array
@@ -200,6 +185,15 @@ void _PyThreadState_PopFrame(PyThreadState *tstate, _PyInterpreterFrame *frame);
 _PyInterpreterFrame *
 _PyFrame_Push(PyThreadState *tstate, PyFunctionObject *func);
 
+
+static inline
+PyGenObject *_PyFrame_GetGenerator(_PyInterpreterFrame *frame)
+{
+    assert(frame->owner == FRAME_OWNED_BY_GENERATOR);
+    size_t offset_in_gen = offsetof(PyGenObject, gi_iframe);
+    return (PyGenObject *)(((char *)frame) - offset_in_gen);
+}
+
 #ifdef __cplusplus
 }
 #endif

Misc/NEWS.d/next/Core and Builtins/2022-03-17-16-25-57.bpo-47045.xQgHul.rst

@@ -0,0 +1,3 @@
+Remove the ``f_state`` field from the _PyInterpreterFrame struct. Add the
+``owner`` field to the _PyInterpreterFrame struct to make ownership explicit
+to simplify clearing and deallocing frames and generators.

Modules/_xxsubinterpretersmodule.c

@@ -1843,10 +1843,7 @@ _is_running(PyInterpreterState *interp)
     if (frame == NULL) {
         return 0;
     }
-
-    int executing = _PyFrame_IsExecuting(frame);
-
-    return executing;
+    return 1;
 }
 
 static int

Objects/frameobject.c

@@ -412,6 +412,42 @@ frame_stack_pop(PyFrameObject *f)
     Py_DECREF(v);
 }
 
+static PyFrameState
+_PyFrame_GetState(PyFrameObject *frame)
+{
+    if (frame->f_frame->stacktop == 0) {
+        return FRAME_CLEARED;
+    }
+    switch(frame->f_frame->owner) {
+        case FRAME_OWNED_BY_GENERATOR:
+        {
+            PyGenObject *gen = _PyFrame_GetGenerator(frame->f_frame);
+            return gen->gi_frame_state;
+        }
+        case FRAME_OWNED_BY_THREAD:
+        {
+            if (frame->f_frame->f_lasti < 0) {
+                return FRAME_CREATED;
+            }
+            uint8_t *code = (uint8_t *)frame->f_frame->f_code->co_code_adaptive;
+            int opcode = code[frame->f_frame->f_lasti*sizeof(_Py_CODEUNIT)];
+            switch(_PyOpcode_Deopt[opcode]) {
+                case COPY_FREE_VARS:
+                case MAKE_CELL:
+                case RETURN_GENERATOR:
+                    /* Frame not fully initialized */
+                    return FRAME_CREATED;
+                default:
+                    return FRAME_EXECUTING;
+            }
+        }
+        case FRAME_OWNED_BY_FRAME_OBJECT:
+            return FRAME_COMPLETED;
+    }
+    Py_UNREACHABLE();
+}
+
+
 /* Setter for f_lineno - you can set f_lineno from within a trace function in
  * order to jump to a given line of code, subject to some restrictions.  Most
  * lines are OK to jump to because they don't make any assumptions about the
@@ -440,6 +476,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
         return -1;
     }
 
+    PyFrameState state = _PyFrame_GetState(f);
     /*
      * This code preserves the historical restrictions on
      * setting the line number of a frame.
@@ -448,28 +485,31 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
      * In addition, jumps are forbidden when not tracing,
      * as this is a debugging feature.
      */
-    switch(f->f_frame->f_state) {
-        case FRAME_CREATED:
+    switch(PyThreadState_GET()->tracing_what) {
+        case PyTrace_EXCEPTION:
+            PyErr_SetString(PyExc_ValueError,
+                "can only jump from a 'line' trace event");
+            return -1;
+        case PyTrace_CALL:
             PyErr_Format(PyExc_ValueError,
                      "can't jump from the 'call' trace event of a new frame");
             return -1;
-        case FRAME_RETURNED:
-        case FRAME_UNWINDING:
-        case FRAME_RAISED:
-        case FRAME_CLEARED:
+        case PyTrace_LINE:
+            break;
+        case PyTrace_RETURN:
+            if (state == FRAME_SUSPENDED) {
+                break;
+            }
+            /* fall through */
+        default:
             PyErr_SetString(PyExc_ValueError,
                 "can only jump from a 'line' trace event");
             return -1;
-        case FRAME_EXECUTING:
-        case FRAME_SUSPENDED:
-            /* You can only do this from within a trace function, not via
-            * _getframe or similar hackery. */
-            if (!f->f_trace) {
-                PyErr_Format(PyExc_ValueError,
-                            "f_lineno can only be set by a trace function");
-                return -1;
-            }
-            break;
+    }
+    if (!f->f_trace) {
+        PyErr_Format(PyExc_ValueError,
+                    "f_lineno can only be set by a trace function");
+        return -1;
     }
 
     int new_lineno;
@@ -555,8 +595,7 @@ frame_setlineno(PyFrameObject *f, PyObject* p_new_lineno, void *Py_UNUSED(ignore
         PyErr_SetString(PyExc_ValueError, msg);
         return -1;
     }
-    /* Unwind block stack. */
-    if (f->f_frame->f_state == FRAME_SUSPENDED) {
+    if (state == FRAME_SUSPENDED) {
         /* Account for value popped by yield */
         start_stack = pop_value(start_stack);
     }
@@ -623,7 +662,9 @@ frame_dealloc(PyFrameObject *f)
 {
     /* It is the responsibility of the owning generator/coroutine
      * to have cleared the generator pointer */
-    assert(!f->f_frame->is_generator);
+
+    assert(f->f_frame->owner != FRAME_OWNED_BY_GENERATOR ||
+        _PyFrame_GetGenerator(f->f_frame)->gi_frame_state == FRAME_CLEARED);
 
     if (_PyObject_GC_IS_TRACKED(f)) {
         _PyObject_GC_UNTRACK(f);
@@ -633,8 +674,7 @@ frame_dealloc(PyFrameObject *f)
     PyCodeObject *co = NULL;
 
     /* Kill all local variables including specials, if we own them */
-    if (f->f_owns_frame) {
-        f->f_owns_frame = 0;
+    if (f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT) {
         assert(f->f_frame == (_PyInterpreterFrame *)f->_f_frame_data);
         _PyInterpreterFrame *frame = (_PyInterpreterFrame *)f->_f_frame_data;
         /* Don't clear code object until the end */
@@ -659,7 +699,7 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
 {
     Py_VISIT(f->f_back);
     Py_VISIT(f->f_trace);
-    if (f->f_owns_frame == 0) {
+    if (f->f_frame->owner != FRAME_OWNED_BY_FRAME_OBJECT) {
         return 0;
     }
     assert(f->f_frame->frame_obj == NULL);
@@ -669,13 +709,6 @@ frame_traverse(PyFrameObject *f, visitproc visit, void *arg)
 static int
 frame_tp_clear(PyFrameObject *f)
 {
-    /* Before anything else, make sure that this frame is clearly marked
-     * as being defunct!  Else, e.g., a generator reachable from this
-     * frame may also point to this frame, believe itself to still be
-     * active, and try cleaning up this frame again.
-     */
-    f->f_frame->f_state = FRAME_CLEARED;
-
     Py_CLEAR(f->f_trace);
 
     /* locals and stack */
@@ -691,19 +724,25 @@ frame_tp_clear(PyFrameObject *f)
 static PyObject *
 frame_clear(PyFrameObject *f, PyObject *Py_UNUSED(ignored))
 {
-    if (_PyFrame_IsExecuting(f->f_frame)) {
-        PyErr_SetString(PyExc_RuntimeError,
-                        "cannot clear an executing frame");
-        return NULL;
+    if (f->f_frame->owner == FRAME_OWNED_BY_GENERATOR) {
+        PyGenObject *gen = _PyFrame_GetGenerator(f->f_frame);
+        if (gen->gi_frame_state == FRAME_EXECUTING) {
+            goto running;
+        }
+        _PyGen_Finalize((PyObject *)gen);
     }
-    if (f->f_frame->is_generator) {
-        assert(!f->f_owns_frame);
-        size_t offset_in_gen = offsetof(PyGenObject, gi_iframe);
-        PyObject *gen = (PyObject *)(((char *)f->f_frame) - offset_in_gen);
-        _PyGen_Finalize(gen);
+    else if (f->f_frame->owner == FRAME_OWNED_BY_THREAD) {
+        goto running;
+    }
+    else {
+        assert(f->f_frame->owner == FRAME_OWNED_BY_FRAME_OBJECT);
+        (void)frame_tp_clear(f);
     }
-    (void)frame_tp_clear(f);
     Py_RETURN_NONE;
+running:
+    PyErr_SetString(PyExc_RuntimeError,
+                    "cannot clear an executing frame");
+    return NULL;
 }
 
 PyDoc_STRVAR(clear__doc__,
@@ -835,7 +874,7 @@ PyFrame_New(PyThreadState *tstate, PyCodeObject *code,
     }
     init_frame((_PyInterpreterFrame *)f->_f_frame_data, func, locals);
     f->f_frame = (_PyInterpreterFrame *)f->_f_frame_data;
-    f->f_owns_frame = 1;
+    f->f_frame->owner = FRAME_OWNED_BY_FRAME_OBJECT;
     Py_DECREF(func);
     _PyObject_GC_TRACK(f);
     return f;
@@ -912,7 +951,7 @@ _PyFrame_FastToLocalsWithError(_PyInterpreterFrame *frame) {
 
         PyObject *name = PyTuple_GET_ITEM(co->co_localsplusnames, i);
         PyObject *value = fast[i];
-        if (frame->f_state != FRAME_CLEARED) {
+        if (frame->stacktop) {
             if (kind & CO_FAST_FREE) {
                 // The cell was set by COPY_FREE_VARS.
                 assert(value != NULL && PyCell_Check(value));
@@ -1049,7 +1088,7 @@ _PyFrame_LocalsToFast(_PyInterpreterFrame *frame, int clear)
 void
 PyFrame_LocalsToFast(PyFrameObject *f, int clear)
 {
-    if (f == NULL || f->f_frame->f_state == FRAME_CLEARED) {
+    if (f == NULL || _PyFrame_GetState(f) == FRAME_CLEARED) {
         return;
     }
     _PyFrame_LocalsToFast(f->f_frame, clear);
@@ -1096,3 +1135,5 @@ _PyEval_BuiltinsFromGlobals(PyThreadState *tstate, PyObject *globals)
 
     return _PyEval_GetBuiltins(tstate);
 }
+
+