bpo-42917: Made block stack for frame objects dynamically sizable

Doc/c-api/code.rst

@@ -35,17 +35,23 @@ bound into a function.
 
 .. c:function:: PyCodeObject* PyCode_New(int argcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab)
 
-   Return a new code object.  If you need a dummy code object to create a frame,
-   use :c:func:`PyCode_NewEmpty` instead.  Calling :c:func:`PyCode_New` directly
-   can bind you to a precise Python version since the definition of the bytecode
-   changes often.
+   New code should use PyCode_NewWithBlockStackSize instead.
 
 .. c:function:: PyCodeObject* PyCode_NewWithPosOnlyArgs(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab)
 
-   Similar to :c:func:`PyCode_New`, but with an extra "posonlyargcount" for positional-only arguments.
+   New code should use PyCode_NewWithBlockStackSize instead.
 
    .. versionadded:: 3.8
 
+.. c:function:: PyCodeObject* PyCode_NewWithBlockStackSize(int argcount, int posonlyargcount, int kwonlyargcount, int nlocals, int stacksize, int blockstacksize, int flags, PyObject *code, PyObject *consts, PyObject *names, PyObject *varnames, PyObject *freevars, PyObject *cellvars, PyObject *filename, PyObject *name, int firstlineno, PyObject *lnotab)
+
+   Return a new code object.  If you need a dummy code object to create a frame,
+   use :c:func:`PyCode_NewEmpty` instead.  Calling :c:func:`PyCode_New` directly
+   can bind you to a precise Python version since the definition of the bytecode
+   changes often.
+
+   .. versionadded:: 3.10
+
 .. c:function:: PyCodeObject* PyCode_NewEmpty(const char *filename, const char *funcname, int firstlineno)
 
    Return a new empty code object with the specified filename,

Doc/data/refcounts.dat

@@ -252,6 +252,25 @@ PyCode_NewWithPosOnlyArgs:PyObject*:name:0:
 PyCode_NewWithPosOnlyArgs:int:firstlineno::
 PyCode_NewWithPosOnlyArgs:PyObject*:lnotab:0:
 
+PyCode_NewWithBlockStackSize:PyCodeObject*::+1:
+PyCode_NewWithBlockStackSize:int:argcount::
+PyCode_NewWithBlockStackSize:int:posonlyargcount::
+PyCode_NewWithBlockStackSize:int:kwonlyargcount::
+PyCode_NewWithBlockStackSize:int:nlocals::
+PyCode_NewWithBlockStackSize:int:stacksize::
+PyCode_NewWithBlockStackSize:int:blockstacksize::
+PyCode_NewWithBlockStackSize:int:flags::
+PyCode_NewWithBlockStackSize:PyObject*:code:0:
+PyCode_NewWithBlockStackSize:PyObject*:consts:0:
+PyCode_NewWithBlockStackSize:PyObject*:names:0:
+PyCode_NewWithBlockStackSize:PyObject*:varnames:0:
+PyCode_NewWithBlockStackSize:PyObject*:freevars:0:
+PyCode_NewWithBlockStackSize:PyObject*:cellvars:0:
+PyCode_NewWithBlockStackSize:PyObject*:filename:0:
+PyCode_NewWithBlockStackSize:PyObject*:name:0:
+PyCode_NewWithBlockStackSize:int:firstlineno::
+PyCode_NewWithBlockStackSize:PyObject*:lnotab:0:
+
 PyCode_New:PyCodeObject*::+1:
 PyCode_New:int:argcount::
 PyCode_New:int:kwonlyargcount::

Doc/whatsnew/3.10.rst

@@ -578,6 +578,9 @@ Other Language Changes
   ``__globals__["__builtins__"]`` if it exists, else from the current builtins.
   (Contributed by Mark Shannon in :issue:`42990`.)
 
+* The block stack size for :class:`frame` objects is now unlimited (previously
+  was limited to 20).
+  (Contributed by Thomas Anderson in :issue:`42917`.)
 
 New Modules
 ===========
@@ -914,7 +917,6 @@ Optimizations
   for more details. (Contributed by Victor Stinner and Pablo Galindo in
   :issue:`38980`.)
 
-
 * Function parameters and their annotations are no longer computed at runtime,
   but rather at compilation time.  They are stored as a tuple of strings at the
   bytecode level. It is now around 2 times faster to create a function with
@@ -926,6 +928,11 @@ Optimizations
   algorithm to avoid quadratic behavior on long strings.  (Contributed
   by Dennis Sweeney in :issue:`41972`)
 
+* Reduced size of stack :class:`frame` objects by more than half for typical
+  functions.
+  (Contributed by Thomas Anderson in :issue:`42917`.)
+
+
 Deprecated
 ==========
 
@@ -1112,6 +1119,9 @@ Changes in the Python API
   also inherits the current builtins.
   (Contributed by Victor Stinner in :issue:`42990`.)
 
+* :meth:`code.__new__` now takes a *co_blockstacksize* argument.
+  (Contributed by Thomas Anderson in :issue:`42917`.)
+
 CPython bytecode changes
 ========================
 
@@ -1279,6 +1289,10 @@ Porting to Python 3.10
   been included directly, consider including ``Python.h`` instead.
   (Contributed by Nicholas Sim in :issue:`35134`)
 
+* Usages of :c:func:`PyCode_New` and :c:func:`PyCode_NewWithPosOnlyArgs` should
+  be updated to use :c:func:`PyCode_NewWithBlockSize`.
+  (Contributed by Thomas Anderson in :issue:`42917`.)
+
 Deprecated
 ----------
 

Include/cpython/code.h

@@ -22,6 +22,7 @@ struct PyCodeObject {
     int co_kwonlyargcount;      /* #keyword only arguments */
     int co_nlocals;             /* #local variables */
     int co_stacksize;           /* #entries needed for evaluation stack */
+    int co_blockstacksize;      /* #entries needed for block stack */
     int co_flags;               /* CO_..., see below */
     int co_firstlineno;         /* first source line number */
     PyObject *co_code;          /* instruction opcodes */
@@ -106,24 +107,32 @@ struct PyCodeObject {
 */
 #define PY_PARSER_REQUIRES_FUTURE_KEYWORD
 
-#define CO_MAXBLOCKS 20 /* Max static block nesting within a function */
-
 PyAPI_DATA(PyTypeObject) PyCode_Type;
 
 #define PyCode_Check(op) Py_IS_TYPE(op, &PyCode_Type)
 #define PyCode_GetNumFree(op) (PyTuple_GET_SIZE((op)->co_freevars))
 
 /* Public interface */
+
+/* Used in 3.7 and earlier.  New code should use PyCode_NewWithBlockStackSize
+   instead. */
 PyAPI_FUNC(PyCodeObject *) PyCode_New(
         int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
         PyObject *, PyObject *, int, PyObject *);
 
+/* Used between 3.8 and 3.9.  New code should use PyCode_NewWithBlockStackSize
+   instead. */
 PyAPI_FUNC(PyCodeObject *) PyCode_NewWithPosOnlyArgs(
         int, int, int, int, int, int, PyObject *, PyObject *,
         PyObject *, PyObject *, PyObject *, PyObject *,
         PyObject *, PyObject *, int, PyObject *);
-        /* same as struct above */
+
+/* Used in 3.10 and later */
+PyAPI_FUNC(PyCodeObject *) PyCode_NewWithBlockStackSize(
+        int, int, int, int, int, int, int, PyObject *, PyObject *,
+        PyObject *, PyObject *, PyObject *, PyObject *,
+        PyObject *, PyObject *, int, PyObject *);
 
 /* Creates a new empty code object with the specified source location. */
 PyAPI_FUNC(PyCodeObject *)

Include/cpython/frameobject.h

@@ -45,8 +45,34 @@ struct _frame {
     int f_lineno;               /* Current line number. Only valid if non-zero */
     int f_iblock;               /* index in f_blockstack */
     PyFrameState f_state;       /* What state the frame is in */
-    PyTryBlock f_blockstack[CO_MAXBLOCKS]; /* for try and loop blocks */
-    PyObject *f_localsplus[1];  /* locals+stack, dynamically sized */
+    PyTryBlock *f_blockstack;   /* points after the value stack */
+    PyObject *f_localsplus[1];  /* locals+stacks, dynamically sized */
+    /* The size of a PyFrameObject depends on the number of local
+     * variables and the size of the value and block stacks.
+     *                              _________________________________
+     *                             |            GC Header            |
+     *          PyFrameObject * -> |---------------------------------|
+     *                             | Fixed size PyFrameObject fields |
+     *                             |                                 |
+     *                             |      PyVarObject ob_base        |
+     *                             |              ...                |
+     *                             |    PyTryBlock* f_blockstack     |
+     *   PyObject *f_localsplus -> |---------------------------------|
+     *                             |    Dynamically sized locals     |
+     *                             |                                 |
+     *                             |        Local variables          |
+     *                             |         Cell variables          |
+     *                             |         Free variables          |
+     *  PyObject **f_valuestack -> |---------------------------------|
+     *                             |  Dynamically sized value stack  |
+     *                             |               |                 |
+     *                             |               V                 |
+     * PyTryBlock *f_blockstack -> |---------------------------------|
+     *                             |  Dynamically sized block stack  |
+     *                             |               |                 |
+     *                             |               V                 |
+     *                             |_________________________________|
+     */
 };
 
 static inline int _PyFrame_IsRunnable(struct _frame *f) {

Lib/ctypes/test/test_values.py

@@ -80,9 +80,9 @@ class struct_frozen(Structure):
                 continue
             items.append((entry.name.decode("ascii"), entry.size))
 
-        expected = [("__hello__", 139),
-                    ("__phello__", -139),
-                    ("__phello__.spam", 139),
+        expected = [("__hello__", 143),
+                    ("__phello__", -143),
+                    ("__phello__.spam", 143),
                     ]
         self.assertEqual(items, expected, "PyImport_FrozenModules example "
             "in Doc/library/ctypes.rst may be out of date")

Lib/test/test_code.py

@@ -218,6 +218,7 @@ def func(): pass
                         co.co_kwonlyargcount,
                         co.co_nlocals,
                         co.co_stacksize,
+                        co.co_blockstacksize,
                         co.co_flags,
                         co.co_code,
                         co.co_consts,

Lib/test/test_syntax.py

@@ -571,35 +571,6 @@
 This raises a SyntaxError, it used to raise a SystemError.
 Context for this change can be found on issue #27514
 
-In 2.5 there was a missing exception and an assert was triggered in a debug
-build.  The number of blocks must be greater than CO_MAXBLOCKS.  SF #1565514
-
-   >>> while 1:
-   ...  while 2:
-   ...   while 3:
-   ...    while 4:
-   ...     while 5:
-   ...      while 6:
-   ...       while 8:
-   ...        while 9:
-   ...         while 10:
-   ...          while 11:
-   ...           while 12:
-   ...            while 13:
-   ...             while 14:
-   ...              while 15:
-   ...               while 16:
-   ...                while 17:
-   ...                 while 18:
-   ...                  while 19:
-   ...                   while 20:
-   ...                    while 21:
-   ...                     while 22:
-   ...                      break
-   Traceback (most recent call last):
-     ...
-   SyntaxError: too many statically nested blocks
-
 Misuse of the nonlocal and global statement can lead to a few unique syntax errors.
 
    >>> def f():
@@ -1117,16 +1088,6 @@ def test_empty_line_after_linecont(self):
         except SyntaxError:
             self.fail("Empty line after a line continuation character is valid.")
 
-    @support.cpython_only
-    def test_nested_named_except_blocks(self):
-        code = ""
-        for i in range(12):
-            code += f"{'    '*i}try:\n"
-            code += f"{'    '*(i+1)}raise Exception\n"
-            code += f"{'    '*i}except Exception as e:\n"
-        code += f"{' '*4*12}pass"
-        self._check_error(code, "too many statically nested blocks")
-
     def test_barry_as_flufl_with_syntax_errors(self):
         # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if
         # is reading the wrong token in the presence of syntax errors later

Lib/test/test_sys.py

@@ -1271,13 +1271,13 @@ class C(object): pass
         check(sys.float_info, vsize('') + self.P * len(sys.float_info))
         # frame
         import inspect
-        CO_MAXBLOCKS = 20
         x = inspect.currentframe()
         ncells = len(x.f_code.co_cellvars)
         nfrees = len(x.f_code.co_freevars)
+        nblocks = x.f_code.co_blockstacksize
         extras = x.f_code.co_stacksize + x.f_code.co_nlocals +\
-                  ncells + nfrees - 1
-        check(x, vsize('4Pi2c4P3ic' + CO_MAXBLOCKS*'3i' + 'P' + extras*'P'))
+                  ncells + nfrees
+        check(x, vsize('8Pi2cP3icP' + extras*'P' + nblocks*'3i'))
         # function
         def func(): pass
         check(func, size('14P'))

Misc/NEWS.d/next/Core and Builtins/2021-01-15-06-09-02.bpo-42917.F5HYBY.rst

@@ -0,0 +1 @@
+Make block stack for frame objects dynamically sizable and lift the old limit of 20 blocks.