From f02fa64bf2d03ef7a28650c164e17a5fb5d8543d Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Tue, 24 Jan 2023 17:25:37 +0000 Subject: [PATCH] GH-100762: Don't call `gen.throw()` in `gen.close()`, unless necessary. (GH-101013) * Store exception stack depth in YIELD_VALUE's oparg and use it avoid expensive gen.throw() in gen.close() where possible. --- Doc/library/dis.rst | 5 +++- Lib/importlib/_bootstrap_external.py | 5 ++-- Lib/test/test_dis.py | 4 ++-- ...-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst | 3 +++ Objects/genobject.c | 24 ++++++++++++++++++- Python/bytecodes.c | 1 - Python/compile.c | 6 ++--- Python/generated_cases.c.h | 1 - Python/opcode_metadata.h | 2 +- 9 files changed, 39 insertions(+), 12 deletions(-) create mode 100644 Misc/NEWS.d/next/Core and Builtins/2023-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst diff --git a/Doc/library/dis.rst b/Doc/library/dis.rst index 7f309ce56ddec0..6a68ec4b14be31 100644 --- a/Doc/library/dis.rst +++ b/Doc/library/dis.rst @@ -700,7 +700,10 @@ iterations of the loop. Yields ``STACK.pop()`` from a :term:`generator`. .. versionchanged:: 3.11 - oparg set to be the stack depth, for efficient handling on frames. + oparg set to be the stack depth. + + .. versionchanged:: 3.12 + oparg set to be the exception block depth, for efficient closing of generators. .. opcode:: SETUP_ANNOTATIONS diff --git a/Lib/importlib/_bootstrap_external.py b/Lib/importlib/_bootstrap_external.py index 5a8cbb98ed3f56..cd44d0050ede82 100644 --- a/Lib/importlib/_bootstrap_external.py +++ b/Lib/importlib/_bootstrap_external.py @@ -429,7 +429,8 @@ def _write_atomic(path, data, mode=0o666): # Python 3.12a1 3513 (Add CALL_INTRINSIC_1 instruction, removed STOPITERATION_ERROR, PRINT_EXPR, IMPORT_STAR) # Python 3.12a1 3514 (Remove ASYNC_GEN_WRAP, LIST_TO_TUPLE, and UNARY_POSITIVE) # Python 3.12a1 3515 (Embed jump mask in COMPARE_OP oparg) -# Python 3.12a1 3516 (Add COMAPRE_AND_BRANCH instruction) +# Python 3.12a1 3516 (Add COMPARE_AND_BRANCH instruction) +# Python 3.12a1 3517 (Change YIELD_VALUE oparg to exception block depth) # Python 3.13 will start with 3550 @@ -442,7 +443,7 @@ def _write_atomic(path, data, mode=0o666): # Whenever MAGIC_NUMBER is changed, the ranges in the magic_values array # in PC/launcher.c must also be updated. -MAGIC_NUMBER = (3516).to_bytes(2, 'little') + b'\r\n' +MAGIC_NUMBER = (3517).to_bytes(2, 'little') + b'\r\n' _RAW_MAGIC_NUMBER = int.from_bytes(MAGIC_NUMBER, 'little') # For import.c diff --git a/Lib/test/test_dis.py b/Lib/test/test_dis.py index 27487de8fb3d81..bdf48c15309296 100644 --- a/Lib/test/test_dis.py +++ b/Lib/test/test_dis.py @@ -492,7 +492,7 @@ async def _asyncwith(c): GET_AWAITABLE 1 LOAD_CONST 0 (None) >> SEND 3 (to 22) - YIELD_VALUE 3 + YIELD_VALUE 2 RESUME 3 JUMP_BACKWARD_NO_INTERRUPT 4 (to 14) >> POP_TOP @@ -526,7 +526,7 @@ async def _asyncwith(c): GET_AWAITABLE 2 LOAD_CONST 0 (None) >> SEND 4 (to 92) - YIELD_VALUE 6 + YIELD_VALUE 3 RESUME 3 JUMP_BACKWARD_NO_INTERRUPT 4 (to 82) >> CLEANUP_THROW diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst b/Misc/NEWS.d/next/Core and Builtins/2023-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst new file mode 100644 index 00000000000000..2f6b121439a985 --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2023-01-13-12-56-20.gh-issue-100762.YvHaQJ.rst @@ -0,0 +1,3 @@ +Record the (virtual) exception block depth in the oparg of +:opcode:`YIELD_VALUE`. Use this to avoid the expensive ``throw()`` when +closing generators (and coroutines) that can be closed trivially. diff --git a/Objects/genobject.c b/Objects/genobject.c index 2adb1e4bf308e4..35246653c45348 100644 --- a/Objects/genobject.c +++ b/Objects/genobject.c @@ -354,6 +354,13 @@ gen_close(PyGenObject *gen, PyObject *args) PyObject *yf = _PyGen_yf(gen); int err = 0; + if (gen->gi_frame_state == FRAME_CREATED) { + gen->gi_frame_state = FRAME_COMPLETED; + Py_RETURN_NONE; + } + if (gen->gi_frame_state >= FRAME_COMPLETED) { + Py_RETURN_NONE; + } if (yf) { PyFrameState state = gen->gi_frame_state; gen->gi_frame_state = FRAME_EXECUTING; @@ -361,8 +368,23 @@ gen_close(PyGenObject *gen, PyObject *args) gen->gi_frame_state = state; Py_DECREF(yf); } - if (err == 0) + _PyInterpreterFrame *frame = (_PyInterpreterFrame *)gen->gi_iframe; + /* It is possible for the previous instruction to not be a + * YIELD_VALUE if the debugger has changed the lineno. */ + if (err == 0 && frame->prev_instr->opcode == YIELD_VALUE) { + assert(frame->prev_instr[1].opcode == RESUME); + int exception_handler_depth = frame->prev_instr->oparg; + assert(exception_handler_depth > 0); + /* We can safely ignore the outermost try block + * as it automatically generated to handle + * StopIteration. */ + if (exception_handler_depth == 1) { + Py_RETURN_NONE; + } + } + if (err == 0) { PyErr_SetNone(PyExc_GeneratorExit); + } retval = gen_send_ex(gen, Py_None, 1, 1); if (retval) { const char *msg = "generator ignored GeneratorExit"; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 47bbe1a99e3bf0..ac54791d67e439 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -720,7 +720,6 @@ dummy_func( // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. - assert(oparg == STACK_LEVEL()); assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; diff --git a/Python/compile.c b/Python/compile.c index ce714dce6edfe6..9fc997cdf525e9 100644 --- a/Python/compile.c +++ b/Python/compile.c @@ -7162,9 +7162,6 @@ stackdepth(basicblock *entryblock, int code_flags) next = NULL; break; } - if (instr->i_opcode == YIELD_VALUE) { - instr->i_oparg = depth; - } } if (next != NULL) { assert(BB_HAS_FALLTHROUGH(b)); @@ -7332,6 +7329,9 @@ label_exception_targets(basicblock *entryblock) { } } else { + if (instr->i_opcode == YIELD_VALUE) { + instr->i_oparg = except_stack->depth; + } instr->i_except = handler; } } diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index c1eb4000883dc7..5dcc8eeec19f89 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -919,7 +919,6 @@ // NOTE: It's important that YIELD_VALUE never raises an exception! // The compiler treats any exception raised here as a failed close() // or throw() call. - assert(oparg == STACK_LEVEL()); assert(frame != &entry_frame); PyGenObject *gen = _PyFrame_GetGenerator(frame); gen->gi_frame_state = FRAME_SUSPENDED; diff --git a/Python/opcode_metadata.h b/Python/opcode_metadata.h index 3ceaca8c397f6b..1fb0acceb511c7 100644 --- a/Python/opcode_metadata.h +++ b/Python/opcode_metadata.h @@ -59,7 +59,7 @@ static const struct { [GET_ANEXT] = { 1, 2, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [GET_AWAITABLE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [SEND] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, - [YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, + [YIELD_VALUE] = { 1, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [POP_EXCEPT] = { 1, 0, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX }, [RERAISE] = { -1, -1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IB }, [PREP_RERAISE_STAR] = { 2, 1, DIR_NONE, DIR_NONE, DIR_NONE, true, INSTR_FMT_IX },