diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index fcead4d8714870..75325d6edf42ed 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -53,6 +53,13 @@ struct _Py_UOpsAbstractFrame { _Py_UopsSymbol **stack_pointer; _Py_UopsSymbol **stack; _Py_UopsSymbol **locals; + + // Reflects the real localsplus that will be used in the VM. + // This may differ from locals if the frame is inlined. + // For an inlined frame, the inlinee shares the same localsplus + // as the inliner. + _Py_UopsSymbol **real_localsplus; + bool is_inlineable; }; typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; @@ -109,6 +116,7 @@ extern _Py_UOpsAbstractFrame *_Py_uop_frame_new( _Py_UopsSymbol **localsplus_start, int n_locals_already_filled, int curr_stackentries); +extern _Py_UOpsAbstractFrame *_Py_uop_prev_frame(_Py_UOpsContext *ctx); extern int _Py_uop_frame_pop(_Py_UOpsContext *ctx); PyAPI_FUNC(PyObject *) _Py_uop_symbols_test(PyObject *self, PyObject *ignored); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index f1601de6ae75b6..3697b284d8db58 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -101,29 +101,30 @@ extern "C" { #define _GET_ITER GET_ITER #define _GET_LEN GET_LEN #define _GET_YIELD_FROM_ITER GET_YIELD_FROM_ITER -#define _GUARD_BOTH_FLOAT 333 -#define _GUARD_BOTH_INT 334 -#define _GUARD_BOTH_UNICODE 335 -#define _GUARD_BUILTINS_VERSION 336 -#define _GUARD_DORV_VALUES 337 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 338 -#define _GUARD_GLOBALS_VERSION 339 -#define _GUARD_IS_FALSE_POP 340 -#define _GUARD_IS_NONE_POP 341 -#define _GUARD_IS_NOT_NONE_POP 342 -#define _GUARD_IS_TRUE_POP 343 -#define _GUARD_KEYS_VERSION 344 -#define _GUARD_NOT_EXHAUSTED_LIST 345 -#define _GUARD_NOT_EXHAUSTED_RANGE 346 -#define _GUARD_NOT_EXHAUSTED_TUPLE 347 -#define _GUARD_TYPE_VERSION 348 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 349 -#define _INIT_CALL_PY_EXACT_ARGS 350 -#define _INIT_CALL_PY_EXACT_ARGS_0 351 -#define _INIT_CALL_PY_EXACT_ARGS_1 352 -#define _INIT_CALL_PY_EXACT_ARGS_2 353 -#define _INIT_CALL_PY_EXACT_ARGS_3 354 -#define _INIT_CALL_PY_EXACT_ARGS_4 355 +#define _GROW_TIER2_FRAME 333 +#define _GUARD_BOTH_FLOAT 334 +#define _GUARD_BOTH_INT 335 +#define _GUARD_BOTH_UNICODE 336 +#define _GUARD_BUILTINS_VERSION 337 +#define _GUARD_DORV_VALUES 338 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 339 +#define _GUARD_GLOBALS_VERSION 340 +#define _GUARD_IS_FALSE_POP 341 +#define _GUARD_IS_NONE_POP 342 +#define _GUARD_IS_NOT_NONE_POP 343 +#define _GUARD_IS_TRUE_POP 344 +#define _GUARD_KEYS_VERSION 345 +#define _GUARD_NOT_EXHAUSTED_LIST 346 +#define _GUARD_NOT_EXHAUSTED_RANGE 347 +#define _GUARD_NOT_EXHAUSTED_TUPLE 348 +#define _GUARD_TYPE_VERSION 349 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 350 +#define _INIT_CALL_PY_EXACT_ARGS 351 +#define _INIT_CALL_PY_EXACT_ARGS_0 352 +#define _INIT_CALL_PY_EXACT_ARGS_1 353 +#define _INIT_CALL_PY_EXACT_ARGS_2 354 +#define _INIT_CALL_PY_EXACT_ARGS_3 355 +#define _INIT_CALL_PY_EXACT_ARGS_4 356 #define _INSTRUMENTED_CALL INSTRUMENTED_CALL #define _INSTRUMENTED_CALL_FUNCTION_EX INSTRUMENTED_CALL_FUNCTION_EX #define _INSTRUMENTED_CALL_KW INSTRUMENTED_CALL_KW @@ -140,65 +141,65 @@ extern "C" { #define _INSTRUMENTED_RETURN_CONST INSTRUMENTED_RETURN_CONST #define _INSTRUMENTED_RETURN_VALUE INSTRUMENTED_RETURN_VALUE #define _INSTRUMENTED_YIELD_VALUE INSTRUMENTED_YIELD_VALUE -#define _INTERNAL_INCREMENT_OPT_COUNTER 356 -#define _IS_NONE 357 +#define _INTERNAL_INCREMENT_OPT_COUNTER 357 +#define _IS_NONE 358 #define _IS_OP IS_OP -#define _ITER_CHECK_LIST 358 -#define _ITER_CHECK_RANGE 359 -#define _ITER_CHECK_TUPLE 360 -#define _ITER_JUMP_LIST 361 -#define _ITER_JUMP_RANGE 362 -#define _ITER_JUMP_TUPLE 363 -#define _ITER_NEXT_LIST 364 -#define _ITER_NEXT_RANGE 365 -#define _ITER_NEXT_TUPLE 366 -#define _JUMP_TO_TOP 367 +#define _ITER_CHECK_LIST 359 +#define _ITER_CHECK_RANGE 360 +#define _ITER_CHECK_TUPLE 361 +#define _ITER_JUMP_LIST 362 +#define _ITER_JUMP_RANGE 363 +#define _ITER_JUMP_TUPLE 364 +#define _ITER_NEXT_LIST 365 +#define _ITER_NEXT_RANGE 366 +#define _ITER_NEXT_TUPLE 367 +#define _JUMP_TO_TOP 368 #define _LIST_APPEND LIST_APPEND #define _LIST_EXTEND LIST_EXTEND #define _LOAD_ASSERTION_ERROR LOAD_ASSERTION_ERROR -#define _LOAD_ATTR 368 -#define _LOAD_ATTR_CLASS 369 -#define _LOAD_ATTR_CLASS_0 370 -#define _LOAD_ATTR_CLASS_1 371 +#define _LOAD_ATTR 369 +#define _LOAD_ATTR_CLASS 370 +#define _LOAD_ATTR_CLASS_0 371 +#define _LOAD_ATTR_CLASS_1 372 #define _LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN -#define _LOAD_ATTR_INSTANCE_VALUE 372 -#define _LOAD_ATTR_INSTANCE_VALUE_0 373 -#define _LOAD_ATTR_INSTANCE_VALUE_1 374 -#define _LOAD_ATTR_METHOD_LAZY_DICT 375 -#define _LOAD_ATTR_METHOD_NO_DICT 376 -#define _LOAD_ATTR_METHOD_WITH_VALUES 377 -#define _LOAD_ATTR_MODULE 378 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 379 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 380 +#define _LOAD_ATTR_INSTANCE_VALUE 373 +#define _LOAD_ATTR_INSTANCE_VALUE_0 374 +#define _LOAD_ATTR_INSTANCE_VALUE_1 375 +#define _LOAD_ATTR_METHOD_LAZY_DICT 376 +#define _LOAD_ATTR_METHOD_NO_DICT 377 +#define _LOAD_ATTR_METHOD_WITH_VALUES 378 +#define _LOAD_ATTR_MODULE 379 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 380 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 381 #define _LOAD_ATTR_PROPERTY LOAD_ATTR_PROPERTY -#define _LOAD_ATTR_SLOT 381 -#define _LOAD_ATTR_SLOT_0 382 -#define _LOAD_ATTR_SLOT_1 383 -#define _LOAD_ATTR_WITH_HINT 384 +#define _LOAD_ATTR_SLOT 382 +#define _LOAD_ATTR_SLOT_0 383 +#define _LOAD_ATTR_SLOT_1 384 +#define _LOAD_ATTR_WITH_HINT 385 #define _LOAD_BUILD_CLASS LOAD_BUILD_CLASS #define _LOAD_CONST LOAD_CONST -#define _LOAD_CONST_INLINE 385 -#define _LOAD_CONST_INLINE_BORROW 386 -#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 387 -#define _LOAD_CONST_INLINE_WITH_NULL 388 +#define _LOAD_CONST_INLINE 386 +#define _LOAD_CONST_INLINE_BORROW 387 +#define _LOAD_CONST_INLINE_BORROW_WITH_NULL 388 +#define _LOAD_CONST_INLINE_WITH_NULL 389 #define _LOAD_DEREF LOAD_DEREF -#define _LOAD_FAST 389 -#define _LOAD_FAST_0 390 -#define _LOAD_FAST_1 391 -#define _LOAD_FAST_2 392 -#define _LOAD_FAST_3 393 -#define _LOAD_FAST_4 394 -#define _LOAD_FAST_5 395 -#define _LOAD_FAST_6 396 -#define _LOAD_FAST_7 397 +#define _LOAD_FAST 390 +#define _LOAD_FAST_0 391 +#define _LOAD_FAST_1 392 +#define _LOAD_FAST_2 393 +#define _LOAD_FAST_3 394 +#define _LOAD_FAST_4 395 +#define _LOAD_FAST_5 396 +#define _LOAD_FAST_6 397 +#define _LOAD_FAST_7 398 #define _LOAD_FAST_AND_CLEAR LOAD_FAST_AND_CLEAR #define _LOAD_FAST_CHECK LOAD_FAST_CHECK #define _LOAD_FAST_LOAD_FAST LOAD_FAST_LOAD_FAST #define _LOAD_FROM_DICT_OR_DEREF LOAD_FROM_DICT_OR_DEREF #define _LOAD_FROM_DICT_OR_GLOBALS LOAD_FROM_DICT_OR_GLOBALS -#define _LOAD_GLOBAL 398 -#define _LOAD_GLOBAL_BUILTINS 399 -#define _LOAD_GLOBAL_MODULE 400 +#define _LOAD_GLOBAL 399 +#define _LOAD_GLOBAL_BUILTINS 400 +#define _LOAD_GLOBAL_MODULE 401 #define _LOAD_LOCALS LOAD_LOCALS #define _LOAD_NAME LOAD_NAME #define _LOAD_SUPER_ATTR_ATTR LOAD_SUPER_ATTR_ATTR @@ -212,48 +213,51 @@ extern "C" { #define _MATCH_SEQUENCE MATCH_SEQUENCE #define _NOP NOP #define _POP_EXCEPT POP_EXCEPT -#define _POP_FRAME 401 -#define _POP_JUMP_IF_FALSE 402 -#define _POP_JUMP_IF_TRUE 403 +#define _POP_FRAME 402 +#define _POP_JUMP_IF_FALSE 403 +#define _POP_JUMP_IF_TRUE 404 #define _POP_TOP POP_TOP -#define _POP_TOP_LOAD_CONST_INLINE_BORROW 404 +#define _POP_TOP_LOAD_CONST_INLINE_BORROW 405 +#define _POST_INLINE 406 +#define _PRE_INLINE 407 #define _PUSH_EXC_INFO PUSH_EXC_INFO -#define _PUSH_FRAME 405 +#define _PUSH_FRAME 408 +#define _PUSH_FRAME_INLINEABLE 409 #define _PUSH_NULL PUSH_NULL -#define _REPLACE_WITH_TRUE 406 +#define _REPLACE_WITH_TRUE 410 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 407 -#define _SEND 408 +#define _SAVE_RETURN_OFFSET 411 +#define _SEND 412 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 409 -#define _STORE_ATTR 410 -#define _STORE_ATTR_INSTANCE_VALUE 411 -#define _STORE_ATTR_SLOT 412 +#define _START_EXECUTOR 413 +#define _STORE_ATTR 414 +#define _STORE_ATTR_INSTANCE_VALUE 415 +#define _STORE_ATTR_SLOT 416 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 413 -#define _STORE_FAST_0 414 -#define _STORE_FAST_1 415 -#define _STORE_FAST_2 416 -#define _STORE_FAST_3 417 -#define _STORE_FAST_4 418 -#define _STORE_FAST_5 419 -#define _STORE_FAST_6 420 -#define _STORE_FAST_7 421 +#define _STORE_FAST 417 +#define _STORE_FAST_0 418 +#define _STORE_FAST_1 419 +#define _STORE_FAST_2 420 +#define _STORE_FAST_3 421 +#define _STORE_FAST_4 422 +#define _STORE_FAST_5 423 +#define _STORE_FAST_6 424 +#define _STORE_FAST_7 425 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 422 +#define _STORE_SUBSCR 426 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 423 +#define _TO_BOOL 427 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -263,12 +267,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 424 +#define _UNPACK_SEQUENCE 428 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 424 +#define MAX_UOP_ID 428 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index 9452dff9d8797d..12b3049a4b762b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -198,6 +198,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_INIT_CALL_PY_EXACT_ARGS_4] = HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_INIT_CALL_PY_EXACT_ARGS] = HAS_ARG_FLAG | HAS_ESCAPES_FLAG | HAS_PURE_FLAG, [_PUSH_FRAME] = HAS_ESCAPES_FLAG, + [_PUSH_FRAME_INLINEABLE] = HAS_ESCAPES_FLAG, [_CALL_TYPE_1] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, [_CALL_STR_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_CALL_TUPLE_1] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, @@ -242,6 +243,9 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_START_EXECUTOR] = 0, [_FATAL_ERROR] = HAS_ESCAPES_FLAG, [_CHECK_VALIDITY_AND_SET_IP] = HAS_DEOPT_FLAG, + [_PRE_INLINE] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG, + [_POST_INLINE] = HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ESCAPES_FLAG, + [_GROW_TIER2_FRAME] = HAS_ARG_FLAG | HAS_DEOPT_FLAG, }; const uint8_t _PyUop_Replication[MAX_UOP_ID+1] = { @@ -334,6 +338,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_GET_ITER] = "_GET_ITER", [_GET_LEN] = "_GET_LEN", [_GET_YIELD_FROM_ITER] = "_GET_YIELD_FROM_ITER", + [_GROW_TIER2_FRAME] = "_GROW_TIER2_FRAME", [_GUARD_BOTH_FLOAT] = "_GUARD_BOTH_FLOAT", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", @@ -427,8 +432,11 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_POP_FRAME] = "_POP_FRAME", [_POP_TOP] = "_POP_TOP", [_POP_TOP_LOAD_CONST_INLINE_BORROW] = "_POP_TOP_LOAD_CONST_INLINE_BORROW", + [_POST_INLINE] = "_POST_INLINE", + [_PRE_INLINE] = "_PRE_INLINE", [_PUSH_EXC_INFO] = "_PUSH_EXC_INFO", [_PUSH_FRAME] = "_PUSH_FRAME", + [_PUSH_FRAME_INLINEABLE] = "_PUSH_FRAME_INLINEABLE", [_PUSH_NULL] = "_PUSH_NULL", [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index b0859a382de523..466947c4d4308c 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -278,8 +278,8 @@ def many_vars(): ex = get_first_executor(many_vars) self.assertIsNotNone(ex) - self.assertTrue(any((opcode, oparg, operand) == ("_LOAD_FAST", 259, 0) - for opcode, oparg, _, operand in list(ex))) + self.assertTrue(any((opcode, oparg) == ("_LOAD_FAST", 259) + for opcode, oparg, _, _ in list(ex))) def test_unspecialized_unpack(self): # An example of an unspecialized opcode @@ -496,7 +496,7 @@ def dummy(x): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_PRE_INLINE", uops) self.assertIn("_BINARY_OP_ADD_INT", uops) def test_branch_taken(self): @@ -682,7 +682,7 @@ def dummy(x): res, ex = self._run_with_optimizer(testfunc, 32) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_PUSH_FRAME", uops) + self.assertIn("_PRE_INLINE", uops) self.assertIn("_BINARY_OP_ADD_INT", uops) self.assertNotIn("_CHECK_PEP_523", uops) @@ -925,6 +925,39 @@ def testfunc(n): self.assertLessEqual(len(guard_both_float_count), 1) self.assertIn("_COMPARE_OP_STR", uops) + def test_function_inlining(self): + def testfunc(n): + for y in range(n): + x = foo(y, y) + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertEqual(len(guard_count), 0) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertIn("_POST_INLINE", uops) + self.assertNotIn("_PUSH_FRAME", uops) + + def test_method_inlining(self): + thing = Bar() + def testfunc(n): + for y in range(n): + x = thing.foo(y, y) + return x + + res, ex = self._run_with_optimizer(testfunc, 32) + self.assertTrue(res) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + guard_count = [opname for opname in iter_opnames(ex) if opname == "_GUARD_BOTH_INT"] + self.assertEqual(len(guard_count), 0) + self.assertIn("_BINARY_OP_ADD_INT", uops) + self.assertIn("_POST_INLINE", uops) + self.assertNotIn("_PUSH_FRAME", uops) + def test_type_inconsistency(self): ns = {} src = textwrap.dedent(""" @@ -955,6 +988,49 @@ def testfunc(n): _, ex = self._run_with_optimizer(testfunc, 16) self.assertIsNone(ex) + def test_function_call_inline(self): + def cast(typ, val): + return val + def testfunc(n): + x = 0 + for i in range(n): + x = cast(int, i) + 1 + return x + x, ex = self._run_with_optimizer(testfunc, 20) + self.assertEqual(x, 20) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # print() + # print(list(iter_opnames(ex))) + self.assertNotIn("_PUSH_FRAME", uops) + self.assertNotIn("_POP_FRALE", uops) + + def test_method_call_inline(self): + class Caster: + def cast(self, typ, val): + return val + def testfunc(n): + cast = Caster().cast + x = 0 + for i in range(n): + x = cast(int, i) + 1 + return x + x, ex = self._run_with_optimizer(testfunc, 20) + self.assertEqual(x, 20) + self.assertIsNotNone(ex) + uops = get_opnames(ex) + # print() + # print(list(iter_opnames(ex))) + self.assertNotIn("_PUSH_FRAME", uops) + self.assertNotIn("_POP_FRALE", uops) + +def foo(x, y): + return x + y + +class Bar: + def foo(self, x, y): + return x + y + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_generated_cases.py b/Lib/test/test_generated_cases.py index 32c2c2fca05c4e..7b9dd36f85454f 100644 --- a/Lib/test/test_generated_cases.py +++ b/Lib/test/test_generated_cases.py @@ -908,7 +908,7 @@ def test_overridden_abstract_args(self): case OP2: { _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); + out = sym_new_not_null(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; @@ -933,7 +933,7 @@ def test_no_overridden_case(self): output = """ case OP: { _Py_UopsSymbol *out; - out = sym_new_unknown(ctx); + out = sym_new_not_null(ctx); if (out == NULL) goto out_of_space; stack_pointer[-1] = out; break; diff --git a/Misc/NEWS.d/next/Core and Builtins/2024-03-03-20-37-10.gh-issue-116291.W1Z7Oy.rst b/Misc/NEWS.d/next/Core and Builtins/2024-03-03-20-37-10.gh-issue-116291.W1Z7Oy.rst new file mode 100644 index 00000000000000..f2ba585212eedd --- /dev/null +++ b/Misc/NEWS.d/next/Core and Builtins/2024-03-03-20-37-10.gh-issue-116291.W1Z7Oy.rst @@ -0,0 +1 @@ +Add true function inlining to the tier 2 optimizer. Patch by Ken Jin and Guido van Rossum. diff --git a/Python/bytecodes.c b/Python/bytecodes.c index 15794cb017f9ca..f95dee2d0e903c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -3144,6 +3144,29 @@ dummy_func( #endif } + // Exact same as _PUSH_FRAME. But marks a frame as inlineable + // to the tier 2 redundancy eliminator. + // TODO: add support to pseudo for uops. + op(_PUSH_FRAME_INLINEABLE, (new_frame: _PyInterpreterFrame* -- unused if (0))) { + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + SYNC_SP(); + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); +#if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } +#endif + } + macro(CALL_BOUND_METHOD_EXACT_ARGS) = unused/1 + // Skip over the counter _CHECK_PEP_523 + @@ -4126,6 +4149,40 @@ dummy_func( frame->instr_ptr = (_Py_CODEUNIT *)instr_ptr; } + // Inlining prelude. + // Not too easy to express the stack effect. + op(_PRE_INLINE, (--)) { + // NULL out locals of the new inlined frame. + PyObject **end = frame->localsplus + oparg; + while (stack_pointer < end) { + *stack_pointer = NULL; + stack_pointer++; + } + CHECK_EVAL_BREAKER(); + } + + // Inlining postlude + op(_POST_INLINE, ( -- retval)) { + // clear the locals + PyObject *ret = PEEK(1); + stack_pointer--; + PyObject **end = stack_pointer - oparg; + while (stack_pointer > end) { + Py_CLEAR(stack_pointer[-1]); + stack_pointer--; + } + retval = ret; + CHECK_EVAL_BREAKER(); + } + + op(_GROW_TIER2_FRAME, (--)) { + DEOPT_IF(frame->owner != FRAME_OWNED_BY_THREAD); + DEOPT_IF(stack_pointer + oparg > tstate->datastack_limit); + assert(stack_pointer <= tstate->datastack_top); + tstate->datastack_top = stack_pointer + oparg; + } + + // END BYTECODES // } diff --git a/Python/ceval.c b/Python/ceval.c index f817f288903694..f9eb7c2889d9d2 100644 --- a/Python/ceval.c +++ b/Python/ceval.c @@ -1675,10 +1675,10 @@ static void clear_thread_frame(PyThreadState *tstate, _PyInterpreterFrame * frame) { assert(frame->owner == FRAME_OWNED_BY_THREAD); - // Make sure that this is, indeed, the top frame. We can't check this in - // _PyThreadState_PopFrame, since f_code is already cleared at that point: - assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == - tstate->datastack_top); + // // Make sure that this is, indeed, the top frame. We can't check this in + // // _PyThreadState_PopFrame, since f_code is already cleared at that point: + // assert((PyObject **)frame + _PyFrame_GetCode(frame)->co_framesize == + // tstate->datastack_top); tstate->c_recursion_remaining--; assert(frame->frame_obj == NULL || frame->frame_obj->f_frame == frame); _PyFrame_ClearExceptCode(frame); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 806f748b05ebbf..c3eb7184a9f8ab 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -3015,6 +3015,29 @@ break; } + case _PUSH_FRAME_INLINEABLE: { + _PyInterpreterFrame *new_frame; + new_frame = (_PyInterpreterFrame *)stack_pointer[-1]; + // Write it out explicitly because it's subtly different. + // Eventually this should be the only occurrence of this code. + assert(tstate->interp->eval_frame == NULL); + stack_pointer += -1; + _PyFrame_SetStackPointer(frame, stack_pointer); + new_frame->previous = frame; + CALL_STAT_INC(inlined_py_calls); + frame = tstate->current_frame = new_frame; + tstate->py_recursion_remaining--; + LOAD_SP(); + LOAD_IP(0); + #if LLTRACE && TIER_ONE + lltrace = maybe_lltrace_resume_frame(frame, &entry_frame, GLOBALS()); + if (lltrace < 0) { + goto exit_unwind; + } + #endif + break; + } + /* _CALL_PY_WITH_DEFAULTS is not a viable micro-op for tier 2 */ case _CALL_TYPE_1: { @@ -3868,4 +3891,43 @@ break; } + case _PRE_INLINE: { + oparg = CURRENT_OPARG(); + // NULL out locals of the new inlined frame. + PyObject **end = frame->localsplus + oparg; + while (stack_pointer < end) { + *stack_pointer = NULL; + stack_pointer++; + } + CHECK_EVAL_BREAKER(); + break; + } + + case _POST_INLINE: { + PyObject *retval; + oparg = CURRENT_OPARG(); + // clear the locals + PyObject *ret = PEEK(1); + stack_pointer--; + PyObject **end = stack_pointer - oparg; + while (stack_pointer > end) { + Py_CLEAR(stack_pointer[-1]); + stack_pointer--; + } + retval = ret; + stack_pointer[0] = retval; + stack_pointer += 1; + CHECK_EVAL_BREAKER(); + break; + } + + case _GROW_TIER2_FRAME: { + oparg = CURRENT_OPARG(); + if (frame->owner != FRAME_OWNED_BY_THREAD) goto deoptimize; + if (stack_pointer + oparg > tstate->datastack_limit) goto deoptimize; + assert(stack_pointer <= tstate->datastack_top); + tstate->datastack_top = stack_pointer + oparg; + break; + } + #undef TIER_TWO diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 18e3382839a92f..392b49d8862813 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -258,15 +258,14 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, } - #define STACK_LEVEL() ((int)(stack_pointer - ctx->frame->stack)) #define GETLOCAL(idx) ((ctx->frame->locals[idx])) #define REPLACE_OP(INST, OP, ARG, OPERAND) \ - INST->opcode = OP; \ - INST->oparg = ARG; \ - INST->operand = OPERAND; + (INST)->opcode = OP; \ + (INST)->oparg = ARG; \ + (INST)->operand = OPERAND; #define OUT_OF_SPACE_IF_NULL(EXPR) \ do { \ @@ -281,6 +280,14 @@ remove_globals(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, OUT_OF_SPACE_IF_NULL(null = _Py_uop_sym_new_null(ctx)); \ } while (0); +static int +real_localsplus_idx(_Py_UOpsContext *ctx, int oparg) +{ + assert(ctx->frame->real_localsplus != NULL); + int target = (int)(&GETLOCAL(oparg) - ctx->frame->real_localsplus); + assert(target >= 0); + return target; +} /* Shortened forms for convenience, used in optimizer_bytecodes.c */ #define sym_is_not_null _Py_uop_sym_is_not_null @@ -358,6 +365,8 @@ optimize_uops( } ctx->curr_frame_depth++; ctx->frame = frame; + // Root frame should never be inlined. + frame->real_localsplus = frame->locals; for (_PyUOpInstruction *this_instr = trace; this_instr < trace + trace_len && !op_is_end(this_instr->opcode); @@ -492,7 +501,11 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) static void peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_size) { + _PyUOpInstruction *push_frame[MAX_ABSTRACT_FRAME_DEPTH]; + int frame_depth = 1; PyCodeObject *co = (PyCodeObject *)frame->f_executable; + PyFunctionObject *func = NULL; + bool is_leaf_frame = false; for (int pc = 0; pc < buffer_size; pc++) { int opcode = buffer[pc].opcode; switch(opcode) { @@ -513,9 +526,30 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s break; } case _PUSH_FRAME: + { + is_leaf_frame = true; + push_frame[frame_depth] = &buffer[pc]; + frame_depth++; + func = (PyFunctionObject *)buffer[pc].operand; + if (func == NULL) { + co = NULL; + } + else { + assert(PyFunction_Check(func)); + co = (PyCodeObject *)func->func_code; + } + assert(frame_depth <= MAX_ABSTRACT_FRAME_DEPTH); + break; + } case _POP_FRAME: { - PyFunctionObject *func = (PyFunctionObject *)buffer[pc].operand; + frame_depth--; + if (is_leaf_frame) { + push_frame[frame_depth]->opcode = _PUSH_FRAME_INLINEABLE; + } + is_leaf_frame = false; + assert(frame_depth >= 1); + func = (PyFunctionObject *)buffer[pc].operand; if (func == NULL) { co = NULL; } @@ -532,6 +566,99 @@ peephole_opt(_PyInterpreterFrame *frame, _PyUOpInstruction *buffer, int buffer_s } } + +static bool +function_decide_simple_inlineable( + _PyUOpInstruction *func_body_start, + _PyUOpInstruction *func_body_end) +{ + // Usually means MAKE_CELL or something + if (func_body_start->opcode != _RESUME_CHECK) { + return false; + } + func_body_start++; + _PyUOpInstruction *curr = func_body_start; + while (curr < func_body_end) { + int opcode = curr->opcode; + // We should be the leaf frame. + assert(opcode != _PUSH_FRAME && opcode != _PUSH_FRAME_INLINEABLE); + if (opcode == _POP_FRAME) { + return true; + } + if (_PyUop_Flags[curr->opcode] & (HAS_ESCAPES_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG)) { + // Pure overrides error flag. + if (!(_PyUop_Flags[curr->opcode] & HAS_PURE_FLAG)) { + return false; + } + } + curr++; + } + Py_UNREACHABLE(); +} + +static void +inline_simple_frames(_PyUOpInstruction *buffer, int buffer_size) +{ + bool did_inline = false; + for (int pc = 0; pc < buffer_size; pc++) { + int opcode = buffer[pc].opcode; + switch (opcode) { + case _PUSH_FRAME_INLINEABLE: { + assert(buffer[pc - 3].opcode == _CHECK_STACK_SPACE); + assert(buffer[pc - 2].opcode == _INIT_CALL_PY_EXACT_ARGS); + assert(buffer[pc - 1].opcode == _SAVE_RETURN_OFFSET); + assert(buffer[pc + 1].opcode == _CHECK_VALIDITY_AND_SET_IP || + buffer[pc + 1].opcode == _CHECK_VALIDITY); + // Skip over the CHECK_VALIDITY when deciding, + // as those can be optimized away later. + if (!function_decide_simple_inlineable(&buffer[pc + 2], buffer + buffer_size)) { + buffer[pc].opcode = _PUSH_FRAME; + break; + } + assert(buffer[pc + 2].opcode == _RESUME_CHECK); + did_inline = true; + uint64_t operand = buffer[pc].operand; + int locals_len = (int)(operand >> 32); + int stack_len = (int)(operand & 0xFFFFFFFF); + REPLACE_OP(&buffer[pc - 3], _GROW_TIER2_FRAME, locals_len + stack_len, 0); + REPLACE_OP(&buffer[pc - 2], _NOP, 0, 0); + REPLACE_OP(&buffer[pc - 1], _NOP, 0, 0); + REPLACE_OP(&buffer[pc], _PRE_INLINE, locals_len, 0); + REPLACE_OP(&buffer[pc + 1], _NOP, 0, 0); + REPLACE_OP(&buffer[pc + 2], _NOP, 0, 0); + break; + } + case _POP_FRAME: { + if (did_inline) { + buffer[pc].oparg = (int)buffer[pc].operand; + buffer[pc].opcode = _POST_INLINE; + } + did_inline = false; + break; + } + case _LOAD_FAST: + case _STORE_FAST: + case _LOAD_FAST_AND_CLEAR: + if (did_inline) { + buffer[pc].oparg = (int)buffer[pc].operand; + buffer[pc].operand = 0; + } + break; + case _SET_IP: { + if (did_inline) { + REPLACE_OP(&buffer[pc], _NOP, 0, 0); + } + break; + } + case _JUMP_TO_TOP: + case _EXIT_TRACE: + return; + default: + break; + } + } +} + // 0 - failure, no error raised, just fall back to Tier 1 // -1 - failure, and raise error // 1 - optimizer success @@ -566,6 +693,8 @@ _Py_uop_analyze_and_optimize( assert(err == 1); remove_unneeded_uops(buffer, buffer_size); + inline_simple_frames(buffer, buffer_size); + // remove_unneeded_uops(buffer, buffer_size); OPT_STAT_INC(optimizer_successes); return 1; diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index ee67a2b075b015..7c1da58172e1ff 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -71,6 +71,7 @@ dummy_func(void) { op(_LOAD_FAST, (-- value)) { value = GETLOCAL(oparg); + REPLACE_OP(this_instr, _LOAD_FAST, oparg, real_localsplus_idx(ctx, oparg)); } op(_LOAD_FAST_AND_CLEAR, (-- value)) { @@ -78,10 +79,12 @@ dummy_func(void) { _Py_UopsSymbol *temp; OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); GETLOCAL(oparg) = temp; + REPLACE_OP(this_instr, _LOAD_FAST_AND_CLEAR, oparg, real_localsplus_idx(ctx, oparg)); } op(_STORE_FAST, (value --)) { GETLOCAL(oparg) = value; + REPLACE_OP(this_instr, _STORE_FAST, oparg, real_localsplus_idx(ctx, oparg)); } op(_PUSH_NULL, (-- res)) { @@ -408,6 +411,12 @@ dummy_func(void) { } } + op(_LOAD_ATTR, (owner -- attr, self_or_null if (oparg & 1))) { + (void)owner; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); + } + op(_LOAD_ATTR_MODULE, (index/1, owner -- attr, null if (oparg & 1))) { (void)index; OUT_OF_SPACE_IF_NULL(null = sym_new_null(ctx)); @@ -492,6 +501,7 @@ dummy_func(void) { op(_INIT_CALL_PY_EXACT_ARGS, (callable, self_or_null, args[oparg] -- new_frame: _Py_UOpsAbstractFrame *)) { int argcount = oparg; + bool is_inlineable = false; (void)callable; @@ -517,13 +527,18 @@ dummy_func(void) { if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { localsplus_start = args; n_locals_already_filled = argcount; + is_inlineable = true; } OUT_OF_SPACE_IF_NULL(new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + new_frame->is_inlineable = is_inlineable; } op(_POP_FRAME, (retval -- res)) { SYNC_SP(); + REPLACE_OP(this_instr, _POP_FRAME, + this_instr->oparg, + (stack_pointer - _Py_uop_prev_frame(ctx)->stack_pointer)); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -532,10 +547,26 @@ dummy_func(void) { op(_PUSH_FRAME, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { SYNC_SP(); + new_frame->real_localsplus = new_frame->locals; + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + } + + op(_PUSH_FRAME_INLINEABLE, (new_frame: _Py_UOpsAbstractFrame * -- unused if (0))) { + SYNC_SP(); + new_frame->real_localsplus = ctx->frame->real_localsplus; ctx->frame->stack_pointer = stack_pointer; ctx->frame = new_frame; ctx->curr_frame_depth++; stack_pointer = new_frame->stack_pointer; + // First 32 bits set to locals_len, last 32 bits set to stack_len. + uint64_t operand = (((uint64_t)(new_frame->locals_len)) << 32) | (new_frame->stack_len); + REPLACE_OP(this_instr, _PUSH_FRAME_INLINEABLE, oparg, operand); + if (!new_frame->is_inlineable) { + REPLACE_OP(this_instr, _PUSH_FRAME, oparg, 0); + } } op(_UNPACK_SEQUENCE, (seq -- values[oparg])) { diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index 4ed1ac90c1e4bd..2543cc750025c6 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -28,6 +28,7 @@ case _LOAD_FAST: { _Py_UopsSymbol *value; value = GETLOCAL(oparg); + REPLACE_OP(this_instr, _LOAD_FAST, oparg, real_localsplus_idx(ctx, oparg)); stack_pointer[0] = value; stack_pointer += 1; break; @@ -39,6 +40,7 @@ _Py_UopsSymbol *temp; OUT_OF_SPACE_IF_NULL(temp = sym_new_null(ctx)); GETLOCAL(oparg) = temp; + REPLACE_OP(this_instr, _LOAD_FAST_AND_CLEAR, oparg, real_localsplus_idx(ctx, oparg)); stack_pointer[0] = value; stack_pointer += 1; break; @@ -58,6 +60,7 @@ _Py_UopsSymbol *value; value = stack_pointer[-1]; GETLOCAL(oparg) = value; + REPLACE_OP(this_instr, _STORE_FAST, oparg, real_localsplus_idx(ctx, oparg)); stack_pointer += -1; break; } @@ -80,7 +83,7 @@ case _END_SEND: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-2] = value; stack_pointer += -1; @@ -89,7 +92,7 @@ case _UNARY_NEGATIVE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -97,7 +100,7 @@ case _UNARY_NOT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -205,7 +208,7 @@ case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -213,7 +216,7 @@ case _UNARY_INVERT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -482,7 +485,7 @@ case _BINARY_SUBSCR: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -491,7 +494,7 @@ case _BINARY_SLICE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-3] = res; stack_pointer += -2; @@ -505,7 +508,7 @@ case _BINARY_SUBSCR_LIST_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -514,7 +517,7 @@ case _BINARY_SUBSCR_STR_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -523,7 +526,7 @@ case _BINARY_SUBSCR_TUPLE_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -532,7 +535,7 @@ case _BINARY_SUBSCR_DICT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -573,7 +576,7 @@ case _CALL_INTRINSIC_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -581,7 +584,7 @@ case _CALL_INTRINSIC_2: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -593,6 +596,9 @@ _Py_UopsSymbol *res; retval = stack_pointer[-1]; stack_pointer += -1; + REPLACE_OP(this_instr, _POP_FRAME, + this_instr->oparg, + (stack_pointer - _Py_uop_prev_frame(ctx)->stack_pointer)); ctx->frame->stack_pointer = stack_pointer; frame_pop(ctx); stack_pointer = ctx->frame->stack_pointer; @@ -608,7 +614,7 @@ case _GET_AITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -616,7 +622,7 @@ case _GET_ANEXT: { _Py_UopsSymbol *awaitable; - awaitable = sym_new_unknown(ctx); + awaitable = sym_new_not_null(ctx); if (awaitable == NULL) goto out_of_space; stack_pointer[0] = awaitable; stack_pointer += 1; @@ -625,7 +631,7 @@ case _GET_AWAITABLE: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -644,7 +650,7 @@ case _LOAD_ASSERTION_ERROR: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -653,7 +659,7 @@ case _LOAD_BUILD_CLASS: { _Py_UopsSymbol *bc; - bc = sym_new_unknown(ctx); + bc = sym_new_not_null(ctx); if (bc == NULL) goto out_of_space; stack_pointer[0] = bc; stack_pointer += 1; @@ -687,7 +693,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = sym_new_not_null(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -698,7 +704,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = sym_new_not_null(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -709,7 +715,7 @@ _Py_UopsSymbol **values; values = &stack_pointer[-1]; for (int _i = oparg; --_i >= 0;) { - values[_i] = sym_new_unknown(ctx); + values[_i] = sym_new_not_null(ctx); if (values[_i] == NULL) goto out_of_space; } stack_pointer += -1 + oparg; @@ -752,7 +758,7 @@ case _LOAD_LOCALS: { _Py_UopsSymbol *locals; - locals = sym_new_unknown(ctx); + locals = sym_new_not_null(ctx); if (locals == NULL) goto out_of_space; stack_pointer[0] = locals; stack_pointer += 1; @@ -761,7 +767,7 @@ case _LOAD_FROM_DICT_OR_GLOBALS: { _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); + v = sym_new_not_null(ctx); if (v == NULL) goto out_of_space; stack_pointer[-1] = v; break; @@ -769,7 +775,7 @@ case _LOAD_NAME: { _Py_UopsSymbol *v; - v = sym_new_unknown(ctx); + v = sym_new_not_null(ctx); if (v == NULL) goto out_of_space; stack_pointer[0] = v; stack_pointer += 1; @@ -779,7 +785,7 @@ case _LOAD_GLOBAL: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -800,7 +806,7 @@ case _LOAD_GLOBAL_MODULE: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -813,7 +819,7 @@ case _LOAD_GLOBAL_BUILTINS: { _Py_UopsSymbol *res; _Py_UopsSymbol *null = NULL; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; null = sym_new_null(ctx); if (null == NULL) goto out_of_space; @@ -837,7 +843,7 @@ case _LOAD_FROM_DICT_OR_DEREF: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; @@ -845,7 +851,7 @@ case _LOAD_DEREF: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[0] = value; stack_pointer += 1; @@ -863,7 +869,7 @@ case _BUILD_STRING: { _Py_UopsSymbol *str; - str = sym_new_unknown(ctx); + str = sym_new_not_null(ctx); if (str == NULL) goto out_of_space; stack_pointer[-oparg] = str; stack_pointer += 1 - oparg; @@ -872,7 +878,7 @@ case _BUILD_TUPLE: { _Py_UopsSymbol *tup; - tup = sym_new_unknown(ctx); + tup = sym_new_not_null(ctx); if (tup == NULL) goto out_of_space; stack_pointer[-oparg] = tup; stack_pointer += 1 - oparg; @@ -881,7 +887,7 @@ case _BUILD_LIST: { _Py_UopsSymbol *list; - list = sym_new_unknown(ctx); + list = sym_new_not_null(ctx); if (list == NULL) goto out_of_space; stack_pointer[-oparg] = list; stack_pointer += 1 - oparg; @@ -900,7 +906,7 @@ case _BUILD_SET: { _Py_UopsSymbol *set; - set = sym_new_unknown(ctx); + set = sym_new_not_null(ctx); if (set == NULL) goto out_of_space; stack_pointer[-oparg] = set; stack_pointer += 1 - oparg; @@ -909,7 +915,7 @@ case _BUILD_MAP: { _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); + map = sym_new_not_null(ctx); if (map == NULL) goto out_of_space; stack_pointer[-oparg*2] = map; stack_pointer += 1 - oparg*2; @@ -922,7 +928,7 @@ case _BUILD_CONST_KEY_MAP: { _Py_UopsSymbol *map; - map = sym_new_unknown(ctx); + map = sym_new_not_null(ctx); if (map == NULL) goto out_of_space; stack_pointer[-1 - oparg] = map; stack_pointer += -oparg; @@ -948,7 +954,7 @@ case _LOAD_SUPER_ATTR_ATTR: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer += -2; @@ -958,9 +964,9 @@ case _LOAD_SUPER_ATTR_METHOD: { _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); + self_or_null = sym_new_not_null(ctx); if (self_or_null == NULL) goto out_of_space; stack_pointer[-3] = attr; stack_pointer[-2] = self_or_null; @@ -969,12 +975,13 @@ } case _LOAD_ATTR: { + _Py_UopsSymbol *owner; _Py_UopsSymbol *attr; _Py_UopsSymbol *self_or_null = NULL; - attr = sym_new_unknown(ctx); - if (attr == NULL) goto out_of_space; - self_or_null = sym_new_unknown(ctx); - if (self_or_null == NULL) goto out_of_space; + owner = stack_pointer[-1]; + (void)owner; + OUT_OF_SPACE_IF_NULL(attr = sym_new_not_null(ctx)); + OUT_OF_SPACE_IF_NULL(self_or_null = sym_new_unknown(ctx)); stack_pointer[-1] = attr; if (oparg & 1) stack_pointer[0] = self_or_null; stack_pointer += (oparg & 1); @@ -1131,7 +1138,7 @@ case _COMPARE_OP: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1140,7 +1147,7 @@ case _COMPARE_OP_FLOAT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1149,7 +1156,7 @@ case _COMPARE_OP_INT: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1158,7 +1165,7 @@ case _COMPARE_OP_STR: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1167,7 +1174,7 @@ case _IS_OP: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1176,7 +1183,7 @@ case _CONTAINS_OP: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-2] = b; stack_pointer += -1; @@ -1186,9 +1193,9 @@ case _CHECK_EG_MATCH: { _Py_UopsSymbol *rest; _Py_UopsSymbol *match; - rest = sym_new_unknown(ctx); + rest = sym_new_not_null(ctx); if (rest == NULL) goto out_of_space; - match = sym_new_unknown(ctx); + match = sym_new_not_null(ctx); if (match == NULL) goto out_of_space; stack_pointer[-2] = rest; stack_pointer[-1] = match; @@ -1197,7 +1204,7 @@ case _CHECK_EXC_MATCH: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1209,7 +1216,7 @@ case _IS_NONE: { _Py_UopsSymbol *b; - b = sym_new_unknown(ctx); + b = sym_new_not_null(ctx); if (b == NULL) goto out_of_space; stack_pointer[-1] = b; break; @@ -1217,7 +1224,7 @@ case _GET_LEN: { _Py_UopsSymbol *len_o; - len_o = sym_new_unknown(ctx); + len_o = sym_new_not_null(ctx); if (len_o == NULL) goto out_of_space; stack_pointer[0] = len_o; stack_pointer += 1; @@ -1226,7 +1233,7 @@ case _MATCH_CLASS: { _Py_UopsSymbol *attrs; - attrs = sym_new_unknown(ctx); + attrs = sym_new_not_null(ctx); if (attrs == NULL) goto out_of_space; stack_pointer[-3] = attrs; stack_pointer += -2; @@ -1235,7 +1242,7 @@ case _MATCH_MAPPING: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1244,7 +1251,7 @@ case _MATCH_SEQUENCE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1253,7 +1260,7 @@ case _MATCH_KEYS: { _Py_UopsSymbol *values_or_none; - values_or_none = sym_new_unknown(ctx); + values_or_none = sym_new_not_null(ctx); if (values_or_none == NULL) goto out_of_space; stack_pointer[0] = values_or_none; stack_pointer += 1; @@ -1262,7 +1269,7 @@ case _GET_ITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1270,7 +1277,7 @@ case _GET_YIELD_FROM_ITER: { _Py_UopsSymbol *iter; - iter = sym_new_unknown(ctx); + iter = sym_new_not_null(ctx); if (iter == NULL) goto out_of_space; stack_pointer[-1] = iter; break; @@ -1280,7 +1287,7 @@ case _FOR_ITER_TIER_TWO: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1301,7 +1308,7 @@ case _ITER_NEXT_LIST: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1320,7 +1327,7 @@ case _ITER_NEXT_TUPLE: { _Py_UopsSymbol *next; - next = sym_new_unknown(ctx); + next = sym_new_not_null(ctx); if (next == NULL) goto out_of_space; stack_pointer[0] = next; stack_pointer += 1; @@ -1353,9 +1360,9 @@ case _BEFORE_ASYNC_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); + exit = sym_new_not_null(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1366,9 +1373,9 @@ case _BEFORE_WITH: { _Py_UopsSymbol *exit; _Py_UopsSymbol *res; - exit = sym_new_unknown(ctx); + exit = sym_new_not_null(ctx); if (exit == NULL) goto out_of_space; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = exit; stack_pointer[0] = res; @@ -1378,7 +1385,7 @@ case _WITH_EXCEPT_START: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[0] = res; stack_pointer += 1; @@ -1388,9 +1395,9 @@ case _PUSH_EXC_INFO: { _Py_UopsSymbol *prev_exc; _Py_UopsSymbol *new_exc; - prev_exc = sym_new_unknown(ctx); + prev_exc = sym_new_not_null(ctx); if (prev_exc == NULL) goto out_of_space; - new_exc = sym_new_unknown(ctx); + new_exc = sym_new_not_null(ctx); if (new_exc == NULL) goto out_of_space; stack_pointer[-1] = prev_exc; stack_pointer[0] = new_exc; @@ -1438,7 +1445,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1446,7 +1453,7 @@ case _LOAD_ATTR_NONDESCRIPTOR_NO_DICT: { _Py_UopsSymbol *attr; - attr = sym_new_unknown(ctx); + attr = sym_new_not_null(ctx); if (attr == NULL) goto out_of_space; stack_pointer[-1] = attr; break; @@ -1533,6 +1540,7 @@ self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; int argcount = oparg; + bool is_inlineable = false; (void)callable; PyFunctionObject *func = (PyFunctionObject *)(this_instr + 2)->operand; if (func == NULL) { @@ -1554,9 +1562,11 @@ if (sym_is_null(self_or_null) || sym_is_not_null(self_or_null)) { localsplus_start = args; n_locals_already_filled = argcount; + is_inlineable = true; } OUT_OF_SPACE_IF_NULL(new_frame = frame_new(ctx, co, localsplus_start, n_locals_already_filled, 0)); + new_frame->is_inlineable = is_inlineable; stack_pointer[-2 - oparg] = (_Py_UopsSymbol *)new_frame; stack_pointer += -1 - oparg; break; @@ -1566,10 +1576,29 @@ _Py_UOpsAbstractFrame *new_frame; new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; stack_pointer += -1; + new_frame->real_localsplus = new_frame->locals; + ctx->frame->stack_pointer = stack_pointer; + ctx->frame = new_frame; + ctx->curr_frame_depth++; + stack_pointer = new_frame->stack_pointer; + break; + } + + case _PUSH_FRAME_INLINEABLE: { + _Py_UOpsAbstractFrame *new_frame; + new_frame = (_Py_UOpsAbstractFrame *)stack_pointer[-1]; + stack_pointer += -1; + new_frame->real_localsplus = ctx->frame->real_localsplus; ctx->frame->stack_pointer = stack_pointer; ctx->frame = new_frame; ctx->curr_frame_depth++; stack_pointer = new_frame->stack_pointer; + // First 32 bits set to locals_len, last 32 bits set to stack_len. + uint64_t operand = (((uint64_t)(new_frame->locals_len)) << 32) | (new_frame->stack_len); + REPLACE_OP(this_instr, _PUSH_FRAME_INLINEABLE, oparg, operand); + if (!new_frame->is_inlineable) { + REPLACE_OP(this_instr, _PUSH_FRAME, oparg, 0); + } break; } @@ -1577,7 +1606,7 @@ case _CALL_TYPE_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1586,7 +1615,7 @@ case _CALL_STR_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1595,7 +1624,7 @@ case _CALL_TUPLE_1: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1611,7 +1640,7 @@ case _CALL_BUILTIN_CLASS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1620,7 +1649,7 @@ case _CALL_BUILTIN_O: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1629,7 +1658,7 @@ case _CALL_BUILTIN_FAST: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1638,7 +1667,7 @@ case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1647,7 +1676,7 @@ case _CALL_LEN: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1656,7 +1685,7 @@ case _CALL_ISINSTANCE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1665,7 +1694,7 @@ case _CALL_METHOD_DESCRIPTOR_O: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1674,7 +1703,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1683,7 +1712,7 @@ case _CALL_METHOD_DESCRIPTOR_NOARGS: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1692,7 +1721,7 @@ case _CALL_METHOD_DESCRIPTOR_FAST: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2 - oparg] = res; stack_pointer += -1 - oparg; @@ -1709,7 +1738,7 @@ case _MAKE_FUNCTION: { _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); + func = sym_new_not_null(ctx); if (func == NULL) goto out_of_space; stack_pointer[-1] = func; break; @@ -1717,7 +1746,7 @@ case _SET_FUNCTION_ATTRIBUTE: { _Py_UopsSymbol *func; - func = sym_new_unknown(ctx); + func = sym_new_not_null(ctx); if (func == NULL) goto out_of_space; stack_pointer[-2] = func; stack_pointer += -1; @@ -1726,7 +1755,7 @@ case _BUILD_SLICE: { _Py_UopsSymbol *slice; - slice = sym_new_unknown(ctx); + slice = sym_new_not_null(ctx); if (slice == NULL) goto out_of_space; stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; stack_pointer += -1 - ((oparg == 3) ? 1 : 0); @@ -1735,7 +1764,7 @@ case _CONVERT_VALUE: { _Py_UopsSymbol *result; - result = sym_new_unknown(ctx); + result = sym_new_not_null(ctx); if (result == NULL) goto out_of_space; stack_pointer[-1] = result; break; @@ -1743,7 +1772,7 @@ case _FORMAT_SIMPLE: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-1] = res; break; @@ -1751,7 +1780,7 @@ case _FORMAT_WITH_SPEC: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1771,7 +1800,7 @@ case _BINARY_OP: { _Py_UopsSymbol *res; - res = sym_new_unknown(ctx); + res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; stack_pointer[-2] = res; stack_pointer += -1; @@ -1898,7 +1927,7 @@ case _POP_TOP_LOAD_CONST_INLINE_BORROW: { _Py_UopsSymbol *value; - value = sym_new_unknown(ctx); + value = sym_new_not_null(ctx); if (value == NULL) goto out_of_space; stack_pointer[-1] = value; break; @@ -1957,3 +1986,20 @@ break; } + case _PRE_INLINE: { + break; + } + + case _POST_INLINE: { + _Py_UopsSymbol *retval; + retval = sym_new_not_null(ctx); + if (retval == NULL) goto out_of_space; + stack_pointer[0] = retval; + stack_pointer += 1; + break; + } + + case _GROW_TIER2_FRAME: { + break; + } + diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 86b0d4d395afa2..c7903231cf30a8 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -302,6 +302,8 @@ _Py_uop_frame_new( frame->locals = localsplus_start; frame->stack = frame->locals + co->co_nlocalsplus; frame->stack_pointer = frame->stack + curr_stackentries; + frame->real_localsplus = NULL; + frame->is_inlineable = false; ctx->n_consumed = localsplus_start + (co->co_nlocalsplus + co->co_stacksize); if (ctx->n_consumed >= ctx->limit) { return NULL; @@ -364,6 +366,12 @@ _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx) return 0; } +_Py_UOpsAbstractFrame * +_Py_uop_prev_frame(_Py_UOpsContext *ctx) +{ + return &ctx->frames[ctx->curr_frame_depth - 2]; +} + int _Py_uop_frame_pop(_Py_UOpsContext *ctx) { diff --git a/Tools/cases_generator/optimizer_generator.py b/Tools/cases_generator/optimizer_generator.py index fca42b51fbd689..707cfb93a13eb1 100644 --- a/Tools/cases_generator/optimizer_generator.py +++ b/Tools/cases_generator/optimizer_generator.py @@ -83,14 +83,14 @@ def emit_default(out: CWriter, uop: Uop) -> None: if var.name != "unused" and not var.peek: if var.is_array(): out.emit(f"for (int _i = {var.size}; --_i >= 0;) {{\n") - out.emit(f"{var.name}[_i] = sym_new_unknown(ctx);\n") + out.emit(f"{var.name}[_i] = sym_new_not_null(ctx);\n") out.emit(f"if ({var.name}[_i] == NULL) goto out_of_space;\n") out.emit("}\n") elif var.name == "null": out.emit(f"{var.name} = sym_new_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n") else: - out.emit(f"{var.name} = sym_new_unknown(ctx);\n") + out.emit(f"{var.name} = sym_new_not_null(ctx);\n") out.emit(f"if ({var.name} == NULL) goto out_of_space;\n")