From b14e882428ceda1e5852a1c22772e7f88927bded Mon Sep 17 00:00:00 2001 From: Mark Shannon Date: Wed, 1 Nov 2023 10:53:27 +0000 Subject: [PATCH] GH-111485: Use micro-ops to split specialization code from base action (GH-111561) --- Include/internal/pycore_opcode_metadata.h | 331 ++++++--- Lib/test/test_capi/test_misc.py | 2 +- Python/abstract_interp_cases.c.h | 22 +- Python/bytecodes.c | 152 +++-- Python/executor_cases.c.h | 107 +-- Python/generated_cases.c.h | 792 ++++++++++++---------- Tools/cases_generator/analysis.py | 7 +- Tools/cases_generator/generate_cases.py | 20 +- Tools/cases_generator/instructions.py | 2 +- 9 files changed, 858 insertions(+), 577 deletions(-) diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index 7b89f89b3f69c3..ed38ed0f39f94c 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -27,68 +27,94 @@ #define _EXIT_TRACE 300 #define _SET_IP 301 -#define _GUARD_BOTH_INT 302 -#define _BINARY_OP_MULTIPLY_INT 303 -#define _BINARY_OP_ADD_INT 304 -#define _BINARY_OP_SUBTRACT_INT 305 -#define _GUARD_BOTH_FLOAT 306 -#define _BINARY_OP_MULTIPLY_FLOAT 307 -#define _BINARY_OP_ADD_FLOAT 308 -#define _BINARY_OP_SUBTRACT_FLOAT 309 -#define _GUARD_BOTH_UNICODE 310 -#define _BINARY_OP_ADD_UNICODE 311 -#define _BINARY_OP_INPLACE_ADD_UNICODE 312 -#define _POP_FRAME 313 -#define _GUARD_GLOBALS_VERSION 314 -#define _GUARD_BUILTINS_VERSION 315 -#define _LOAD_GLOBAL_MODULE 316 -#define _LOAD_GLOBAL_BUILTINS 317 -#define _GUARD_TYPE_VERSION 318 -#define _CHECK_MANAGED_OBJECT_HAS_VALUES 319 -#define _LOAD_ATTR_INSTANCE_VALUE 320 -#define _CHECK_ATTR_MODULE 321 -#define _LOAD_ATTR_MODULE 322 -#define _CHECK_ATTR_WITH_HINT 323 -#define _LOAD_ATTR_WITH_HINT 324 -#define _LOAD_ATTR_SLOT 325 -#define _CHECK_ATTR_CLASS 326 -#define _LOAD_ATTR_CLASS 327 -#define _GUARD_DORV_VALUES 328 -#define _STORE_ATTR_INSTANCE_VALUE 329 -#define _STORE_ATTR_SLOT 330 -#define _IS_NONE 331 -#define _ITER_CHECK_LIST 332 -#define _ITER_JUMP_LIST 333 -#define _IS_ITER_EXHAUSTED_LIST 334 -#define _ITER_NEXT_LIST 335 -#define _ITER_CHECK_TUPLE 336 -#define _ITER_JUMP_TUPLE 337 -#define _IS_ITER_EXHAUSTED_TUPLE 338 -#define _ITER_NEXT_TUPLE 339 -#define _ITER_CHECK_RANGE 340 -#define _ITER_JUMP_RANGE 341 -#define _IS_ITER_EXHAUSTED_RANGE 342 -#define _ITER_NEXT_RANGE 343 -#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 344 -#define _GUARD_KEYS_VERSION 345 -#define _LOAD_ATTR_METHOD_WITH_VALUES 346 -#define _LOAD_ATTR_METHOD_NO_DICT 347 -#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 348 -#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 349 -#define _CHECK_ATTR_METHOD_LAZY_DICT 350 -#define _LOAD_ATTR_METHOD_LAZY_DICT 351 -#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 352 -#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 353 -#define _CHECK_PEP_523 354 -#define _CHECK_FUNCTION_EXACT_ARGS 355 -#define _CHECK_STACK_SPACE 356 -#define _INIT_CALL_PY_EXACT_ARGS 357 -#define _PUSH_FRAME 358 -#define _POP_JUMP_IF_FALSE 359 -#define _POP_JUMP_IF_TRUE 360 -#define _JUMP_TO_TOP 361 -#define _SAVE_RETURN_OFFSET 362 -#define _INSERT 363 +#define _SPECIALIZE_TO_BOOL 302 +#define _TO_BOOL 303 +#define _GUARD_BOTH_INT 304 +#define _BINARY_OP_MULTIPLY_INT 305 +#define _BINARY_OP_ADD_INT 306 +#define _BINARY_OP_SUBTRACT_INT 307 +#define _GUARD_BOTH_FLOAT 308 +#define _BINARY_OP_MULTIPLY_FLOAT 309 +#define _BINARY_OP_ADD_FLOAT 310 +#define _BINARY_OP_SUBTRACT_FLOAT 311 +#define _GUARD_BOTH_UNICODE 312 +#define _BINARY_OP_ADD_UNICODE 313 +#define _BINARY_OP_INPLACE_ADD_UNICODE 314 +#define _SPECIALIZE_BINARY_SUBSCR 315 +#define _BINARY_SUBSCR 316 +#define _SPECIALIZE_STORE_SUBSCR 317 +#define _STORE_SUBSCR 318 +#define _POP_FRAME 319 +#define _SPECIALIZE_SEND 320 +#define _SEND 321 +#define _SPECIALIZE_UNPACK_SEQUENCE 322 +#define _UNPACK_SEQUENCE 323 +#define _SPECIALIZE_STORE_ATTR 324 +#define _STORE_ATTR 325 +#define _SPECIALIZE_LOAD_GLOBAL 326 +#define _LOAD_GLOBAL 327 +#define _GUARD_GLOBALS_VERSION 328 +#define _GUARD_BUILTINS_VERSION 329 +#define _LOAD_GLOBAL_MODULE 330 +#define _LOAD_GLOBAL_BUILTINS 331 +#define _SPECIALIZE_LOAD_SUPER_ATTR 332 +#define _LOAD_SUPER_ATTR 333 +#define _SPECIALIZE_LOAD_ATTR 334 +#define _LOAD_ATTR 335 +#define _GUARD_TYPE_VERSION 336 +#define _CHECK_MANAGED_OBJECT_HAS_VALUES 337 +#define _LOAD_ATTR_INSTANCE_VALUE 338 +#define _CHECK_ATTR_MODULE 339 +#define _LOAD_ATTR_MODULE 340 +#define _CHECK_ATTR_WITH_HINT 341 +#define _LOAD_ATTR_WITH_HINT 342 +#define _LOAD_ATTR_SLOT 343 +#define _CHECK_ATTR_CLASS 344 +#define _LOAD_ATTR_CLASS 345 +#define _GUARD_DORV_VALUES 346 +#define _STORE_ATTR_INSTANCE_VALUE 347 +#define _STORE_ATTR_SLOT 348 +#define _SPECIALIZE_COMPARE_OP 349 +#define _COMPARE_OP 350 +#define _IS_NONE 351 +#define _SPECIALIZE_FOR_ITER 352 +#define _FOR_ITER 353 +#define _ITER_CHECK_LIST 354 +#define _ITER_JUMP_LIST 355 +#define _IS_ITER_EXHAUSTED_LIST 356 +#define _ITER_NEXT_LIST 357 +#define _ITER_CHECK_TUPLE 358 +#define _ITER_JUMP_TUPLE 359 +#define _IS_ITER_EXHAUSTED_TUPLE 360 +#define _ITER_NEXT_TUPLE 361 +#define _ITER_CHECK_RANGE 362 +#define _ITER_JUMP_RANGE 363 +#define _IS_ITER_EXHAUSTED_RANGE 364 +#define _ITER_NEXT_RANGE 365 +#define _GUARD_DORV_VALUES_INST_ATTR_FROM_DICT 366 +#define _GUARD_KEYS_VERSION 367 +#define _LOAD_ATTR_METHOD_WITH_VALUES 368 +#define _LOAD_ATTR_METHOD_NO_DICT 369 +#define _LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES 370 +#define _LOAD_ATTR_NONDESCRIPTOR_NO_DICT 371 +#define _CHECK_ATTR_METHOD_LAZY_DICT 372 +#define _LOAD_ATTR_METHOD_LAZY_DICT 373 +#define _SPECIALIZE_CALL 374 +#define _CALL 375 +#define _CHECK_CALL_BOUND_METHOD_EXACT_ARGS 376 +#define _INIT_CALL_BOUND_METHOD_EXACT_ARGS 377 +#define _CHECK_PEP_523 378 +#define _CHECK_FUNCTION_EXACT_ARGS 379 +#define _CHECK_STACK_SPACE 380 +#define _INIT_CALL_PY_EXACT_ARGS 381 +#define _PUSH_FRAME 382 +#define _SPECIALIZE_BINARY_OP 383 +#define _BINARY_OP 384 +#define _POP_JUMP_IF_FALSE 385 +#define _POP_JUMP_IF_TRUE 386 +#define _JUMP_TO_TOP 387 +#define _SAVE_RETURN_OFFSET 388 +#define _INSERT 389 extern int _PyOpcode_num_popped(int opcode, int oparg, bool jump); #ifdef NEED_OPCODE_METADATA @@ -138,6 +164,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case UNARY_NOT: return 1; + case _SPECIALIZE_TO_BOOL: + return 1; + case _TO_BOOL: + return 1; case TO_BOOL: return 1; case TO_BOOL_BOOL: @@ -192,6 +222,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case BINARY_OP_INPLACE_ADD_UNICODE: return 2; + case _SPECIALIZE_BINARY_SUBSCR: + return 2; + case _BINARY_SUBSCR: + return 2; case BINARY_SUBSCR: return 2; case BINARY_SLICE: @@ -212,6 +246,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return (oparg-1) + 2; case SET_ADD: return (oparg-1) + 2; + case _SPECIALIZE_STORE_SUBSCR: + return 2; + case _STORE_SUBSCR: + return 3; case STORE_SUBSCR: return 3; case STORE_SUBSCR_LIST_INT: @@ -244,6 +282,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case GET_AWAITABLE: return 1; + case _SPECIALIZE_SEND: + return 2; + case _SEND: + return 2; case SEND: return 2; case SEND_GEN: @@ -268,6 +310,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case DELETE_NAME: return 0; + case _SPECIALIZE_UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE: + return 1; case UNPACK_SEQUENCE: return 1; case UNPACK_SEQUENCE_TWO_TUPLE: @@ -278,6 +324,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case UNPACK_EX: return 1; + case _SPECIALIZE_STORE_ATTR: + return 1; + case _STORE_ATTR: + return 2; case STORE_ATTR: return 2; case DELETE_ATTR: @@ -292,6 +342,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case LOAD_NAME: return 0; + case _SPECIALIZE_LOAD_GLOBAL: + return 0; + case _LOAD_GLOBAL: + return 0; case LOAD_GLOBAL: return 0; case _GUARD_GLOBALS_VERSION: @@ -346,6 +400,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return (oparg - 1) + 3; case INSTRUMENTED_LOAD_SUPER_ATTR: return 3; + case _SPECIALIZE_LOAD_SUPER_ATTR: + return 3; + case _LOAD_SUPER_ATTR: + return 3; case LOAD_SUPER_ATTR: return 3; case LOAD_SUPER_METHOD: @@ -358,6 +416,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 3; case LOAD_SUPER_ATTR_METHOD: return 3; + case _SPECIALIZE_LOAD_ATTR: + return 1; + case _LOAD_ATTR: + return 1; case LOAD_ATTR: return 1; case LOAD_METHOD: @@ -408,6 +470,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case STORE_ATTR_SLOT: return 2; + case _SPECIALIZE_COMPARE_OP: + return 2; + case _COMPARE_OP: + return 2; case COMPARE_OP: return 2; case COMPARE_OP_FLOAT: @@ -464,6 +530,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case GET_YIELD_FROM_ITER: return 1; + case _SPECIALIZE_FOR_ITER: + return 1; + case _FOR_ITER: + return 1; case FOR_ITER: return 1; case INSTRUMENTED_FOR_ITER: @@ -544,6 +614,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 1; case INSTRUMENTED_CALL: return 0; + case _SPECIALIZE_CALL: + return oparg + 2; + case _CALL: + return oparg + 2; case CALL: return oparg + 2; case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: @@ -622,6 +696,10 @@ int _PyOpcode_num_popped(int opcode, int oparg, bool jump) { return 2; case COPY: return (oparg-1) + 1; + case _SPECIALIZE_BINARY_OP: + return 2; + case _BINARY_OP: + return 2; case BINARY_OP: return 2; case SWAP: @@ -714,6 +792,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case UNARY_NOT: return 1; + case _SPECIALIZE_TO_BOOL: + return 1; + case _TO_BOOL: + return 1; case TO_BOOL: return 1; case TO_BOOL_BOOL: @@ -768,6 +850,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case BINARY_OP_INPLACE_ADD_UNICODE: return 0; + case _SPECIALIZE_BINARY_SUBSCR: + return 2; + case _BINARY_SUBSCR: + return 1; case BINARY_SUBSCR: return 1; case BINARY_SLICE: @@ -788,6 +874,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return (oparg-1) + 1; case SET_ADD: return (oparg-1) + 1; + case _SPECIALIZE_STORE_SUBSCR: + return 2; + case _STORE_SUBSCR: + return 0; case STORE_SUBSCR: return 0; case STORE_SUBSCR_LIST_INT: @@ -820,6 +910,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 2; case GET_AWAITABLE: return 1; + case _SPECIALIZE_SEND: + return 2; + case _SEND: + return 2; case SEND: return 2; case SEND_GEN: @@ -844,6 +938,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case DELETE_NAME: return 0; + case _SPECIALIZE_UNPACK_SEQUENCE: + return 1; + case _UNPACK_SEQUENCE: + return oparg; case UNPACK_SEQUENCE: return oparg; case UNPACK_SEQUENCE_TWO_TUPLE: @@ -854,6 +952,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return oparg; case UNPACK_EX: return (oparg & 0xFF) + (oparg >> 8) + 1; + case _SPECIALIZE_STORE_ATTR: + return 1; + case _STORE_ATTR: + return 0; case STORE_ATTR: return 0; case DELETE_ATTR: @@ -868,8 +970,12 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case LOAD_NAME: return 1; - case LOAD_GLOBAL: + case _SPECIALIZE_LOAD_GLOBAL: + return 0; + case _LOAD_GLOBAL: return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_GLOBAL: + return (oparg & 1 ? 1 : 0) + 1; case _GUARD_GLOBALS_VERSION: return 0; case _GUARD_BUILTINS_VERSION: @@ -922,22 +1028,30 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return (oparg - 1) + 1; case INSTRUMENTED_LOAD_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; - case LOAD_SUPER_ATTR: + case _SPECIALIZE_LOAD_SUPER_ATTR: + return 3; + case _LOAD_SUPER_ATTR: return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_SUPER_ATTR: + return (oparg & 1 ? 1 : 0) + 1; case LOAD_SUPER_METHOD: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_ZERO_SUPER_METHOD: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_ZERO_SUPER_ATTR: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case LOAD_SUPER_ATTR_ATTR: return 1; case LOAD_SUPER_ATTR_METHOD: return 2; - case LOAD_ATTR: + case _SPECIALIZE_LOAD_ATTR: + return 1; + case _LOAD_ATTR: return ((oparg & 1) ? 1 : 0) + 1; + case LOAD_ATTR: + return (oparg & 1 ? 1 : 0) + 1; case LOAD_METHOD: - return ((oparg & 1) ? 1 : 0) + 1; + return (oparg & 1 ? 1 : 0) + 1; case _GUARD_TYPE_VERSION: return 1; case _CHECK_MANAGED_OBJECT_HAS_VALUES: @@ -984,6 +1098,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 0; case STORE_ATTR_SLOT: return 0; + case _SPECIALIZE_COMPARE_OP: + return 2; + case _COMPARE_OP: + return 1; case COMPARE_OP: return 1; case COMPARE_OP_FLOAT: @@ -1040,6 +1158,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case GET_YIELD_FROM_ITER: return 1; + case _SPECIALIZE_FOR_ITER: + return 1; + case _FOR_ITER: + return 2; case FOR_ITER: return 2; case INSTRUMENTED_FOR_ITER: @@ -1120,6 +1242,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 2; case INSTRUMENTED_CALL: return 0; + case _SPECIALIZE_CALL: + return oparg + 2; + case _CALL: + return 1; case CALL: return 1; case _CHECK_CALL_BOUND_METHOD_EXACT_ARGS: @@ -1198,6 +1324,10 @@ int _PyOpcode_num_pushed(int opcode, int oparg, bool jump) { return 1; case COPY: return (oparg-1) + 2; + case _SPECIALIZE_BINARY_OP: + return 2; + case _BINARY_OP: + return 1; case BINARY_OP: return 1; case SWAP: @@ -1248,6 +1378,7 @@ enum InstructionFormat { INSTR_FMT_IBC0, INSTR_FMT_IBC00, INSTR_FMT_IBC000, + INSTR_FMT_IBC0000000, INSTR_FMT_IBC00000000, INSTR_FMT_IX, INSTR_FMT_IXC, @@ -1331,6 +1462,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [INSTRUMENTED_END_SEND] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [UNARY_NEGATIVE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [UNARY_NOT] = { true, INSTR_FMT_IX, 0 }, + [_SPECIALIZE_TO_BOOL] = { true, INSTR_FMT_IXC, 0 }, + [_TO_BOOL] = { true, INSTR_FMT_IXC0, HAS_ERROR_FLAG }, [TO_BOOL] = { true, INSTR_FMT_IXC00, HAS_ERROR_FLAG }, [TO_BOOL_BOOL] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, [TO_BOOL_INT] = { true, INSTR_FMT_IXC00, HAS_DEOPT_FLAG }, @@ -1358,6 +1491,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_OP_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [_BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [BINARY_OP_INPLACE_ADD_UNICODE] = { true, INSTR_FMT_IXC, HAS_LOCAL_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_BINARY_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, + [_BINARY_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [BINARY_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, [BINARY_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [STORE_SLICE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, @@ -1368,6 +1503,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [BINARY_SUBSCR_GETITEM] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [LIST_APPEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [SET_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_STORE_SUBSCR] = { true, INSTR_FMT_IXC, 0 }, + [_STORE_SUBSCR] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [STORE_SUBSCR] = { true, INSTR_FMT_IXC, HAS_ERROR_FLAG }, [STORE_SUBSCR_LIST_INT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [STORE_SUBSCR_DICT] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, @@ -1384,6 +1521,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [GET_AITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [GET_ANEXT] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [GET_AWAITABLE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_SEND] = { true, INSTR_FMT_IXC, 0 }, + [_SEND] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, [SEND] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, [SEND_GEN] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [INSTRUMENTED_YIELD_VALUE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, @@ -1396,11 +1535,15 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_BUILD_CLASS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [STORE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [DELETE_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_UNPACK_SEQUENCE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [UNPACK_SEQUENCE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [UNPACK_SEQUENCE_TWO_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_TUPLE] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_SEQUENCE_LIST] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [UNPACK_EX] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_STORE_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, + [_STORE_ATTR] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [STORE_ATTR] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [DELETE_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [STORE_GLOBAL] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, @@ -1408,6 +1551,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [LOAD_LOCALS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [LOAD_FROM_DICT_OR_GLOBALS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_NAME] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_LOAD_GLOBAL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, + [_LOAD_GLOBAL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_GLOBAL] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [_GUARD_GLOBALS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, [_GUARD_BUILTINS_VERSION] = { true, INSTR_FMT_IXC, HAS_DEOPT_FLAG }, @@ -1435,12 +1580,16 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [DICT_MERGE] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [MAP_ADD] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [INSTRUMENTED_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_SPECIALIZE_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_LOAD_SUPER_ATTR] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_SUPER_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_ZERO_SUPER_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_ZERO_SUPER_ATTR] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_SUPER_ATTR_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, [LOAD_SUPER_ATTR_METHOD] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_LOAD_ATTR] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_NAME_FLAG }, + [_LOAD_ATTR] = { true, INSTR_FMT_IBC0000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_ATTR] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [LOAD_METHOD] = { true, 0, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_ERROR_FLAG }, [_GUARD_TYPE_VERSION] = { true, INSTR_FMT_IXC0, HAS_DEOPT_FLAG }, @@ -1466,6 +1615,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [STORE_ATTR_WITH_HINT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG | HAS_NAME_FLAG | HAS_DEOPT_FLAG }, [_STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC, 0 }, [STORE_ATTR_SLOT] = { true, INSTR_FMT_IXC000, HAS_DEOPT_FLAG }, + [_SPECIALIZE_COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_COMPARE_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [COMPARE_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [COMPARE_OP_FLOAT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [COMPARE_OP_INT] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, @@ -1494,6 +1645,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [MATCH_KEYS] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [GET_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [GET_YIELD_FROM_ITER] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, + [_SPECIALIZE_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_FOR_ITER] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, [FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_JUMP_FLAG | HAS_ERROR_FLAG }, [INSTRUMENTED_FOR_ITER] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [_ITER_CHECK_LIST] = { true, INSTR_FMT_IX, HAS_DEOPT_FLAG }, @@ -1534,6 +1687,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [_LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC000, HAS_ARG_FLAG }, [LOAD_ATTR_METHOD_LAZY_DICT] = { true, INSTR_FMT_IBC00000000, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [INSTRUMENTED_CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_ERROR_FLAG }, + [_SPECIALIZE_CALL] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_CALL] = { true, INSTR_FMT_IBC0, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [CALL] = { true, INSTR_FMT_IBC00, HAS_ARG_FLAG | HAS_EVAL_BREAK_FLAG | HAS_ERROR_FLAG }, [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_DEOPT_FLAG }, [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, @@ -1573,6 +1728,8 @@ const struct opcode_metadata _PyOpcode_opcode_metadata[OPCODE_METADATA_SIZE] = { [FORMAT_SIMPLE] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [FORMAT_WITH_SPEC] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, [COPY] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, + [_SPECIALIZE_BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG }, + [_BINARY_OP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [BINARY_OP] = { true, INSTR_FMT_IBC, HAS_ARG_FLAG | HAS_ERROR_FLAG }, [SWAP] = { true, INSTR_FMT_IB, HAS_ARG_FLAG }, [INSTRUMENTED_INSTRUCTION] = { true, INSTR_FMT_IX, HAS_ERROR_FLAG }, @@ -1614,7 +1771,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [END_SEND] = { .nuops = 1, .uops = { { END_SEND, 0, 0 } } }, [UNARY_NEGATIVE] = { .nuops = 1, .uops = { { UNARY_NEGATIVE, 0, 0 } } }, [UNARY_NOT] = { .nuops = 1, .uops = { { UNARY_NOT, 0, 0 } } }, - [TO_BOOL] = { .nuops = 1, .uops = { { TO_BOOL, 0, 0 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { TO_BOOL_BOOL, 0, 0 } } }, [TO_BOOL_INT] = { .nuops = 1, .uops = { { TO_BOOL_INT, 0, 0 } } }, [TO_BOOL_LIST] = { .nuops = 1, .uops = { { TO_BOOL_LIST, 0, 0 } } }, @@ -1629,7 +1785,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_OP_ADD_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_ADD_FLOAT, 0, 0 } } }, [BINARY_OP_SUBTRACT_FLOAT] = { .nuops = 2, .uops = { { _GUARD_BOTH_FLOAT, 0, 0 }, { _BINARY_OP_SUBTRACT_FLOAT, 0, 0 } } }, [BINARY_OP_ADD_UNICODE] = { .nuops = 2, .uops = { { _GUARD_BOTH_UNICODE, 0, 0 }, { _BINARY_OP_ADD_UNICODE, 0, 0 } } }, - [BINARY_SUBSCR] = { .nuops = 1, .uops = { { BINARY_SUBSCR, 0, 0 } } }, [BINARY_SLICE] = { .nuops = 1, .uops = { { BINARY_SLICE, 0, 0 } } }, [STORE_SLICE] = { .nuops = 1, .uops = { { STORE_SLICE, 0, 0 } } }, [BINARY_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_LIST_INT, 0, 0 } } }, @@ -1638,7 +1793,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [BINARY_SUBSCR_DICT] = { .nuops = 1, .uops = { { BINARY_SUBSCR_DICT, 0, 0 } } }, [LIST_APPEND] = { .nuops = 1, .uops = { { LIST_APPEND, 0, 0 } } }, [SET_ADD] = { .nuops = 1, .uops = { { SET_ADD, 0, 0 } } }, - [STORE_SUBSCR] = { .nuops = 1, .uops = { { STORE_SUBSCR, 0, 0 } } }, [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { STORE_SUBSCR_LIST_INT, 0, 0 } } }, [STORE_SUBSCR_DICT] = { .nuops = 1, .uops = { { STORE_SUBSCR_DICT, 0, 0 } } }, [DELETE_SUBSCR] = { .nuops = 1, .uops = { { DELETE_SUBSCR, 0, 0 } } }, @@ -1654,19 +1808,17 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_BUILD_CLASS] = { .nuops = 1, .uops = { { LOAD_BUILD_CLASS, 0, 0 } } }, [STORE_NAME] = { .nuops = 1, .uops = { { STORE_NAME, 0, 0 } } }, [DELETE_NAME] = { .nuops = 1, .uops = { { DELETE_NAME, 0, 0 } } }, - [UNPACK_SEQUENCE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE, 0, 0 } } }, + [UNPACK_SEQUENCE] = { .nuops = 2, .uops = { { _SPECIALIZE_UNPACK_SEQUENCE, 1, 0 }, { _UNPACK_SEQUENCE, 0, 0 } } }, [UNPACK_SEQUENCE_TWO_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TWO_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_TUPLE] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_TUPLE, 0, 0 } } }, [UNPACK_SEQUENCE_LIST] = { .nuops = 1, .uops = { { UNPACK_SEQUENCE_LIST, 0, 0 } } }, [UNPACK_EX] = { .nuops = 1, .uops = { { UNPACK_EX, 0, 0 } } }, - [STORE_ATTR] = { .nuops = 1, .uops = { { STORE_ATTR, 0, 0 } } }, [DELETE_ATTR] = { .nuops = 1, .uops = { { DELETE_ATTR, 0, 0 } } }, [STORE_GLOBAL] = { .nuops = 1, .uops = { { STORE_GLOBAL, 0, 0 } } }, [DELETE_GLOBAL] = { .nuops = 1, .uops = { { DELETE_GLOBAL, 0, 0 } } }, [LOAD_LOCALS] = { .nuops = 1, .uops = { { LOAD_LOCALS, 0, 0 } } }, [LOAD_FROM_DICT_OR_GLOBALS] = { .nuops = 1, .uops = { { LOAD_FROM_DICT_OR_GLOBALS, 0, 0 } } }, [LOAD_NAME] = { .nuops = 1, .uops = { { LOAD_NAME, 0, 0 } } }, - [LOAD_GLOBAL] = { .nuops = 1, .uops = { { LOAD_GLOBAL, 0, 0 } } }, [LOAD_GLOBAL_MODULE] = { .nuops = 2, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _LOAD_GLOBAL_MODULE, 1, 3 } } }, [LOAD_GLOBAL_BUILTIN] = { .nuops = 3, .uops = { { _GUARD_GLOBALS_VERSION, 1, 1 }, { _GUARD_BUILTINS_VERSION, 1, 2 }, { _LOAD_GLOBAL_BUILTINS, 1, 3 } } }, [DELETE_FAST] = { .nuops = 1, .uops = { { DELETE_FAST, 0, 0 } } }, @@ -1690,7 +1842,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [MAP_ADD] = { .nuops = 1, .uops = { { MAP_ADD, 0, 0 } } }, [LOAD_SUPER_ATTR_ATTR] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_ATTR, 0, 0 } } }, [LOAD_SUPER_ATTR_METHOD] = { .nuops = 1, .uops = { { LOAD_SUPER_ATTR_METHOD, 0, 0 } } }, - [LOAD_ATTR] = { .nuops = 1, .uops = { { LOAD_ATTR, 0, 0 } } }, [LOAD_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_MANAGED_OBJECT_HAS_VALUES, 0, 0 }, { _LOAD_ATTR_INSTANCE_VALUE, 1, 3 } } }, [LOAD_ATTR_MODULE] = { .nuops = 2, .uops = { { _CHECK_ATTR_MODULE, 2, 1 }, { _LOAD_ATTR_MODULE, 1, 3 } } }, [LOAD_ATTR_WITH_HINT] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _CHECK_ATTR_WITH_HINT, 0, 0 }, { _LOAD_ATTR_WITH_HINT, 1, 3 } } }, @@ -1698,7 +1849,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [LOAD_ATTR_CLASS] = { .nuops = 2, .uops = { { _CHECK_ATTR_CLASS, 2, 1 }, { _LOAD_ATTR_CLASS, 4, 5 } } }, [STORE_ATTR_INSTANCE_VALUE] = { .nuops = 3, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _GUARD_DORV_VALUES, 0, 0 }, { _STORE_ATTR_INSTANCE_VALUE, 1, 3 } } }, [STORE_ATTR_SLOT] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _STORE_ATTR_SLOT, 1, 3 } } }, - [COMPARE_OP] = { .nuops = 1, .uops = { { COMPARE_OP, 0, 0 } } }, [COMPARE_OP_FLOAT] = { .nuops = 1, .uops = { { COMPARE_OP_FLOAT, 0, 0 } } }, [COMPARE_OP_INT] = { .nuops = 1, .uops = { { COMPARE_OP_INT, 0, 0 } } }, [COMPARE_OP_STR] = { .nuops = 1, .uops = { { COMPARE_OP_STR, 0, 0 } } }, @@ -1744,7 +1894,6 @@ const struct opcode_macro_expansion _PyOpcode_macro_expansion[OPCODE_MACRO_EXPAN [FORMAT_SIMPLE] = { .nuops = 1, .uops = { { FORMAT_SIMPLE, 0, 0 } } }, [FORMAT_WITH_SPEC] = { .nuops = 1, .uops = { { FORMAT_WITH_SPEC, 0, 0 } } }, [COPY] = { .nuops = 1, .uops = { { COPY, 0, 0 } } }, - [BINARY_OP] = { .nuops = 1, .uops = { { BINARY_OP, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { SWAP, 0, 0 } } }, }; #endif // NEED_OPCODE_METADATA @@ -1754,6 +1903,8 @@ extern const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE]; const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_EXIT_TRACE] = "_EXIT_TRACE", [_SET_IP] = "_SET_IP", + [_SPECIALIZE_TO_BOOL] = "_SPECIALIZE_TO_BOOL", + [_TO_BOOL] = "_TO_BOOL", [_GUARD_BOTH_INT] = "_GUARD_BOTH_INT", [_BINARY_OP_MULTIPLY_INT] = "_BINARY_OP_MULTIPLY_INT", [_BINARY_OP_ADD_INT] = "_BINARY_OP_ADD_INT", @@ -1765,11 +1916,27 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_GUARD_BOTH_UNICODE] = "_GUARD_BOTH_UNICODE", [_BINARY_OP_ADD_UNICODE] = "_BINARY_OP_ADD_UNICODE", [_BINARY_OP_INPLACE_ADD_UNICODE] = "_BINARY_OP_INPLACE_ADD_UNICODE", + [_SPECIALIZE_BINARY_SUBSCR] = "_SPECIALIZE_BINARY_SUBSCR", + [_BINARY_SUBSCR] = "_BINARY_SUBSCR", + [_SPECIALIZE_STORE_SUBSCR] = "_SPECIALIZE_STORE_SUBSCR", + [_STORE_SUBSCR] = "_STORE_SUBSCR", [_POP_FRAME] = "_POP_FRAME", + [_SPECIALIZE_SEND] = "_SPECIALIZE_SEND", + [_SEND] = "_SEND", + [_SPECIALIZE_UNPACK_SEQUENCE] = "_SPECIALIZE_UNPACK_SEQUENCE", + [_UNPACK_SEQUENCE] = "_UNPACK_SEQUENCE", + [_SPECIALIZE_STORE_ATTR] = "_SPECIALIZE_STORE_ATTR", + [_STORE_ATTR] = "_STORE_ATTR", + [_SPECIALIZE_LOAD_GLOBAL] = "_SPECIALIZE_LOAD_GLOBAL", + [_LOAD_GLOBAL] = "_LOAD_GLOBAL", [_GUARD_GLOBALS_VERSION] = "_GUARD_GLOBALS_VERSION", [_GUARD_BUILTINS_VERSION] = "_GUARD_BUILTINS_VERSION", [_LOAD_GLOBAL_MODULE] = "_LOAD_GLOBAL_MODULE", [_LOAD_GLOBAL_BUILTINS] = "_LOAD_GLOBAL_BUILTINS", + [_SPECIALIZE_LOAD_SUPER_ATTR] = "_SPECIALIZE_LOAD_SUPER_ATTR", + [_LOAD_SUPER_ATTR] = "_LOAD_SUPER_ATTR", + [_SPECIALIZE_LOAD_ATTR] = "_SPECIALIZE_LOAD_ATTR", + [_LOAD_ATTR] = "_LOAD_ATTR", [_GUARD_TYPE_VERSION] = "_GUARD_TYPE_VERSION", [_CHECK_MANAGED_OBJECT_HAS_VALUES] = "_CHECK_MANAGED_OBJECT_HAS_VALUES", [_LOAD_ATTR_INSTANCE_VALUE] = "_LOAD_ATTR_INSTANCE_VALUE", @@ -1783,7 +1950,11 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_GUARD_DORV_VALUES] = "_GUARD_DORV_VALUES", [_STORE_ATTR_INSTANCE_VALUE] = "_STORE_ATTR_INSTANCE_VALUE", [_STORE_ATTR_SLOT] = "_STORE_ATTR_SLOT", + [_SPECIALIZE_COMPARE_OP] = "_SPECIALIZE_COMPARE_OP", + [_COMPARE_OP] = "_COMPARE_OP", [_IS_NONE] = "_IS_NONE", + [_SPECIALIZE_FOR_ITER] = "_SPECIALIZE_FOR_ITER", + [_FOR_ITER] = "_FOR_ITER", [_ITER_CHECK_LIST] = "_ITER_CHECK_LIST", [_ITER_JUMP_LIST] = "_ITER_JUMP_LIST", [_IS_ITER_EXHAUSTED_LIST] = "_IS_ITER_EXHAUSTED_LIST", @@ -1804,6 +1975,8 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_LOAD_ATTR_NONDESCRIPTOR_NO_DICT] = "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT", [_CHECK_ATTR_METHOD_LAZY_DICT] = "_CHECK_ATTR_METHOD_LAZY_DICT", [_LOAD_ATTR_METHOD_LAZY_DICT] = "_LOAD_ATTR_METHOD_LAZY_DICT", + [_SPECIALIZE_CALL] = "_SPECIALIZE_CALL", + [_CALL] = "_CALL", [_CHECK_CALL_BOUND_METHOD_EXACT_ARGS] = "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS", [_INIT_CALL_BOUND_METHOD_EXACT_ARGS] = "_INIT_CALL_BOUND_METHOD_EXACT_ARGS", [_CHECK_PEP_523] = "_CHECK_PEP_523", @@ -1811,6 +1984,8 @@ const char * const _PyOpcode_uop_name[OPCODE_UOP_NAME_SIZE] = { [_CHECK_STACK_SPACE] = "_CHECK_STACK_SPACE", [_INIT_CALL_PY_EXACT_ARGS] = "_INIT_CALL_PY_EXACT_ARGS", [_PUSH_FRAME] = "_PUSH_FRAME", + [_SPECIALIZE_BINARY_OP] = "_SPECIALIZE_BINARY_OP", + [_BINARY_OP] = "_BINARY_OP", [_POP_JUMP_IF_FALSE] = "_POP_JUMP_IF_FALSE", [_POP_JUMP_IF_TRUE] = "_POP_JUMP_IF_TRUE", [_JUMP_TO_TOP] = "_JUMP_TO_TOP", diff --git a/Lib/test/test_capi/test_misc.py b/Lib/test/test_capi/test_misc.py index ff34d85416e2df..11dec031d3d79f 100644 --- a/Lib/test/test_capi/test_misc.py +++ b/Lib/test/test_capi/test_misc.py @@ -2595,7 +2595,7 @@ def testfunc(x): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = {opname for opname, _, _ in ex} - self.assertIn("UNPACK_SEQUENCE", uops) + self.assertIn("_UNPACK_SEQUENCE", uops) def test_pop_jump_if_false(self): def testfunc(n): diff --git a/Python/abstract_interp_cases.c.h b/Python/abstract_interp_cases.c.h index 11090ee4dba39f..0e58ed1b0be0f3 100644 --- a/Python/abstract_interp_cases.c.h +++ b/Python/abstract_interp_cases.c.h @@ -38,7 +38,7 @@ break; } - case TO_BOOL: { + case _TO_BOOL: { PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; } @@ -113,7 +113,7 @@ break; } - case BINARY_SUBSCR: { + case _BINARY_SUBSCR: { STACK_SHRINK(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; @@ -164,7 +164,7 @@ break; } - case STORE_SUBSCR: { + case _STORE_SUBSCR: { STACK_SHRINK(3); break; } @@ -242,7 +242,11 @@ break; } - case UNPACK_SEQUENCE: { + case _SPECIALIZE_UNPACK_SEQUENCE: { + break; + } + + case _UNPACK_SEQUENCE: { STACK_SHRINK(1); STACK_GROW(oparg); break; @@ -272,7 +276,7 @@ break; } - case STORE_ATTR: { + case _STORE_ATTR: { STACK_SHRINK(2); break; } @@ -308,7 +312,7 @@ break; } - case LOAD_GLOBAL: { + case _LOAD_GLOBAL: { STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); @@ -456,7 +460,7 @@ break; } - case LOAD_ATTR: { + case _LOAD_ATTR: { STACK_GROW(((oparg & 1) ? 1 : 0)); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1 - (oparg & 1 ? 1 : 0))), true); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-(oparg & 1 ? 1 : 0))), true); @@ -532,7 +536,7 @@ break; } - case COMPARE_OP: { + case _COMPARE_OP: { STACK_SHRINK(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; @@ -901,7 +905,7 @@ break; } - case BINARY_OP: { + case _BINARY_OP: { STACK_SHRINK(1); PARTITIONNODE_OVERWRITE((_Py_PARTITIONNODE_t *)PARTITIONNODE_NULLROOT, PEEK(-(-1)), true); break; diff --git a/Python/bytecodes.c b/Python/bytecodes.c index bb2d491523ca43..dd74f26157f56b 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -311,9 +311,10 @@ dummy_func( TO_BOOL_STR, }; - inst(TO_BOOL, (unused/1, unused/2, value -- res)) { + op(_SPECIALIZE_TO_BOOL, (counter/1, value -- value)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_ToBool(value, next_instr); DISPATCH_SAME_OPARG(); @@ -321,12 +322,17 @@ dummy_func( STAT_INC(TO_BOOL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_TO_BOOL, (unused/2, value -- res)) { int err = PyObject_IsTrue(value); DECREF_INPUTS(); ERROR_IF(err < 0, error); res = err ? Py_True : Py_False; } + macro(TO_BOOL) = _SPECIALIZE_TO_BOOL + _TO_BOOL; + inst(TO_BOOL_BOOL, (unused/1, unused/2, value -- value)) { DEOPT_IF(!PyBool_Check(value)); STAT_INC(TO_BOOL, hit); @@ -530,9 +536,10 @@ dummy_func( BINARY_SUBSCR_TUPLE_INT, }; - inst(BINARY_SUBSCR, (unused/1, container, sub -- res)) { + op(_SPECIALIZE_BINARY_SUBSCR, (counter/1, container, sub -- container, sub)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_BinarySubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -540,11 +547,16 @@ dummy_func( STAT_INC(BINARY_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_BINARY_SUBSCR, (container, sub -- res)) { res = PyObject_GetItem(container, sub); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } + macro(BINARY_SUBSCR) = _SPECIALIZE_BINARY_SUBSCR + _BINARY_SUBSCR; + inst(BINARY_SLICE, (container, start, stop -- res)) { PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); // Can't use ERROR_IF() here, because we haven't @@ -677,9 +689,10 @@ dummy_func( STORE_SUBSCR_LIST_INT, }; - inst(STORE_SUBSCR, (unused/1, v, container, sub -- )) { + op(_SPECIALIZE_STORE_SUBSCR, (counter/1, container, sub -- container, sub)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_StoreSubscr(container, sub, next_instr); DISPATCH_SAME_OPARG(); @@ -687,12 +700,17 @@ dummy_func( STAT_INC(STORE_SUBSCR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_STORE_SUBSCR, (v, container, sub -- )) { /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); DECREF_INPUTS(); ERROR_IF(err, error); } + macro(STORE_SUBSCR) = _SPECIALIZE_STORE_SUBSCR + _STORE_SUBSCR; + inst(STORE_SUBSCR_LIST_INT, (unused/1, value, list, sub -- )) { DEOPT_IF(!PyLong_CheckExact(sub)); DEOPT_IF(!PyList_CheckExact(list)); @@ -956,9 +974,10 @@ dummy_func( SEND_GEN, }; - inst(SEND, (unused/1, receiver, v -- receiver, retval)) { + op(_SPECIALIZE_SEND, (counter/1, receiver, unused -- receiver, unused)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_Send(receiver, next_instr); DISPATCH_SAME_OPARG(); @@ -966,6 +985,9 @@ dummy_func( STAT_INC(SEND, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_SEND, (receiver, v -- receiver, retval)) { assert(frame != &entry_frame); if ((tstate->interp->eval_frame == NULL) && (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && @@ -1004,6 +1026,8 @@ dummy_func( Py_DECREF(v); } + macro(SEND) = _SPECIALIZE_SEND + _SEND; + inst(SEND_GEN, (unused/1, receiver, v -- receiver, unused)) { DEOPT_IF(tstate->interp->eval_frame); PyGenObject *gen = (PyGenObject *)receiver; @@ -1182,9 +1206,9 @@ dummy_func( UNPACK_SEQUENCE_LIST, }; - inst(UNPACK_SEQUENCE, (unused/1, seq -- unused[oparg])) { + op(_SPECIALIZE_UNPACK_SEQUENCE, (counter/1, seq -- seq)) { #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -1192,12 +1216,17 @@ dummy_func( STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_UNPACK_SEQUENCE, (seq -- unused[oparg])) { PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); DECREF_INPUTS(); ERROR_IF(res == 0, error); } + macro(UNPACK_SEQUENCE) = _SPECIALIZE_UNPACK_SEQUENCE + _UNPACK_SEQUENCE; + inst(UNPACK_SEQUENCE_TWO_TUPLE, (unused/1, seq -- values[oparg])) { DEOPT_IF(!PyTuple_CheckExact(seq)); DEOPT_IF(PyTuple_GET_SIZE(seq) != 2); @@ -1244,9 +1273,10 @@ dummy_func( STORE_ATTR_WITH_HINT, }; - inst(STORE_ATTR, (unused/1, unused/3, v, owner --)) { + op(_SPECIALIZE_STORE_ATTR, (counter/1, owner -- owner)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); next_instr = this_instr; _Py_Specialize_StoreAttr(owner, next_instr, name); @@ -1255,12 +1285,17 @@ dummy_func( STAT_INC(STORE_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_STORE_ATTR, (unused/3, v, owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); DECREF_INPUTS(); ERROR_IF(err, error); } + macro(STORE_ATTR) = _SPECIALIZE_STORE_ATTR + _STORE_ATTR; + inst(DELETE_ATTR, (owner --)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_DelAttr(owner, name); @@ -1365,9 +1400,10 @@ dummy_func( LOAD_GLOBAL_BUILTIN, }; - inst(LOAD_GLOBAL, (unused/1, unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { + op(_SPECIALIZE_LOAD_GLOBAL, (counter/1 -- )) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); @@ -1376,6 +1412,9 @@ dummy_func( STAT_INC(LOAD_GLOBAL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_LOAD_GLOBAL, (unused/1, unused/1, unused/1 -- res, null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -1396,7 +1435,6 @@ dummy_func( } else { /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ ERROR_IF(PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0, error); if (res == NULL) { @@ -1413,6 +1451,8 @@ dummy_func( null = NULL; } + macro(LOAD_GLOBAL) = _SPECIALIZE_LOAD_GLOBAL + _LOAD_GLOBAL; + op(_GUARD_GLOBALS_VERSION, (version/1 --)) { PyDictObject *dict = (PyDictObject *)GLOBALS(); DEOPT_IF(!PyDict_CheckExact(dict)); @@ -1701,12 +1741,11 @@ dummy_func( LOAD_SUPER_ATTR_METHOD, }; - inst(LOAD_SUPER_ATTR, (unused/1, global_super, class, self -- attr, null if (oparg & 1))) { + op(_SPECIALIZE_LOAD_SUPER_ATTR, (counter/1, global_super, class, unused -- global_super, class, unused)) { TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); #if ENABLE_SPECIALIZATION int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); DISPATCH_SAME_OPARG(); @@ -1714,7 +1753,10 @@ dummy_func( STAT_INC(LOAD_SUPER_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + op(_LOAD_SUPER_ATTR, (global_super, class, self -- attr, null if (oparg & 1))) { + TIER_ONE_ONLY if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( @@ -1722,7 +1764,6 @@ dummy_func( frame, this_instr, global_super, arg); ERROR_IF(err, error); } - // we make no attempt to optimize here; specializations should // handle any case whose performance we care about PyObject *stack[] = {class, self}; @@ -1745,12 +1786,15 @@ dummy_func( } DECREF_INPUTS(); ERROR_IF(super == NULL, error); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); attr = PyObject_GetAttr(super, name); Py_DECREF(super); ERROR_IF(attr == NULL, error); null = NULL; } + macro(LOAD_SUPER_ATTR) = _SPECIALIZE_LOAD_SUPER_ATTR + _LOAD_SUPER_ATTR; + pseudo(LOAD_SUPER_METHOD) = { LOAD_SUPER_ATTR, }; @@ -1813,9 +1857,10 @@ dummy_func( LOAD_ATTR_NONDESCRIPTOR_NO_DICT, }; - inst(LOAD_ATTR, (unused/9, owner -- attr, self_or_null if (oparg & 1))) { + op(_SPECIALIZE_LOAD_ATTR, (counter/1, owner -- owner)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); next_instr = this_instr; _Py_Specialize_LoadAttr(owner, next_instr, name); @@ -1824,6 +1869,9 @@ dummy_func( STAT_INC(LOAD_ATTR, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_LOAD_ATTR, (unused/8, owner -- attr, self_or_null if (oparg & 1))) { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ @@ -1831,7 +1879,6 @@ dummy_func( if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. - meth | self | arg1 | ... | argN */ assert(attr != NULL); // No errors on this branch @@ -1842,7 +1889,6 @@ dummy_func( something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. - NULL | meth | arg1 | ... | argN */ DECREF_INPUTS(); @@ -1858,6 +1904,8 @@ dummy_func( } } + macro(LOAD_ATTR) = _SPECIALIZE_LOAD_ATTR + _LOAD_ATTR; + pseudo(LOAD_METHOD) = { LOAD_ATTR, }; @@ -2133,9 +2181,10 @@ dummy_func( COMPARE_OP_STR, }; - inst(COMPARE_OP, (unused/1, left, right -- res)) { + op(_SPECIALIZE_COMPARE_OP, (counter/1, left, right -- left, right)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_CompareOp(left, right, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2143,6 +2192,9 @@ dummy_func( STAT_INC(COMPARE_OP, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_COMPARE_OP, (left, right -- res)) { assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); DECREF_INPUTS(); @@ -2155,6 +2207,8 @@ dummy_func( } } + macro(COMPARE_OP) = _SPECIALIZE_COMPARE_OP + _COMPARE_OP; + inst(COMPARE_OP_FLOAT, (unused/1, left, right -- res)) { DEOPT_IF(!PyFloat_CheckExact(left)); DEOPT_IF(!PyFloat_CheckExact(right)); @@ -2448,9 +2502,10 @@ dummy_func( FOR_ITER_GEN, }; - inst(FOR_ITER, (unused/1, iter -- iter, next)) { + op(_SPECIALIZE_FOR_ITER, (counter/1, iter -- iter)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_ForIter(iter, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -2458,6 +2513,9 @@ dummy_func( STAT_INC(FOR_ITER, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + } + + op(_FOR_ITER, (iter -- iter, next)) { /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ next = (*Py_TYPE(iter)->tp_iternext)(iter); if (next == NULL) { @@ -2480,6 +2538,8 @@ dummy_func( // Common case: no jump, leave it to the code generator } + macro(FOR_ITER) = _SPECIALIZE_FOR_ITER + _FOR_ITER; + inst(INSTRUMENTED_FOR_ITER, (unused/1 -- )) { _Py_CODEUNIT *target; PyObject *iter = TOP(); @@ -2937,24 +2997,28 @@ dummy_func( CALL_ALLOC_AND_ENTER_INIT, }; - // When calling Python, inline the call using DISPATCH_INLINED(). - inst(CALL, (unused/1, unused/2, callable, self_or_null, args[oparg] -- res)) { - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } + op(_SPECIALIZE_CALL, (counter/1, callable, self_or_null, args[oparg] -- callable, self_or_null, args[oparg])) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, total_args); + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); DISPATCH_SAME_OPARG(); } STAT_INC(CALL, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { + } + + // When calling Python, inline the call using DISPATCH_INLINED(). + op(_CALL, (unused/2, callable, self_or_null, args[oparg] -- res)) { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + else if (Py_TYPE(callable) == &PyMethod_Type) { args--; total_args++; PyObject *self = ((PyMethodObject *)callable)->im_self; @@ -3017,6 +3081,8 @@ dummy_func( CHECK_EVAL_BREAKER(); } + macro(CALL) = _SPECIALIZE_CALL + _CALL; + op(_CHECK_CALL_BOUND_METHOD_EXACT_ARGS, (callable, null, unused[oparg] -- callable, null, unused[oparg])) { DEOPT_IF(null != NULL); DEOPT_IF(Py_TYPE(callable) != &PyMethod_Type); @@ -3799,9 +3865,10 @@ dummy_func( top = Py_NewRef(bottom); } - inst(BINARY_OP, (unused/1, lhs, rhs -- res)) { + op(_SPECIALIZE_BINARY_OP, (counter/1, lhs, rhs -- lhs, rhs)) { + TIER_ONE_ONLY #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); DISPATCH_SAME_OPARG(); @@ -3811,12 +3878,17 @@ dummy_func( #endif /* ENABLE_SPECIALIZATION */ assert(NB_ADD <= oparg); assert(oparg <= NB_INPLACE_XOR); + } + + op(_BINARY_OP, (lhs, rhs -- res)) { assert(_PyEval_BinaryOps[oparg]); res = _PyEval_BinaryOps[oparg](lhs, rhs); DECREF_INPUTS(); ERROR_IF(res == NULL, error); } + macro(BINARY_OP) = _SPECIALIZE_BINARY_OP + _BINARY_OP; + inst(SWAP, (bottom, unused[oparg-2], top -- top, unused[oparg-2], bottom)) { assert(oparg >= 2); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 4d5b6dc7f01051..3477733c8c4402 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -114,19 +114,10 @@ break; } - case TO_BOOL: { + case _TO_BOOL: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ int err = PyObject_IsTrue(value); Py_DECREF(value); if (err < 0) goto pop_1_error; @@ -372,21 +363,12 @@ break; } - case BINARY_SUBSCR: { + case _BINARY_SUBSCR: { PyObject *sub; PyObject *container; PyObject *res; sub = stack_pointer[-1]; container = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ res = PyObject_GetItem(container, sub); Py_DECREF(container); Py_DECREF(sub); @@ -564,22 +546,13 @@ break; } - case STORE_SUBSCR: { + case _STORE_SUBSCR: { PyObject *sub; PyObject *container; PyObject *v; sub = stack_pointer[-1]; container = stack_pointer[-2]; v = stack_pointer[-3]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ /* container[sub] = v */ int err = PyObject_SetItem(container, sub, v); Py_DECREF(v); @@ -893,11 +866,12 @@ break; } - case UNPACK_SEQUENCE: { + case _SPECIALIZE_UNPACK_SEQUENCE: { PyObject *seq; seq = stack_pointer[-1]; + uint16_t counter = (uint16_t)operand; #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { next_instr = this_instr; _Py_Specialize_UnpackSequence(seq, next_instr, oparg); DISPATCH_SAME_OPARG(); @@ -905,6 +879,12 @@ STAT_INC(UNPACK_SEQUENCE, deferred); DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); #endif /* ENABLE_SPECIALIZATION */ + break; + } + + case _UNPACK_SEQUENCE: { + PyObject *seq; + seq = stack_pointer[-1]; PyObject **top = stack_pointer + oparg - 1; int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); Py_DECREF(seq); @@ -979,21 +959,11 @@ break; } - case STORE_ATTR: { + case _STORE_ATTR: { PyObject *owner; PyObject *v; owner = stack_pointer[-1]; v = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); int err = PyObject_SetAttr(owner, name, v); Py_DECREF(v); @@ -1124,19 +1094,9 @@ break; } - case LOAD_GLOBAL: { + case _LOAD_GLOBAL: { PyObject *res; PyObject *null = NULL; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); if (PyDict_CheckExact(GLOBALS()) && PyDict_CheckExact(BUILTINS())) @@ -1157,7 +1117,6 @@ } else { /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; if (res == NULL) { @@ -1624,21 +1583,11 @@ break; } - case LOAD_ATTR: { + case _LOAD_ATTR: { PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; owner = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); if (oparg & 1) { /* Designed to work in tandem with CALL, pushes two values. */ @@ -1646,7 +1595,6 @@ if (_PyObject_GetMethod(owner, name, &attr)) { /* We can bypass temporary bound method object. meth is unbound method and obj is self. - meth | self | arg1 | ... | argN */ assert(attr != NULL); // No errors on this branch @@ -1657,7 +1605,6 @@ something was returned by a descriptor protocol). Set the second element of the stack to NULL, to signal CALL that it's not a method call. - NULL | meth | arg1 | ... | argN */ Py_DECREF(owner); @@ -1885,21 +1832,12 @@ break; } - case COMPARE_OP: { + case _COMPARE_OP: { PyObject *right; PyObject *left; PyObject *res; right = stack_pointer[-1]; left = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); - } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ assert((oparg >> 5) <= Py_GE); res = PyObject_RichCompare(left, right, oparg >> 5); Py_DECREF(left); @@ -3246,23 +3184,12 @@ break; } - case BINARY_OP: { + case _BINARY_OP: { PyObject *rhs; PyObject *lhs; PyObject *res; rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - DISPATCH_SAME_OPARG(); - } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); assert(_PyEval_BinaryOps[oparg]); res = _PyEval_BinaryOps[oparg](lhs, rhs); Py_DECREF(lhs); diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index 8b6ebe4f8ab4af..1c05e2d50867da 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -332,20 +332,28 @@ static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); PyObject *value; PyObject *res; + // _SPECIALIZE_TO_BOOL value = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_ToBool(value, next_instr); - DISPATCH_SAME_OPARG(); + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ToBool(value, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(TO_BOOL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _TO_BOOL + { + int err = PyObject_IsTrue(value); + Py_DECREF(value); + if (err < 0) goto pop_1_error; + res = err ? Py_True : Py_False; } - STAT_INC(TO_BOOL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - int err = PyObject_IsTrue(value); - Py_DECREF(value); - if (err < 0) goto pop_1_error; - res = err ? Py_True : Py_False; stack_pointer[-1] = res; DISPATCH(); } @@ -708,21 +716,29 @@ PyObject *sub; PyObject *container; PyObject *res; + // _SPECIALIZE_BINARY_SUBSCR sub = stack_pointer[-1]; container = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_BinarySubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinarySubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _BINARY_SUBSCR + { + res = PyObject_GetItem(container, sub); + Py_DECREF(container); + Py_DECREF(sub); + if (res == NULL) goto pop_2_error; } - STAT_INC(BINARY_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - res = PyObject_GetItem(container, sub); - Py_DECREF(container); - Py_DECREF(sub); - if (res == NULL) goto pop_2_error; STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); @@ -962,24 +978,32 @@ PyObject *sub; PyObject *container; PyObject *v; + // _SPECIALIZE_STORE_SUBSCR sub = stack_pointer[-1]; container = stack_pointer[-2]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_StoreSubscr(container, sub, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_SUBSCR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_SUBSCR v = stack_pointer[-3]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_StoreSubscr(container, sub, next_instr); - DISPATCH_SAME_OPARG(); + { + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) goto pop_3_error; } - STAT_INC(STORE_SUBSCR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) goto pop_3_error; STACK_SHRINK(3); DISPATCH(); } @@ -1378,56 +1402,64 @@ PREDICTED(SEND); _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_SEND == 1, "incorrect cache size"); - PyObject *v; PyObject *receiver; + PyObject *v; PyObject *retval; - v = stack_pointer[-1]; + // _SPECIALIZE_SEND receiver = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_Send(receiver, next_instr); - DISPATCH_SAME_OPARG(); - } - STAT_INC(SEND, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - assert(frame != &entry_frame); - if ((tstate->interp->eval_frame == NULL) && - (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && - ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) { - PyGenObject *gen = (PyGenObject *)receiver; - _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; - STACK_SHRINK(1); - _PyFrame_StackPush(gen_frame, v); - gen->gi_frame_state = FRAME_EXECUTING; - gen->gi_exc_state.previous_item = tstate->exc_info; - tstate->exc_info = &gen->gi_exc_state; - assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); - frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); - DISPATCH_INLINED(gen_frame); - } - if (Py_IsNone(v) && PyIter_Check(receiver)) { - retval = Py_TYPE(receiver)->tp_iternext(receiver); - } - else { - retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Send(receiver, next_instr); + DISPATCH_SAME_OPARG(); + } + STAT_INC(SEND, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - if (retval == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) - ) { - monitor_raise(tstate, frame, this_instr); + // _SEND + v = stack_pointer[-1]; + { + assert(frame != &entry_frame); + if ((tstate->interp->eval_frame == NULL) && + (Py_TYPE(receiver) == &PyGen_Type || Py_TYPE(receiver) == &PyCoro_Type) && + ((PyGenObject *)receiver)->gi_frame_state < FRAME_EXECUTING) + { + PyGenObject *gen = (PyGenObject *)receiver; + _PyInterpreterFrame *gen_frame = (_PyInterpreterFrame *)gen->gi_iframe; + STACK_SHRINK(1); + _PyFrame_StackPush(gen_frame, v); + gen->gi_frame_state = FRAME_EXECUTING; + gen->gi_exc_state.previous_item = tstate->exc_info; + tstate->exc_info = &gen->gi_exc_state; + assert(1 + INLINE_CACHE_ENTRIES_SEND == next_instr - this_instr); + frame->return_offset = (uint16_t)(1 + INLINE_CACHE_ENTRIES_SEND + oparg); + DISPATCH_INLINED(gen_frame); } - if (_PyGen_FetchStopIterationValue(&retval) == 0) { - assert(retval != NULL); - JUMPBY(oparg); + if (Py_IsNone(v) && PyIter_Check(receiver)) { + retval = Py_TYPE(receiver)->tp_iternext(receiver); } else { - goto error; + retval = PyObject_CallMethodOneArg(receiver, &_Py_ID(send), v); } + if (retval == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_StopIteration) + ) { + monitor_raise(tstate, frame, this_instr); + } + if (_PyGen_FetchStopIterationValue(&retval) == 0) { + assert(retval != NULL); + JUMPBY(oparg); + } + else { + goto error; + } + } + Py_DECREF(v); } - Py_DECREF(v); stack_pointer[-1] = retval; DISPATCH(); } @@ -1694,20 +1726,27 @@ _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_UNPACK_SEQUENCE == 1, "incorrect cache size"); PyObject *seq; + // _SPECIALIZE_UNPACK_SEQUENCE seq = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_UnpackSequence(seq, next_instr, oparg); - DISPATCH_SAME_OPARG(); + { + uint16_t counter = read_u16(&this_instr[1].cache); + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_UnpackSequence(seq, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(UNPACK_SEQUENCE, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _UNPACK_SEQUENCE + { + PyObject **top = stack_pointer + oparg - 1; + int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); + Py_DECREF(seq); + if (res == 0) goto pop_1_error; } - STAT_INC(UNPACK_SEQUENCE, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - PyObject **top = stack_pointer + oparg - 1; - int res = _PyEval_UnpackIterable(tstate, seq, oparg, -1, top); - Py_DECREF(seq); - if (res == 0) goto pop_1_error; STACK_SHRINK(1); STACK_GROW(oparg); DISPATCH(); @@ -1799,23 +1838,31 @@ static_assert(INLINE_CACHE_ENTRIES_STORE_ATTR == 4, "incorrect cache size"); PyObject *owner; PyObject *v; + // _SPECIALIZE_STORE_ATTR owner = stack_pointer[-1]; + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + next_instr = this_instr; + _Py_Specialize_StoreAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(STORE_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _STORE_ATTR v = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { + { PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - next_instr = this_instr; - _Py_Specialize_StoreAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); + int err = PyObject_SetAttr(owner, name, v); + Py_DECREF(v); + Py_DECREF(owner); + if (err) goto pop_2_error; } - STAT_INC(STORE_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - int err = PyObject_SetAttr(owner, name, v); - Py_DECREF(v); - Py_DECREF(owner); - if (err) goto pop_2_error; STACK_SHRINK(2); DISPATCH(); } @@ -1968,51 +2015,58 @@ static_assert(INLINE_CACHE_ENTRIES_LOAD_GLOBAL == 4, "incorrect cache size"); PyObject *res; PyObject *null = NULL; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_GLOBAL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) + // _SPECIALIZE_LOAD_GLOBAL { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) goto error; + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadGlobal(GLOBALS(), BUILTINS(), next_instr, name); + DISPATCH_SAME_OPARG(); } - Py_INCREF(res); + STAT_INC(LOAD_GLOBAL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - else { - /* Slow-path if globals or builtins is not a dict */ - - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + // _LOAD_GLOBAL + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } if (true) goto error; } + Py_INCREF(res); + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) goto error; + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) goto error; + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) goto error; + } + } } + null = NULL; } - null = NULL; STACK_GROW(1); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = res; @@ -2498,63 +2552,69 @@ PREDICTED(LOAD_SUPER_ATTR); _Py_CODEUNIT *this_instr = next_instr - 2; static_assert(INLINE_CACHE_ENTRIES_LOAD_SUPER_ATTR == 1, "incorrect cache size"); - PyObject *self; PyObject *class; PyObject *global_super; + PyObject *self; PyObject *attr; PyObject *null = NULL; - self = stack_pointer[-1]; + // _SPECIALIZE_LOAD_SUPER_ATTR class = stack_pointer[-2]; global_super = stack_pointer[-3]; - TIER_ONE_ONLY - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - #if ENABLE_SPECIALIZATION - int load_method = oparg & 1; - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); - DISPATCH_SAME_OPARG(); - } - STAT_INC(LOAD_SUPER_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_CALL, - frame, this_instr, global_super, arg); - if (err) goto pop_3_error; - } - - // we make no attempt to optimize here; specializations should - // handle any case whose performance we care about - PyObject *stack[] = {class, self}; - PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); - if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { - PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; - if (super == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, global_super, arg); + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + int load_method = oparg & 1; + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_LoadSuperAttr(global_super, class, next_instr, load_method); + DISPATCH_SAME_OPARG(); } - else { + STAT_INC(LOAD_SUPER_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + } + // _LOAD_SUPER_ATTR + self = stack_pointer[-1]; + { + TIER_ONE_ONLY + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, global_super, arg); - if (err < 0) { - Py_CLEAR(super); + tstate, PY_MONITORING_EVENT_CALL, + frame, this_instr, global_super, arg); + if (err) goto pop_3_error; + } + // we make no attempt to optimize here; specializations should + // handle any case whose performance we care about + PyObject *stack[] = {class, self}; + PyObject *super = PyObject_Vectorcall(global_super, stack, oparg & 2, NULL); + if (opcode == INSTRUMENTED_LOAD_SUPER_ATTR) { + PyObject *arg = oparg & 2 ? class : &_PyInstrumentation_MISSING; + if (super == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, global_super, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, global_super, arg); + if (err < 0) { + Py_CLEAR(super); + } } } + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (super == NULL) goto pop_3_error; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = PyObject_GetAttr(super, name); + Py_DECREF(super); + if (attr == NULL) goto pop_3_error; + null = NULL; } - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (super == NULL) goto pop_3_error; - attr = PyObject_GetAttr(super, name); - Py_DECREF(super); - if (attr == NULL) goto pop_3_error; - null = NULL; STACK_SHRINK(2); STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; @@ -2637,49 +2697,55 @@ PyObject *owner; PyObject *attr; PyObject *self_or_null = NULL; + // _SPECIALIZE_LOAD_ATTR owner = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - next_instr = this_instr; - _Py_Specialize_LoadAttr(owner, next_instr, name); - DISPATCH_SAME_OPARG(); + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + next_instr = this_instr; + _Py_Specialize_LoadAttr(owner, next_instr, name); + DISPATCH_SAME_OPARG(); + } + STAT_INC(LOAD_ATTR, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - STAT_INC(LOAD_ATTR, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - - meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership + // _LOAD_ATTR + { + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + NULL | meth | arg1 | ... | argN + */ + Py_DECREF(owner); + if (attr == NULL) goto pop_1_error; + self_or_null = NULL; + } } else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - - NULL | meth | arg1 | ... | argN - */ + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); Py_DECREF(owner); if (attr == NULL) goto pop_1_error; - self_or_null = NULL; } } - else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) goto pop_1_error; - } STACK_GROW(((oparg & 1) ? 1 : 0)); stack_pointer[-1 - (oparg & 1 ? 1 : 0)] = attr; if (oparg & 1) { stack_pointer[-(oparg & 1 ? 1 : 0)] = self_or_null; } @@ -3076,27 +3142,35 @@ PyObject *right; PyObject *left; PyObject *res; + // _SPECIALIZE_COMPARE_OP right = stack_pointer[-1]; left = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_CompareOp(left, right, next_instr, oparg); - DISPATCH_SAME_OPARG(); + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_CompareOp(left, right, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(COMPARE_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - STAT_INC(COMPARE_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) goto pop_2_error; - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) goto pop_2_error; - res = res_bool ? Py_True : Py_False; + // _COMPARE_OP + { + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) goto pop_2_error; + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) goto pop_2_error; + res = res_bool ? Py_True : Py_False; + } } STACK_SHRINK(1); stack_pointer[-1] = res; @@ -3630,36 +3704,44 @@ static_assert(INLINE_CACHE_ENTRIES_FOR_ITER == 1, "incorrect cache size"); PyObject *iter; PyObject *next; + // _SPECIALIZE_FOR_ITER iter = stack_pointer[-1]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_ForIter(iter, next_instr, oparg); - DISPATCH_SAME_OPARG(); + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_ForIter(iter, next_instr, oparg); + DISPATCH_SAME_OPARG(); + } + STAT_INC(FOR_ITER, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - STAT_INC(FOR_ITER, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - goto error; + // _FOR_ITER + { + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + goto error; + } + monitor_raise(tstate, frame, this_instr); + _PyErr_Clear(tstate); } - monitor_raise(tstate, frame, this_instr); - _PyErr_Clear(tstate); + /* iterator ended normally */ + assert(next_instr[oparg].op.code == END_FOR || + next_instr[oparg].op.code == INSTRUMENTED_END_FOR); + Py_DECREF(iter); + STACK_SHRINK(1); + /* Jump forward oparg, then skip following END_FOR instruction */ + JUMPBY(oparg + 1); + DISPATCH(); } - /* iterator ended normally */ - assert(next_instr[oparg].op.code == END_FOR || - next_instr[oparg].op.code == INSTRUMENTED_END_FOR); - Py_DECREF(iter); - STACK_SHRINK(1); - /* Jump forward oparg, then skip following END_FOR instruction */ - JUMPBY(oparg + 1); - DISPATCH(); + // Common case: no jump, leave it to the code generator } - // Common case: no jump, leave it to the code generator STACK_GROW(1); stack_pointer[-1] = next; DISPATCH(); @@ -4211,84 +4293,92 @@ PyObject *self_or_null; PyObject *callable; PyObject *res; + // _SPECIALIZE_CALL args = stack_pointer - oparg; self_or_null = stack_pointer[-1 - oparg]; callable = stack_pointer[-2 - oparg]; - // oparg counts all of the args, but *not* self: - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_Call(callable, next_instr, total_args); - DISPATCH_SAME_OPARG(); - } - STAT_INC(CALL, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - if (self_or_null == NULL && Py_TYPE(callable) == &PyMethod_Type) { - args--; - total_args++; - PyObject *self = ((PyMethodObject *)callable)->im_self; - args[0] = Py_NewRef(self); - PyObject *method = ((PyMethodObject *)callable)->im_func; - args[-1] = Py_NewRef(method); - Py_DECREF(callable); - callable = method; - } - // Check if the call can be inlined or not - if (Py_TYPE(callable) == &PyFunction_Type && - tstate->interp->eval_frame == NULL && - ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) { - int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; - PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); - _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( - tstate, (PyFunctionObject *)callable, locals, - args, total_args, NULL - ); - // Manipulate stack directly since we leave using DISPATCH_INLINED(). - STACK_SHRINK(oparg + 2); - // The frame has stolen all the arguments from the stack, - // so there is no need to clean them up. - if (new_frame == NULL) { - goto error; + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_Call(callable, next_instr, oparg + (self_or_null != NULL)); + DISPATCH_SAME_OPARG(); } - assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); - frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; - DISPATCH_INLINED(new_frame); + STAT_INC(CALL, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ } - /* Callable is not a normal Python function */ - res = PyObject_Vectorcall( - callable, args, - total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, - NULL); - if (opcode == INSTRUMENTED_CALL) { - PyObject *arg = total_args == 0 ? - &_PyInstrumentation_MISSING : args[0]; - if (res == NULL) { - _Py_call_instrumentation_exc2( - tstate, PY_MONITORING_EVENT_C_RAISE, - frame, this_instr, callable, arg); + // _CALL + { + // oparg counts all of the args, but *not* self: + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; } - else { - int err = _Py_call_instrumentation_2args( - tstate, PY_MONITORING_EVENT_C_RETURN, - frame, this_instr, callable, arg); - if (err < 0) { - Py_CLEAR(res); + else if (Py_TYPE(callable) == &PyMethod_Type) { + args--; + total_args++; + PyObject *self = ((PyMethodObject *)callable)->im_self; + args[0] = Py_NewRef(self); + PyObject *method = ((PyMethodObject *)callable)->im_func; + args[-1] = Py_NewRef(method); + Py_DECREF(callable); + callable = method; + } + // Check if the call can be inlined or not + if (Py_TYPE(callable) == &PyFunction_Type && + tstate->interp->eval_frame == NULL && + ((PyFunctionObject *)callable)->vectorcall == _PyFunction_Vectorcall) + { + int code_flags = ((PyCodeObject*)PyFunction_GET_CODE(callable))->co_flags; + PyObject *locals = code_flags & CO_OPTIMIZED ? NULL : Py_NewRef(PyFunction_GET_GLOBALS(callable)); + _PyInterpreterFrame *new_frame = _PyEvalFramePushAndInit( + tstate, (PyFunctionObject *)callable, locals, + args, total_args, NULL + ); + // Manipulate stack directly since we leave using DISPATCH_INLINED(). + STACK_SHRINK(oparg + 2); + // The frame has stolen all the arguments from the stack, + // so there is no need to clean them up. + if (new_frame == NULL) { + goto error; } + assert(1 + INLINE_CACHE_ENTRIES_CALL == next_instr - this_instr); + frame->return_offset = 1 + INLINE_CACHE_ENTRIES_CALL; + DISPATCH_INLINED(new_frame); } + /* Callable is not a normal Python function */ + res = PyObject_Vectorcall( + callable, args, + total_args | PY_VECTORCALL_ARGUMENTS_OFFSET, + NULL); + if (opcode == INSTRUMENTED_CALL) { + PyObject *arg = total_args == 0 ? + &_PyInstrumentation_MISSING : args[0]; + if (res == NULL) { + _Py_call_instrumentation_exc2( + tstate, PY_MONITORING_EVENT_C_RAISE, + frame, this_instr, callable, arg); + } + else { + int err = _Py_call_instrumentation_2args( + tstate, PY_MONITORING_EVENT_C_RETURN, + frame, this_instr, callable, arg); + if (err < 0) { + Py_CLEAR(res); + } + } + } + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(callable); + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } } - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(callable); - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); - } - if (res == NULL) { STACK_SHRINK(oparg); goto pop_2_error; } STACK_SHRINK(oparg); STACK_SHRINK(1); stack_pointer[-1] = res; @@ -5505,24 +5595,32 @@ PyObject *rhs; PyObject *lhs; PyObject *res; + // _SPECIALIZE_BINARY_OP rhs = stack_pointer[-1]; lhs = stack_pointer[-2]; - #if ENABLE_SPECIALIZATION - if (ADAPTIVE_COUNTER_IS_ZERO(this_instr[1].cache)) { - next_instr = this_instr; - _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); - DISPATCH_SAME_OPARG(); + { + uint16_t counter = read_u16(&this_instr[1].cache); + TIER_ONE_ONLY + #if ENABLE_SPECIALIZATION + if (ADAPTIVE_COUNTER_IS_ZERO(counter)) { + next_instr = this_instr; + _Py_Specialize_BinaryOp(lhs, rhs, next_instr, oparg, LOCALS_ARRAY); + DISPATCH_SAME_OPARG(); + } + STAT_INC(BINARY_OP, deferred); + DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); + #endif /* ENABLE_SPECIALIZATION */ + assert(NB_ADD <= oparg); + assert(oparg <= NB_INPLACE_XOR); + } + // _BINARY_OP + { + assert(_PyEval_BinaryOps[oparg]); + res = _PyEval_BinaryOps[oparg](lhs, rhs); + Py_DECREF(lhs); + Py_DECREF(rhs); + if (res == NULL) goto pop_2_error; } - STAT_INC(BINARY_OP, deferred); - DECREMENT_ADAPTIVE_COUNTER(this_instr[1].cache); - #endif /* ENABLE_SPECIALIZATION */ - assert(NB_ADD <= oparg); - assert(oparg <= NB_INPLACE_XOR); - assert(_PyEval_BinaryOps[oparg]); - res = _PyEval_BinaryOps[oparg](lhs, rhs); - Py_DECREF(lhs); - Py_DECREF(rhs); - if (res == NULL) goto pop_2_error; STACK_SHRINK(1); stack_pointer[-1] = res; DISPATCH(); diff --git a/Tools/cases_generator/analysis.py b/Tools/cases_generator/analysis.py index 027d953fe47398..bd08e9de481e09 100644 --- a/Tools/cases_generator/analysis.py +++ b/Tools/cases_generator/analysis.py @@ -383,7 +383,12 @@ def analyze_macro(self, macro: parsing.Macro) -> MacroInstruction: return MacroInstruction(macro.name, format, flags, macro, parts, offset) def analyze_pseudo(self, pseudo: parsing.Pseudo) -> PseudoInstruction: - targets = [self.instrs[target] for target in pseudo.targets] + targets: list[Instruction | MacroInstruction] = [] + for target_name in pseudo.targets: + if target_name in self.instrs: + targets.append(self.instrs[target_name]) + else: + targets.append(self.macro_instrs[target_name]) assert targets ignored_flags = {"HAS_EVAL_BREAK_FLAG", "HAS_DEOPT_FLAG", "HAS_ERROR_FLAG"} assert len({t.instr_flags.bitmap(ignore=ignored_flags) for t in targets}) == 1 diff --git a/Tools/cases_generator/generate_cases.py b/Tools/cases_generator/generate_cases.py index ed4fc8d6130cec..f5853703481d8a 100644 --- a/Tools/cases_generator/generate_cases.py +++ b/Tools/cases_generator/generate_cases.py @@ -178,17 +178,17 @@ def effect_str(effects: list[StackEffect]) -> str: # for all targets. for target in self.pseudos[thing.name].targets: target_instr = self.instrs.get(target) - # Currently target is always an instr. This could change - # in the future, e.g., if we have a pseudo targetting a - # macro instruction. - assert target_instr - target_popped = effect_str(target_instr.input_effects) - target_pushed = effect_str(target_instr.output_effects) - if popped is None: - popped, pushed = target_popped, target_pushed + if target_instr is None: + macro_instr = self.macro_instrs[target] + popped, pushed = stacking.get_stack_effect_info_for_macro(macro_instr) else: - assert popped == target_popped - assert pushed == target_pushed + target_popped = effect_str(target_instr.input_effects) + target_pushed = effect_str(target_instr.output_effects) + if popped is None: + popped, pushed = target_popped, target_pushed + else: + assert popped == target_popped + assert pushed == target_pushed case _: assert_never(thing) assert popped is not None and pushed is not None diff --git a/Tools/cases_generator/instructions.py b/Tools/cases_generator/instructions.py index 636a6beb818a1d..4ecb73921952ad 100644 --- a/Tools/cases_generator/instructions.py +++ b/Tools/cases_generator/instructions.py @@ -285,7 +285,7 @@ class PseudoInstruction: """A pseudo instruction.""" name: str - targets: list[Instruction] + targets: list[Instruction | MacroInstruction] instr_flags: InstructionFlags