diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index c8b8fe3ad581f0..cf16d9cd496940 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -593,117 +593,67 @@ } case _STORE_SLICE: { - PyObject *stop; - PyObject *start; - PyObject *container; - PyObject *v; - stop = stack_pointer[-1]; - start = stack_pointer[-2]; - container = stack_pointer[-3]; - v = stack_pointer[-4]; - PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); - int err; - if (slice == NULL) { - err = 1; - } - else { - err = PyObject_SetItem(container, slice, v); - Py_DECREF(slice); + int result = _STORE_SLICE_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(v); - Py_DECREF(container); - if (err) JUMP_TO_ERROR(); - stack_pointer += -4; break; } case _BINARY_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *res; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); - // Deopt unless 0 <= sub < PyList_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); - STAT_INC(BINARY_SUBSCR, hit); - res = PyList_GET_ITEM(list, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer[-2] = res; - stack_pointer += -1; + int result = _BINARY_SUBSCR_LIST_INT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_SUBSCR_STR_INT: { - PyObject *sub; - PyObject *str; - PyObject *res; - sub = stack_pointer[-1]; - str = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET(); - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET(); - // Specialize for reading an ASCII character from any string: - Py_UCS4 c = PyUnicode_READ_CHAR(str, index); - if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET(); - STAT_INC(BINARY_SUBSCR, hit); - res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(str); - stack_pointer[-2] = res; - stack_pointer += -1; + int result = _BINARY_SUBSCR_STR_INT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_SUBSCR_TUPLE_INT: { - PyObject *sub; - PyObject *tuple; - PyObject *res; - sub = stack_pointer[-1]; - tuple = stack_pointer[-2]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET(); - // Deopt unless 0 <= sub < PyTuple_Size(list) - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET(); - STAT_INC(BINARY_SUBSCR, hit); - res = PyTuple_GET_ITEM(tuple, index); - assert(res != NULL); - Py_INCREF(res); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(tuple); - stack_pointer[-2] = res; - stack_pointer += -1; + int result = _BINARY_SUBSCR_TUPLE_INT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _BINARY_SUBSCR_DICT: { - PyObject *sub; - PyObject *dict; - PyObject *res; - sub = stack_pointer[-1]; - dict = stack_pointer[-2]; - if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); - STAT_INC(BINARY_SUBSCR, hit); - int rc = PyDict_GetItemRef(dict, sub, &res); - if (rc == 0) { - _PyErr_SetKeyError(sub); + int result = _BINARY_SUBSCR_DICT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(dict); - Py_DECREF(sub); - if (rc <= 0) JUMP_TO_ERROR(); - // not found or error - stack_pointer[-2] = res; - stack_pointer += -1; break; } @@ -734,44 +684,28 @@ } case _STORE_SUBSCR: { - PyObject *sub; - PyObject *container; - PyObject *v; - sub = stack_pointer[-1]; - container = stack_pointer[-2]; - v = stack_pointer[-3]; - /* container[sub] = v */ - int err = PyObject_SetItem(container, sub, v); - Py_DECREF(v); - Py_DECREF(container); - Py_DECREF(sub); - if (err) JUMP_TO_ERROR(); - stack_pointer += -3; + int result = _STORE_SUBSCR_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _STORE_SUBSCR_LIST_INT: { - PyObject *sub; - PyObject *list; - PyObject *value; - sub = stack_pointer[-1]; - list = stack_pointer[-2]; - value = stack_pointer[-3]; - if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); - if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); - // Ensure nonnegative, zero-or-one-digit ints. - if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); - Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; - // Ensure index < len(list) - if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); - STAT_INC(STORE_SUBSCR, hit); - PyObject *old_value = PyList_GET_ITEM(list, index); - PyList_SET_ITEM(list, index, value); - assert(old_value != NULL); - Py_DECREF(old_value); - _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); - Py_DECREF(list); - stack_pointer += -3; + int result = _STORE_SUBSCR_LIST_INT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } @@ -866,110 +800,41 @@ /* _INSTRUMENTED_RETURN_CONST is not a viable micro-op for tier 2 because it is instrumented */ case _GET_AITER: { - PyObject *obj; - PyObject *iter; - obj = stack_pointer[-1]; - unaryfunc getter = NULL; - PyTypeObject *type = Py_TYPE(obj); - if (type->tp_as_async != NULL) { - getter = type->tp_as_async->am_aiter; - } - if (getter == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an object with " - "__aiter__ method, got %.100s", - type->tp_name); - Py_DECREF(obj); - if (true) JUMP_TO_ERROR(); - } - iter = (*getter)(obj); - Py_DECREF(obj); - if (iter == NULL) JUMP_TO_ERROR(); - if (Py_TYPE(iter)->tp_as_async == NULL || - Py_TYPE(iter)->tp_as_async->am_anext == NULL) { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' received an object from __aiter__ " - "that does not implement __anext__: %.100s", - Py_TYPE(iter)->tp_name); - Py_DECREF(iter); - if (true) JUMP_TO_ERROR(); + int result = _GET_AITER_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-1] = iter; break; } case _GET_ANEXT: { - PyObject *aiter; - PyObject *awaitable; - aiter = stack_pointer[-1]; - unaryfunc getter = NULL; - PyObject *next_iter = NULL; - PyTypeObject *type = Py_TYPE(aiter); - if (PyAsyncGen_CheckExact(aiter)) { - awaitable = type->tp_as_async->am_anext(aiter); - if (awaitable == NULL) { - JUMP_TO_ERROR(); - } - } else { - if (type->tp_as_async != NULL){ - getter = type->tp_as_async->am_anext; - } - if (getter != NULL) { - next_iter = (*getter)(aiter); - if (next_iter == NULL) { - JUMP_TO_ERROR(); - } - } - else { - _PyErr_Format(tstate, PyExc_TypeError, - "'async for' requires an iterator with " - "__anext__ method, got %.100s", - type->tp_name); - JUMP_TO_ERROR(); - } - awaitable = _PyCoro_GetAwaitableIter(next_iter); - if (awaitable == NULL) { - _PyErr_FormatFromCause( - PyExc_TypeError, - "'async for' received an invalid object " - "from __anext__: %.100s", - Py_TYPE(next_iter)->tp_name); - Py_DECREF(next_iter); - JUMP_TO_ERROR(); - } else { - Py_DECREF(next_iter); - } + int result = _GET_ANEXT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - stack_pointer[0] = awaitable; - stack_pointer += 1; break; } case _GET_AWAITABLE: { - PyObject *iterable; - PyObject *iter; - oparg = CURRENT_OPARG(); - iterable = stack_pointer[-1]; - iter = _PyCoro_GetAwaitableIter(iterable); - if (iter == NULL) { - _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); - } - Py_DECREF(iterable); - if (iter != NULL && PyCoro_CheckExact(iter)) { - PyObject *yf = _PyGen_yf((PyGenObject*)iter); - if (yf != NULL) { - /* `iter` is a coroutine object that is being - awaited, `yf` is a pointer to the current awaitable - being awaited on. */ - Py_DECREF(yf); - Py_CLEAR(iter); - _PyErr_SetString(tstate, PyExc_RuntimeError, - "coroutine is being awaited already"); - /* The code below jumps to `error` if `iter` is NULL. */ - } + int result = _GET_AWAITABLE_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (iter == NULL) JUMP_TO_ERROR(); - stack_pointer[-1] = iter; break; } @@ -1010,25 +875,15 @@ } case _STORE_NAME: { - PyObject *v; - oparg = CURRENT_OPARG(); - v = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - PyObject *ns = LOCALS(); - int err; - if (ns == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when storing %R", name); - Py_DECREF(v); - if (true) JUMP_TO_ERROR(); + int result = _STORE_NAME_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (PyDict_CheckExact(ns)) - err = PyDict_SetItem(ns, name, v); - else - err = PyObject_SetItem(ns, name, v); - Py_DECREF(v); - if (err) JUMP_TO_ERROR(); - stack_pointer += -1; break; } @@ -1203,78 +1058,30 @@ } case _LOAD_FROM_DICT_OR_GLOBALS: { - PyObject *mod_or_class_dict; - PyObject *v; - oparg = CURRENT_OPARG(); - mod_or_class_dict = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); - if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + int result = _LOAD_FROM_DICT_OR_GLOBALS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (v == NULL) { - if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { - JUMP_TO_ERROR(); - } - if (v == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - JUMP_TO_ERROR(); - } - } - } - Py_DECREF(mod_or_class_dict); - stack_pointer[-1] = v; break; } /* _LOAD_NAME is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _LOAD_GLOBAL: { - PyObject *res; - PyObject *null = NULL; - oparg = CURRENT_OPARG(); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); - if (PyDict_CheckExact(GLOBALS()) - && PyDict_CheckExact(BUILTINS())) - { - res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), - (PyDictObject *)BUILTINS(), - name); - if (res == NULL) { - if (!_PyErr_Occurred(tstate)) { - /* _PyDict_LoadGlobal() returns NULL without raising - * an exception if the key doesn't exist */ - _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - } - if (true) JUMP_TO_ERROR(); - } - Py_INCREF(res); - } - else { - /* Slow-path if globals or builtins is not a dict */ - /* namespace 1: globals */ - if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR(); - if (res == NULL) { - /* namespace 2: builtins */ - if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR(); - if (res == NULL) { - _PyEval_FormatExcCheckArg( - tstate, PyExc_NameError, - NAME_ERROR_MSG, name); - if (true) JUMP_TO_ERROR(); - } - } + int result = _LOAD_GLOBAL_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - null = NULL; - stack_pointer[0] = res; - if (oparg & 1) stack_pointer[1] = null; - stack_pointer += 1 + (oparg & 1); break; } @@ -1375,28 +1182,15 @@ } case _LOAD_FROM_DICT_OR_DEREF: { - PyObject *class_dict; - PyObject *value; - oparg = CURRENT_OPARG(); - class_dict = stack_pointer[-1]; - PyObject *name; - assert(class_dict); - assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); - name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); - if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + int result = _LOAD_FROM_DICT_OR_DEREF_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (!value) { - PyObject *cell = GETLOCAL(oparg); - value = PyCell_GET(cell); - if (value == NULL) { - _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); - JUMP_TO_ERROR(); - } - Py_INCREF(value); - } - Py_DECREF(class_dict); - stack_pointer[-1] = value; break; } @@ -1443,17 +1237,15 @@ } case _BUILD_STRING: { - PyObject **pieces; - PyObject *str; - oparg = CURRENT_OPARG(); - pieces = &stack_pointer[-oparg]; - str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(pieces[_i]); + int result = _BUILD_STRING_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (str == NULL) JUMP_TO_ERROR(); - stack_pointer[-oparg] = str; - stack_pointer += 1 - oparg; break; } @@ -1482,27 +1274,15 @@ } case _LIST_EXTEND: { - PyObject *iterable; - PyObject *list; - oparg = CURRENT_OPARG(); - iterable = stack_pointer[-1]; - list = stack_pointer[-2 - (oparg-1)]; - PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); - if (none_val == NULL) { - if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && - (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) - { - _PyErr_Clear(tstate); - _PyErr_Format(tstate, PyExc_TypeError, - "Value after * must be an iterable, not %.200s", - Py_TYPE(iterable)->tp_name); - } - Py_DECREF(iterable); - if (true) JUMP_TO_ERROR(); + int result = _LIST_EXTEND_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - assert(Py_IsNone(none_val)); - Py_DECREF(iterable); - stack_pointer += -1; break; } @@ -1522,86 +1302,54 @@ /* _BUILD_SET is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _BUILD_MAP: { - PyObject **values; - PyObject *map; - oparg = CURRENT_OPARG(); - values = &stack_pointer[-oparg*2]; - map = _PyDict_FromItems( - values, 2, - values+1, 2, - oparg); - for (int _i = oparg*2; --_i >= 0;) { - Py_DECREF(values[_i]); + int result = _BUILD_MAP_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (map == NULL) JUMP_TO_ERROR(); - stack_pointer[-oparg*2] = map; - stack_pointer += 1 - oparg*2; break; } case _SETUP_ANNOTATIONS: { - int err; - PyObject *ann_dict; - if (LOCALS() == NULL) { - _PyErr_Format(tstate, PyExc_SystemError, - "no locals found when setting up annotations"); - if (true) JUMP_TO_ERROR(); - } - /* check if __annotations__ in locals()... */ - if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); - if (ann_dict == NULL) { - ann_dict = PyDict_New(); - if (ann_dict == NULL) JUMP_TO_ERROR(); - err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), - ann_dict); - Py_DECREF(ann_dict); - if (err) JUMP_TO_ERROR(); - } - else { - Py_DECREF(ann_dict); + int result = _SETUP_ANNOTATIONS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } break; } case _BUILD_CONST_KEY_MAP: { - PyObject *keys; - PyObject **values; - PyObject *map; - oparg = CURRENT_OPARG(); - keys = stack_pointer[-1]; - values = &stack_pointer[-1 - oparg]; - assert(PyTuple_CheckExact(keys)); - assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); - map = _PyDict_FromItems( - &PyTuple_GET_ITEM(keys, 0), 1, - values, 1, oparg); - for (int _i = oparg; --_i >= 0;) { - Py_DECREF(values[_i]); - } - Py_DECREF(keys); - if (map == NULL) JUMP_TO_ERROR(); - stack_pointer[-1 - oparg] = map; - stack_pointer += -oparg; + int result = _BUILD_CONST_KEY_MAP_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _DICT_UPDATE: { - PyObject *update; - PyObject *dict; - oparg = CURRENT_OPARG(); - update = stack_pointer[-1]; - dict = stack_pointer[-2 - (oparg - 1)]; - if (PyDict_Update(dict, update) < 0) { - if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { - _PyErr_Format(tstate, PyExc_TypeError, - "'%.200s' object is not a mapping", - Py_TYPE(update)->tp_name); - } - Py_DECREF(update); - if (true) JUMP_TO_ERROR(); + int result = _DICT_UPDATE_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(update); - stack_pointer += -1; break; } @@ -1642,105 +1390,41 @@ /* _INSTRUMENTED_LOAD_SUPER_ATTR is not a viable micro-op for tier 2 because it is instrumented */ case _LOAD_SUPER_ATTR_ATTR: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - oparg = CURRENT_OPARG(); - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(!(oparg & 1)); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); - Py_DECREF(global_super); - Py_DECREF(class); - Py_DECREF(self); - if (attr == NULL) JUMP_TO_ERROR(); - stack_pointer[-3] = attr; - stack_pointer += -2; + int result = _LOAD_SUPER_ATTR_ATTR_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _LOAD_SUPER_ATTR_METHOD: { - PyObject *self; - PyObject *class; - PyObject *global_super; - PyObject *attr; - PyObject *self_or_null; - oparg = CURRENT_OPARG(); - self = stack_pointer[-1]; - class = stack_pointer[-2]; - global_super = stack_pointer[-3]; - assert(oparg & 1); - if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); - if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); - STAT_INC(LOAD_SUPER_ATTR, hit); - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); - PyTypeObject *cls = (PyTypeObject *)class; - int method_found = 0; - attr = _PySuper_Lookup(cls, self, name, - Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); - Py_DECREF(global_super); - Py_DECREF(class); - if (attr == NULL) { - Py_DECREF(self); - if (true) JUMP_TO_ERROR(); - } - if (method_found) { - self_or_null = self; // transfer ownership - } else { - Py_DECREF(self); - self_or_null = NULL; + int result = _LOAD_SUPER_ATTR_METHOD_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-3] = attr; - stack_pointer[-2] = self_or_null; - stack_pointer += -1; break; } case _LOAD_ATTR: { - PyObject *owner; - PyObject *attr; - PyObject *self_or_null = NULL; - oparg = CURRENT_OPARG(); - owner = stack_pointer[-1]; - PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); - if (oparg & 1) { - /* Designed to work in tandem with CALL, pushes two values. */ - attr = NULL; - if (_PyObject_GetMethod(owner, name, &attr)) { - /* We can bypass temporary bound method object. - meth is unbound method and obj is self. - meth | self | arg1 | ... | argN - */ - assert(attr != NULL); // No errors on this branch - self_or_null = owner; // Transfer ownership - } - else { - /* meth is not an unbound method (but a regular attr, or - something was returned by a descriptor protocol). Set - the second element of the stack to NULL, to signal - CALL that it's not a method call. - meth | NULL | arg1 | ... | argN - */ - Py_DECREF(owner); - if (attr == NULL) JUMP_TO_ERROR(); - self_or_null = NULL; - } - } - else { - /* Classic, pushes one value. */ - attr = PyObject_GetAttr(owner, name); - Py_DECREF(owner); - if (attr == NULL) JUMP_TO_ERROR(); + int result = _LOAD_ATTR_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-1] = attr; - if (oparg & 1) stack_pointer[0] = self_or_null; - stack_pointer += (oparg & 1); break; } @@ -2022,71 +1706,41 @@ } case _COMPARE_OP: { - PyObject *right; - PyObject *left; - PyObject *res; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - assert((oparg >> 5) <= Py_GE); - res = PyObject_RichCompare(left, right, oparg >> 5); - Py_DECREF(left); - Py_DECREF(right); - if (res == NULL) JUMP_TO_ERROR(); - if (oparg & 16) { - int res_bool = PyObject_IsTrue(res); - Py_DECREF(res); - if (res_bool < 0) JUMP_TO_ERROR(); - res = res_bool ? Py_True : Py_False; + int result = _COMPARE_OP_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-2] = res; - stack_pointer += -1; break; } case _COMPARE_OP_FLOAT: { - PyObject *right; - PyObject *left; - PyObject *res; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - STAT_INC(COMPARE_OP, hit); - double dleft = PyFloat_AS_DOUBLE(left); - double dright = PyFloat_AS_DOUBLE(right); - // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg - int sign_ish = COMPARISON_BIT(dleft, dright); - _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); - _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; + int result = _COMPARE_OP_FLOAT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _COMPARE_OP_INT: { - PyObject *right; - PyObject *left; - PyObject *res; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET(); - if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET(); - STAT_INC(COMPARE_OP, hit); - assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && - _PyLong_DigitCount((PyLongObject *)right) <= 1); - Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); - Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); - // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg - int sign_ish = COMPARISON_BIT(ileft, iright); - _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); - _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); - res = (sign_ish & oparg) ? Py_True : Py_False; - // It's always a bool, so we don't care about oparg & 16. - stack_pointer[-2] = res; - stack_pointer += -1; + int result = _COMPARE_OP_INT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2146,70 +1800,41 @@ } case _CONTAINS_OP_SET: { - PyObject *right; - PyObject *left; - PyObject *b; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET(); - STAT_INC(CONTAINS_OP, hit); - // Note: both set and frozenset use the same seq_contains method! - int res = _PySet_Contains((PySetObject *)right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) JUMP_TO_ERROR(); - b = (res ^ oparg) ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; + int result = _CONTAINS_OP_SET_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _CONTAINS_OP_DICT: { - PyObject *right; - PyObject *left; - PyObject *b; - oparg = CURRENT_OPARG(); - right = stack_pointer[-1]; - left = stack_pointer[-2]; - if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET(); - STAT_INC(CONTAINS_OP, hit); - int res = PyDict_Contains(right, left); - Py_DECREF(left); - Py_DECREF(right); - if (res < 0) JUMP_TO_ERROR(); - b = (res ^ oparg) ? Py_True : Py_False; - stack_pointer[-2] = b; - stack_pointer += -1; + int result = _CONTAINS_OP_DICT_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _CHECK_EG_MATCH: { - PyObject *match_type; - PyObject *exc_value; - PyObject *rest; - PyObject *match; - match_type = stack_pointer[-1]; - exc_value = stack_pointer[-2]; - if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (true) JUMP_TO_ERROR(); - } - match = NULL; - rest = NULL; - int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, - &match, &rest); - Py_DECREF(exc_value); - Py_DECREF(match_type); - if (res < 0) JUMP_TO_ERROR(); - assert((match == NULL) == (rest == NULL)); - if (match == NULL) JUMP_TO_ERROR(); - if (!Py_IsNone(match)) { - PyErr_SetHandledException(match); + int result = _CHECK_EG_MATCH_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-2] = rest; - stack_pointer[-1] = match; break; } @@ -2265,31 +1890,15 @@ } case _MATCH_CLASS: { - PyObject *names; - PyObject *type; - PyObject *subject; - PyObject *attrs; - oparg = CURRENT_OPARG(); - names = stack_pointer[-1]; - type = stack_pointer[-2]; - subject = stack_pointer[-3]; - // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or - // None on failure. - assert(PyTuple_CheckExact(names)); - attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); - Py_DECREF(subject); - Py_DECREF(type); - Py_DECREF(names); - if (attrs) { - assert(PyTuple_CheckExact(attrs)); // Success! - } - else { - if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); - // Error! - attrs = Py_None; // Failure! + int result = _MATCH_CLASS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - stack_pointer[-3] = attrs; - stack_pointer += -2; break; } @@ -2376,27 +1985,15 @@ /* _FOR_ITER is not a viable micro-op for tier 2 because it is replaced */ case _FOR_ITER_TIER_TWO: { - PyObject *iter; - PyObject *next; - iter = stack_pointer[-1]; - /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ - next = (*Py_TYPE(iter)->tp_iternext)(iter); - if (next == NULL) { - if (_PyErr_Occurred(tstate)) { - if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { - JUMP_TO_ERROR(); - } - _PyErr_Clear(tstate); - } - /* iterator ended normally */ - Py_DECREF(iter); - STACK_SHRINK(1); - /* The translator sets the deopt target just past END_FOR */ - if (true) JUMP_TO_JUMP_TARGET(); + int result = _FOR_ITER_TIER_TWO_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - // Common case: no jump, leave it to the code generator - stack_pointer[0] = next; - stack_pointer += 1; break; } @@ -2515,39 +2112,15 @@ /* _BEFORE_WITH is not a viable micro-op for tier 2 because it has both popping and not-popping errors */ case _WITH_EXCEPT_START: { - PyObject *val; - PyObject *lasti; - PyObject *exit_func; - PyObject *res; - val = stack_pointer[-1]; - lasti = stack_pointer[-3]; - exit_func = stack_pointer[-4]; - /* At the top of the stack are 4 values: - - val: TOP = exc_info() - - unused: SECOND = previous exception - - lasti: THIRD = lasti of exception in exc_info() - - exit_func: FOURTH = the context.__exit__ bound method - We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). - Then we push the __exit__ return value. - */ - PyObject *exc, *tb; - assert(val && PyExceptionInstance_Check(val)); - exc = PyExceptionInstance_Class(val); - tb = PyException_GetTraceback(val); - if (tb == NULL) { - tb = Py_None; - } - else { - Py_DECREF(tb); + int result = _WITH_EXCEPT_START_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - assert(PyLong_Check(lasti)); - (void)lasti; // Shut up compiler warning if asserts are off - PyObject *stack[4] = {NULL, exc, val, tb}; - res = PyObject_Vectorcall(exit_func, stack + 1, - 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[0] = res; - stack_pointer += 1; break; } @@ -2759,32 +2332,80 @@ } case _INIT_CALL_PY_EXACT_ARGS_0: { - stack_pointer = _INIT_CALL_PY_EXACT_ARGS_0_func(tstate, frame, stack_pointer); + int result = _INIT_CALL_PY_EXACT_ARGS_0_func(tstate, frame, &stack_pointer); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _INIT_CALL_PY_EXACT_ARGS_1: { - stack_pointer = _INIT_CALL_PY_EXACT_ARGS_1_func(tstate, frame, stack_pointer); + int result = _INIT_CALL_PY_EXACT_ARGS_1_func(tstate, frame, &stack_pointer); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _INIT_CALL_PY_EXACT_ARGS_2: { - stack_pointer = _INIT_CALL_PY_EXACT_ARGS_2_func(tstate, frame, stack_pointer); + int result = _INIT_CALL_PY_EXACT_ARGS_2_func(tstate, frame, &stack_pointer); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _INIT_CALL_PY_EXACT_ARGS_3: { - stack_pointer = _INIT_CALL_PY_EXACT_ARGS_3_func(tstate, frame, stack_pointer); + int result = _INIT_CALL_PY_EXACT_ARGS_3_func(tstate, frame, &stack_pointer); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _INIT_CALL_PY_EXACT_ARGS_4: { - stack_pointer = _INIT_CALL_PY_EXACT_ARGS_4_func(tstate, frame, stack_pointer); + int result = _INIT_CALL_PY_EXACT_ARGS_4_func(tstate, frame, &stack_pointer); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } case _INIT_CALL_PY_EXACT_ARGS: { - stack_pointer = _INIT_CALL_PY_EXACT_ARGS_func(tstate, frame, stack_pointer, CURRENT_OPARG()); + int result = _INIT_CALL_PY_EXACT_ARGS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } @@ -2892,138 +2513,54 @@ } case _CALL_BUILTIN_CLASS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET(); - PyTypeObject *tp = (PyTypeObject *)callable; - if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + int result = _CALL_BUILTIN_CLASS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(tp); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } case _CALL_BUILTIN_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_O functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + int result = _CALL_BUILTIN_O_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET(); - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - PyObject *arg = args[0]; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } case _CALL_BUILTIN_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL functions, without keywords */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); - /* res = func(self, args, nargs) */ - res = ((PyCFunctionFast)(void(*)(void))cfunc)( - PyCFunction_GET_SELF(callable), - args, - total_args); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + int result = _CALL_BUILTIN_FAST_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } case _CALL_BUILTIN_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); - if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - /* res = func(self, args, nargs, kwnames) */ - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void)) - PyCFunction_GET_FUNCTION(callable); - res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + int result = _CALL_BUILTIN_FAST_WITH_KEYWORDS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } @@ -3102,155 +2639,54 @@ } case _CALL_METHOD_DESCRIPTOR_O: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + int result = _CALL_METHOD_DESCRIPTOR_O_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (total_args != 2) JUMP_TO_JUMP_TARGET(); - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET(); - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); - PyObject *arg = args[1]; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, arg); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(arg); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } case _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); - PyTypeObject *d_type = method->d_common.d_type; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - int nargs = total_args - 1; - PyCFunctionFastWithKeywords cfunc = - (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; - res = cfunc(self, args + 1, nargs, NULL); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Free the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + int result = _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } case _CALL_METHOD_DESCRIPTOR_NOARGS: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - assert(oparg == 0 || oparg == 1); - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; + int result = _CALL_METHOD_DESCRIPTOR_NOARGS_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - if (total_args != 1) JUMP_TO_JUMP_TARGET(); - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); - PyMethodDef *meth = method->d_method; - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); - if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET(); - // CPython promises to check all non-vectorcall function calls. - if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - PyCFunction cfunc = meth->ml_meth; - _Py_EnterRecursiveCallTstateUnchecked(tstate); - res = _PyCFunction_TrampolineCall(cfunc, self, NULL); - _Py_LeaveRecursiveCallTstate(tstate); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - Py_DECREF(self); - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } case _CALL_METHOD_DESCRIPTOR_FAST: { - PyObject **args; - PyObject *self_or_null; - PyObject *callable; - PyObject *res; - oparg = CURRENT_OPARG(); - args = &stack_pointer[-oparg]; - self_or_null = stack_pointer[-1 - oparg]; - callable = stack_pointer[-2 - oparg]; - int total_args = oparg; - if (self_or_null != NULL) { - args--; - total_args++; - } - PyMethodDescrObject *method = (PyMethodDescrObject *)callable; - /* Builtin METH_FASTCALL methods, without keywords */ - if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); - PyMethodDef *meth = method->d_method; - if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); - PyObject *self = args[0]; - if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); - STAT_INC(CALL, hit); - PyCFunctionFast cfunc = - (PyCFunctionFast)(void(*)(void))meth->ml_meth; - int nargs = total_args - 1; - res = cfunc(self, args + 1, nargs); - assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); - /* Clear the stack of the arguments. */ - for (int i = 0; i < total_args; i++) { - Py_DECREF(args[i]); + int result = _CALL_METHOD_DESCRIPTOR_FAST_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); } - Py_DECREF(callable); - if (res == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - oparg] = res; - stack_pointer += -1 - oparg; break; } @@ -3315,21 +2751,15 @@ } case _BUILD_SLICE: { - PyObject *step = NULL; - PyObject *stop; - PyObject *start; - PyObject *slice; - oparg = CURRENT_OPARG(); - if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } - stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; - start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; - slice = PySlice_New(start, stop, step); - Py_DECREF(start); - Py_DECREF(stop); - Py_XDECREF(step); - if (slice == NULL) JUMP_TO_ERROR(); - stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; - stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + int result = _BUILD_SLICE_func(tstate, frame, &stack_pointer, CURRENT_OPARG()); + switch (result) { + case 0: + break; + case 1: + JUMP_TO_ERROR(); + case 2: + JUMP_TO_JUMP_TARGET(); + } break; } diff --git a/Python/executor_stubs.c.h b/Python/executor_stubs.c.h index 2d5107e44d9d65..ac616d41713d51 100644 --- a/Python/executor_stubs.c.h +++ b/Python/executor_stubs.c.h @@ -8,8 +8,903 @@ #endif #define TIER_TWO 2 - PyObject ** _INIT_CALL_PY_EXACT_ARGS_0_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer) { +#undef JUMP_TO_ERROR +#define JUMP_TO_ERROR() return 1; + +#undef JUMP_TO_JUMP_TARGET +#define JUMP_TO_JUMP_TARGET() return 2; + + int _STORE_SLICE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *stop; + PyObject *start; + PyObject *container; + PyObject *v; + stop = stack_pointer[-1]; + start = stack_pointer[-2]; + container = stack_pointer[-3]; + v = stack_pointer[-4]; + PyObject *slice = _PyBuildSlice_ConsumeRefs(start, stop); + int err; + if (slice == NULL) { + err = 1; + } + else { + err = PyObject_SetItem(container, slice, v); + Py_DECREF(slice); + } + Py_DECREF(v); + Py_DECREF(container); + if (err) JUMP_TO_ERROR(); + stack_pointer += -4; + *_stack_pointer = stack_pointer; + return 0; + } + + int _BINARY_SUBSCR_LIST_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *sub; + PyObject *list; + PyObject *res; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); + // Deopt unless 0 <= sub < PyList_Size(list) + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); + STAT_INC(BINARY_SUBSCR, hit); + res = PyList_GET_ITEM(list, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer[-2] = res; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _BINARY_SUBSCR_STR_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *sub; + PyObject *str; + PyObject *res; + sub = stack_pointer[-1]; + str = stack_pointer[-2]; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyUnicode_CheckExact(str)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (PyUnicode_GET_LENGTH(str) <= index) JUMP_TO_JUMP_TARGET(); + // Specialize for reading an ASCII character from any string: + Py_UCS4 c = PyUnicode_READ_CHAR(str, index); + if (Py_ARRAY_LENGTH(_Py_SINGLETON(strings).ascii) <= c) JUMP_TO_JUMP_TARGET(); + STAT_INC(BINARY_SUBSCR, hit); + res = (PyObject*)&_Py_SINGLETON(strings).ascii[c]; + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(str); + stack_pointer[-2] = res; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _BINARY_SUBSCR_TUPLE_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *sub; + PyObject *tuple; + PyObject *res; + sub = stack_pointer[-1]; + tuple = stack_pointer[-2]; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyTuple_CheckExact(tuple)) JUMP_TO_JUMP_TARGET(); + // Deopt unless 0 <= sub < PyTuple_Size(list) + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + if (index >= PyTuple_GET_SIZE(tuple)) JUMP_TO_JUMP_TARGET(); + STAT_INC(BINARY_SUBSCR, hit); + res = PyTuple_GET_ITEM(tuple, index); + assert(res != NULL); + Py_INCREF(res); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(tuple); + stack_pointer[-2] = res; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _BINARY_SUBSCR_DICT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *sub; + PyObject *dict; + PyObject *res; + sub = stack_pointer[-1]; + dict = stack_pointer[-2]; + if (!PyDict_CheckExact(dict)) JUMP_TO_JUMP_TARGET(); + STAT_INC(BINARY_SUBSCR, hit); + int rc = PyDict_GetItemRef(dict, sub, &res); + if (rc == 0) { + _PyErr_SetKeyError(sub); + } + Py_DECREF(dict); + Py_DECREF(sub); + if (rc <= 0) JUMP_TO_ERROR(); + // not found or error + stack_pointer[-2] = res; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _STORE_SUBSCR_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *sub; + PyObject *container; + PyObject *v; + sub = stack_pointer[-1]; + container = stack_pointer[-2]; + v = stack_pointer[-3]; + /* container[sub] = v */ + int err = PyObject_SetItem(container, sub, v); + Py_DECREF(v); + Py_DECREF(container); + Py_DECREF(sub); + if (err) JUMP_TO_ERROR(); + stack_pointer += -3; + *_stack_pointer = stack_pointer; + return 0; + } + + int _STORE_SUBSCR_LIST_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *sub; + PyObject *list; + PyObject *value; + sub = stack_pointer[-1]; + list = stack_pointer[-2]; + value = stack_pointer[-3]; + if (!PyLong_CheckExact(sub)) JUMP_TO_JUMP_TARGET(); + if (!PyList_CheckExact(list)) JUMP_TO_JUMP_TARGET(); + // Ensure nonnegative, zero-or-one-digit ints. + if (!_PyLong_IsNonNegativeCompact((PyLongObject *)sub)) JUMP_TO_JUMP_TARGET(); + Py_ssize_t index = ((PyLongObject*)sub)->long_value.ob_digit[0]; + // Ensure index < len(list) + if (index >= PyList_GET_SIZE(list)) JUMP_TO_JUMP_TARGET(); + STAT_INC(STORE_SUBSCR, hit); + PyObject *old_value = PyList_GET_ITEM(list, index); + PyList_SET_ITEM(list, index, value); + assert(old_value != NULL); + Py_DECREF(old_value); + _Py_DECREF_SPECIALIZED(sub, (destructor)PyObject_Free); + Py_DECREF(list); + stack_pointer += -3; + *_stack_pointer = stack_pointer; + return 0; + } + + int _GET_AITER_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *obj; + PyObject *iter; + obj = stack_pointer[-1]; + unaryfunc getter = NULL; + PyTypeObject *type = Py_TYPE(obj); + if (type->tp_as_async != NULL) { + getter = type->tp_as_async->am_aiter; + } + if (getter == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an object with " + "__aiter__ method, got %.100s", + type->tp_name); + Py_DECREF(obj); + if (true) JUMP_TO_ERROR(); + } + iter = (*getter)(obj); + Py_DECREF(obj); + if (iter == NULL) JUMP_TO_ERROR(); + if (Py_TYPE(iter)->tp_as_async == NULL || + Py_TYPE(iter)->tp_as_async->am_anext == NULL) { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' received an object from __aiter__ " + "that does not implement __anext__: %.100s", + Py_TYPE(iter)->tp_name); + Py_DECREF(iter); + if (true) JUMP_TO_ERROR(); + } + stack_pointer[-1] = iter; + *_stack_pointer = stack_pointer; + return 0; + } + + int _GET_ANEXT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *aiter; + PyObject *awaitable; + aiter = stack_pointer[-1]; + unaryfunc getter = NULL; + PyObject *next_iter = NULL; + PyTypeObject *type = Py_TYPE(aiter); + if (PyAsyncGen_CheckExact(aiter)) { + awaitable = type->tp_as_async->am_anext(aiter); + if (awaitable == NULL) { + JUMP_TO_ERROR(); + } + } else { + if (type->tp_as_async != NULL){ + getter = type->tp_as_async->am_anext; + } + if (getter != NULL) { + next_iter = (*getter)(aiter); + if (next_iter == NULL) { + JUMP_TO_ERROR(); + } + } + else { + _PyErr_Format(tstate, PyExc_TypeError, + "'async for' requires an iterator with " + "__anext__ method, got %.100s", + type->tp_name); + JUMP_TO_ERROR(); + } + awaitable = _PyCoro_GetAwaitableIter(next_iter); + if (awaitable == NULL) { + _PyErr_FormatFromCause( + PyExc_TypeError, + "'async for' received an invalid object " + "from __anext__: %.100s", + Py_TYPE(next_iter)->tp_name); + Py_DECREF(next_iter); + JUMP_TO_ERROR(); + } else { + Py_DECREF(next_iter); + } + } + stack_pointer[0] = awaitable; + stack_pointer += 1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _GET_AWAITABLE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *iterable; + PyObject *iter; + iterable = stack_pointer[-1]; + iter = _PyCoro_GetAwaitableIter(iterable); + if (iter == NULL) { + _PyEval_FormatAwaitableError(tstate, Py_TYPE(iterable), oparg); + } + Py_DECREF(iterable); + if (iter != NULL && PyCoro_CheckExact(iter)) { + PyObject *yf = _PyGen_yf((PyGenObject*)iter); + if (yf != NULL) { + /* `iter` is a coroutine object that is being + awaited, `yf` is a pointer to the current awaitable + being awaited on. */ + Py_DECREF(yf); + Py_CLEAR(iter); + _PyErr_SetString(tstate, PyExc_RuntimeError, + "coroutine is being awaited already"); + /* The code below jumps to `error` if `iter` is NULL. */ + } + } + if (iter == NULL) JUMP_TO_ERROR(); + stack_pointer[-1] = iter; + *_stack_pointer = stack_pointer; + return 0; + } + + int _STORE_NAME_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *v; + v = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + PyObject *ns = LOCALS(); + int err; + if (ns == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when storing %R", name); + Py_DECREF(v); + if (true) JUMP_TO_ERROR(); + } + if (PyDict_CheckExact(ns)) + err = PyDict_SetItem(ns, name, v); + else + err = PyObject_SetItem(ns, name, v); + Py_DECREF(v); + if (err) JUMP_TO_ERROR(); + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _LOAD_FROM_DICT_OR_GLOBALS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *mod_or_class_dict; + PyObject *v; + mod_or_class_dict = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg); + if (PyMapping_GetOptionalItem(mod_or_class_dict, name, &v) < 0) { + JUMP_TO_ERROR(); + } + if (v == NULL) { + if (PyDict_GetItemRef(GLOBALS(), name, &v) < 0) { + JUMP_TO_ERROR(); + } + if (v == NULL) { + if (PyMapping_GetOptionalItem(BUILTINS(), name, &v) < 0) { + JUMP_TO_ERROR(); + } + if (v == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + JUMP_TO_ERROR(); + } + } + } + Py_DECREF(mod_or_class_dict); + stack_pointer[-1] = v; + *_stack_pointer = stack_pointer; + return 0; + } + + int _LOAD_GLOBAL_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *res; + PyObject *null = NULL; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg>>1); + if (PyDict_CheckExact(GLOBALS()) + && PyDict_CheckExact(BUILTINS())) + { + res = _PyDict_LoadGlobal((PyDictObject *)GLOBALS(), + (PyDictObject *)BUILTINS(), + name); + if (res == NULL) { + if (!_PyErr_Occurred(tstate)) { + /* _PyDict_LoadGlobal() returns NULL without raising + * an exception if the key doesn't exist */ + _PyEval_FormatExcCheckArg(tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + } + if (true) JUMP_TO_ERROR(); + } + Py_INCREF(res); + } + else { + /* Slow-path if globals or builtins is not a dict */ + /* namespace 1: globals */ + if (PyMapping_GetOptionalItem(GLOBALS(), name, &res) < 0) JUMP_TO_ERROR(); + if (res == NULL) { + /* namespace 2: builtins */ + if (PyMapping_GetOptionalItem(BUILTINS(), name, &res) < 0) JUMP_TO_ERROR(); + if (res == NULL) { + _PyEval_FormatExcCheckArg( + tstate, PyExc_NameError, + NAME_ERROR_MSG, name); + if (true) JUMP_TO_ERROR(); + } + } + } + null = NULL; + stack_pointer[0] = res; + if (oparg & 1) stack_pointer[1] = null; + stack_pointer += 1 + (oparg & 1); + *_stack_pointer = stack_pointer; + return 0; + } + + int _LOAD_FROM_DICT_OR_DEREF_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *class_dict; + PyObject *value; + class_dict = stack_pointer[-1]; + PyObject *name; + assert(class_dict); + assert(oparg >= 0 && oparg < _PyFrame_GetCode(frame)->co_nlocalsplus); + name = PyTuple_GET_ITEM(_PyFrame_GetCode(frame)->co_localsplusnames, oparg); + if (PyMapping_GetOptionalItem(class_dict, name, &value) < 0) { + JUMP_TO_ERROR(); + } + if (!value) { + PyObject *cell = GETLOCAL(oparg); + value = PyCell_GET(cell); + if (value == NULL) { + _PyEval_FormatExcUnbound(tstate, _PyFrame_GetCode(frame), oparg); + JUMP_TO_ERROR(); + } + Py_INCREF(value); + } + Py_DECREF(class_dict); + stack_pointer[-1] = value; + *_stack_pointer = stack_pointer; + return 0; + } + + int _BUILD_STRING_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **pieces; + PyObject *str; + pieces = &stack_pointer[-oparg]; + str = _PyUnicode_JoinArray(&_Py_STR(empty), pieces, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(pieces[_i]); + } + if (str == NULL) JUMP_TO_ERROR(); + stack_pointer[-oparg] = str; + stack_pointer += 1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _LIST_EXTEND_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *iterable; + PyObject *list; + iterable = stack_pointer[-1]; + list = stack_pointer[-2 - (oparg-1)]; + PyObject *none_val = _PyList_Extend((PyListObject *)list, iterable); + if (none_val == NULL) { + if (_PyErr_ExceptionMatches(tstate, PyExc_TypeError) && + (Py_TYPE(iterable)->tp_iter == NULL && !PySequence_Check(iterable))) + { + _PyErr_Clear(tstate); + _PyErr_Format(tstate, PyExc_TypeError, + "Value after * must be an iterable, not %.200s", + Py_TYPE(iterable)->tp_name); + } + Py_DECREF(iterable); + if (true) JUMP_TO_ERROR(); + } + assert(Py_IsNone(none_val)); + Py_DECREF(iterable); + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _BUILD_MAP_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **values; + PyObject *map; + values = &stack_pointer[-oparg*2]; + map = _PyDict_FromItems( + values, 2, + values+1, 2, + oparg); + for (int _i = oparg*2; --_i >= 0;) { + Py_DECREF(values[_i]); + } + if (map == NULL) JUMP_TO_ERROR(); + stack_pointer[-oparg*2] = map; + stack_pointer += 1 - oparg*2; + *_stack_pointer = stack_pointer; + return 0; + } + + int _SETUP_ANNOTATIONS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + int err; + PyObject *ann_dict; + if (LOCALS() == NULL) { + _PyErr_Format(tstate, PyExc_SystemError, + "no locals found when setting up annotations"); + if (true) JUMP_TO_ERROR(); + } + /* check if __annotations__ in locals()... */ + if (PyMapping_GetOptionalItem(LOCALS(), &_Py_ID(__annotations__), &ann_dict) < 0) JUMP_TO_ERROR(); + if (ann_dict == NULL) { + ann_dict = PyDict_New(); + if (ann_dict == NULL) JUMP_TO_ERROR(); + err = PyObject_SetItem(LOCALS(), &_Py_ID(__annotations__), + ann_dict); + Py_DECREF(ann_dict); + if (err) JUMP_TO_ERROR(); + } + else { + Py_DECREF(ann_dict); + } + *_stack_pointer = stack_pointer; + return 0; + } + + int _BUILD_CONST_KEY_MAP_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *keys; + PyObject **values; + PyObject *map; + keys = stack_pointer[-1]; + values = &stack_pointer[-1 - oparg]; + assert(PyTuple_CheckExact(keys)); + assert(PyTuple_GET_SIZE(keys) == (Py_ssize_t)oparg); + map = _PyDict_FromItems( + &PyTuple_GET_ITEM(keys, 0), 1, + values, 1, oparg); + for (int _i = oparg; --_i >= 0;) { + Py_DECREF(values[_i]); + } + Py_DECREF(keys); + if (map == NULL) JUMP_TO_ERROR(); + stack_pointer[-1 - oparg] = map; + stack_pointer += -oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _DICT_UPDATE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *update; + PyObject *dict; + update = stack_pointer[-1]; + dict = stack_pointer[-2 - (oparg - 1)]; + if (PyDict_Update(dict, update) < 0) { + if (_PyErr_ExceptionMatches(tstate, PyExc_AttributeError)) { + _PyErr_Format(tstate, PyExc_TypeError, + "'%.200s' object is not a mapping", + Py_TYPE(update)->tp_name); + } + Py_DECREF(update); + if (true) JUMP_TO_ERROR(); + } + Py_DECREF(update); + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _LOAD_SUPER_ATTR_ATTR_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(!(oparg & 1)); + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + attr = _PySuper_Lookup((PyTypeObject *)class, self, name, NULL); + Py_DECREF(global_super); + Py_DECREF(class); + Py_DECREF(self); + if (attr == NULL) JUMP_TO_ERROR(); + stack_pointer[-3] = attr; + stack_pointer += -2; + *_stack_pointer = stack_pointer; + return 0; + } + + int _LOAD_SUPER_ATTR_METHOD_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *self; + PyObject *class; + PyObject *global_super; + PyObject *attr; + PyObject *self_or_null; + self = stack_pointer[-1]; + class = stack_pointer[-2]; + global_super = stack_pointer[-3]; + assert(oparg & 1); + if (global_super != (PyObject *)&PySuper_Type) JUMP_TO_JUMP_TARGET(); + if (!PyType_Check(class)) JUMP_TO_JUMP_TARGET(); + STAT_INC(LOAD_SUPER_ATTR, hit); + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 2); + PyTypeObject *cls = (PyTypeObject *)class; + int method_found = 0; + attr = _PySuper_Lookup(cls, self, name, + Py_TYPE(self)->tp_getattro == PyObject_GenericGetAttr ? &method_found : NULL); + Py_DECREF(global_super); + Py_DECREF(class); + if (attr == NULL) { + Py_DECREF(self); + if (true) JUMP_TO_ERROR(); + } + if (method_found) { + self_or_null = self; // transfer ownership + } else { + Py_DECREF(self); + self_or_null = NULL; + } + stack_pointer[-3] = attr; + stack_pointer[-2] = self_or_null; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _LOAD_ATTR_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *owner; + PyObject *attr; + PyObject *self_or_null = NULL; + owner = stack_pointer[-1]; + PyObject *name = GETITEM(FRAME_CO_NAMES, oparg >> 1); + if (oparg & 1) { + /* Designed to work in tandem with CALL, pushes two values. */ + attr = NULL; + if (_PyObject_GetMethod(owner, name, &attr)) { + /* We can bypass temporary bound method object. + meth is unbound method and obj is self. + meth | self | arg1 | ... | argN + */ + assert(attr != NULL); // No errors on this branch + self_or_null = owner; // Transfer ownership + } + else { + /* meth is not an unbound method (but a regular attr, or + something was returned by a descriptor protocol). Set + the second element of the stack to NULL, to signal + CALL that it's not a method call. + meth | NULL | arg1 | ... | argN + */ + Py_DECREF(owner); + if (attr == NULL) JUMP_TO_ERROR(); + self_or_null = NULL; + } + } + else { + /* Classic, pushes one value. */ + attr = PyObject_GetAttr(owner, name); + Py_DECREF(owner); + if (attr == NULL) JUMP_TO_ERROR(); + } + stack_pointer[-1] = attr; + if (oparg & 1) stack_pointer[0] = self_or_null; + stack_pointer += (oparg & 1); + *_stack_pointer = stack_pointer; + return 0; + } + + int _COMPARE_OP_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + assert((oparg >> 5) <= Py_GE); + res = PyObject_RichCompare(left, right, oparg >> 5); + Py_DECREF(left); + Py_DECREF(right); + if (res == NULL) JUMP_TO_ERROR(); + if (oparg & 16) { + int res_bool = PyObject_IsTrue(res); + Py_DECREF(res); + if (res_bool < 0) JUMP_TO_ERROR(); + res = res_bool ? Py_True : Py_False; + } + stack_pointer[-2] = res; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _COMPARE_OP_FLOAT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + STAT_INC(COMPARE_OP, hit); + double dleft = PyFloat_AS_DOUBLE(left); + double dright = PyFloat_AS_DOUBLE(right); + // 1 if NaN, 2 if <, 4 if >, 8 if ==; this matches low four bits of the oparg + int sign_ish = COMPARISON_BIT(dleft, dright); + _Py_DECREF_SPECIALIZED(left, _PyFloat_ExactDealloc); + _Py_DECREF_SPECIALIZED(right, _PyFloat_ExactDealloc); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _COMPARE_OP_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *right; + PyObject *left; + PyObject *res; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (!_PyLong_IsCompact((PyLongObject *)left)) JUMP_TO_JUMP_TARGET(); + if (!_PyLong_IsCompact((PyLongObject *)right)) JUMP_TO_JUMP_TARGET(); + STAT_INC(COMPARE_OP, hit); + assert(_PyLong_DigitCount((PyLongObject *)left) <= 1 && + _PyLong_DigitCount((PyLongObject *)right) <= 1); + Py_ssize_t ileft = _PyLong_CompactValue((PyLongObject *)left); + Py_ssize_t iright = _PyLong_CompactValue((PyLongObject *)right); + // 2 if <, 4 if >, 8 if ==; this matches the low 4 bits of the oparg + int sign_ish = COMPARISON_BIT(ileft, iright); + _Py_DECREF_SPECIALIZED(left, (destructor)PyObject_Free); + _Py_DECREF_SPECIALIZED(right, (destructor)PyObject_Free); + res = (sign_ish & oparg) ? Py_True : Py_False; + // It's always a bool, so we don't care about oparg & 16. + stack_pointer[-2] = res; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CONTAINS_OP_SET_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (!(PySet_CheckExact(right) || PyFrozenSet_CheckExact(right))) JUMP_TO_JUMP_TARGET(); + STAT_INC(CONTAINS_OP, hit); + // Note: both set and frozenset use the same seq_contains method! + int res = _PySet_Contains((PySetObject *)right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CONTAINS_OP_DICT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *right; + PyObject *left; + PyObject *b; + right = stack_pointer[-1]; + left = stack_pointer[-2]; + if (!PyDict_CheckExact(right)) JUMP_TO_JUMP_TARGET(); + STAT_INC(CONTAINS_OP, hit); + int res = PyDict_Contains(right, left); + Py_DECREF(left); + Py_DECREF(right); + if (res < 0) JUMP_TO_ERROR(); + b = (res ^ oparg) ? Py_True : Py_False; + stack_pointer[-2] = b; + stack_pointer += -1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CHECK_EG_MATCH_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *match_type; + PyObject *exc_value; + PyObject *rest; + PyObject *match; + match_type = stack_pointer[-1]; + exc_value = stack_pointer[-2]; + if (_PyEval_CheckExceptStarTypeValid(tstate, match_type) < 0) { + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (true) JUMP_TO_ERROR(); + } + match = NULL; + rest = NULL; + int res = _PyEval_ExceptionGroupMatch(exc_value, match_type, + &match, &rest); + Py_DECREF(exc_value); + Py_DECREF(match_type); + if (res < 0) JUMP_TO_ERROR(); + assert((match == NULL) == (rest == NULL)); + if (match == NULL) JUMP_TO_ERROR(); + if (!Py_IsNone(match)) { + PyErr_SetHandledException(match); + } + stack_pointer[-2] = rest; + stack_pointer[-1] = match; + *_stack_pointer = stack_pointer; + return 0; + } + + int _MATCH_CLASS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *names; + PyObject *type; + PyObject *subject; + PyObject *attrs; + names = stack_pointer[-1]; + type = stack_pointer[-2]; + subject = stack_pointer[-3]; + // Pop TOS and TOS1. Set TOS to a tuple of attributes on success, or + // None on failure. + assert(PyTuple_CheckExact(names)); + attrs = _PyEval_MatchClass(tstate, subject, type, oparg, names); + Py_DECREF(subject); + Py_DECREF(type); + Py_DECREF(names); + if (attrs) { + assert(PyTuple_CheckExact(attrs)); // Success! + } + else { + if (_PyErr_Occurred(tstate)) JUMP_TO_ERROR(); + // Error! + attrs = Py_None; // Failure! + } + stack_pointer[-3] = attrs; + stack_pointer += -2; + *_stack_pointer = stack_pointer; + return 0; + } + + int _FOR_ITER_TIER_TWO_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *iter; + PyObject *next; + iter = stack_pointer[-1]; + /* before: [iter]; after: [iter, iter()] *or* [] (and jump over END_FOR.) */ + next = (*Py_TYPE(iter)->tp_iternext)(iter); + if (next == NULL) { + if (_PyErr_Occurred(tstate)) { + if (!_PyErr_ExceptionMatches(tstate, PyExc_StopIteration)) { + JUMP_TO_ERROR(); + } + _PyErr_Clear(tstate); + } + /* iterator ended normally */ + Py_DECREF(iter); + STACK_SHRINK(1); + /* The translator sets the deopt target just past END_FOR */ + if (true) JUMP_TO_JUMP_TARGET(); + } + // Common case: no jump, leave it to the code generator + stack_pointer[0] = next; + stack_pointer += 1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _WITH_EXCEPT_START_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *val; + PyObject *lasti; + PyObject *exit_func; + PyObject *res; + val = stack_pointer[-1]; + lasti = stack_pointer[-3]; + exit_func = stack_pointer[-4]; + /* At the top of the stack are 4 values: + - val: TOP = exc_info() + - unused: SECOND = previous exception + - lasti: THIRD = lasti of exception in exc_info() + - exit_func: FOURTH = the context.__exit__ bound method + We call FOURTH(type(TOP), TOP, GetTraceback(TOP)). + Then we push the __exit__ return value. + */ + PyObject *exc, *tb; + assert(val && PyExceptionInstance_Check(val)); + exc = PyExceptionInstance_Class(val); + tb = PyException_GetTraceback(val); + if (tb == NULL) { + tb = Py_None; + } + else { + Py_DECREF(tb); + } + assert(PyLong_Check(lasti)); + (void)lasti; // Shut up compiler warning if asserts are off + PyObject *stack[4] = {NULL, exc, val, tb}; + res = PyObject_Vectorcall(exit_func, stack + 1, + 3 | PY_VECTORCALL_ARGUMENTS_OFFSET, NULL); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[0] = res; + stack_pointer += 1; + *_stack_pointer = stack_pointer; + return 0; + } + + int _INIT_CALL_PY_EXACT_ARGS_0_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer) { int oparg; + PyObject **stack_pointer = *_stack_pointer; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -29,11 +924,13 @@ } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; - return stack_pointer; + *_stack_pointer = stack_pointer; + return 0; } - PyObject ** _INIT_CALL_PY_EXACT_ARGS_1_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer) { + int _INIT_CALL_PY_EXACT_ARGS_1_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer) { int oparg; + PyObject **stack_pointer = *_stack_pointer; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -53,11 +950,13 @@ } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; - return stack_pointer; + *_stack_pointer = stack_pointer; + return 0; } - PyObject ** _INIT_CALL_PY_EXACT_ARGS_2_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer) { + int _INIT_CALL_PY_EXACT_ARGS_2_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer) { int oparg; + PyObject **stack_pointer = *_stack_pointer; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -77,11 +976,13 @@ } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; - return stack_pointer; + *_stack_pointer = stack_pointer; + return 0; } - PyObject ** _INIT_CALL_PY_EXACT_ARGS_3_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer) { + int _INIT_CALL_PY_EXACT_ARGS_3_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer) { int oparg; + PyObject **stack_pointer = *_stack_pointer; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -101,11 +1002,13 @@ } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; - return stack_pointer; + *_stack_pointer = stack_pointer; + return 0; } - PyObject ** _INIT_CALL_PY_EXACT_ARGS_4_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer) { + int _INIT_CALL_PY_EXACT_ARGS_4_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer) { int oparg; + PyObject **stack_pointer = *_stack_pointer; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -125,10 +1028,12 @@ } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; - return stack_pointer; + *_stack_pointer = stack_pointer; + return 0; } - PyObject ** _INIT_CALL_PY_EXACT_ARGS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer, int oparg) { + int _INIT_CALL_PY_EXACT_ARGS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; PyObject **args; PyObject *self_or_null; PyObject *callable; @@ -147,7 +1052,329 @@ } stack_pointer[-2 - oparg] = (PyObject *)new_frame; stack_pointer += -1 - oparg; - return stack_pointer; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_BUILTIN_CLASS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (!PyType_Check(callable)) JUMP_TO_JUMP_TARGET(); + PyTypeObject *tp = (PyTypeObject *)callable; + if (tp->tp_vectorcall == NULL) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + res = tp->tp_vectorcall((PyObject *)tp, args, total_args, NULL); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(tp); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_BUILTIN_O_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_O functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (total_args != 1) JUMP_TO_JUMP_TARGET(); + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != METH_O) JUMP_TO_JUMP_TARGET(); + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + PyObject *arg = args[0]; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + res = _PyCFunction_TrampolineCall(cfunc, PyCFunction_GET_SELF(callable), arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_BUILTIN_FAST_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL functions, without keywords */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + PyCFunction cfunc = PyCFunction_GET_FUNCTION(callable); + /* res = func(self, args, nargs) */ + res = ((PyCFunctionFast)(void(*)(void))cfunc)( + PyCFunction_GET_SELF(callable), + args, + total_args); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_BUILTIN_FAST_WITH_KEYWORDS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + /* Builtin METH_FASTCALL | METH_KEYWORDS functions */ + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (!PyCFunction_CheckExact(callable)) JUMP_TO_JUMP_TARGET(); + if (PyCFunction_GET_FLAGS(callable) != (METH_FASTCALL | METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + /* res = func(self, args, nargs, kwnames) */ + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void)) + PyCFunction_GET_FUNCTION(callable); + res = cfunc(PyCFunction_GET_SELF(callable), args, total_args, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_METHOD_DESCRIPTOR_O_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + if (total_args != 2) JUMP_TO_JUMP_TARGET(); + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_O) JUMP_TO_JUMP_TARGET(); + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + PyObject *arg = args[1]; + PyObject *self = args[0]; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + res = _PyCFunction_TrampolineCall(cfunc, self, arg); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(arg); + Py_DECREF(callable); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != (METH_FASTCALL|METH_KEYWORDS)) JUMP_TO_JUMP_TARGET(); + PyTypeObject *d_type = method->d_common.d_type; + PyObject *self = args[0]; + if (!Py_IS_TYPE(self, d_type)) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + int nargs = total_args - 1; + PyCFunctionFastWithKeywords cfunc = + (PyCFunctionFastWithKeywords)(void(*)(void))meth->ml_meth; + res = cfunc(self, args + 1, nargs, NULL); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Free the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_METHOD_DESCRIPTOR_NOARGS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + assert(oparg == 0 || oparg == 1); + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + if (total_args != 1) JUMP_TO_JUMP_TARGET(); + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + PyMethodDef *meth = method->d_method; + PyObject *self = args[0]; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + if (meth->ml_flags != METH_NOARGS) JUMP_TO_JUMP_TARGET(); + // CPython promises to check all non-vectorcall function calls. + if (tstate->c_recursion_remaining <= 0) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + PyCFunction cfunc = meth->ml_meth; + _Py_EnterRecursiveCallTstateUnchecked(tstate); + res = _PyCFunction_TrampolineCall(cfunc, self, NULL); + _Py_LeaveRecursiveCallTstate(tstate); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + Py_DECREF(self); + Py_DECREF(callable); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _CALL_METHOD_DESCRIPTOR_FAST_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject **args; + PyObject *self_or_null; + PyObject *callable; + PyObject *res; + args = &stack_pointer[-oparg]; + self_or_null = stack_pointer[-1 - oparg]; + callable = stack_pointer[-2 - oparg]; + int total_args = oparg; + if (self_or_null != NULL) { + args--; + total_args++; + } + PyMethodDescrObject *method = (PyMethodDescrObject *)callable; + /* Builtin METH_FASTCALL methods, without keywords */ + if (!Py_IS_TYPE(method, &PyMethodDescr_Type)) JUMP_TO_JUMP_TARGET(); + PyMethodDef *meth = method->d_method; + if (meth->ml_flags != METH_FASTCALL) JUMP_TO_JUMP_TARGET(); + PyObject *self = args[0]; + if (!Py_IS_TYPE(self, method->d_common.d_type)) JUMP_TO_JUMP_TARGET(); + STAT_INC(CALL, hit); + PyCFunctionFast cfunc = + (PyCFunctionFast)(void(*)(void))meth->ml_meth; + int nargs = total_args - 1; + res = cfunc(self, args + 1, nargs); + assert((res != NULL) ^ (_PyErr_Occurred(tstate) != NULL)); + /* Clear the stack of the arguments. */ + for (int i = 0; i < total_args; i++) { + Py_DECREF(args[i]); + } + Py_DECREF(callable); + if (res == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - oparg] = res; + stack_pointer += -1 - oparg; + *_stack_pointer = stack_pointer; + return 0; + } + + int _BUILD_SLICE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg) { + PyObject **stack_pointer = *_stack_pointer; + PyObject *step = NULL; + PyObject *stop; + PyObject *start; + PyObject *slice; + if (oparg == 3) { step = stack_pointer[-((oparg == 3) ? 1 : 0)]; } + stop = stack_pointer[-1 - ((oparg == 3) ? 1 : 0)]; + start = stack_pointer[-2 - ((oparg == 3) ? 1 : 0)]; + slice = PySlice_New(start, stop, step); + Py_DECREF(start); + Py_DECREF(stop); + Py_XDECREF(step); + if (slice == NULL) JUMP_TO_ERROR(); + stack_pointer[-2 - ((oparg == 3) ? 1 : 0)] = slice; + stack_pointer += -1 - ((oparg == 3) ? 1 : 0); + *_stack_pointer = stack_pointer; + return 0; } +#undef JUMP_TO_ERROR +#undef JUMP_TO_JUMP_TARGET +#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target +#define JUMP_TO_ERROR() goto jump_to_error_target #undef TIER_TWO diff --git a/Python/executor_stubs.h b/Python/executor_stubs.h index 38075ff8ad412f..112d74c29d1106 100644 --- a/Python/executor_stubs.h +++ b/Python/executor_stubs.h @@ -3,15 +3,97 @@ // Python/bytecodes.c // Do not edit! - PyObject ** _INIT_CALL_PY_EXACT_ARGS_0_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer); + int _STORE_SLICE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); - PyObject ** _INIT_CALL_PY_EXACT_ARGS_1_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer); + int _BINARY_SUBSCR_LIST_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); - PyObject ** _INIT_CALL_PY_EXACT_ARGS_2_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer); + int _BINARY_SUBSCR_STR_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); - PyObject ** _INIT_CALL_PY_EXACT_ARGS_3_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer); + int _BINARY_SUBSCR_TUPLE_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); - PyObject ** _INIT_CALL_PY_EXACT_ARGS_4_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer); + int _BINARY_SUBSCR_DICT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); - PyObject ** _INIT_CALL_PY_EXACT_ARGS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject **stack_pointer, int oparg); + int _STORE_SUBSCR_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _STORE_SUBSCR_LIST_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _GET_AITER_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _GET_ANEXT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _GET_AWAITABLE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _STORE_NAME_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _LOAD_FROM_DICT_OR_GLOBALS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _LOAD_GLOBAL_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _LOAD_FROM_DICT_OR_DEREF_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _BUILD_STRING_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _LIST_EXTEND_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _BUILD_MAP_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _SETUP_ANNOTATIONS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _BUILD_CONST_KEY_MAP_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _DICT_UPDATE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _LOAD_SUPER_ATTR_ATTR_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _LOAD_SUPER_ATTR_METHOD_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _LOAD_ATTR_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _COMPARE_OP_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _COMPARE_OP_FLOAT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _COMPARE_OP_INT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CONTAINS_OP_SET_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CONTAINS_OP_DICT_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CHECK_EG_MATCH_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _MATCH_CLASS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _FOR_ITER_TIER_TWO_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _WITH_EXCEPT_START_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _INIT_CALL_PY_EXACT_ARGS_0_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer); + + int _INIT_CALL_PY_EXACT_ARGS_1_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer); + + int _INIT_CALL_PY_EXACT_ARGS_2_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer); + + int _INIT_CALL_PY_EXACT_ARGS_3_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer); + + int _INIT_CALL_PY_EXACT_ARGS_4_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer); + + int _INIT_CALL_PY_EXACT_ARGS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_BUILTIN_CLASS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_BUILTIN_O_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_BUILTIN_FAST_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_BUILTIN_FAST_WITH_KEYWORDS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_METHOD_DESCRIPTOR_O_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_METHOD_DESCRIPTOR_NOARGS_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _CALL_METHOD_DESCRIPTOR_FAST_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); + + int _BUILD_SLICE_func(PyThreadState *tstate, _PyInterpreterFrame *frame, PyObject ***_stack_pointer, int oparg); diff --git a/Tools/cases_generator/analyzer.py b/Tools/cases_generator/analyzer.py index 51d1209f73afa4..b731f8314ef5df 100644 --- a/Tools/cases_generator/analyzer.py +++ b/Tools/cases_generator/analyzer.py @@ -5,6 +5,242 @@ from typing import Optional +TEMPLATE_SIZES = [ + (651, "_INIT_CALL_PY_EXACT_ARGS"), + (411, "_CALL_ISINSTANCE"), + (399, "_CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS"), + (383, "_CALL_BUILTIN_FAST_WITH_KEYWORDS"), + (383, "_CALL_BUILTIN_FAST"), + (383, "_CALL_BUILTIN_CLASS"), + (380, "_CALL_METHOD_DESCRIPTOR_O"), + (379, "_CALL_METHOD_DESCRIPTOR_FAST"), + (372, "_CALL_LEN"), + (371, "_BINARY_SUBSCR_STR_INT"), + (369, "_CALL_METHOD_DESCRIPTOR_NOARGS"), + (358, "_LOAD_SUPER_ATTR_METHOD"), + (358, "_COLD_EXIT"), + (349, "_CALL_BUILTIN_O"), + (327, "_CHECK_EG_MATCH"), + (313, "_LOAD_GLOBAL"), + (311, "_BUILD_SLICE"), + (303, "_INIT_CALL_PY_EXACT_ARGS_4"), + (294, "_INIT_CALL_PY_EXACT_ARGS_3"), + (292, "_STORE_SLICE"), + (287, "_GET_AITER"), + (285, "_INIT_CALL_PY_EXACT_ARGS_2"), + (283, "_GET_ANEXT"), + (282, "_LOAD_SUPER_ATTR_ATTR"), + (281, "_BUILD_CONST_KEY_MAP"), + (278, "_LOAD_ATTR_WITH_HINT"), + (277, "_COMPARE_OP_INT"), + (276, "_INIT_CALL_PY_EXACT_ARGS_1"), + (275, "_GET_AWAITABLE"), + (258, "_LOAD_FROM_DICT_OR_GLOBALS"), + (258, "_COMPARE_OP"), + (257, "_INIT_CALL_PY_EXACT_ARGS_0"), + (256, "_LOAD_ATTR"), + (251, "_LIST_EXTEND"), + (245, "_BUILD_MAP"), + (244, "_CONTAINS_OP_SET"), + (240, "_BINARY_SUBSCR_DICT"), + (238, "_MATCH_CLASS"), + (235, "_CONTAINS_OP_DICT"), + (233, "_SETUP_ANNOTATIONS"), + (231, "_BUILD_STRING"), + (231, "_BINARY_OP_SUBTRACT_FLOAT"), + (231, "_BINARY_OP_MULTIPLY_FLOAT"), + (231, "_BINARY_OP_ADD_FLOAT"), + (228, "_STORE_NAME"), + (225, "_STORE_SUBSCR_LIST_INT"), + (221, "_DICT_UPDATE"), + (219, "_COMPARE_OP_FLOAT"), + (212, "_WITH_EXCEPT_START"), + (208, "_BINARY_SUBSCR_LIST_INT"), + (204, "_BINARY_SUBSCR_TUPLE_INT"), + (203, "_LOAD_FROM_DICT_OR_DEREF"), + (203, "_FOR_ITER_TIER_TWO"), + (202, "_STORE_SUBSCR"), + (195, "_LOAD_ATTR_MODULE"), + (194, "_CALL_INTRINSIC_2"), + (194, "_BINARY_SLICE"), + (192, "_CONTAINS_OP"), + (190, "_CHECK_EXC_MATCH"), + (188, "_DICT_MERGE"), + (187, "_BINARY_OP"), + (185, "_GET_YIELD_FROM_ITER"), + (183, "_UNPACK_SEQUENCE_LIST"), + (183, "_STORE_ATTR"), + (176, "_FORMAT_WITH_SPEC"), + (176, "_BINARY_SUBSCR"), + (176, "_BINARY_OP_SUBTRACT_INT"), + (176, "_BINARY_OP_MULTIPLY_INT"), + (176, "_BINARY_OP_ADD_UNICODE"), + (176, "_BINARY_OP_ADD_INT"), + (175, "_DELETE_FAST"), + (174, "_COPY_FREE_VARS"), + (169, "_UNPACK_SEQUENCE_TWO_TUPLE"), + (166, "_DELETE_NAME"), + (165, "_UNPACK_SEQUENCE_TUPLE"), + (165, "_CALL_TUPLE_1"), + (165, "_CALL_STR_1"), + (164, "_INIT_CALL_BOUND_METHOD_EXACT_ARGS"), + (164, "_DELETE_SUBSCR"), + (163, "_COMPARE_OP_STR"), + (155, "_STORE_SUBSCR_DICT"), + (153, "_LOAD_FAST_CHECK"), + (151, "_UNPACK_EX"), + (150, "_UNPACK_SEQUENCE"), + (150, "_STORE_ATTR_INSTANCE_VALUE"), + (147, "_IS_OP"), + (144, "_FORMAT_SIMPLE"), + (142, "_LOAD_ATTR_INSTANCE_VALUE_1"), + (140, "_MAKE_FUNCTION"), + (140, "_LOAD_BUILD_CLASS"), + (140, "_CALL_INTRINSIC_1"), + (138, "_TO_BOOL_STR"), + (138, "_DELETE_GLOBAL"), + (137, "_LOAD_ATTR_SLOT_1"), + (135, "_TO_BOOL_LIST"), + (135, "_SET_UPDATE"), + (134, "_TO_BOOL"), + (133, "_STORE_GLOBAL"), + (133, "_CONVERT_VALUE"), + (132, "_TO_BOOL_INT"), + (131, "_LOAD_ATTR_INSTANCE_VALUE_0"), + (130, "_STORE_ATTR_SLOT"), + (129, "_SET_ADD"), + (129, "_DELETE_ATTR"), + (129, "_CALL_TYPE_1"), + (126, "_LOAD_ATTR_SLOT_0"), + (123, "_DELETE_DEREF"), + (122, "_UNARY_NEGATIVE"), + (122, "_UNARY_INVERT"), + (122, "_GET_ITER"), + (120, "_MAKE_CELL"), + (120, "_BUILD_TUPLE"), + (120, "_BUILD_LIST"), + (118, "_LIST_APPEND"), + (114, "_SET_FUNCTION_ATTRIBUTE"), + (109, "_LOAD_DEREF"), + (104, "_LOAD_LOCALS"), + (100, "_LOAD_GLOBAL_MODULE"), + (100, "_LOAD_GLOBAL_BUILTINS"), + (98, "_EXIT_INIT_CHECK"), + (96, "_LOAD_ATTR_CLASS_1"), + (95, "_ITER_NEXT_RANGE"), + (93, "_LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES"), + (93, "_LOAD_ATTR_NONDESCRIPTOR_NO_DICT"), + (92, "_POP_FRAME"), + (92, "_JUMP_TO_TOP"), + (92, "_GUARD_IS_NOT_NONE_POP"), + (91, "_MAP_ADD"), + (91, "_IS_NONE"), + (90, "_GET_LEN"), + (88, "_CHECK_FUNCTION_EXACT_ARGS"), + (86, "_GUARD_IS_NONE_POP"), + (86, "_CHECK_PERIODIC"), + (85, "_POP_EXCEPT"), + (85, "_LOAD_ATTR_CLASS_0"), + (83, "_MATCH_KEYS"), + (82, "_STORE_DEREF"), + (79, "_STORE_FAST"), + (78, "_GUARD_DORV_VALUES_INST_ATTR_FROM_DICT"), + (78, "_CHECK_MANAGED_OBJECT_HAS_VALUES"), + (76, "_STORE_FAST_7"), + (74, "_END_SEND"), + (74, "_CHECK_STACK_SPACE"), + (70, "_STORE_FAST_6"), + (70, "_STORE_FAST_5"), + (70, "_STORE_FAST_4"), + (70, "_STORE_FAST_3"), + (70, "_STORE_FAST_2"), + (70, "_STORE_FAST_1"), + (70, "_STORE_FAST_0"), + (69, "_SIDE_EXIT"), + (67, "_START_EXECUTOR"), + (64, "_REPLACE_WITH_TRUE"), + (64, "_POP_TOP_LOAD_CONST_INLINE_BORROW"), + (62, "_CHECK_CALL_BOUND_METHOD_EXACT_ARGS"), + (57, "_POP_TOP"), + (55, "_PUSH_FRAME"), + (51, "_MATCH_SEQUENCE"), + (51, "_MATCH_MAPPING"), + (48, "_PUSH_EXC_INFO"), + (48, "_DEOPT"), + (48, "_CHECK_ATTR_MODULE"), + (45, "_GUARD_GLOBALS_VERSION"), + (45, "_GUARD_BUILTINS_VERSION"), + (44, "_ERROR_POP_N"), + (43, "_ITER_NEXT_LIST"), + (43, "_CHECK_ATTR_CLASS"), + (40, "_ITER_NEXT_TUPLE"), + (40, "_GUARD_BOTH_UNICODE"), + (40, "_GUARD_BOTH_INT"), + (40, "_GUARD_BOTH_FLOAT"), + (38, "_GUARD_KEYS_VERSION"), + (36, "_CHECK_VALIDITY_AND_SET_IP"), + (35, "_SWAP"), + (35, "_GUARD_NOT_EXHAUSTED_TUPLE"), + (35, "_GUARD_NOT_EXHAUSTED_LIST"), + (35, "_COPY"), + (34, "_LOAD_CONST"), + (34, "_GUARD_TYPE_VERSION"), + (33, "_LOAD_ATTR_METHOD_WITH_VALUES"), + (33, "_LOAD_ATTR_METHOD_NO_DICT"), + (33, "_LOAD_ATTR_METHOD_LAZY_DICT"), + (30, "_TO_BOOL_NONE"), + (30, "_LOAD_CONST_INLINE_WITH_NULL"), + (30, "_CHECK_FUNCTION"), + (29, "_CHECK_ATTR_WITH_HINT"), + (28, "_RESUME_CHECK"), + (28, "_LOAD_FAST_AND_CLEAR"), + (28, "_CHECK_ATTR_METHOD_LAZY_DICT"), + (27, "_LOAD_FAST"), + (25, "_LOAD_ASSERTION_ERROR"), + (25, "_CHECK_VALIDITY"), + (24, "_UNARY_NOT"), + (23, "_TO_BOOL_BOOL"), + (23, "_ITER_CHECK_TUPLE"), + (23, "_ITER_CHECK_RANGE"), + (23, "_ITER_CHECK_LIST"), + (23, "_GUARD_NOT_EXHAUSTED_RANGE"), + (23, "_GUARD_IS_TRUE_POP"), + (23, "_GUARD_IS_FALSE_POP"), + (22, "_LOAD_FAST_7"), + (22, "_LOAD_CONST_INLINE_BORROW_WITH_NULL"), + (22, "_LOAD_CONST_INLINE"), + (21, "_FATAL_ERROR"), + (20, "_CHECK_PEP_523"), + (19, "_LOAD_FAST_6"), + (19, "_LOAD_FAST_5"), + (19, "_LOAD_FAST_4"), + (19, "_LOAD_FAST_3"), + (19, "_LOAD_FAST_2"), + (19, "_LOAD_FAST_1"), + (19, "_LOAD_FAST_0"), + (16, "_GUARD_DORV_VALUES"), + (14, "_LOAD_CONST_INLINE_BORROW"), + (12, "_INTERNAL_INCREMENT_OPT_COUNTER"), + (11, "_SET_IP"), + (11, "_SAVE_RETURN_OFFSET"), + (11, "_PUSH_NULL"), + (6, "_EXIT_TRACE"), + (0, "_NOP"), +] +TEMPLATE_SIZES = {x[1]: x[0] for x in TEMPLATE_SIZES} + + +BROKEN = { + "_BINARY_OP_MULTIPLY_FLOAT", + "_LOAD_ATTR_WITH_HINT", + "CALL_LEN", + "CALL_ISINSTANCE", + "_COLD_EXIT", + "_BINARY_OP_ADD_FLOAT", + "_BINARY_OP_SUBTRACT_FLOAT", +} + + @dataclass class Properties: escapes: bool @@ -64,7 +300,6 @@ def infallible(self) -> bool: return not self.error_with_pop and not self.error_without_pop - SKIP_PROPERTIES = Properties( escapes=False, error_with_pop=False, @@ -153,7 +388,7 @@ class Uop: _size: int = -1 implicitly_created: bool = False replicated = 0 - replicates : "Uop | None" = None + replicates: "Uop | None" = None def dump(self, indent: str) -> None: print( @@ -296,19 +531,26 @@ def override_error( ) -def convert_stack_item(item: parser.StackEffect, replace_op_arg_1: str | None) -> StackItem: +def convert_stack_item( + item: parser.StackEffect, replace_op_arg_1: str | None +) -> StackItem: cond = item.cond if replace_op_arg_1 and OPARG_AND_1.match(item.cond): cond = replace_op_arg_1 - return StackItem( - item.name, item.type, cond, (item.size or "1") - ) + return StackItem(item.name, item.type, cond, (item.size or "1")) + -def analyze_stack(op: parser.InstDef, replace_op_arg_1: str | None = None) -> StackEffect: +def analyze_stack( + op: parser.InstDef, replace_op_arg_1: str | None = None +) -> StackEffect: inputs: list[StackItem] = [ - convert_stack_item(i, replace_op_arg_1) for i in op.inputs if isinstance(i, parser.StackEffect) + convert_stack_item(i, replace_op_arg_1) + for i in op.inputs + if isinstance(i, parser.StackEffect) + ] + outputs: list[StackItem] = [ + convert_stack_item(i, replace_op_arg_1) for i in op.outputs ] - outputs: list[StackItem] = [convert_stack_item(i, replace_op_arg_1) for i in op.outputs] for input, output in zip(inputs, outputs): if input.name == output.name: input.peek = output.peek = True @@ -333,6 +575,7 @@ def variable_used(node: parser.InstDef, name: str) -> bool: token.kind == "IDENTIFIER" and token.text == name for token in node.tokens ) + def tier_variable(node: parser.InstDef) -> int | None: """Determine whether a tier variable is used in a node.""" for token in node.tokens: @@ -343,6 +586,7 @@ def tier_variable(node: parser.InstDef) -> int | None: return int(token.text[-1]) return None + def has_error_with_pop(op: parser.InstDef) -> bool: return ( variable_used(op, "ERROR_IF") @@ -351,6 +595,7 @@ def has_error_with_pop(op: parser.InstDef) -> bool: or variable_used(op, "resume_with_error") ) + def has_error_without_pop(op: parser.InstDef) -> bool: return ( variable_used(op, "ERROR_NO_POP") @@ -503,8 +748,10 @@ def stack_effect_only_peeks(instr: parser.InstDef) -> bool: for s, other in zip(stack_inputs, instr.outputs) ) + OPARG_AND_1 = re.compile("\\(*oparg *& *1") + def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: for effect in op.inputs: if isinstance(effect, parser.CacheEffect): @@ -520,6 +767,7 @@ def effect_depends_on_oparg_1(op: parser.InstDef) -> bool: return True return False + def compute_properties(op: parser.InstDef) -> Properties: has_free = ( variable_used(op, "PyCell_New") @@ -562,11 +810,16 @@ def compute_properties(op: parser.InstDef) -> Properties: pure="pure" in op.annotations, passthrough=passthrough, tier=tier_variable(op), - stub="stub" in op.annotations, + stub=((TEMPLATE_SIZES.get(op.name, 0) > 200 or TEMPLATE_SIZES.get("_" + op.name, 0) > 200) and op.name not in BROKEN), ) -def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uops: dict[str, Uop]) -> Uop: +def make_uop( + name: str, + op: parser.InstDef, + inputs: list[parser.InputEffect], + uops: dict[str, Uop], +) -> Uop: result = Uop( name=name, context=op.context, @@ -583,7 +836,9 @@ def make_uop(name: str, op: parser.InstDef, inputs: list[parser.InputEffect], uo properties = compute_properties(op) if properties.oparg: # May not need oparg anymore - properties.oparg = any(token.text == "oparg" for token in op.block.tokens) + properties.oparg = any( + token.text == "oparg" for token in op.block.tokens + ) rep = Uop( name=name_x, context=op.context, diff --git a/Tools/cases_generator/tier2_generator.py b/Tools/cases_generator/tier2_generator.py index e38824238e86ae..4acdc753cc2bf9 100644 --- a/Tools/cases_generator/tier2_generator.py +++ b/Tools/cases_generator/tier2_generator.py @@ -213,10 +213,18 @@ def generate_tier2( continue out.emit(f"case {uop.name}: {{\n") if uop.properties.stub: - out.emit(f"stack_pointer = {uop.name}_func(tstate, frame, stack_pointer") + out.emit(f"int result = {uop.name}_func(tstate, frame, &stack_pointer") if uop.properties.const_oparg < 0: out.emit(", CURRENT_OPARG()") out.emit(");\n") + out.emit("switch (result) {\n") + out.emit("case 0:\n") + out.emit("break;\n") + out.emit("case 1:\n") + out.emit("JUMP_TO_ERROR();\n"); + out.emit("case 2:\n") + out.emit("JUMP_TO_JUMP_TARGET();\n"); + out.emit("}\n") stack = None else: declare_variables(uop, out) @@ -239,11 +247,13 @@ def get_stub_signature(name: str, uop: Uop) -> str: args = [ "PyThreadState *tstate", "_PyInterpreterFrame *frame", - "PyObject **stack_pointer" + "PyObject ***_stack_pointer" ] if uop.properties.const_oparg < 0: args.append("int oparg") - return f"PyObject ** {name}_func({', '.join(args)})" + if not name.startswith("_"): + name = "_" + name + return f"int {name}_func({', '.join(args)})" def generate_tier2_stubs( @@ -256,6 +266,12 @@ def generate_tier2_stubs( #error "This file is for Tier 2 only" #endif #define TIER_TWO 2 + +#undef JUMP_TO_ERROR +#define JUMP_TO_ERROR() return 1; + +#undef JUMP_TO_JUMP_TARGET +#define JUMP_TO_JUMP_TARGET() return 2; """ ) @@ -270,7 +286,7 @@ def generate_tier2_stubs( if uop.properties.const_oparg >= 0: out.emit("int oparg;\n") - + out.emit("PyObject **stack_pointer = *_stack_pointer;\n") declare_variables(uop, out) stack = Stack() write_uop(uop, out, stack) @@ -278,9 +294,15 @@ def generate_tier2_stubs( if uop.properties.ends_with_eval_breaker: out.emit("CHECK_EVAL_BREAKER();\n") out.start_line() - out.emit("return stack_pointer;\n") + out.emit("*_stack_pointer = stack_pointer;\n") + out.emit("return 0;\n") out.emit("}\n\n") + outfile.write("#undef JUMP_TO_ERROR\n") + outfile.write("#undef JUMP_TO_JUMP_TARGET\n") + # TODO: Handle these macros better + outfile.write("#define JUMP_TO_JUMP_TARGET() goto jump_to_jump_target\n") + outfile.write("#define JUMP_TO_ERROR() goto jump_to_error_target\n") outfile.write("#undef TIER_TWO\n")