diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index 9f808af38e69df..365756458bba30 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,7 +6,7 @@ ENV WASI_SDK_VERSION=20 ENV WASI_SDK_PATH=/opt/wasi-sdk ENV WASMTIME_HOME=/opt/wasmtime -ENV WASMTIME_VERSION=14.0.4 +ENV WASMTIME_VERSION=18.0.2 ENV WASMTIME_CPU_ARCH=x86_64 RUN dnf -y --nodocs --setopt=install_weak_deps=False install /usr/bin/{blurb,clang,curl,git,ln,tar,xz} 'dnf-command(builddep)' && \ diff --git a/Include/internal/pycore_opcode_metadata.h b/Include/internal/pycore_opcode_metadata.h index ab34366ab1066c..f355a496cf2e70 100644 --- a/Include/internal/pycore_opcode_metadata.h +++ b/Include/internal/pycore_opcode_metadata.h @@ -1320,7 +1320,7 @@ _PyOpcode_macro_expansion[256] = { [STORE_SUBSCR_LIST_INT] = { .nuops = 1, .uops = { { _STORE_SUBSCR_LIST_INT, 0, 0 } } }, [SWAP] = { .nuops = 1, .uops = { { _SWAP, 0, 0 } } }, [TO_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL, 0, 0 } } }, - [TO_BOOL_ALWAYS_TRUE] = { .nuops = 1, .uops = { { _TO_BOOL_ALWAYS_TRUE, 2, 1 } } }, + [TO_BOOL_ALWAYS_TRUE] = { .nuops = 2, .uops = { { _GUARD_TYPE_VERSION, 2, 1 }, { _REPLACE_WITH_TRUE, 0, 0 } } }, [TO_BOOL_BOOL] = { .nuops = 1, .uops = { { _TO_BOOL_BOOL, 0, 0 } } }, [TO_BOOL_INT] = { .nuops = 1, .uops = { { _TO_BOOL_INT, 0, 0 } } }, [TO_BOOL_LIST] = { .nuops = 1, .uops = { { _TO_BOOL_LIST, 0, 0 } } }, diff --git a/Include/internal/pycore_optimizer.h b/Include/internal/pycore_optimizer.h index 8c36534f38030e..75325d6edf42ed 100644 --- a/Include/internal/pycore_optimizer.h +++ b/Include/internal/pycore_optimizer.h @@ -97,12 +97,14 @@ extern _Py_UopsSymbol *_Py_uop_sym_new_type( _Py_UOpsContext *ctx, PyTypeObject *typ); extern _Py_UopsSymbol *_Py_uop_sym_new_const(_Py_UOpsContext *ctx, PyObject *const_val); extern _Py_UopsSymbol *_Py_uop_sym_new_null(_Py_UOpsContext *ctx); +extern bool _Py_uop_sym_has_type(_Py_UopsSymbol *sym); extern bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ); extern bool _Py_uop_sym_set_null(_Py_UopsSymbol *sym); extern bool _Py_uop_sym_set_non_null(_Py_UopsSymbol *sym); extern bool _Py_uop_sym_set_type(_Py_UopsSymbol *sym, PyTypeObject *typ); extern bool _Py_uop_sym_set_const(_Py_UopsSymbol *sym, PyObject *const_val); extern bool _Py_uop_sym_is_bottom(_Py_UopsSymbol *sym); +extern int _Py_uop_sym_truthiness(_Py_UopsSymbol *sym); extern int _Py_uop_abstractcontext_init(_Py_UOpsContext *ctx); diff --git a/Include/internal/pycore_uop_ids.h b/Include/internal/pycore_uop_ids.h index 0274cb6badd42a..3697b284d8db58 100644 --- a/Include/internal/pycore_uop_ids.h +++ b/Include/internal/pycore_uop_ids.h @@ -224,40 +224,40 @@ extern "C" { #define _PUSH_FRAME 408 #define _PUSH_FRAME_INLINEABLE 409 #define _PUSH_NULL PUSH_NULL +#define _REPLACE_WITH_TRUE 410 #define _RESUME_CHECK RESUME_CHECK -#define _SAVE_RETURN_OFFSET 410 -#define _SEND 411 +#define _SAVE_RETURN_OFFSET 411 +#define _SEND 412 #define _SEND_GEN SEND_GEN #define _SETUP_ANNOTATIONS SETUP_ANNOTATIONS #define _SET_ADD SET_ADD #define _SET_FUNCTION_ATTRIBUTE SET_FUNCTION_ATTRIBUTE #define _SET_UPDATE SET_UPDATE -#define _START_EXECUTOR 412 -#define _STORE_ATTR 413 -#define _STORE_ATTR_INSTANCE_VALUE 414 -#define _STORE_ATTR_SLOT 415 +#define _START_EXECUTOR 413 +#define _STORE_ATTR 414 +#define _STORE_ATTR_INSTANCE_VALUE 415 +#define _STORE_ATTR_SLOT 416 #define _STORE_ATTR_WITH_HINT STORE_ATTR_WITH_HINT #define _STORE_DEREF STORE_DEREF -#define _STORE_FAST 416 -#define _STORE_FAST_0 417 -#define _STORE_FAST_1 418 -#define _STORE_FAST_2 419 -#define _STORE_FAST_3 420 -#define _STORE_FAST_4 421 -#define _STORE_FAST_5 422 -#define _STORE_FAST_6 423 -#define _STORE_FAST_7 424 +#define _STORE_FAST 417 +#define _STORE_FAST_0 418 +#define _STORE_FAST_1 419 +#define _STORE_FAST_2 420 +#define _STORE_FAST_3 421 +#define _STORE_FAST_4 422 +#define _STORE_FAST_5 423 +#define _STORE_FAST_6 424 +#define _STORE_FAST_7 425 #define _STORE_FAST_LOAD_FAST STORE_FAST_LOAD_FAST #define _STORE_FAST_STORE_FAST STORE_FAST_STORE_FAST #define _STORE_GLOBAL STORE_GLOBAL #define _STORE_NAME STORE_NAME #define _STORE_SLICE STORE_SLICE -#define _STORE_SUBSCR 425 +#define _STORE_SUBSCR 426 #define _STORE_SUBSCR_DICT STORE_SUBSCR_DICT #define _STORE_SUBSCR_LIST_INT STORE_SUBSCR_LIST_INT #define _SWAP SWAP -#define _TO_BOOL 426 -#define _TO_BOOL_ALWAYS_TRUE TO_BOOL_ALWAYS_TRUE +#define _TO_BOOL 427 #define _TO_BOOL_BOOL TO_BOOL_BOOL #define _TO_BOOL_INT TO_BOOL_INT #define _TO_BOOL_LIST TO_BOOL_LIST @@ -267,12 +267,12 @@ extern "C" { #define _UNARY_NEGATIVE UNARY_NEGATIVE #define _UNARY_NOT UNARY_NOT #define _UNPACK_EX UNPACK_EX -#define _UNPACK_SEQUENCE 427 +#define _UNPACK_SEQUENCE 428 #define _UNPACK_SEQUENCE_LIST UNPACK_SEQUENCE_LIST #define _UNPACK_SEQUENCE_TUPLE UNPACK_SEQUENCE_TUPLE #define _UNPACK_SEQUENCE_TWO_TUPLE UNPACK_SEQUENCE_TWO_TUPLE #define _WITH_EXCEPT_START WITH_EXCEPT_START -#define MAX_UOP_ID 427 +#define MAX_UOP_ID 428 #ifdef __cplusplus } diff --git a/Include/internal/pycore_uop_metadata.h b/Include/internal/pycore_uop_metadata.h index cb8b427c7239f8..12b3049a4b762b 100644 --- a/Include/internal/pycore_uop_metadata.h +++ b/Include/internal/pycore_uop_metadata.h @@ -54,7 +54,7 @@ const uint16_t _PyUop_Flags[MAX_UOP_ID+1] = { [_TO_BOOL_LIST] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_TO_BOOL_NONE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, [_TO_BOOL_STR] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, - [_TO_BOOL_ALWAYS_TRUE] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG, + [_REPLACE_WITH_TRUE] = 0, [_UNARY_INVERT] = HAS_ERROR_FLAG | HAS_ESCAPES_FLAG, [_GUARD_BOTH_INT] = HAS_DEOPT_FLAG | HAS_EXIT_FLAG | HAS_PASSTHROUGH_FLAG, [_BINARY_OP_MULTIPLY_INT] = HAS_ERROR_FLAG | HAS_PURE_FLAG, @@ -438,6 +438,7 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_PUSH_FRAME] = "_PUSH_FRAME", [_PUSH_FRAME_INLINEABLE] = "_PUSH_FRAME_INLINEABLE", [_PUSH_NULL] = "_PUSH_NULL", + [_REPLACE_WITH_TRUE] = "_REPLACE_WITH_TRUE", [_RESUME_CHECK] = "_RESUME_CHECK", [_SAVE_RETURN_OFFSET] = "_SAVE_RETURN_OFFSET", [_SETUP_ANNOTATIONS] = "_SETUP_ANNOTATIONS", @@ -469,7 +470,6 @@ const char *const _PyOpcode_uop_name[MAX_UOP_ID+1] = { [_STORE_SUBSCR_LIST_INT] = "_STORE_SUBSCR_LIST_INT", [_SWAP] = "_SWAP", [_TO_BOOL] = "_TO_BOOL", - [_TO_BOOL_ALWAYS_TRUE] = "_TO_BOOL_ALWAYS_TRUE", [_TO_BOOL_BOOL] = "_TO_BOOL_BOOL", [_TO_BOOL_INT] = "_TO_BOOL_INT", [_TO_BOOL_LIST] = "_TO_BOOL_LIST", diff --git a/Lib/test/libregrtest/main.py b/Lib/test/libregrtest/main.py index 6fbe3d00981ddd..126daca388fd7f 100644 --- a/Lib/test/libregrtest/main.py +++ b/Lib/test/libregrtest/main.py @@ -384,15 +384,10 @@ def run_tests_sequentially(self, runtests) -> None: result = self.run_test(test_name, runtests, tracer) - # Unload the newly imported test modules (best effort - # finalization). To work around gh-115490, don't unload - # test.support.interpreters and its submodules even if they - # weren't loaded before. - keep = "test.support.interpreters" + # Unload the newly imported test modules (best effort finalization) new_modules = [module for module in sys.modules if module not in save_modules and - module.startswith(("test.", "test_")) - and not module.startswith(keep)] + module.startswith(("test.", "test_"))] for module in new_modules: sys.modules.pop(module, None) # Remove the attribute of the parent module. diff --git a/Lib/test/test__xxinterpchannels.py b/Lib/test/test__xxinterpchannels.py index cc2ed7849b0c0f..c5d29bd2dd911f 100644 --- a/Lib/test/test__xxinterpchannels.py +++ b/Lib/test/test__xxinterpchannels.py @@ -18,6 +18,11 @@ channels = import_helper.import_module('_xxinterpchannels') +# Additional tests are found in Lib/test/test_interpreters/test_channels.py. +# New tests should be added there. +# XXX The tests here should be moved there. See the note under LowLevelTests. + + ################################## # helpers diff --git a/Lib/test/test_capi/test_opt.py b/Lib/test/test_capi/test_opt.py index 2e8b6fbb6cc06c..466947c4d4308c 100644 --- a/Lib/test/test_capi/test_opt.py +++ b/Lib/test/test_capi/test_opt.py @@ -331,7 +331,8 @@ def testfunc(a): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_GUARD_IS_NOT_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_not_none(self): def testfunc(a): @@ -347,7 +348,8 @@ def testfunc(a): ex = get_first_executor(testfunc) self.assertIsNotNone(ex) uops = get_opnames(ex) - self.assertIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NONE_POP", uops) + self.assertNotIn("_GUARD_IS_NOT_NONE_POP", uops) def test_pop_jump_if_true(self): def testfunc(n): diff --git a/Lib/test/test_interpreters/test_channels.py b/Lib/test/test_interpreters/test_channels.py index 07e503837bcf75..57204e2776468d 100644 --- a/Lib/test/test_interpreters/test_channels.py +++ b/Lib/test/test_interpreters/test_channels.py @@ -1,3 +1,4 @@ +import importlib import threading from textwrap import dedent import unittest @@ -11,6 +12,24 @@ from .utils import _run_output, TestBase +class LowLevelTests(TestBase): + + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + # Additional tests are found in Lib/test/test__xxinterpchannels.py. + # XXX Those should be either moved to LowLevelTests or eliminated + # in favor of high-level tests in this file. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(channels) + + class TestChannels(TestBase): def test_create(self): diff --git a/Lib/test/test_interpreters/test_queues.py b/Lib/test/test_interpreters/test_queues.py index 905ef035d43feb..0a1fdb41f73166 100644 --- a/Lib/test/test_interpreters/test_queues.py +++ b/Lib/test/test_interpreters/test_queues.py @@ -1,3 +1,4 @@ +import importlib import threading from textwrap import dedent import unittest @@ -20,6 +21,20 @@ def tearDown(self): pass +class LowLevelTests(TestBase): + + # The behaviors in the low-level module is important in as much + # as it is exercised by the high-level module. Therefore the + # most # important testing happens in the high-level tests. + # These low-level tests cover corner cases that are not + # encountered by the high-level module, thus they + # mostly shouldn't matter as much. + + def test_highlevel_reloaded(self): + # See gh-115490 (https://github.com/python/cpython/issues/115490). + importlib.reload(queues) + + class QueueTests(TestBase): def test_create(self): diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index 2969c6e2f98a23..0601b33e79ebb6 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -96,6 +96,11 @@ def imul(a, b): a *= b self.assertRaises((MemoryError, OverflowError), mul, lst, n) self.assertRaises((MemoryError, OverflowError), imul, lst, n) + def test_empty_slice(self): + x = [] + x[:] = x + self.assertEqual(x, []) + def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index dbbbe63e95ad46..912384ab6bfe84 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -5867,6 +5867,12 @@ def foo(a: 'Node[T'): with self.assertRaises(SyntaxError): get_type_hints(foo) + def test_syntax_error_empty_string(self): + for form in [typing.List, typing.Set, typing.Type, typing.Deque]: + with self.subTest(form=form): + with self.assertRaises(SyntaxError): + form[''] + def test_name_error(self): def foo(a: 'Noode[T]'): diff --git a/Lib/typing.py b/Lib/typing.py index 2cb4a714ea3a85..cca9525d632ea5 100644 --- a/Lib/typing.py +++ b/Lib/typing.py @@ -880,7 +880,7 @@ def __init__(self, arg, is_argument=True, module=None, *, is_class=False): # If we do `def f(*args: *Ts)`, then we'll have `arg = '*Ts'`. # Unfortunately, this isn't a valid expression on its own, so we # do the unpacking manually. - if arg[0] == '*': + if arg.startswith('*'): arg_to_compile = f'({arg},)[0]' # E.g. (*Ts,)[0] or (*tuple[int, int],)[0] else: arg_to_compile = arg diff --git a/Misc/NEWS.d/next/Build/2024-03-04-12-43-42.gh-issue-116313.cLLb8S.rst b/Misc/NEWS.d/next/Build/2024-03-04-12-43-42.gh-issue-116313.cLLb8S.rst new file mode 100644 index 00000000000000..61501549060024 --- /dev/null +++ b/Misc/NEWS.d/next/Build/2024-03-04-12-43-42.gh-issue-116313.cLLb8S.rst @@ -0,0 +1 @@ +Get WASI builds to work under wasmtime 18 w/ WASI 0.2/preview2 primitives. diff --git a/Misc/NEWS.d/next/Library/2024-03-05-02-09-18.gh-issue-116325.FmlBYv.rst b/Misc/NEWS.d/next/Library/2024-03-05-02-09-18.gh-issue-116325.FmlBYv.rst new file mode 100644 index 00000000000000..aec4ee79b59cf2 --- /dev/null +++ b/Misc/NEWS.d/next/Library/2024-03-05-02-09-18.gh-issue-116325.FmlBYv.rst @@ -0,0 +1,2 @@ +:mod:`typing`: raise :exc:`SyntaxError` instead of :exc:`AttributeError` +on forward references as empty strings. diff --git a/Modules/_xxinterpchannelsmodule.c b/Modules/_xxinterpchannelsmodule.c index 82d2ae7fc4c963..0ad184a78e8c1a 100644 --- a/Modules/_xxinterpchannelsmodule.c +++ b/Modules/_xxinterpchannelsmodule.c @@ -93,38 +93,6 @@ API.. The module does not create any objects that are shared globally. PyMem_RawFree(VAR) -struct xid_class_registry { - size_t count; -#define MAX_XID_CLASSES 5 - struct { - PyTypeObject *cls; - } added[MAX_XID_CLASSES]; -}; - -static int -register_xid_class(PyTypeObject *cls, crossinterpdatafunc shared, - struct xid_class_registry *classes) -{ - int res = ensure_xid_class(cls, shared); - if (res == 0) { - assert(classes->count < MAX_XID_CLASSES); - // The class has refs elsewhere, so we need to incref here. - classes->added[classes->count].cls = cls; - classes->count += 1; - } - return res; -} - -static void -clear_xid_class_registry(struct xid_class_registry *classes) -{ - while (classes->count > 0) { - classes->count -= 1; - PyTypeObject *cls = classes->added[classes->count].cls; - _PyCrossInterpreterData_UnregisterClass(cls); - } -} - #define XID_IGNORE_EXC 1 #define XID_FREE 2 @@ -223,28 +191,6 @@ add_new_exception(PyObject *mod, const char *name, PyObject *base) #define ADD_NEW_EXCEPTION(MOD, NAME, BASE) \ add_new_exception(MOD, MODULE_NAME_STR "." Py_STRINGIFY(NAME), BASE) -static PyTypeObject * -add_new_type(PyObject *mod, PyType_Spec *spec, crossinterpdatafunc shared, - struct xid_class_registry *classes) -{ - PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( - mod, spec, NULL); - if (cls == NULL) { - return NULL; - } - if (PyModule_AddType(mod, cls) < 0) { - Py_DECREF(cls); - return NULL; - } - if (shared != NULL) { - if (register_xid_class(cls, shared, classes)) { - Py_DECREF(cls); - return NULL; - } - } - return cls; -} - static int wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) { @@ -269,8 +215,6 @@ wait_for_lock(PyThread_type_lock mutex, PY_TIMEOUT_T timeout) /* module state *************************************************************/ typedef struct { - struct xid_class_registry xid_classes; - /* Added at runtime by interpreters module. */ PyTypeObject *send_channel_type; PyTypeObject *recv_channel_type; @@ -332,19 +276,33 @@ traverse_module_state(module_state *state, visitproc visit, void *arg) return 0; } -static int -clear_module_state(module_state *state) +static void +clear_xid_types(module_state *state) { /* external types */ - Py_CLEAR(state->send_channel_type); - Py_CLEAR(state->recv_channel_type); + if (state->send_channel_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + } + if (state->recv_channel_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->recv_channel_type); + Py_CLEAR(state->recv_channel_type); + } /* heap types */ - Py_CLEAR(state->ChannelInfoType); if (state->ChannelIDType != NULL) { (void)_PyCrossInterpreterData_UnregisterClass(state->ChannelIDType); + Py_CLEAR(state->ChannelIDType); } - Py_CLEAR(state->ChannelIDType); +} + +static int +clear_module_state(module_state *state) +{ + clear_xid_types(state); + + /* heap types */ + Py_CLEAR(state->ChannelInfoType); /* exceptions */ Py_CLEAR(state->ChannelError); @@ -2614,6 +2572,25 @@ static PyType_Spec channelid_typespec = { .slots = channelid_typeslots, }; +static PyTypeObject * +add_channelid_type(PyObject *mod) +{ + PyTypeObject *cls = (PyTypeObject *)PyType_FromModuleAndSpec( + mod, &channelid_typespec, NULL); + if (cls == NULL) { + return NULL; + } + if (PyModule_AddType(mod, cls) < 0) { + Py_DECREF(cls); + return NULL; + } + if (ensure_xid_class(cls, _channelid_shared) < 0) { + Py_DECREF(cls); + return NULL; + } + return cls; +} + /* SendChannel and RecvChannel classes */ @@ -2697,21 +2674,29 @@ set_channelend_types(PyObject *mod, PyTypeObject *send, PyTypeObject *recv) if (state == NULL) { return -1; } - struct xid_class_registry *xid_classes = &state->xid_classes; - if (state->send_channel_type != NULL - || state->recv_channel_type != NULL) - { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + // Clear the old values if the .py module was reloaded. + if (state->send_channel_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + } + if (state->recv_channel_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->recv_channel_type); + Py_CLEAR(state->recv_channel_type); } + + // Add and register the types. state->send_channel_type = (PyTypeObject *)Py_NewRef(send); state->recv_channel_type = (PyTypeObject *)Py_NewRef(recv); - - if (register_xid_class(send, _channelend_shared, xid_classes)) { + if (ensure_xid_class(send, _channelend_shared) < 0) { + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } - if (register_xid_class(recv, _channelend_shared, xid_classes)) { + if (ensure_xid_class(recv, _channelend_shared) < 0) { + (void)_PyCrossInterpreterData_UnregisterClass(state->send_channel_type); + Py_CLEAR(state->send_channel_type); + Py_CLEAR(state->recv_channel_type); return -1; } @@ -3294,13 +3279,11 @@ module_exec(PyObject *mod) if (_globals_init() != 0) { return -1; } - struct xid_class_registry *xid_classes = NULL; module_state *state = get_module_state(mod); if (state == NULL) { goto error; } - xid_classes = &state->xid_classes; /* Add exception types */ if (exceptions_init(mod) != 0) { @@ -3319,8 +3302,7 @@ module_exec(PyObject *mod) } // ChannelID - state->ChannelIDType = add_new_type( - mod, &channelid_typespec, _channelid_shared, xid_classes); + state->ChannelIDType = add_channelid_type(mod); if (state->ChannelIDType == NULL) { goto error; } @@ -3332,8 +3314,8 @@ module_exec(PyObject *mod) return 0; error: - if (xid_classes != NULL) { - clear_xid_class_registry(xid_classes); + if (state != NULL) { + clear_xid_types(state); } _globals_fini(); return -1; @@ -3360,9 +3342,6 @@ module_clear(PyObject *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); return 0; @@ -3374,9 +3353,6 @@ module_free(void *mod) module_state *state = get_module_state(mod); assert(state != NULL); - // Before clearing anything, we unregister the various XID types. */ - clear_xid_class_registry(&state->xid_classes); - // Now we clear the module state. clear_module_state(state); diff --git a/Modules/_xxinterpqueuesmodule.c b/Modules/_xxinterpqueuesmodule.c index e35d1699cfea89..1b76b6963ae0f1 100644 --- a/Modules/_xxinterpqueuesmodule.c +++ b/Modules/_xxinterpqueuesmodule.c @@ -169,6 +169,9 @@ static int clear_module_state(module_state *state) { /* external types */ + if (state->queue_type != NULL) { + (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); + } Py_CLEAR(state->queue_type); /* QueueError */ @@ -1078,15 +1081,18 @@ set_external_queue_type(PyObject *module, PyTypeObject *queue_type) { module_state *state = get_module_state(module); + // Clear the old value if the .py module was reloaded. if (state->queue_type != NULL) { - PyErr_SetString(PyExc_TypeError, "already registered"); - return -1; + (void)_PyCrossInterpreterData_UnregisterClass( + state->queue_type); + Py_CLEAR(state->queue_type); } - state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); + // Add and register the new type. if (ensure_xid_class(queue_type, _queueobj_shared) < 0) { return -1; } + state->queue_type = (PyTypeObject *)Py_NewRef(queue_type); return 0; } @@ -1703,10 +1709,6 @@ module_clear(PyObject *mod) { module_state *state = get_module_state(mod); - if (state->queue_type != NULL) { - (void)_PyCrossInterpreterData_UnregisterClass(state->queue_type); - } - // Now we clear the module state. clear_module_state(state); return 0; diff --git a/Objects/dictobject.c b/Objects/dictobject.c index 39778df603f43f..536746ca41eed5 100644 --- a/Objects/dictobject.c +++ b/Objects/dictobject.c @@ -183,9 +183,6 @@ set_values(PyDictObject *mp, PyDictValues *values) _Py_atomic_store_ptr_release(&mp->ma_values, values); } -// Defined until we get QSBR -#define _PyMem_FreeQsbr PyMem_Free - #define LOCK_KEYS(keys) PyMutex_LockFlags(&keys->dk_mutex, _Py_LOCK_DONT_DETACH) #define UNLOCK_KEYS(keys) PyMutex_Unlock(&keys->dk_mutex) @@ -806,7 +803,7 @@ free_keys_object(PyDictKeysObject *keys, bool use_qsbr) { #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeQsbr(keys); + _PyMem_FreeDelayed(keys); return; } #endif @@ -846,7 +843,7 @@ free_values(PyDictValues *values, bool use_qsbr) int prefix_size = DICT_VALUES_SIZE(values); #ifdef Py_GIL_DISABLED if (use_qsbr) { - _PyMem_FreeQsbr(((char *)values)-prefix_size); + _PyMem_FreeDelayed(((char *)values)-prefix_size); return; } #endif @@ -1238,10 +1235,10 @@ _Py_dict_lookup(PyDictObject *mp, PyObject *key, Py_hash_t hash, PyObject **valu return ix; } +#ifdef Py_GIL_DISABLED static inline void ensure_shared_on_read(PyDictObject *mp) { -#ifdef Py_GIL_DISABLED if (!_Py_IsOwnedByCurrentThread((PyObject *)mp) && !IS_DICT_SHARED(mp)) { // The first time we access a dict from a non-owning thread we mark it // as shared. This ensures that a concurrent resize operation will @@ -1253,8 +1250,8 @@ ensure_shared_on_read(PyDictObject *mp) } Py_END_CRITICAL_SECTION(); } -#endif } +#endif static inline void ensure_shared_on_resize(PyDictObject *mp) @@ -6694,7 +6691,7 @@ _PyObject_StoreInstanceAttribute(PyObject *obj, PyDictValues *values, #ifdef Py_GIL_DISABLED // Try a thread-safe lookup to see if the index is already allocated ix = unicodekeys_lookup_unicode_threadsafe(keys, name, hash); - if (ix == DKIX_EMPTY) { + if (ix == DKIX_EMPTY || ix == DKIX_KEY_CHANGED) { // Lock keys and do insert LOCK_KEYS(keys); ix = insert_into_splitdictkeys(keys, name, hash); diff --git a/Objects/listobject.c b/Objects/listobject.c index 87effb1b3a65fa..79d61878c381bc 100644 --- a/Objects/listobject.c +++ b/Objects/listobject.c @@ -515,7 +515,7 @@ list_item(PyObject *aa, Py_ssize_t i) } static PyObject * -list_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) +list_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) { PyListObject *np; PyObject **src, **dest; @@ -559,7 +559,7 @@ PyList_GetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh) else if (ihigh > Py_SIZE(a)) { ihigh = Py_SIZE(a); } - ret = list_slice((PyListObject *)a, ilow, ihigh); + ret = list_slice_lock_held((PyListObject *)a, ilow, ihigh); Py_END_CRITICAL_SECTION(); return ret; } @@ -584,13 +584,13 @@ list_concat_lock_held(PyListObject *a, PyListObject *b) dest = np->ob_item; for (i = 0; i < Py_SIZE(a); i++) { PyObject *v = src[i]; - FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); + dest[i] = Py_NewRef(v); } src = b->ob_item; dest = np->ob_item + Py_SIZE(a); for (i = 0; i < Py_SIZE(b); i++) { PyObject *v = src[i]; - FT_ATOMIC_STORE_PTR_RELAXED(dest[i], Py_NewRef(v)); + dest[i] = Py_NewRef(v); } Py_SET_SIZE(np, size); return (PyObject *)np; @@ -636,7 +636,7 @@ list_repeat_lock_held(PyListObject *a, Py_ssize_t n) _Py_RefcntAdd(elem, n); PyObject **dest_end = dest + output_size; while (dest < dest_end) { - FT_ATOMIC_STORE_PTR_RELAXED(*dest++, elem); + *dest++ = elem; } } else { @@ -644,7 +644,7 @@ list_repeat_lock_held(PyListObject *a, Py_ssize_t n) PyObject **src_end = src + input_size; while (src < src_end) { _Py_RefcntAdd(*src, n); - FT_ATOMIC_STORE_PTR_RELAXED(*dest++, *src++); + *dest++ = *src++; } _Py_memory_repeat((char *)np->ob_item, sizeof(PyObject *)*output_size, @@ -718,7 +718,7 @@ list_clear_slot(PyObject *self) * guaranteed the call cannot fail. */ static int -list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +list_ass_slice_lock_held(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { /* Because [X]DECREF can recursively invoke list operations on this list, we must postpone all [X]DECREF activity until @@ -741,15 +741,6 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) if (v == NULL) n = 0; else { - if (a == b) { - /* Special case "a[i:j] = a" -- copy b first */ - v = list_slice(b, 0, Py_SIZE(b)); - if (v == NULL) - return result; - result = list_ass_slice(a, ilow, ihigh, v); - Py_DECREF(v); - return result; - } v_as_SF = PySequence_Fast(v, "can only assign an iterable"); if(v_as_SF == NULL) goto Error; @@ -823,6 +814,34 @@ list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) #undef b } +static int +list_ass_slice(PyListObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) +{ + int ret; + if (a == (PyListObject *)v) { + Py_BEGIN_CRITICAL_SECTION(a); + Py_ssize_t n = PyList_GET_SIZE(a); + PyObject *copy = list_slice_lock_held(a, 0, n); + if (copy == NULL) { + return -1; + } + ret = list_ass_slice_lock_held(a, ilow, ihigh, copy); + Py_DECREF(copy); + Py_END_CRITICAL_SECTION(); + } + else if (v != NULL && PyList_CheckExact(v)) { + Py_BEGIN_CRITICAL_SECTION2(a, v); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION2(); + } + else { + Py_BEGIN_CRITICAL_SECTION(a); + ret = list_ass_slice_lock_held(a, ilow, ihigh, v); + Py_END_CRITICAL_SECTION(); + } + return ret; +} + int PyList_SetSlice(PyObject *a, Py_ssize_t ilow, Py_ssize_t ihigh, PyObject *v) { @@ -956,7 +975,7 @@ static PyObject * list_copy_impl(PyListObject *self) /*[clinic end generated code: output=ec6b72d6209d418e input=81c54b0c7bb4f73d]*/ { - return list_slice(self, 0, Py_SIZE(self)); + return list_slice_lock_held(self, 0, Py_SIZE(self)); } /*[clinic input] @@ -2884,8 +2903,7 @@ list_remove_impl(PyListObject *self, PyObject *value) int cmp = PyObject_RichCompareBool(obj, value, Py_EQ); Py_DECREF(obj); if (cmp > 0) { - if (list_ass_slice(self, i, i+1, - (PyObject *)NULL) == 0) + if (list_ass_slice_lock_held(self, i, i+1, NULL) == 0) Py_RETURN_NONE; return NULL; } @@ -3085,6 +3103,45 @@ static PySequenceMethods list_as_sequence = { list_inplace_repeat, /* sq_inplace_repeat */ }; +static inline PyObject * +list_slice_step_lock_held(PyListObject *a, Py_ssize_t start, Py_ssize_t step, Py_ssize_t len) +{ + PyListObject *np = (PyListObject *)list_new_prealloc(len); + if (np == NULL) { + return NULL; + } + size_t cur; + Py_ssize_t i; + PyObject **src = a->ob_item; + PyObject **dest = np->ob_item; + for (cur = start, i = 0; i < len; + cur += (size_t)step, i++) { + PyObject *v = src[cur]; + dest[i] = Py_NewRef(v); + } + Py_SET_SIZE(np, len); + return (PyObject *)np; +} + +static PyObject * +list_slice_wrap(PyListObject *aa, Py_ssize_t start, Py_ssize_t stop, Py_ssize_t step) +{ + PyObject *res = NULL; + Py_BEGIN_CRITICAL_SECTION(aa); + Py_ssize_t len = PySlice_AdjustIndices(Py_SIZE(aa), &start, &stop, step); + if (len <= 0) { + res = PyList_New(0); + } + else if (step == 1) { + res = list_slice_lock_held(aa, start, stop); + } + else { + res = list_slice_step_lock_held(aa, start, step, len); + } + Py_END_CRITICAL_SECTION(); + return res; +} + static PyObject * list_subscript(PyObject* _self, PyObject* item) { @@ -3099,38 +3156,11 @@ list_subscript(PyObject* _self, PyObject* item) return list_item((PyObject *)self, i); } else if (PySlice_Check(item)) { - Py_ssize_t start, stop, step, slicelength, i; - size_t cur; - PyObject* result; - PyObject* it; - PyObject **src, **dest; - + Py_ssize_t start, stop, step; if (PySlice_Unpack(item, &start, &stop, &step) < 0) { return NULL; } - slicelength = PySlice_AdjustIndices(Py_SIZE(self), &start, &stop, - step); - - if (slicelength <= 0) { - return PyList_New(0); - } - else if (step == 1) { - return list_slice(self, start, stop); - } - else { - result = list_new_prealloc(slicelength); - if (!result) return NULL; - - src = self->ob_item; - dest = ((PyListObject *)result)->ob_item; - for (cur = start, i = 0; i < slicelength; - cur += (size_t)step, i++) { - it = Py_NewRef(src[cur]); - dest[i] = it; - } - Py_SET_SIZE(result, slicelength); - return result; - } + return list_slice_wrap(self, start, stop, step); } else { PyErr_Format(PyExc_TypeError, @@ -3241,8 +3271,10 @@ list_ass_subscript(PyObject* _self, PyObject* item, PyObject* value) /* protect against a[::-1] = a */ if (self == (PyListObject*)value) { - seq = list_slice((PyListObject*)value, 0, - PyList_GET_SIZE(value)); + Py_BEGIN_CRITICAL_SECTION(value); + seq = list_slice_lock_held((PyListObject*)value, 0, + Py_SIZE(value)); + Py_END_CRITICAL_SECTION(); } else { seq = PySequence_Fast(value, diff --git a/Python/bytecodes.c b/Python/bytecodes.c index d2f104ac5a5666..f95dee2d0e903c 100644 --- a/Python/bytecodes.c +++ b/Python/bytecodes.c @@ -382,15 +382,16 @@ dummy_func( } } - inst(TO_BOOL_ALWAYS_TRUE, (unused/1, version/2, value -- res)) { - // This one is a bit weird, because we expect *some* failures: - assert(version); - EXIT_IF(Py_TYPE(value)->tp_version_tag != version); - STAT_INC(TO_BOOL, hit); - DECREF_INPUTS(); + op(_REPLACE_WITH_TRUE, (value -- res)) { + Py_DECREF(value); res = Py_True; } + macro(TO_BOOL_ALWAYS_TRUE) = + unused/1 + + _GUARD_TYPE_VERSION + + _REPLACE_WITH_TRUE; + inst(UNARY_INVERT, (value -- res)) { res = PyNumber_Invert(value); DECREF_INPUTS(); diff --git a/Python/executor_cases.c.h b/Python/executor_cases.c.h index 0121479ac6e4b7..c3eb7184a9f8ab 100644 --- a/Python/executor_cases.c.h +++ b/Python/executor_cases.c.h @@ -383,15 +383,10 @@ break; } - case _TO_BOOL_ALWAYS_TRUE: { + case _REPLACE_WITH_TRUE: { PyObject *value; PyObject *res; value = stack_pointer[-1]; - uint32_t version = (uint32_t)CURRENT_OPERAND(); - // This one is a bit weird, because we expect *some* failures: - assert(version); - if (Py_TYPE(value)->tp_version_tag != version) goto side_exit; - STAT_INC(TO_BOOL, hit); Py_DECREF(value); res = Py_True; stack_pointer[-1] = res; diff --git a/Python/generated_cases.c.h b/Python/generated_cases.c.h index e612c9e4c37632..4947c919b4ca20 100644 --- a/Python/generated_cases.c.h +++ b/Python/generated_cases.c.h @@ -5580,17 +5580,24 @@ next_instr += 4; INSTRUCTION_STATS(TO_BOOL_ALWAYS_TRUE); static_assert(INLINE_CACHE_ENTRIES_TO_BOOL == 3, "incorrect cache size"); + PyObject *owner; PyObject *value; PyObject *res; /* Skip 1 cache entry */ - value = stack_pointer[-1]; - uint32_t version = read_u32(&this_instr[2].cache); - // This one is a bit weird, because we expect *some* failures: - assert(version); - DEOPT_IF(Py_TYPE(value)->tp_version_tag != version, TO_BOOL); - STAT_INC(TO_BOOL, hit); - Py_DECREF(value); - res = Py_True; + // _GUARD_TYPE_VERSION + owner = stack_pointer[-1]; + { + uint32_t type_version = read_u32(&this_instr[2].cache); + PyTypeObject *tp = Py_TYPE(owner); + assert(type_version != 0); + DEOPT_IF(tp->tp_version_tag != type_version, TO_BOOL); + } + // _REPLACE_WITH_TRUE + value = owner; + { + Py_DECREF(value); + res = Py_True; + } stack_pointer[-1] = res; DISPATCH(); } diff --git a/Python/optimizer_analysis.c b/Python/optimizer_analysis.c index 6e75f77d8e4deb..392b49d8862813 100644 --- a/Python/optimizer_analysis.c +++ b/Python/optimizer_analysis.c @@ -299,15 +299,48 @@ real_localsplus_idx(_Py_UOpsContext *ctx, int oparg) #define sym_is_null _Py_uop_sym_is_null #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null +#define sym_has_type _Py_uop_sym_has_type #define sym_matches_type _Py_uop_sym_matches_type #define sym_set_null _Py_uop_sym_set_null #define sym_set_non_null _Py_uop_sym_set_non_null #define sym_set_type _Py_uop_sym_set_type #define sym_set_const _Py_uop_sym_set_const #define sym_is_bottom _Py_uop_sym_is_bottom +#define sym_truthiness _Py_uop_sym_truthiness #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop +static int +optimize_to_bool( + _PyUOpInstruction *this_instr, + _Py_UOpsContext *ctx, + _Py_UopsSymbol *value, + _Py_UopsSymbol **result_ptr) +{ + if (sym_matches_type(value, &PyBool_Type)) { + REPLACE_OP(this_instr, _NOP, 0, 0); + *result_ptr = value; + return 1; + } + int truthiness = sym_truthiness(value); + if (truthiness >= 0) { + PyObject *load = truthiness ? Py_True : Py_False; + REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); + *result_ptr = sym_new_const(ctx, load); + return 1; + } + return 0; +} + +static void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) +{ + REPLACE_OP(this_instr, _POP_TOP, 0, 0); + if (exit) { + REPLACE_OP((this_instr+1), _EXIT_TRACE, 0, 0); + this_instr[1].target = this_instr->target; + } +} /* 1 for success, 0 for not ready, cannot error at the moment. */ static int @@ -432,6 +465,9 @@ remove_unneeded_uops(_PyUOpInstruction *buffer, int buffer_size) last->opcode = _NOP; buffer[pc].opcode = NOP; } + if (last->opcode == _REPLACE_WITH_TRUE) { + last->opcode = _NOP; + } break; } case _JUMP_TO_TOP: diff --git a/Python/optimizer_bytecodes.c b/Python/optimizer_bytecodes.c index 96af5794b5578c..7c1da58172e1ff 100644 --- a/Python/optimizer_bytecodes.c +++ b/Python/optimizer_bytecodes.c @@ -21,6 +21,7 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define sym_new_const _Py_uop_sym_new_const #define sym_new_null _Py_uop_sym_new_null #define sym_matches_type _Py_uop_sym_matches_type +#define sym_has_type _Py_uop_sym_has_type #define sym_set_null _Py_uop_sym_set_null #define sym_set_non_null _Py_uop_sym_set_non_null #define sym_set_type _Py_uop_sym_set_type @@ -29,6 +30,16 @@ typedef struct _Py_UOpsAbstractFrame _Py_UOpsAbstractFrame; #define frame_new _Py_uop_frame_new #define frame_pop _Py_uop_frame_pop +extern int +optimize_to_bool( + _PyUOpInstruction *this_instr, + _Py_UOpsContext *ctx, + _Py_UopsSymbol *value, + _Py_UopsSymbol **result_ptr); + +extern void +eliminate_pop_guard(_PyUOpInstruction *this_instr, bool exit) + static int dummy_func(void) { @@ -274,63 +285,72 @@ dummy_func(void) { } op(_TO_BOOL, (value -- res)) { - (void)value; - res = sym_new_type(ctx, &PyBool_Type); - OUT_OF_SPACE_IF_NULL(res); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); + } + else { + res = sym_new_type(ctx, &PyBool_Type); + OUT_OF_SPACE_IF_NULL(res); + } } - op(_TO_BOOL_BOOL, (value -- value)) { - if (sym_matches_type(value, &PyBool_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + op(_TO_BOOL_BOOL, (value -- res)) { + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); } else { if(!sym_set_type(value, &PyBool_Type)) { goto hit_bottom; } + res = value; } } op(_TO_BOOL_INT, (value -- res)) { - if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { - PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) - ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); } else { + if(!sym_set_type(value, &PyLong_Type)) { + goto hit_bottom; + } OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); } - if(!sym_set_type(value, &PyLong_Type)) { - goto hit_bottom; - } } op(_TO_BOOL_LIST, (value -- res)) { - if(!sym_set_type(value, &PyList_Type)) { - goto hit_bottom; + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); + } + else { + if(!sym_set_type(value, &PyList_Type)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); } op(_TO_BOOL_NONE, (value -- res)) { - if (sym_get_const(value) == Py_None) { - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); + } + else { + if (!sym_set_const(value, Py_None)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); } - sym_set_const(value, Py_None); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); } op(_TO_BOOL_STR, (value -- res)) { - if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { - PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); } else { OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); - } - if(!sym_set_type(value, &PyUnicode_Type)) { - goto hit_bottom; + if(!sym_set_type(value, &PyUnicode_Type)) { + goto hit_bottom; + } } } @@ -571,7 +591,45 @@ dummy_func(void) { (void)iter; } + op(_GUARD_IS_TRUE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_True); + } + } + op(_GUARD_IS_FALSE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_False); + } + } + + op(_GUARD_IS_NONE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, true); + } + } + + op(_GUARD_IS_NOT_NONE_POP, (flag -- )) { + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, false); + } + } // END BYTECODES // diff --git a/Python/optimizer_cases.c.h b/Python/optimizer_cases.c.h index b777462171ead7..2543cc750025c6 100644 --- a/Python/optimizer_cases.c.h +++ b/Python/optimizer_cases.c.h @@ -110,24 +110,31 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - (void)value; - res = sym_new_type(ctx, &PyBool_Type); - OUT_OF_SPACE_IF_NULL(res); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); + } + else { + res = sym_new_type(ctx, &PyBool_Type); + OUT_OF_SPACE_IF_NULL(res); + } stack_pointer[-1] = res; break; } case _TO_BOOL_BOOL: { _Py_UopsSymbol *value; + _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (sym_matches_type(value, &PyBool_Type)) { - REPLACE_OP(this_instr, _NOP, 0, 0); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); } else { if(!sym_set_type(value, &PyBool_Type)) { goto hit_bottom; } + res = value; } + stack_pointer[-1] = res; break; } @@ -135,18 +142,15 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (sym_is_const(value) && sym_matches_type(value, &PyLong_Type)) { - PyObject *load = _PyLong_IsZero((PyLongObject *)sym_get_const(value)) - ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); } else { + if(!sym_set_type(value, &PyLong_Type)) { + goto hit_bottom; + } OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); } - if(!sym_set_type(value, &PyLong_Type)) { - goto hit_bottom; - } stack_pointer[-1] = res; break; } @@ -155,10 +159,15 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if(!sym_set_type(value, &PyList_Type)) { - goto hit_bottom; + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); + } + else { + if(!sym_set_type(value, &PyList_Type)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); } - OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); stack_pointer[-1] = res; break; } @@ -167,11 +176,15 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (sym_get_const(value) == Py_None) { - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)Py_False); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); + } + else { + if (!sym_set_const(value, Py_None)) { + goto hit_bottom; + } + OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); } - sym_set_const(value, Py_None); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, Py_False)); stack_pointer[-1] = res; break; } @@ -180,22 +193,20 @@ _Py_UopsSymbol *value; _Py_UopsSymbol *res; value = stack_pointer[-1]; - if (sym_is_const(value) && sym_matches_type(value, &PyUnicode_Type)) { - PyObject *load = sym_get_const(value) == &_Py_STR(empty) ? Py_False : Py_True; - REPLACE_OP(this_instr, _POP_TOP_LOAD_CONST_INLINE_BORROW, 0, (uintptr_t)load); - OUT_OF_SPACE_IF_NULL(res = sym_new_const(ctx, load)); + if (optimize_to_bool(this_instr, ctx, value, &res)) { + OUT_OF_SPACE_IF_NULL(res); } else { OUT_OF_SPACE_IF_NULL(res = sym_new_type(ctx, &PyBool_Type)); - } - if(!sym_set_type(value, &PyUnicode_Type)) { - goto hit_bottom; + if(!sym_set_type(value, &PyUnicode_Type)) { + goto hit_bottom; + } } stack_pointer[-1] = res; break; } - case _TO_BOOL_ALWAYS_TRUE: { + case _REPLACE_WITH_TRUE: { _Py_UopsSymbol *res; res = sym_new_not_null(ctx); if (res == NULL) goto out_of_space; @@ -1821,21 +1832,57 @@ /* _INSTRUMENTED_POP_JUMP_IF_NOT_NONE is not a viable micro-op for tier 2 */ case _GUARD_IS_TRUE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_True); + } stack_pointer += -1; break; } case _GUARD_IS_FALSE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, value != Py_False); + } stack_pointer += -1; break; } case _GUARD_IS_NONE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, !Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, true); + } stack_pointer += -1; break; } case _GUARD_IS_NOT_NONE_POP: { + _Py_UopsSymbol *flag; + flag = stack_pointer[-1]; + if (sym_is_const(flag)) { + PyObject *value = sym_get_const(flag); + assert(value != NULL); + eliminate_pop_guard(this_instr, Py_IsNone(value)); + } + else if (sym_has_type(flag)) { + assert(!sym_matches_type(flag, &_PyNone_Type)); + eliminate_pop_guard(this_instr, false); + } stack_pointer += -1; break; } diff --git a/Python/optimizer_symbols.c b/Python/optimizer_symbols.c index 280c10da028eb8..c7903231cf30a8 100644 --- a/Python/optimizer_symbols.c +++ b/Python/optimizer_symbols.c @@ -4,6 +4,7 @@ #include "cpython/optimizer.h" #include "pycore_code.h" #include "pycore_frame.h" +#include "pycore_long.h" #include "pycore_optimizer.h" #include @@ -230,6 +231,15 @@ _Py_uop_sym_new_null(_Py_UOpsContext *ctx) return null_sym; } +bool +_Py_uop_sym_has_type(_Py_UopsSymbol *sym) +{ + if (_Py_uop_sym_is_bottom(sym)) { + return false; + } + return sym->typ != NULL; +} + bool _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) { @@ -240,6 +250,40 @@ _Py_uop_sym_matches_type(_Py_UopsSymbol *sym, PyTypeObject *typ) return sym->typ == typ; } +int +_Py_uop_sym_truthiness(_Py_UopsSymbol *sym) +{ + /* There are some non-constant values for + * which `bool(val)` always evaluates to + * True or False, such as tuples with known + * length, but unknown contents, or bound-methods. + * This function will need updating + * should we support those values. + */ + if (_Py_uop_sym_is_bottom(sym)) { + return -1; + } + if (!_Py_uop_sym_is_const(sym)) { + return -1; + } + PyObject *value = _Py_uop_sym_get_const(sym); + if (value == Py_None) { + return 0; + } + /* Only handle a few known safe types */ + PyTypeObject *tp = Py_TYPE(value); + if (tp == &PyLong_Type) { + return !_PyLong_IsZero((PyLongObject *)value); + } + if (tp == &PyUnicode_Type) { + return value != &_Py_STR(empty); + } + if (tp == &PyBool_Type) { + return value == Py_True; + } + return -1; +} + // 0 on success, -1 on error. _Py_UOpsAbstractFrame * _Py_uop_frame_new( @@ -421,6 +465,7 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) goto fail; } _Py_uop_sym_set_const(sym, val_42); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 1, "bool(42) is not True"); TEST_PREDICATE(!_Py_uop_sym_is_null(sym), "42 is NULL"); TEST_PREDICATE(_Py_uop_sym_is_not_null(sym), "42 isn't not NULL"); TEST_PREDICATE(_Py_uop_sym_matches_type(sym, &PyLong_Type), "42 isn't an int"); @@ -444,6 +489,14 @@ _Py_uop_symbols_test(PyObject *Py_UNUSED(self), PyObject *Py_UNUSED(ignored)) _Py_uop_sym_set_const(sym, val_43); // Should make it bottom TEST_PREDICATE(_Py_uop_sym_is_bottom(sym), "(42 and 43) isn't bottom"); + + sym = _Py_uop_sym_new_const(ctx, Py_None); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(None) is not False"); + sym = _Py_uop_sym_new_const(ctx, Py_False); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(False) is not False"); + sym = _Py_uop_sym_new_const(ctx, PyLong_FromLong(0)); + TEST_PREDICATE(_Py_uop_sym_truthiness(sym) == 0, "bool(0) is not False"); + _Py_uop_abstractcontext_fini(ctx); Py_DECREF(val_42); Py_DECREF(val_43); diff --git a/Python/sysmodule.c b/Python/sysmodule.c index 1bfd031fdd26b2..a4161da02980a7 100644 --- a/Python/sysmodule.c +++ b/Python/sysmodule.c @@ -1498,31 +1498,33 @@ get_hash_info(PyThreadState *tstate) int field = 0; PyHash_FuncDef *hashfunc; hash_info = PyStructSequence_New(&Hash_InfoType); - if (hash_info == NULL) - return NULL; - hashfunc = PyHash_GetFuncDef(); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(8*sizeof(Py_hash_t))); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromSsize_t(_PyHASH_MODULUS)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(_PyHASH_INF)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(0)); // This is no longer used - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(_PyHASH_IMAG)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyUnicode_FromString(hashfunc->name)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(hashfunc->hash_bits)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(hashfunc->seed_bits)); - PyStructSequence_SET_ITEM(hash_info, field++, - PyLong_FromLong(Py_HASH_CUTOFF)); - if (_PyErr_Occurred(tstate)) { - Py_CLEAR(hash_info); + if (hash_info == NULL) { return NULL; } + hashfunc = PyHash_GetFuncDef(); + +#define SET_HASH_INFO_ITEM(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + Py_CLEAR(hash_info); \ + return NULL; \ + } \ + PyStructSequence_SET_ITEM(hash_info, field++, item); \ + } while(0) + + SET_HASH_INFO_ITEM(PyLong_FromLong(8 * sizeof(Py_hash_t))); + SET_HASH_INFO_ITEM(PyLong_FromSsize_t(_PyHASH_MODULUS)); + SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_INF)); + SET_HASH_INFO_ITEM(PyLong_FromLong(0)); // This is no longer used + SET_HASH_INFO_ITEM(PyLong_FromLong(_PyHASH_IMAG)); + SET_HASH_INFO_ITEM(PyUnicode_FromString(hashfunc->name)); + SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->hash_bits)); + SET_HASH_INFO_ITEM(PyLong_FromLong(hashfunc->seed_bits)); + SET_HASH_INFO_ITEM(PyLong_FromLong(Py_HASH_CUTOFF)); + +#undef SET_HASH_INFO_ITEM + return hash_info; } /*[clinic input] @@ -1641,12 +1643,13 @@ sys_getwindowsversion_impl(PyObject *module) int pos = 0; OSVERSIONINFOEXW ver; - version = PyObject_GetAttrString(module, "_cached_windows_version"); + if (PyObject_GetOptionalAttrString(module, "_cached_windows_version", &version) < 0) { + return NULL; + }; if (version && PyObject_TypeCheck(version, &WindowsVersionType)) { return version; } Py_XDECREF(version); - PyErr_Clear(); ver.dwOSVersionInfoSize = sizeof(ver); if (!GetVersionExW((OSVERSIONINFOW*) &ver)) @@ -1656,15 +1659,24 @@ sys_getwindowsversion_impl(PyObject *module) if (version == NULL) return NULL; - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMajorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwMinorVersion)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwBuildNumber)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.dwPlatformId)); - PyStructSequence_SET_ITEM(version, pos++, PyUnicode_FromWideChar(ver.szCSDVersion, -1)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMajor)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wServicePackMinor)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wSuiteMask)); - PyStructSequence_SET_ITEM(version, pos++, PyLong_FromLong(ver.wProductType)); +#define SET_VERSION_INFO(CALL) \ + do { \ + PyObject *item = (CALL); \ + if (item == NULL) { \ + goto error; \ + } \ + PyStructSequence_SET_ITEM(version, pos++, item); \ + } while(0) + + SET_VERSION_INFO(PyLong_FromLong(ver.dwMajorVersion)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwMinorVersion)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwBuildNumber)); + SET_VERSION_INFO(PyLong_FromLong(ver.dwPlatformId)); + SET_VERSION_INFO(PyUnicode_FromWideChar(ver.szCSDVersion, -1)); + SET_VERSION_INFO(PyLong_FromLong(ver.wServicePackMajor)); + SET_VERSION_INFO(PyLong_FromLong(ver.wServicePackMinor)); + SET_VERSION_INFO(PyLong_FromLong(ver.wSuiteMask)); + SET_VERSION_INFO(PyLong_FromLong(ver.wProductType)); // GetVersion will lie if we are running in a compatibility mode. // We need to read the version info from a system file resource @@ -1672,6 +1684,10 @@ sys_getwindowsversion_impl(PyObject *module) // just return whatever GetVersion said. PyObject *realVersion = _sys_getwindowsversion_from_kernel32(); if (!realVersion) { + if (!PyErr_ExceptionMatches(PyExc_WindowsError)) { + return NULL; + } + PyErr_Clear(); realVersion = Py_BuildValue("(kkk)", ver.dwMajorVersion, @@ -1680,21 +1696,19 @@ sys_getwindowsversion_impl(PyObject *module) ); } - if (realVersion) { - PyStructSequence_SET_ITEM(version, pos++, realVersion); - } + SET_VERSION_INFO(realVersion); - if (PyErr_Occurred()) { - Py_DECREF(version); - return NULL; - } +#undef SET_VERSION_INFO if (PyObject_SetAttrString(module, "_cached_windows_version", version) < 0) { - Py_DECREF(version); - return NULL; + goto error; } return version; + +error: + Py_DECREF(version); + return NULL; } #pragma warning(pop) diff --git a/Tools/c-analyzer/README b/Tools/c-analyzer/README index 86bf1e77e0bfea..41ea13280347a7 100644 --- a/Tools/c-analyzer/README +++ b/Tools/c-analyzer/README @@ -11,9 +11,8 @@ falls into one of several categories: * module state * Python runtime state -The ignored-globals.txt file is organized similarly. Of the different -categories, the last two are problematic and generally should not exist -in the codebase. +Of the different categories, the last two are problematic and +generally should not exist in the codebase. Globals that hold module state (i.e. in Modules/*.c) cause problems when multiple interpreters are in use. For more info, see PEP 3121, @@ -42,4 +41,3 @@ You can also use the more generic tool: If it reports any globals then they should be resolved. If the globals are runtime state then they should be folded into _PyRuntimeState. -Otherwise they should be added to ignored-globals.txt. diff --git a/Tools/wasm/wasi.py b/Tools/wasm/wasi.py index 1e75db5c7b8329..efb005e53ab989 100644 --- a/Tools/wasm/wasi.py +++ b/Tools/wasm/wasi.py @@ -283,8 +283,10 @@ def main(): # The 8388608 value comes from `ulimit -s` under Linux # which equates to 8291 KiB. "--wasm max-wasm-stack=8388608 " - # Enable thread support. - "--wasm threads=y --wasi threads=y " + # Use WASI 0.2 primitives. + "--wasi preview2 " + # Enable thread support; causes use of preview1. + #"--wasm threads=y --wasi threads=y " # Map the checkout to / to load the stdlib from /Lib. "--dir {HOST_DIR}::{GUEST_DIR} " # Set PYTHONPATH to the sysconfig data. diff --git a/Tools/wasm/wasm_build.py b/Tools/wasm/wasm_build.py index c0b9999a5dad03..47a0abb8b5feef 100755 --- a/Tools/wasm/wasm_build.py +++ b/Tools/wasm/wasm_build.py @@ -329,8 +329,10 @@ def _check_wasi() -> None: # workaround for https://github.com/python/cpython/issues/95952 "HOSTRUNNER": ( "wasmtime run " - "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib " - "--mapdir /::{srcdir} --" + "--wasm max-wasm-stack=8388608 " + "--wasi preview2 " + "--dir {srcdir}::/ " + "--env PYTHONPATH=/{relbuilddir}/build/lib.wasi-wasm32-{version}:/Lib" ), "PATH": [WASI_SDK_PATH / "bin", os.environ["PATH"]], }, diff --git a/configure b/configure index 4a980fea453697..c758749cd371ed 100755 --- a/configure +++ b/configure @@ -7655,7 +7655,7 @@ then : fi ;; #( WASI/*) : - HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --' ;; #( + HOSTRUNNER='wasmtime run --wasm max-wasm-stack=8388608 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/' ;; #( *) : HOSTRUNNER='' ;; diff --git a/configure.ac b/configure.ac index 103c24962b7b42..80d0a7edc7cad4 100644 --- a/configure.ac +++ b/configure.ac @@ -1537,7 +1537,7 @@ then dnl TODO: support other WASI runtimes dnl wasmtime starts the proces with "/" as CWD. For OOT builds add the dnl directory containing _sysconfigdata to PYTHONPATH. - [WASI/*], [HOSTRUNNER='wasmtime run --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --mapdir /::$(srcdir) --'], + [WASI/*], [HOSTRUNNER='wasmtime run --wasm max-wasm-stack=8388608 --wasi preview2 --env PYTHONPATH=/$(shell realpath --relative-to $(abs_srcdir) $(abs_builddir))/$(shell cat pybuilddir.txt):/Lib --dir $(srcdir)::/'], [HOSTRUNNER=''] ) fi