Skip to content

Commit

Permalink
pythongh-119213: Be More Careful About _PyArg_Parser.kwtuple Across I…
Browse files Browse the repository at this point in the history
…nterpreters (pythongh-119331)

_PyArg_Parser holds static global data generated for modules by Argument Clinic.  The _PyArg_Parser.kwtuple field is a tuple object, even though it's stored within a static global.  In some cases the tuple is statically allocated and thus it's okay that it gets shared by multiple interpreters.  However, in other cases the tuple is set lazily, allocated from the heap using the active interprepreter at the point the tuple is needed.

This is a problem once that interpreter is destroyed since _PyArg_Parser.kwtuple becomes at dangling pointer, leading to crashes.  It isn't a problem if the tuple is allocated under the main interpreter, since its lifetime is bound to the lifetime of the runtime.  The solution here is to temporarily switch to the main interpreter.  The alternative would be to always statically allocate the tuple.

This change also fixes a bug where only the most recent parser was added to the global linked list.
(cherry picked from commit 8186500)

Co-authored-by: Eric Snow <ericsnowcurrently@gmail.com>
  • Loading branch information
ericsnowcurrently authored and miss-islington committed May 22, 2024
1 parent ac91636 commit 05a00aa
Show file tree
Hide file tree
Showing 10 changed files with 144 additions and 3 deletions.
1 change: 1 addition & 0 deletions Include/internal/pycore_global_objects_fini_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Include/internal/pycore_global_strings.h
Original file line number Diff line number Diff line change
Expand Up @@ -710,6 +710,7 @@ struct _Py_global_strings {
STRUCT_FOR_ID(sort)
STRUCT_FOR_ID(source)
STRUCT_FOR_ID(source_traceback)
STRUCT_FOR_ID(spam)
STRUCT_FOR_ID(src)
STRUCT_FOR_ID(src_dir_fd)
STRUCT_FOR_ID(stacklevel)
Expand Down
1 change: 1 addition & 0 deletions Include/internal/pycore_runtime_init_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

3 changes: 3 additions & 0 deletions Include/internal/pycore_unicodeobject_generated.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

33 changes: 33 additions & 0 deletions Lib/test/test_capi/test_getargs.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,17 @@
import sys
from test import support
from test.support import import_helper
from test.support import script_helper
from test.support import warnings_helper
# Skip this test if the _testcapi module isn't available.
_testcapi = import_helper.import_module('_testcapi')
from _testcapi import getargs_keywords, getargs_keyword_only

try:
import _testinternalcapi
except ImportError:
_testinternalcapi = NULL

# > How about the following counterproposal. This also changes some of
# > the other format codes to be a little more regular.
# >
Expand Down Expand Up @@ -1346,6 +1352,33 @@ def test_nested_tuple(self):
"argument 1 must be sequence of length 1, not 0"):
parse(((),), {}, '(' + f + ')', ['a'])

@unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi')
def test_gh_119213(self):
rc, out, err = script_helper.assert_python_ok("-c", """if True:
from test import support
script = '''if True:
import _testinternalcapi
_testinternalcapi.gh_119213_getargs(spam='eggs')
'''
config = dict(
allow_fork=False,
allow_exec=False,
allow_threads=True,
allow_daemon_threads=False,
use_main_obmalloc=False,
gil=2,
check_multi_interp_extensions=True,
)
rc = support.run_in_subinterp_with_config(script, **config)
assert rc == 0
# The crash is different if the interpreter was not destroyed first.
#interpid = _testinternalcapi.create_interpreter()
#rc = _testinternalcapi.exec_interpreter(interpid, script)
#assert rc == 0
""")
self.assertEqual(rc, 0)


if __name__ == "__main__":
unittest.main()
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
Non-builtin modules built with argument clinic were crashing if used in a
subinterpreter before the main interpreter. The objects that were causing
the problem by leaking between interpreters carelessly have been fixed.
20 changes: 20 additions & 0 deletions Modules/_testinternalcapi.c
Original file line number Diff line number Diff line change
Expand Up @@ -2006,6 +2006,25 @@ has_inline_values(PyObject *self, PyObject *obj)
Py_RETURN_FALSE;
}


/*[clinic input]
gh_119213_getargs
spam: object = None
Test _PyArg_Parser.kwtuple
[clinic start generated code]*/

static PyObject *
gh_119213_getargs_impl(PyObject *module, PyObject *spam)
/*[clinic end generated code: output=d8d9c95d5b446802 input=65ef47511da80fc2]*/
{
// It must never have been called in the main interprer
assert(!_Py_IsMainInterpreter(PyInterpreterState_Get()));
return Py_NewRef(spam);
}


static PyMethodDef module_functions[] = {
{"get_configs", get_configs, METH_NOARGS},
{"get_recursion_depth", get_recursion_depth, METH_NOARGS},
Expand Down Expand Up @@ -2096,6 +2115,7 @@ static PyMethodDef module_functions[] = {
#ifdef _Py_TIER2
{"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS},
#endif
GH_119213_GETARGS_METHODDEF
{NULL, NULL} /* sentinel */
};

Expand Down
62 changes: 61 additions & 1 deletion Modules/clinic/_testinternalcapi.c.h

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 19 additions & 2 deletions Python/getargs.c
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "pycore_dict.h" // _PyDict_HasOnlyStringKeys()
#include "pycore_modsupport.h" // export _PyArg_NoKeywords()
#include "pycore_pylifecycle.h" // _PyArg_Fini
#include "pycore_pystate.h" // _Py_IsMainInterpreter()
#include "pycore_tuple.h" // _PyTuple_ITEMS()
#include "pycore_pyerrors.h" // _Py_CalculateSuggestions()

Expand Down Expand Up @@ -1947,7 +1948,23 @@ _parser_init(void *arg)
int owned;
PyObject *kwtuple = parser->kwtuple;
if (kwtuple == NULL) {
/* We may temporarily switch to the main interpreter to avoid
* creating a tuple that could outlive its owning interpreter. */
PyThreadState *save_tstate = NULL;
PyThreadState *temp_tstate = NULL;
if (!_Py_IsMainInterpreter(PyInterpreterState_Get())) {
temp_tstate = PyThreadState_New(_PyInterpreterState_Main());
if (temp_tstate == NULL) {
return -1;
}
save_tstate = PyThreadState_Swap(temp_tstate);
}
kwtuple = new_kwtuple(keywords, len, pos);
if (temp_tstate != NULL) {
PyThreadState_Clear(temp_tstate);
(void)PyThreadState_Swap(save_tstate);
PyThreadState_Delete(temp_tstate);
}
if (kwtuple == NULL) {
return -1;
}
Expand All @@ -1969,8 +1986,8 @@ _parser_init(void *arg)
parser->next = _Py_atomic_load_ptr(&_PyRuntime.getargs.static_parsers);
do {
// compare-exchange updates parser->next on failure
} while (_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers,
&parser->next, parser));
} while (!_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers,
&parser->next, parser));
return 0;
}

Expand Down
2 changes: 2 additions & 0 deletions Tools/clinic/libclinic/parse_args.py
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,8 @@ def declare_parser(
#endif
"""
else:
# XXX Why do we not statically allocate the tuple
# for non-builtin modules?
declarations = """
#if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)
Expand Down

0 comments on commit 05a00aa

Please sign in to comment.