gh-119213: Be More Careful About _PyArg_Parser.kwtuple Across Interpreters

Include/internal/pycore_global_strings.h

@@ -710,6 +710,7 @@ struct _Py_global_strings {
  STRUCT_FOR_ID(sort)
  STRUCT_FOR_ID(source)
  STRUCT_FOR_ID(source_traceback)
+ STRUCT_FOR_ID(spam)
  STRUCT_FOR_ID(src)
  STRUCT_FOR_ID(src_dir_fd)
  STRUCT_FOR_ID(stacklevel)

Lib/test/test_capi/test_getargs.py

@@ -4,11 +4,17 @@
 import sys
 from test import support
 from test.support import import_helper
+from test.support import script_helper
 from test.support import warnings_helper
 # Skip this test if the _testcapi module isn't available.
 _testcapi = import_helper.import_module('_testcapi')
 from _testcapi import getargs_keywords, getargs_keyword_only
 
+try:
+ import _testinternalcapi
+except ImportError:
+ _testinternalcapi = NULL
+
 # > How about the following counterproposal. This also changes some of
 # > the other format codes to be a little more regular.
 # >
@@ -1346,6 +1352,33 @@ def test_nested_tuple(self):
  "argument 1 must be sequence of length 1, not 0"):
  parse(((),), {}, '(' + f + ')', ['a'])
 
+ @unittest.skipIf(_testinternalcapi is None, 'needs _testinternalcapi')
+ def test_gh_119213(self):
+ rc, out, err = script_helper.assert_python_ok("-c", """if True:
+ from test import support
+ script = '''if True:
+ import _testinternalcapi
+ _testinternalcapi.gh_119213_getargs(spam='eggs')
+ '''
+ config = dict(
+ allow_fork=False,
+ allow_exec=False,
+ allow_threads=True,
+ allow_daemon_threads=False,
+ use_main_obmalloc=False,
+ gil=2,
+ check_multi_interp_extensions=True,
+ )
+ rc = support.run_in_subinterp_with_config(script, **config)
+ assert rc == 0
+
+ # The crash is different if the interpreter was not destroyed first.
+ #interpid = _testinternalcapi.create_interpreter()
+ #rc = _testinternalcapi.exec_interpreter(interpid, script)
+ #assert rc == 0
+ """)
+ self.assertEqual(rc, 0)
+
 
 if __name__ == "__main__":
  unittest.main()

Misc/NEWS.d/next/Core and Builtins/2024-05-21-11-27-14.gh-issue-119213.nxjxrt.rst

@@ -0,0 +1,3 @@
+Non-builtin modules built with argument clinic were crashing if used in a
+subinterpreter before the main interpreter. The objects that were causing
+the problem by leaking between interpreters carelessly have been fixed.

Modules/_testinternalcapi.c

@@ -2006,6 +2006,25 @@ has_inline_values(PyObject *self, PyObject *obj)
  Py_RETURN_FALSE;
 }
 
+
+/*[clinic input]
+gh_119213_getargs
+
+ spam: object = None
+
+Test _PyArg_Parser.kwtuple
+[clinic start generated code]*/
+
+static PyObject *
+gh_119213_getargs_impl(PyObject *module, PyObject *spam)
+/*[clinic end generated code: output=d8d9c95d5b446802 input=65ef47511da80fc2]*/
+{
+ // It must never have been called in the main interprer
+ assert(!_Py_IsMainInterpreter(PyInterpreterState_Get()));
+ return Py_NewRef(spam);
+}
+
+
 static PyMethodDef module_functions[] = {
  {"get_configs", get_configs, METH_NOARGS},
  {"get_recursion_depth", get_recursion_depth, METH_NOARGS},
@@ -2096,6 +2115,7 @@ static PyMethodDef module_functions[] = {
 #ifdef _Py_TIER2
  {"uop_symbols_test", _Py_uop_symbols_test, METH_NOARGS},
 #endif
+ GH_119213_GETARGS_METHODDEF
  {NULL, NULL} /* sentinel */
 };
 

Python/getargs.c

@@ -7,6 +7,7 @@
 #include "pycore_dict.h" // _PyDict_HasOnlyStringKeys()
 #include "pycore_modsupport.h" // export _PyArg_NoKeywords()
 #include "pycore_pylifecycle.h" // _PyArg_Fini
+#include "pycore_pystate.h" // _Py_IsMainInterpreter()
 #include "pycore_tuple.h" // _PyTuple_ITEMS()
 #include "pycore_pyerrors.h" // _Py_CalculateSuggestions()
 
@@ -1947,7 +1948,23 @@ _parser_init(void *arg)
  int owned;
  PyObject *kwtuple = parser->kwtuple;
  if (kwtuple == NULL) {
+ /* We may temporarily switch to the main interpreter to avoid
+ * creating a tuple that could outlive its owning interpreter. */
+ PyThreadState *save_tstate = NULL;
+ PyThreadState *temp_tstate = NULL;
+ if (!_Py_IsMainInterpreter(PyInterpreterState_Get())) {
+ temp_tstate = PyThreadState_New(_PyInterpreterState_Main());
+ if (temp_tstate == NULL) {
+ return -1;
+ }
+ save_tstate = PyThreadState_Swap(temp_tstate);
+ }
  kwtuple = new_kwtuple(keywords, len, pos);
+ if (temp_tstate != NULL) {
+ PyThreadState_Clear(temp_tstate);
+ (void)PyThreadState_Swap(save_tstate);
+ PyThreadState_Delete(temp_tstate);
+ }
  if (kwtuple == NULL) {
  return -1;
  }
@@ -1969,8 +1986,8 @@ _parser_init(void *arg)
  parser->next = _Py_atomic_load_ptr(&_PyRuntime.getargs.static_parsers);
  do {
  // compare-exchange updates parser->next on failure
- } while (_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers,
- &parser->next, parser));
+ } while (!_Py_atomic_compare_exchange_ptr(&_PyRuntime.getargs.static_parsers,
+  &parser->next, parser));
  return 0;
 }
 

Tools/clinic/libclinic/parse_args.py

@@ -51,6 +51,8 @@ def declare_parser(
  #endif
  """
  else:
+ # XXX Why do we not statically allocate the tuple
+ # for non-builtin modules?
  declarations = """
  #if defined(Py_BUILD_CORE) && !defined(Py_BUILD_CORE_MODULE)