gh-99113: A Per-Interpreter GIL!

Include/cpython/initconfig.h

@@ -252,6 +252,7 @@ typedef struct {
  int allow_threads;
  int allow_daemon_threads;
  int check_multi_interp_extensions;
+ int own_gil;
 } PyInterpreterConfig;
 
 #define _PyInterpreterConfig_INIT \
@@ -262,6 +263,7 @@ typedef struct {
  .allow_threads = 1, \
  .allow_daemon_threads = 0, \
  .check_multi_interp_extensions = 1, \
+ .own_gil = 1, \
  }
 
 #define _PyInterpreterConfig_LEGACY_INIT \
@@ -272,6 +274,7 @@ typedef struct {
  .allow_threads = 1, \
  .allow_daemon_threads = 1, \
  .check_multi_interp_extensions = 0, \
+ .own_gil = 0, \
  }
 
 /* --- Helper functions --------------------------------------- */

Include/internal/pycore_ceval.h

@@ -21,8 +21,7 @@ struct _ceval_runtime_state;
 
 
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
-extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *);
-extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock);
+extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);
 PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp);
 PyAPI_FUNC(int) _PyEval_AddPendingCall(
@@ -96,11 +95,13 @@ _PyEval_Vector(PyThreadState *tstate,
  PyObject* const* args, size_t argcount,
  PyObject *kwnames);
 
-extern int _PyEval_ThreadsInitialized(struct pyruntimestate *runtime);
-extern PyStatus _PyEval_InitGIL(PyThreadState *tstate);
+extern int _PyEval_ThreadsInitialized(void);
+extern PyStatus _PyEval_InitGIL(PyThreadState *tstate, int own_gil);
 extern void _PyEval_FiniGIL(PyInterpreterState *interp);
 
+extern void _PyEval_AcquireLock(PyThreadState *tstate);
 extern void _PyEval_ReleaseLock(PyThreadState *tstate);
+extern PyThreadState * _PyThreadState_SwapNoGIL(PyThreadState *);
 
 extern void _PyEval_DeactivateOpCache(void);
 

Include/internal/pycore_ceval_state.h

@@ -49,7 +49,6 @@ struct _ceval_runtime_state {
  the main thread of the main interpreter can handle signals: see
  _Py_ThreadCanHandleSignals(). */
  _Py_atomic_int signals_pending;
- struct _gil_runtime_state gil;
 };
 
 #ifdef PY_HAVE_PERF_TRAMPOLINE
@@ -83,6 +82,8 @@ struct _pending_calls {
 
 struct _ceval_state {
  int recursion_limit;
+ struct _gil_runtime_state *gil;
+ int own_gil;
  /* This single variable consolidates all requests to break out of
  the fast path in the eval loop. */
  _Py_atomic_int eval_breaker;

Include/internal/pycore_interp.h

@@ -178,6 +178,9 @@ struct _is {
  basis. Also see _PyRuntimeState regarding the various mutex fields.
  */
 
+ /* The per-interpreter GIL, which might not be used. */
+ struct _gil_runtime_state _gil;
+
  /* the initial PyInterpreterState.threads.head */
  PyThreadState _initial_thread;
 };

Include/internal/pycore_runtime.h

@@ -32,8 +32,6 @@ struct _getargs_runtime_state {
  struct _PyArg_Parser *static_parsers;
 };
 
-/* ceval state */
-
 /* GIL state */
 
 struct _gilstate_runtime_state {

Include/moduleobject.h

@@ -78,11 +78,17 @@ struct PyModuleDef_Slot {
 
 #define Py_mod_create 1
 #define Py_mod_exec 2
+#define Py_mod_multiple_interpreters 3
 
 #ifndef Py_LIMITED_API
-#define _Py_mod_LAST_SLOT 2
+#define _Py_mod_LAST_SLOT 3
 #endif
 
+/* for Py_mod_multiple_interpreters: */
+#define Py_MOD_MULTIPLE_INTERPRETERS_NOT_SUPPORTED ((void *)0)
+#define Py_MOD_MULTIPLE_INTERPRETERS_SUPPORTED ((void *)1)
+#define Py_MOD_PER_INTERPRETER_GIL_SUPPORTED ((void *)2)
+
 #endif /* New in 3.5 */
 
 struct PyModuleDef {

Lib/test/test__xxsubinterpreters.py

@@ -10,6 +10,7 @@
 import _testcapi
 from test import support
 from test.support import import_helper
+from test.support import os_helper
 from test.support import script_helper
 
 
@@ -917,7 +918,7 @@ def f():
  t = threading.Thread(target=f)
  t.start()
  """)
- with support.temp_dir() as dirname:
+ with os_helper.temp_dir() as dirname:
  filename = script_helper.make_script(dirname, 'interp', script)
  with script_helper.spawn_python(filename) as proc:
  retcode = proc.wait()

Lib/test/test_capi/test_misc.py

@@ -1247,21 +1247,25 @@ def test_configured_settings(self):
  EXEC = 1<<16
 
  features = ['obmalloc', 'fork', 'exec', 'threads', 'daemon_threads',
- 'extensions']
+ 'extensions', 'own_gil']
  kwlist = [f'allow_{n}' for n in features]
  kwlist[0] = 'use_main_obmalloc'
- kwlist[-1] = 'check_multi_interp_extensions'
-
+ kwlist[-2] = 'check_multi_interp_extensions'
+ kwlist[-1] = 'own_gil'
  # expected to work
  for config, expected in {
- (True, True, True, True, True, True):
- OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS,
- (True, False, False, False, False, False): OBMALLOC,
- (False, False, False, True, False, True): THREADS | EXTENSIONS,
+ (True, True, True, True, True, True, True):
+ (OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS | EXTENSIONS, True),
+ (True, False, False, False, False, False, False):
+ (OBMALLOC, False),
+ (False, False, False, True, False, True, False):
+ (THREADS | EXTENSIONS, False),
  }.items():
  kwargs = dict(zip(kwlist, config))
+ exp_flags, exp_gil = expected
  expected = {
- 'feature_flags': expected,
+ 'feature_flags': exp_flags,
+ 'own_gil': exp_gil,
  }
  with self.subTest(config):
  r, w = os.pipe()
@@ -1281,7 +1285,7 @@ def test_configured_settings(self):
 
  # expected to fail
  for config in [
- (False, False, False, False, False, False),
+ (False, False, False, False, False, False, False),
  ]:
  kwargs = dict(zip(kwlist, config))
  with self.subTest(config):
@@ -1317,6 +1321,7 @@ def test_overridden_setting_extensions_subinterp_check(self):
  'allow_exec': True,
  'allow_threads': True,
  'allow_daemon_threads': True,
+ 'own_gil': False,
  }
 
  def check(enabled, override):
@@ -1327,6 +1332,7 @@ def check(enabled, override):
  flags = BASE_FLAGS | EXTENSIONS if enabled else BASE_FLAGS
  settings = {
  'feature_flags': flags,
+ 'own_gil': False,
  }
 
  expected = {

Lib/test/test_embed.py

@@ -1666,6 +1666,7 @@ def test_init_main_interpreter_settings(self):
  # All optional features should be enabled.
  'feature_flags':
  OBMALLOC | FORK | EXEC | THREADS | DAEMON_THREADS,
+ 'own_gil': True,
  }
  out, err = self.run_embedded_interpreter(
  'test_init_main_interpreter_settings',

Lib/test/test_import/__init__.py

@@ -1639,6 +1639,7 @@ class SubinterpImportTests(unittest.TestCase):
  # Isolation-related config values aren't included here.
  )
  ISOLATED = dict(
+ own_gil=True,
  use_main_obmalloc=False,
  )
  NOT_ISOLATED = {k: not v for k, v in ISOLATED.items()}

Lib/test/test_threading.py

@@ -1349,6 +1349,7 @@ def func():
  allow_threads={allowed},
  allow_daemon_threads={daemon_allowed},
  check_multi_interp_extensions=False,
+ own_gil=False,
  )
  """)
  with test.support.SuppressCrashReport():

Misc/NEWS.d/next/Core and Builtins/2023-05-03-17-46-47.gh-issue-104108.GOxAYt.rst

@@ -0,0 +1,6 @@
+Multi-phase init extension modules may now indicate whether or not they
+actually support multiple interpreters. By default such modules are
+expected to support use in multiple interpreters. In the uncommon case that
+one does not, it may use the new ``Py_mod_multiple_interpreters`` module def
+slot. A value of ``0`` means the module does not support them. ``1`` means
+it does. The default is ``1``.

Modules/_abc.c

@@ -944,6 +944,7 @@ _abcmodule_free(void *module)
 
 static PyModuleDef_Slot _abcmodule_slots[] = {
  {Py_mod_exec, _abcmodule_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_asynciomodule.c

@@ -3805,6 +3805,8 @@ module_exec(PyObject *mod)
 
 static struct PyModuleDef_Slot module_slots[] = {
  {Py_mod_exec, module_exec},
+ // XXX gh-103092: fix isolation.
+ //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL},
 };
 

Modules/_bisectmodule.c

@@ -457,6 +457,7 @@ bisect_modexec(PyObject *m)
 
 static PyModuleDef_Slot bisect_slots[] = {
  {Py_mod_exec, bisect_modexec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_blake2/blake2module.c

@@ -127,6 +127,7 @@ blake2_exec(PyObject *m)
 
 static PyModuleDef_Slot _blake2_slots[] = {
  {Py_mod_exec, blake2_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 
@@ -146,4 +147,4 @@ PyMODINIT_FUNC
 PyInit__blake2(void)
 {
  return PyModuleDef_Init(&blake2_module);
-}
+}

Modules/_bz2module.c

@@ -799,6 +799,7 @@ _bz2_free(void *module)
 
 static struct PyModuleDef_Slot _bz2_slots[] = {
  {Py_mod_exec, _bz2_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_codecsmodule.c

@@ -1049,6 +1049,7 @@ static PyMethodDef _codecs_functions[] = {
 };
 
 static PyModuleDef_Slot _codecs_slots[] = {
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_collectionsmodule.c

@@ -2571,6 +2571,7 @@ collections_exec(PyObject *module) {
 
 static struct PyModuleDef_Slot collections_slots[] = {
  {Py_mod_exec, collections_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_contextvarsmodule.c

@@ -44,6 +44,7 @@ _contextvars_exec(PyObject *m)
 
 static struct PyModuleDef_Slot _contextvars_slots[] = {
  {Py_mod_exec, _contextvars_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_cryptmodule.c

@@ -58,6 +58,7 @@ static PyMethodDef crypt_methods[] = {
 };
 
 static PyModuleDef_Slot _crypt_slots[] = {
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_csv.c

@@ -1798,6 +1798,7 @@ csv_exec(PyObject *module) {
 
 static PyModuleDef_Slot csv_slots[] = {
  {Py_mod_exec, csv_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_ctypes/_ctypes_test.c

@@ -1054,6 +1054,7 @@ _testfunc_pylist_append(PyObject *list, PyObject *item)
 }
 
 static struct PyModuleDef_Slot _ctypes_test_slots[] = {
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_curses_panel.c

@@ -690,6 +690,8 @@ _curses_panel_exec(PyObject *mod)
 
 static PyModuleDef_Slot _curses_slots[] = {
  {Py_mod_exec, _curses_panel_exec},
+ // XXX gh-103092: fix isolation.
+ //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_dbmmodule.c

@@ -583,6 +583,7 @@ _dbm_module_free(void *module)
 
 static PyModuleDef_Slot _dbmmodule_slots[] = {
  {Py_mod_exec, _dbm_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_elementtree.c

@@ -4419,6 +4419,8 @@ module_exec(PyObject *m)
 
 static struct PyModuleDef_Slot elementtree_slots[] = {
  {Py_mod_exec, module_exec},
+ // XXX gh-103092: fix isolation.
+ //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL},
 };
 

Modules/_functoolsmodule.c

@@ -1520,6 +1520,7 @@ _functools_free(void *module)
 
 static struct PyModuleDef_Slot _functools_slots[] = {
  {Py_mod_exec, _functools_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_gdbmmodule.c

@@ -793,6 +793,7 @@ _gdbm_module_free(void *module)
 
 static PyModuleDef_Slot _gdbm_module_slots[] = {
  {Py_mod_exec, _gdbm_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_hashopenssl.c

@@ -2260,6 +2260,7 @@ static PyModuleDef_Slot hashlib_slots[] = {
  {Py_mod_exec, hashlib_md_meth_names},
  {Py_mod_exec, hashlib_init_constructors},
  {Py_mod_exec, hashlib_exception},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_heapqmodule.c

@@ -682,6 +682,7 @@ heapq_exec(PyObject *m)
 
 static struct PyModuleDef_Slot heapq_slots[] = {
  {Py_mod_exec, heapq_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_json.c

@@ -1801,6 +1801,7 @@ _json_exec(PyObject *module)
 
 static PyModuleDef_Slot _json_slots[] = {
  {Py_mod_exec, _json_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_localemodule.c

@@ -874,6 +874,7 @@ _locale_exec(PyObject *module)
 
 static struct PyModuleDef_Slot _locale_slots[] = {
  {Py_mod_exec, _locale_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_lsprof.c

@@ -885,6 +885,8 @@ _lsprof_exec(PyObject *module)
 
 static PyModuleDef_Slot _lsprofslots[] = {
  {Py_mod_exec, _lsprof_exec},
+ // XXX gh-103092: fix isolation.
+ //{Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };
 

Modules/_lzmamodule.c

@@ -1611,6 +1611,7 @@ static PyMethodDef lzma_methods[] = {
 
 static PyModuleDef_Slot lzma_slots[] = {
  {Py_mod_exec, lzma_exec},
+ {Py_mod_multiple_interpreters, Py_MOD_PER_INTERPRETER_GIL_SUPPORTED},
  {0, NULL}
 };