gh-81057: Move More Globals to _PyRuntimeState (gh-100092)

#81057

Author: ericsnowcurrently

Include/internal/pycore_dtoa.h

@@ -25,7 +25,7 @@ Bigint {
 #ifdef Py_USING_MEMORY_DEBUGGER
 
 struct _dtoa_runtime_state {
-        int _not_used;
+    int _not_used;
 };
 #define _dtoa_runtime_state_INIT {0}
 
@@ -41,9 +41,12 @@ struct _dtoa_runtime_state {
     ((PRIVATE_MEM+sizeof(double)-1)/sizeof(double))
 
 struct _dtoa_runtime_state {
-        struct Bigint *freelist[Bigint_Kmax+1];
-        double preallocated[Bigint_PREALLOC_SIZE];
-        double *preallocated_next;
+    /* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */
+    // XXX This should be freed during runtime fini.
+    struct Bigint *p5s;
+    struct Bigint *freelist[Bigint_Kmax+1];
+    double preallocated[Bigint_PREALLOC_SIZE];
+    double *preallocated_next;
 };
 #define _dtoa_runtime_state_INIT(runtime) \
     { \

Include/internal/pycore_obmalloc.h

@@ -658,6 +658,7 @@ struct _obmalloc_usage {
 
 
 struct _obmalloc_state {
+    int dump_debug_stats;
     struct _obmalloc_pools pools;
     struct _obmalloc_mgmt mgmt;
     struct _obmalloc_usage usage;

Include/internal/pycore_obmalloc_init.h

@@ -56,6 +56,7 @@ extern "C" {
 
 #define _obmalloc_state_INIT(obmalloc) \
     { \
+        .dump_debug_stats = -1, \
         .pools = { \
             .used = _obmalloc_pools_INIT(obmalloc.pools), \
         }, \

Include/internal/pycore_parser.h

@@ -8,12 +8,31 @@ extern "C" {
 #  error "this header requires Py_BUILD_CORE define"
 #endif
 
+
+#include "pycore_pyarena.h"         // PyArena
+
+
+#ifdef Py_DEBUG
+#define _PYPEGEN_NSTATISTICS 2000
+#endif
+
+struct _parser_runtime_state {
+#ifdef Py_DEBUG
+    long memo_statistics[_PYPEGEN_NSTATISTICS];
+#else
+    int _not_used;
+#endif
+};
+
+
+
 extern struct _mod* _PyParser_ASTFromString(
     const char *str,
     PyObject* filename,
     int mode,
     PyCompilerFlags *flags,
     PyArena *arena);
+
 extern struct _mod* _PyParser_ASTFromFile(
     FILE *fp,
     PyObject *filename_ob,
@@ -25,6 +44,7 @@ extern struct _mod* _PyParser_ASTFromFile(
     int *errcode,
     PyArena *arena);
 
+
 #ifdef __cplusplus
 }
 #endif

Include/internal/pycore_runtime.h

@@ -17,6 +17,7 @@ extern "C" {
 #include "pycore_global_objects.h"  // struct _Py_global_objects
 #include "pycore_import.h"          // struct _import_runtime_state
 #include "pycore_interp.h"          // PyInterpreterState
+#include "pycore_parser.h"          // struct _parser_runtime_state
 #include "pycore_pymem.h"           // struct _pymem_allocators
 #include "pycore_pyhash.h"          // struct pyhash_runtime_state
 #include "pycore_obmalloc.h"        // struct obmalloc_state
@@ -129,6 +130,10 @@ typedef struct pyruntimestate {
 
     unsigned long main_thread;
 
+    PyWideStringList orig_argv;
+
+    struct _parser_runtime_state parser;
+
 #define NEXITFUNCS 32
     void (*exitfuncs[NEXITFUNCS])(void);
     int nexitfuncs;

Include/internal/pycore_unicodeobject.h

@@ -9,6 +9,7 @@ extern "C" {
 #endif
 
 #include "pycore_fileutils.h"     // _Py_error_handler
+#include "pycore_ucnhash.h"       // _PyUnicode_Name_CAPI
 
 void _PyUnicode_ExactDealloc(PyObject *op);
 
@@ -52,6 +53,8 @@ struct _Py_unicode_ids {
 struct _Py_unicode_state {
     struct _Py_unicode_fs_codec fs_codec;
 
+    _PyUnicode_Name_CAPI *ucnhash_capi;
+
     // Unicode identifiers (_Py_Identifier): see _PyUnicode_FromId()
     struct _Py_unicode_ids ids;
 };

Modules/_io/bufferedio.c

@@ -746,26 +746,26 @@ _buffered_init(buffered *self)
 int
 _PyIO_trap_eintr(void)
 {
-    static PyObject *eintr_int = NULL;
     PyObject *typ, *val, *tb;
     PyOSErrorObject *env_err;
-
-    if (eintr_int == NULL) {
-        eintr_int = PyLong_FromLong(EINTR);
-        assert(eintr_int != NULL);
-    }
     if (!PyErr_ExceptionMatches(PyExc_OSError))
         return 0;
     PyErr_Fetch(&typ, &val, &tb);
     PyErr_NormalizeException(&typ, &val, &tb);
     env_err = (PyOSErrorObject *) val;
     assert(env_err != NULL);
-    if (env_err->myerrno != NULL &&
-        PyObject_RichCompareBool(env_err->myerrno, eintr_int, Py_EQ) > 0) {
-        Py_DECREF(typ);
-        Py_DECREF(val);
-        Py_XDECREF(tb);
-        return 1;
+    if (env_err->myerrno != NULL) {
+        assert(EINTR > 0 && EINTR < INT_MAX);
+        assert(PyLong_CheckExact(env_err->myerrno));
+        int overflow;
+        int myerrno = PyLong_AsLongAndOverflow(env_err->myerrno, &overflow);
+        PyErr_Clear();
+        if (myerrno == EINTR) {
+            Py_DECREF(typ);
+            Py_DECREF(val);
+            Py_XDECREF(tb);
+            return 1;
+        }
     }
     /* This silences any error set by PyObject_RichCompareBool() */
     PyErr_Restore(typ, val, tb);

Objects/obmalloc.c

@@ -908,11 +908,12 @@ new_arena(void)
     struct arena_object* arenaobj;
     uint excess;        /* number of bytes above pool alignment */
     void *address;
-    static int debug_stats = -1;
 
+    int debug_stats = _PyRuntime.obmalloc.dump_debug_stats;
     if (debug_stats == -1) {
         const char *opt = Py_GETENV("PYTHONMALLOCSTATS");
         debug_stats = (opt != NULL && *opt != '\0');
+        _PyRuntime.obmalloc.dump_debug_stats = debug_stats;
     }
     if (debug_stats) {
         _PyObject_DebugMallocStats(stderr);

Objects/unicodeobject.c

@@ -5697,8 +5697,6 @@ PyUnicode_AsUTF16String(PyObject *unicode)
 
 /* --- Unicode Escape Codec ----------------------------------------------- */
 
-static _PyUnicode_Name_CAPI *ucnhash_capi = NULL;
-
 PyObject *
 _PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
                                Py_ssize_t size,
@@ -5711,6 +5709,8 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
     const char *end;
     PyObject *errorHandler = NULL;
     PyObject *exc = NULL;
+    _PyUnicode_Name_CAPI *ucnhash_capi;
+    PyInterpreterState *interp = _PyInterpreterState_Get();
 
     // so we can remember if we've seen an invalid escape char or not
     *first_invalid_escape = NULL;
@@ -5858,6 +5858,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
 
             /* \N{name} */
         case 'N':
+            ucnhash_capi = interp->unicode.ucnhash_capi;
             if (ucnhash_capi == NULL) {
                 /* load the unicode data module */
                 ucnhash_capi = (_PyUnicode_Name_CAPI *)PyCapsule_Import(
@@ -5869,6 +5870,7 @@ _PyUnicode_DecodeUnicodeEscapeInternal(const char *s,
                         );
                     goto onError;
                 }
+                interp->unicode.ucnhash_capi = ucnhash_capi;
             }
 
             message = "malformed \\N character escape";
@@ -15128,10 +15130,10 @@ _PyUnicode_Fini(PyInterpreterState *interp)
         assert(get_interned_dict() == NULL);
         // bpo-47182: force a unicodedata CAPI capsule re-import on
         // subsequent initialization of main interpreter.
-        ucnhash_capi = NULL;
     }
 
     _PyUnicode_FiniEncodings(&state->fs_codec);
+    interp->unicode.ucnhash_capi = NULL;
 
     unicode_clear_identifiers(state);
 }

Parser/pegen.c

@@ -246,8 +246,8 @@ _PyPegen_fill_token(Parser *p)
 // The array counts the number of tokens skipped by memoization,
 // indexed by type.
 
-#define NSTATISTICS 2000
-static long memo_statistics[NSTATISTICS];
+#define NSTATISTICS _PYPEGEN_NSTATISTICS
+#define memo_statistics _PyRuntime.parser.memo_statistics
 
 void
 _PyPegen_clear_memo_statistics()

Python/dtoa.c

@@ -673,10 +673,6 @@ mult(Bigint *a, Bigint *b)
 
 #ifndef Py_USING_MEMORY_DEBUGGER
 
-/* p5s is a linked list of powers of 5 of the form 5**(2**i), i >= 2 */
-
-static Bigint *p5s;
-
 /* multiply the Bigint b by 5**k.  Returns a pointer to the result, or NULL on
    failure; if the returned pointer is distinct from b then the original
    Bigint b will have been Bfree'd.   Ignores the sign of b. */
@@ -696,15 +692,15 @@ pow5mult(Bigint *b, int k)
 
     if (!(k >>= 2))
         return b;
-    p5 = p5s;
+    p5 = _PyRuntime.dtoa.p5s;
     if (!p5) {
         /* first time */
         p5 = i2b(625);
         if (p5 == NULL) {
             Bfree(b);
             return NULL;
         }
-        p5s = p5;
+        _PyRuntime.dtoa.p5s = p5;
         p5->next = 0;
     }
     for(;;) {

Python/initconfig.c

@@ -595,17 +595,13 @@ _Py_ClearStandardStreamEncoding(void)
 
 /* --- Py_GetArgcArgv() ------------------------------------------- */
 
-/* For Py_GetArgcArgv(); set by _Py_SetArgcArgv() */
-static PyWideStringList orig_argv = {.length = 0, .items = NULL};
-
-
 void
 _Py_ClearArgcArgv(void)
 {
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    _PyWideStringList_Clear(&orig_argv);
+    _PyWideStringList_Clear(&_PyRuntime.orig_argv);
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 }
@@ -620,7 +616,9 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv)
     PyMemAllocatorEx old_alloc;
     _PyMem_SetDefaultAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
 
-    res = _PyWideStringList_Copy(&orig_argv, &argv_list);
+    // XXX _PyRuntime.orig_argv only gets cleared by Py_Main(),
+    // so it it currently leaks for embedders.
+    res = _PyWideStringList_Copy(&_PyRuntime.orig_argv, &argv_list);
 
     PyMem_SetAllocator(PYMEM_DOMAIN_RAW, &old_alloc);
     return res;
@@ -631,8 +629,8 @@ _Py_SetArgcArgv(Py_ssize_t argc, wchar_t * const *argv)
 void
 Py_GetArgcArgv(int *argc, wchar_t ***argv)
 {
-    *argc = (int)orig_argv.length;
-    *argv = orig_argv.items;
+    *argc = (int)_PyRuntime.orig_argv.length;
+    *argv = _PyRuntime.orig_argv.items;
 }
 
 

Tools/c-analyzer/cpython/globals-to-fix.tsv

@@ -305,20 +305,12 @@ Objects/sliceobject.c	-	_Py_EllipsisObject	-
 ##-----------------------
 ## effectively-const but initialized lazily
 
-## idempotent
-Python/dtoa.c	-	p5s	-
-Objects/obmalloc.c	new_arena	debug_stats	-
-
 ## others
 Python/perf_trampoline.c	-	perf_map_file	-
-Objects/unicodeobject.c	-	ucnhash_capi	-
 
 ##-----------------------
 ## state
 
-## local buffer
-Python/suggestions.c	levenshtein_distance	buffer	-
-
 ## other
 Objects/object.c	-	_Py_RefTotal	-
 Python/perf_trampoline.c	-	perf_status	-
@@ -398,7 +390,6 @@ Modules/faulthandler.c	-	old_stack	-
 ##-----------------------
 ## initialized once
 
-Modules/_io/bufferedio.c	_PyIO_trap_eintr	eintr_int	-
 Modules/posixmodule.c	os_dup2_impl	dup3_works	-
 Modules/posixmodule.c	-	structseq_new	-
 Modules/posixmodule.c	-	ticks_per_second	-

Tools/c-analyzer/cpython/ignored.tsv

@@ -87,8 +87,6 @@ Parser/myreadline.c	-	PyOS_ReadlineFunctionPointer	-
 
 Python/initconfig.c	-	_Py_StandardStreamEncoding	-
 Python/initconfig.c	-	_Py_StandardStreamErrors	-
-# XXX This only gets cleared by Py_Main().
-Python/initconfig.c	-	orig_argv	-
 
 ##-----------------------
 ## public C-API
@@ -143,15 +141,6 @@ Python/pylifecycle.c	-	runtime_initialized	-
 Modules/syslogmodule.c	-	S_ident_o	-
 Modules/syslogmodule.c	-	S_log_open	-
 
-##-----------------------
-## *not* tied to init/fini cycle
-
-# These do not ge reset with each init/fini cycle.
-# XXX These should probably be tied to init/fini.  Move to _PyRuntimeState?
-
-# special-use diagnistic state
-Parser/pegen.c	-	memo_statistics	-
-
 ##-----------------------
 ## one-off temporary state