Merge branch 'main' into pythongh-127389

alonme · Nov 29, 2024 · 152a74a · 152a74a
2 parents 6645fb5 + 762c603
commit 152a74a
Show file tree

Hide file tree

Showing 7 changed files with 17 additions and 3 deletions.
diff --git a/Doc/library/token.rst b/Doc/library/token.rst
@@ -79,6 +79,13 @@ the :mod:`tokenize` module.
    ``type_comments=True``.
 
 
+.. data:: EXACT_TOKEN_TYPES
+
+   A dictionary mapping the string representation of a token to its numeric code.
+
+   .. versionadded:: 3.8
+
+
 .. versionchanged:: 3.5
    Added :data:`!AWAIT` and :data:`!ASYNC` tokens.
 

diff --git a/Lib/token.py b/Lib/token.py
diff --git a/Misc/ACKS b/Misc/ACKS
@@ -1154,6 +1154,7 @@ Mark Lutz
 Taras Lyapun
 Jim Lynch
 Mikael Lyngvig
+Ilya Lyubavski
 Jeff MacDonald
 John Machin
 Andrew I MacIntyre

diff --git a/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst b/Misc/NEWS.d/next/Core_and_Builtins/2024-11-16-11-11-35.gh-issue-126881.ijofLZ.rst
@@ -0,0 +1 @@
+Fix crash in finalization of dtoa state. Patch by Kumar Aditya.
diff --git a/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst b/Misc/NEWS.d/next/Library/2024-11-27-16-06-10.gh-issue-127303.asqkgh.rst
@@ -0,0 +1 @@
+Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`.
diff --git a/Python/pylifecycle.c b/Python/pylifecycle.c
@@ -1888,7 +1888,6 @@ finalize_interp_clear(PyThreadState *tstate)
     _PyXI_Fini(tstate->interp);
     _PyExc_ClearExceptionGroupType(tstate->interp);
     _Py_clear_generic_types(tstate->interp);
-    _PyDtoa_Fini(tstate->interp);
 
     /* Clear interpreter state and all thread states */
     _PyInterpreterState_Clear(tstate);
@@ -1910,6 +1909,9 @@ finalize_interp_clear(PyThreadState *tstate)
 
     finalize_interp_types(tstate->interp);
 
+    /* Finalize dtoa at last so that finalizers calling repr of float doesn't crash */
+    _PyDtoa_Fini(tstate->interp);
+
     /* Free any delayed free requests immediately */
     _PyMem_FiniDelayed(tstate->interp);
 

diff --git a/Tools/build/generate_token.py b/Tools/build/generate_token.py
@@ -226,7 +226,8 @@ def make_rst(infile, outfile='Doc/library/token-list.inc'):
 # {AUTO_GENERATED_BY_SCRIPT}
 '''
 token_py_template += '''
-__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF']
+__all__ = ['tok_name', 'ISTERMINAL', 'ISNONTERMINAL', 'ISEOF',
+           'EXACT_TOKEN_TYPES']
 
 %s
 N_TOKENS = %d
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Fix crash in finalization of dtoa state. Patch by Kumar Aditya.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Publicly expose :data:`~token.EXACT_TOKEN_TYPES` in :attr:`!token.__all__`.