Merge branch 'main' into fixsendfamily

* main: (29 commits)
  pythongh-104276: Make `_struct.unpack_iterator` type use type flag instead of custom constructor (python#104277)
  pythongh-97696: Move around and update the whatsnew entry for asyncio eager task factory (python#104298)
  pythongh-103193: Fix refleaks in `test_inspect` and `test_typing` (python#104320)
  require-pr-label.yml: Add missing "permissions:" (python#104309)
  pythongh-90656: Add platform triplets for 64-bit LoongArch (LA64) (python#30939)
  pythongh-104180: Read SOCKS proxies from macOS System Configuration (python#104181)
  pythongh-97696 Remove unnecessary check for eager_start kwarg (python#104188)
  pythonGH-104308: socket.getnameinfo should release the GIL (python#104307)
  pythongh-104310: Add importlib.util.allowing_all_extensions() (pythongh-104311)
  pythongh-99113: A Per-Interpreter GIL! (pythongh-104210)
  pythonGH-104284: Fix documentation gettext build (python#104296)
  pythongh-89550: Buffer GzipFile.write to reduce execution time by ~15% (python#101251)
  pythongh-104223: Fix issues with inheriting from buffer classes (python#104227)
  pythongh-99108: fix typo in Modules/Setup (python#104293)
  pythonGH-104145: Use fully-qualified cross reference types for the bisect module (python#104172)
  pythongh-103193: Improve `getattr_static` test coverage (python#104286)
  Trim trailing whitespace and test on CI (python#104275)
  pythongh-102500: Remove mention of bytes shorthand (python#104281)
  pythongh-97696: Improve and fix documentation for asyncio eager tasks (python#104256)
  pythongh-99108: Replace SHA3 implementation HACL* version (python#103597)
  ...

Author: carljm

.github/CODEOWNERS

@@ -7,6 +7,9 @@
 # GitHub
 .github/**                    @ezio-melotti @hugovk
 
+# pre-commit
+.pre-commit-config.yaml       @hugovk @AlexWaygood
+
 # Build system
 configure*                    @erlend-aasland @corona10
 

.github/workflows/lint.yml

@@ -0,0 +1,22 @@
+name: Lint
+
+on: [push, pull_request, workflow_dispatch]
+
+permissions:
+  contents: read
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: "3.x"
+      - uses: pre-commit/action@v3.0.0

.github/workflows/require-pr-label.yml

@@ -4,6 +4,10 @@ on:
   pull_request:
     types: [opened, reopened, labeled, unlabeled, synchronize]
 
+permissions:
+  issues: read
+  pull-requests: read
+
 jobs:
   label:
     name: DO-NOT-MERGE / unresolved review

.pre-commit-config.yaml

@@ -0,0 +1,7 @@
+repos:
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v4.4.0
+    hooks:
+      - id: check-yaml
+      - id: trailing-whitespace
+        types_or: [c, python, rst]

Doc/library/asyncio-task.rst

@@ -560,6 +560,13 @@ Eager Task Factory
     using the provided *custom_task_constructor* when creating a new task instead
     of the default :class:`Task`.
 
+    *custom_task_constructor* must be a *callable* with the signature matching
+    the signature of :class:`Task.__init__ <Task>`.
+    The callable must return a :class:`asyncio.Task`-compatible object.
+
+    This function returns a *callable* intended to be used as a task factory of an
+    event loop via :meth:`loop.set_task_factory(factory) <loop.set_task_factory>`).
+
     .. versionadded:: 3.12
 
 
@@ -1014,7 +1021,7 @@ Introspection
 Task Object
 ===========
 
-.. class:: Task(coro, *, loop=None, name=None, context=None)
+.. class:: Task(coro, *, loop=None, name=None, context=None, eager_start=False)
 
    A :class:`Future-like <Future>` object that runs a Python
    :ref:`coroutine <coroutine>`.  Not thread-safe.
@@ -1054,6 +1061,13 @@ Task Object
    If no *context* is provided, the Task copies the current context
    and later runs its coroutine in the copied context.
 
+   An optional keyword-only *eager_start* argument allows eagerly starting
+   the execution of the :class:`asyncio.Task` at task creation time.
+   If set to ``True`` and the event loop is running, the task will start
+   executing the coroutine immediately, until the first time the coroutine
+   blocks. If the coroutine returns or raises without blocking, the task
+   will be finished eagerly and will skip scheduling to the event loop.
+
    .. versionchanged:: 3.7
       Added support for the :mod:`contextvars` module.
 
@@ -1067,6 +1081,9 @@ Task Object
    .. versionchanged:: 3.11
       Added the *context* parameter.
 
+   .. versionchanged:: 3.12
+      Added the *eager_start* parameter.
+
    .. method:: done()
 
       Return ``True`` if the Task is *done*.

Doc/library/bisect.rst

@@ -24,6 +24,8 @@ method to determine whether a value has been found.  Instead, the
 functions only call the :meth:`__lt__` method and will return an insertion
 point between values in an array.
 
+.. _bisect functions:
+
 The following functions are provided:
 
 
@@ -55,7 +57,7 @@ The following functions are provided:
 .. function:: bisect_right(a, x, lo=0, hi=len(a), *, key=None)
               bisect(a, x, lo=0, hi=len(a), *, key=None)
 
-   Similar to :func:`bisect_left`, but returns an insertion point which comes
+   Similar to :py:func:`~bisect.bisect_left`, but returns an insertion point which comes
    after (to the right of) any existing entries of *x* in *a*.
 
    The returned insertion point *ip* partitions the array *a* into two slices
@@ -70,7 +72,7 @@ The following functions are provided:
 
    Insert *x* in *a* in sorted order.
 
-   This function first runs :func:`bisect_left` to locate an insertion point.
+   This function first runs :py:func:`~bisect.bisect_left` to locate an insertion point.
    Next, it runs the :meth:`insert` method on *a* to insert *x* at the
    appropriate position to maintain sort order.
 
@@ -87,10 +89,10 @@ The following functions are provided:
 .. function:: insort_right(a, x, lo=0, hi=len(a), *, key=None)
               insort(a, x, lo=0, hi=len(a), *, key=None)
 
-   Similar to :func:`insort_left`, but inserting *x* in *a* after any existing
+   Similar to :py:func:`~bisect.insort_left`, but inserting *x* in *a* after any existing
    entries of *x*.
 
-   This function first runs :func:`bisect_right` to locate an insertion point.
+   This function first runs :py:func:`~bisect.bisect_right` to locate an insertion point.
    Next, it runs the :meth:`insert` method on *a* to insert *x* at the
    appropriate position to maintain sort order.
 
@@ -120,7 +122,7 @@ thoughts in mind:
   they are used.  Consequently, if the search functions are used in a loop,
   the key function may be called again and again on the same array elements.
   If the key function isn't fast, consider wrapping it with
-  :func:`functools.cache` to avoid duplicate computations.  Alternatively,
+  :py:func:`functools.cache` to avoid duplicate computations.  Alternatively,
   consider searching an array of precomputed keys to locate the insertion
   point (as shown in the examples section below).
 
@@ -140,7 +142,7 @@ thoughts in mind:
 Searching Sorted Lists
 ----------------------
 
-The above :func:`bisect` functions are useful for finding insertion points but
+The above `bisect functions`_ are useful for finding insertion points but
 can be tricky or awkward to use for common searching tasks. The following five
 functions show how to transform them into the standard lookups for sorted
 lists::
@@ -186,8 +188,8 @@ Examples
 
 .. _bisect-example:
 
-The :func:`bisect` function can be useful for numeric table lookups. This
-example uses :func:`bisect` to look up a letter grade for an exam score (say)
+The :py:func:`~bisect.bisect` function can be useful for numeric table lookups. This
+example uses :py:func:`~bisect.bisect` to look up a letter grade for an exam score (say)
 based on a set of ordered numeric breakpoints: 90 and up is an 'A', 80 to 89 is
 a 'B', and so on::
 
@@ -198,8 +200,8 @@ a 'B', and so on::
    >>> [grade(score) for score in [33, 99, 77, 70, 89, 90, 100]]
    ['F', 'A', 'C', 'C', 'B', 'A', 'A']
 
-The :func:`bisect` and :func:`insort` functions also work with lists of
-tuples.  The *key* argument can serve to extract the field used for ordering
+The :py:func:`~bisect.bisect` and :py:func:`~bisect.insort` functions also work with
+lists of tuples.  The *key* argument can serve to extract the field used for ordering
 records in a table::
 
     >>> from collections import namedtuple

Doc/library/typing.rst

@@ -2130,15 +2130,10 @@ Corresponding to collections in :mod:`collections.abc`
 
 .. class:: ByteString(Sequence[int])
 
-   A generic version of :class:`collections.abc.ByteString`.
-
    This type represents the types :class:`bytes`, :class:`bytearray`,
    and :class:`memoryview` of byte sequences.
 
-   As a shorthand for this type, :class:`bytes` can be used to
-   annotate arguments of any of the types mentioned above.
-
-   .. deprecated:: 3.9
+   .. deprecated-removed:: 3.9 3.14
       Prefer :class:`collections.abc.Buffer`, or a union like ``bytes | bytearray | memoryview``.
 
 .. class:: Collection(Sized, Iterable[T_co], Container[T_co])
@@ -2977,6 +2972,8 @@ convenience. This is subject to change, and not all deprecations are listed.
 |  ``typing`` versions of standard | 3.9           | Undecided         | :pep:`585`     |
 |  collections                     |               |                   |                |
 +----------------------------------+---------------+-------------------+----------------+
+|  ``typing.ByteString``           | 3.9           | 3.14              | :gh:`91896`    |
++----------------------------------+---------------+-------------------+----------------+
 |  ``typing.Text``                 | 3.11          | Undecided         | :gh:`92332`    |
 +----------------------------------+---------------+-------------------+----------------+
 |  ``typing.Hashable`` and         | 3.12          | Undecided         | :gh:`94309`    |

Doc/tools/extensions/pyspecific.py

@@ -674,7 +674,14 @@ def process_audit_events(app, doctree, fromdocname):
         node.replace_self(table)
 
 
-def patch_pairindextypes(app) -> None:
+def patch_pairindextypes(app, _env) -> None:
+    """Remove all entries from ``pairindextypes`` before writing POT files.
+
+    We want to run this just before writing output files, as the check to
+    circumvent is in ``I18nBuilder.write_doc()``.
+    As such, we link this to ``env-check-consistency``, even though it has
+    nothing to do with the environment consistency check.
+    """
     if app.builder.name != 'gettext':
         return
 
@@ -688,14 +695,7 @@ def patch_pairindextypes(app) -> None:
         # the Sphinx-translated pairindextypes values. As we intend to move
         # away from this, we need Sphinx to believe that these values don't
         # exist, by deleting them when using the gettext builder.
-
-        pairindextypes.pop('module', None)
-        pairindextypes.pop('keyword', None)
-        pairindextypes.pop('operator', None)
-        pairindextypes.pop('object', None)
-        pairindextypes.pop('exception', None)
-        pairindextypes.pop('statement', None)
-        pairindextypes.pop('builtin', None)
+        pairindextypes.clear()
 
 
 def setup(app):
@@ -719,7 +719,7 @@ def setup(app):
     app.add_directive_to_domain('py', 'awaitablemethod', PyAwaitableMethod)
     app.add_directive_to_domain('py', 'abstractmethod', PyAbstractMethod)
     app.add_directive('miscnews', MiscNews)
-    app.connect('builder-inited', patch_pairindextypes)
+    app.connect('env-check-consistency', patch_pairindextypes)
     app.connect('doctree-resolved', process_audit_events)
     app.connect('env-merge-info', audit_events_merge)
     app.connect('env-purge-doc', audit_events_purge)

Doc/whatsnew/3.12.rst

@@ -282,6 +282,11 @@ asyncio
   writing to sockets and uses :meth:`~socket.socket.sendmsg` if the platform
   supports it. (Contributed by Kumar Aditya in :gh:`91166`.)
 
+* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory`
+  functions to allow opting an event loop in to eager task execution,
+  making some use-cases 2x to 5x faster.
+  (Contributed by Jacob Bower & Itamar O in :gh:`102853`, :gh:`104140`, and :gh:`104138`)
+
 * On Linux, :mod:`asyncio` uses :class:`~asyncio.PidfdChildWatcher` by default
   if :func:`os.pidfd_open` is available and functional instead of
   :class:`~asyncio.ThreadedChildWatcher`.
@@ -342,8 +347,9 @@ inspect
   (Contributed by Thomas Krennwallner in :issue:`35759`.)
 
 * The performance of :func:`inspect.getattr_static` has been considerably
-  improved. Most calls to the function should be around 2x faster than they
-  were in Python 3.11. (Contributed by Alex Waygood in :gh:`103193`.)
+  improved. Most calls to the function should be at least 2x faster than they
+  were in Python 3.11, and some may be 6x faster or more. (Contributed by Alex
+  Waygood in :gh:`103193`.)
 
 pathlib
 -------
@@ -597,7 +603,7 @@ typing
   :func:`runtime-checkable protocols <typing.runtime_checkable>` has changed
   significantly. Most ``isinstance()`` checks against protocols with only a few
   members should be at least 2x faster than in 3.11, and some may be 20x
-  faster or more. However, ``isinstance()`` checks against protocols with seven
+  faster or more. However, ``isinstance()`` checks against protocols with fourteen
   or more members may be slower than in Python 3.11. (Contributed by Alex
   Waygood in :gh:`74690` and :gh:`103193`.)
 
@@ -643,11 +649,6 @@ Optimizations
 * Speed up :class:`asyncio.Task` creation by deferring expensive string formatting.
   (Contributed by Itamar O in :gh:`103793`.)
 
-* Added :func:`asyncio.eager_task_factory` and :func:`asyncio.create_eager_task_factory`
-  functions to allow opting an event loop in to eager task execution,
-  speeding up some use-cases by up to 50%.
-  (Contributed by Jacob Bower & Itamar O in :gh:`102853`)
-
 
 CPython bytecode changes
 ========================
@@ -1168,6 +1169,14 @@ Build Changes
   optimization levels (0, 1, 2) at once.
   (Contributed by Victor Stinner in :gh:`99289`.)
 
+* Add platform triplets for 64-bit LoongArch:
+
+  * loongarch64-linux-gnusf
+  * loongarch64-linux-gnuf32
+  * loongarch64-linux-gnu
+
+  (Contributed by Zhang Na in :gh:`90656`.)
+
 
 C API Changes
 =============

Include/cpython/memoryobject.h

@@ -24,6 +24,7 @@ typedef struct {
 #define _Py_MEMORYVIEW_FORTRAN     0x004  /* Fortran contiguous layout */
 #define _Py_MEMORYVIEW_SCALAR      0x008  /* scalar: ndim = 0 */
 #define _Py_MEMORYVIEW_PIL         0x010  /* PIL-style layout */
+#define _Py_MEMORYVIEW_RESTRICTED  0x020  /* Disallow new references to the memoryview's buffer */
 
 typedef struct {
     PyObject_VAR_HEAD

Include/internal/pycore_ceval.h

@@ -21,8 +21,7 @@ struct _ceval_runtime_state;
 
 
 extern void _Py_FinishPendingCalls(PyThreadState *tstate);
-extern void _PyEval_InitRuntimeState(struct _ceval_runtime_state *);
-extern void _PyEval_InitState(struct _ceval_state *, PyThread_type_lock);
+extern void _PyEval_InitState(PyInterpreterState *, PyThread_type_lock);
 extern void _PyEval_FiniState(struct _ceval_state *ceval);
 PyAPI_FUNC(void) _PyEval_SignalReceived(PyInterpreterState *interp);
 PyAPI_FUNC(int) _PyEval_AddPendingCall(

Include/internal/pycore_ceval_state.h

@@ -49,9 +49,6 @@ struct _ceval_runtime_state {
        the main thread of the main interpreter can handle signals: see
        _Py_ThreadCanHandleSignals(). */
     _Py_atomic_int signals_pending;
-
-    /* This is (only) used indirectly through PyInterpreterState.ceval.gil. */
-    struct _gil_runtime_state gil;
 };
 
 #ifdef PY_HAVE_PERF_TRAMPOLINE

Include/internal/pycore_compile.h

@@ -97,7 +97,8 @@ PyAPI_FUNC(PyObject*) _PyCompile_CodeGen(
         PyObject *ast,
         PyObject *filename,
         PyCompilerFlags *flags,
-        int optimize);
+        int optimize,
+        int compile_mode);
 
 PyAPI_FUNC(PyObject*) _PyCompile_OptimizeCfg(
         PyObject *instructions,

Include/internal/pycore_global_objects_fini_generated.h

@@ -335,6 +335,7 @@ struct _Py_global_strings {
         STRUCT_FOR_ID(code)
         STRUCT_FOR_ID(command)
         STRUCT_FOR_ID(comment_factory)
+        STRUCT_FOR_ID(compile_mode)
         STRUCT_FOR_ID(consts)
         STRUCT_FOR_ID(context)
         STRUCT_FOR_ID(cookie)

Include/internal/pycore_global_strings.h

@@ -178,6 +178,9 @@ struct _is {
        basis.  Also see _PyRuntimeState regarding the various mutex fields.
        */
 
+    /* The per-interpreter GIL, which might not be used. */
+    struct _gil_runtime_state _gil;
+
     /* the initial PyInterpreterState.threads.head */
     PyThreadState _initial_thread;
 };

Include/internal/pycore_interp.h

@@ -9,7 +9,8 @@ extern "C" {
 #endif
 
 PyObject *
-PyMemoryView_FromObjectAndFlags(PyObject *v, int flags);
+_PyMemoryView_FromBufferProc(PyObject *v, int flags,
+                             getbufferproc bufferproc);
 
 #ifdef __cplusplus
 }

Include/internal/pycore_memoryobject.h

@@ -32,8 +32,6 @@ struct _getargs_runtime_state {
     struct _PyArg_Parser *static_parsers;
 };
 
-/* ceval state */
-
 /* GIL state */
 
 struct _gilstate_runtime_state {