Merge branch 'main' into pythongh-73435-pathlib-match-recursive

.github/CODEOWNERS

@@ -97,6 +97,8 @@ Include/pytime.h              @pganssle @abalkin
 /Tools/peg_generator/         @pablogsal @lysnikolaou
 /Lib/test/test_peg_generator/ @pablogsal @lysnikolaou
 /Grammar/python.gram          @pablogsal @lysnikolaou
+/Lib/tokenize.py              @pablogsal @lysnikolaou
+/Lib/test/test_tokenize.py    @pablogsal @lysnikolaou
 
 # AST
 Python/ast.c                  @isidentical

.github/workflows/build.yml

@@ -8,6 +8,7 @@ on:
   push:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'
@@ -16,6 +17,7 @@ on:
   pull_request:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'
@@ -368,6 +370,14 @@ jobs:
         echo "HYPOVENV=${VENV_LOC}" >> $GITHUB_ENV
         echo "VENV_PYTHON=${VENV_PYTHON}" >> $GITHUB_ENV
         ./python -m venv $VENV_LOC && $VENV_PYTHON -m pip install -U hypothesis
+    - name: 'Restore Hypothesis database'
+      id: cache-hypothesis-database
+      uses: actions/cache@v3
+      with:
+        path: ./hypothesis
+        key: hypothesis-database-${{ github.head_ref || github.run_id }}
+        restore-keys: |
+          - hypothesis-database-
     - name: "Run tests"
       working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: |
@@ -388,6 +398,11 @@ jobs:
           -x test_subprocess \
           -x test_signal \
           -x test_sysconfig
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: hypothesis-example-db
+        path: .hypothesis/examples/
 
 
   build_asan:

.github/workflows/doc.yml

@@ -5,6 +5,7 @@ on:
   #push:
   #  branches:
   #  - 'main'
+  #  - '3.12'
   #  - '3.11'
   #  - '3.10'
   #  - '3.9'
@@ -15,6 +16,7 @@ on:
   pull_request:
     branches:
     - 'main'
+    - '3.12'
     - '3.11'
     - '3.10'
     - '3.9'

.gitignore

@@ -23,6 +23,10 @@
 *.gc??
 *.profclang?
 *.profraw
+# Copies of binaries before BOLT optimizations.
+*.prebolt
+# BOLT profile data.
+*.fdata
 *.dyn
 .gdb_history
 .purify
@@ -57,7 +61,6 @@ Doc/.venv/
 Doc/env/
 Doc/.env/
 Include/pydtrace_probes.h
-Lib/lib2to3/*.pickle
 Lib/site-packages/*
 !Lib/site-packages/README.txt
 Lib/test/data/*
@@ -124,6 +127,7 @@ Tools/unicode/data/
 /platform
 /profile-clean-stamp
 /profile-run-stamp
+/profile-bolt-stamp
 /Python/deepfreeze/*.c
 /pybuilddir.txt
 /pyconfig.h

Doc/c-api/long.rst

@@ -322,3 +322,27 @@ distinguished from a number.  Use :c:func:`PyErr_Occurred` to disambiguate.
    with :c:func:`PyLong_FromVoidPtr`.
 
    Returns ``NULL`` on error.  Use :c:func:`PyErr_Occurred` to disambiguate.
+
+
+.. c:function:: int PyUnstable_Long_IsCompact(const PyLongObject* op)
+
+   Return 1 if *op* is compact, 0 otherwise.
+
+   This function makes it possible for performance-critical code to implement
+   a “fast path” for small integers. For compact values use
+   :c:func:`PyUnstable_Long_CompactValue`; for others fall back to a
+   :c:func:`PyLong_As* <PyLong_AsSize_t>` function or
+   :c:func:`calling <PyObject_CallMethod>` :meth:`int.to_bytes`.
+
+   The speedup is expected to be negligible for most users.
+
+   Exactly what values are considered compact is an implementation detail
+   and is subject to change.
+
+.. c:function:: Py_ssize_t PyUnstable_Long_CompactValue(const PyLongObject* op)
+
+   If *op* is compact, as determined by :c:func:`PyUnstable_Long_IsCompact`,
+   return its value.
+
+   Otherwise, the return value is undefined.
+

Doc/c-api/perfmaps.rst

@@ -0,0 +1,50 @@
+.. highlight:: c
+
+.. _perfmaps:
+
+Support for Perf Maps
+----------------------
+
+On supported platforms (as of this writing, only Linux), the runtime can take
+advantage of *perf map files* to make Python functions visible to an external
+profiling tool (such as `perf <https://perf.wiki.kernel.org/index.php/Main_Page>`_).
+A running process may create a file in the ``/tmp`` directory, which contains entries
+that can map a section of executable code to a name. This interface is described in the
+`documentation of the Linux Perf tool <https://git.kernel.org/pub/scm/linux/
+kernel/git/torvalds/linux.git/tree/tools/perf/Documentation/jit-interface.txt>`_.
+
+In Python, these helper APIs can be used by libraries and features that rely
+on generating machine code on the fly.
+
+Note that holding the Global Interpreter Lock (GIL) is not required for these APIs.
+
+.. c:function:: int PyUnstable_PerfMapState_Init(void)
+
+   Open the ``/tmp/perf-$pid.map`` file, unless it's already opened, and create
+   a lock to ensure thread-safe writes to the file (provided the writes are
+   done through :c:func:`PyUnstable_WritePerfMapEntry`). Normally, there's no need
+   to call this explicitly; just use :c:func:`PyUnstable_WritePerfMapEntry`
+   and it will initialize the state on first call.
+
+   Returns ``0`` on success, ``-1`` on failure to create/open the perf map file,
+   or ``-2`` on failure to create a lock. Check ``errno`` for more information
+   about the cause of a failure.
+
+.. c:function:: int PyUnstable_WritePerfMapEntry(const void *code_addr, unsigned int code_size, const char *entry_name)
+
+   Write one single entry to the ``/tmp/perf-$pid.map`` file. This function is
+   thread safe. Here is what an example entry looks like::
+
+      # address      size  name
+      7f3529fcf759 b     py::bar:/run/t.py
+
+   Will call :c:func:`PyUnstable_PerfMapState_Init` before writing the entry, if
+   the perf map file is not already opened. Returns ``0`` on success, or the
+   same error codes as :c:func:`PyUnstable_PerfMapState_Init` on failure.
+
+.. c:function:: void PyUnstable_PerfMapState_Fini(void)
+
+   Close the perf map file opened by :c:func:`PyUnstable_PerfMapState_Init`.
+   This is called by the runtime itself during interpreter shut-down. In
+   general, there shouldn't be a reason to explicitly call this, except to
+   handle specific scenarios such as forking.