Merge branch 'main' into thread-local-interpreter-state

Author: markshannon

.github/workflows/build.yml

@@ -64,6 +64,18 @@ jobs:
       - uses: actions/setup-python@v2
       - name: Install Dependencies
         run: sudo ./.github/workflows/posix-deps-apt.sh
+      - name: Add ccache to PATH
+        run: echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
+      - name: Configure ccache action
+        uses: hendrikmuhs/ccache-action@v1
+      - name: Check Autoconf version 2.69 and aclocal 1.16.3
+        run: |
+          grep "Generated by GNU Autoconf 2.69" configure
+          grep "aclocal 1.16.3" aclocal.m4
+          grep -q "runstatedir" configure
+          grep -q "PKG_PROG_PKG_CONFIG" aclocal.m4
+      - name: Regenerate autoconf files
+        run: docker run --rm -v $(pwd):/src quay.io/tiran/cpython_autoconf:269
       - name: Build CPython
         run: |
           # Build Python with the libpython dynamic library
@@ -75,26 +87,25 @@ jobs:
           git add -u
           changes=$(git status --porcelain)
           # Check for changes in regenerated files
-          if ! test -z "$changes"
-          then
-            echo "Generated files not up to date. Perhaps you forgot to run make regen-all or build.bat --regen ;)"
+          if test -n "$changes"; then
+            echo "Generated files not up to date."
+            echo "Perhaps you forgot to run make regen-all or build.bat --regen. ;)"
+            echo "configure files must be regenerated with a specific, unpatched version of autoconf."
             echo "$changes"
             exit 1
           fi
       - name: Check exported libpython symbols
         run: make smelly
       - name: Check limited ABI symbols
         run: make check-limited-abi
-      - name: Check Autoconf version 2.69
-        run: |
-          grep "Generated by GNU Autoconf 2.69" configure
-          grep "PKG_PROG_PKG_CONFIG" aclocal.m4
 
   build_win32:
     name: 'Windows (x86)'
     runs-on: windows-latest
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
+    env:
+       IncludeUwp: 'true'
     steps:
     - uses: actions/checkout@v2
     - name: Build CPython
@@ -109,6 +120,8 @@ jobs:
     runs-on: windows-latest
     needs: check_source
     if: needs.check_source.outputs.run_tests == 'true'
+    env:
+       IncludeUwp: 'true'
     steps:
     - uses: actions/checkout@v2
     - name: Register MSVC problem matcher
@@ -175,13 +188,28 @@ jobs:
         echo "PATH=/usr/lib/ccache:$PATH" >> $GITHUB_ENV
     - name: Configure ccache action
       uses: hendrikmuhs/ccache-action@v1
-    - name: Configure CPython
-      run: ./configure --with-pydebug --with-openssl=$OPENSSL_DIR
-    - name: Build CPython
+    - name: Setup directory envs for out-of-tree builds
+      run: |
+        echo "CPYTHON_RO_SRCDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-ro-srcdir)" >> $GITHUB_ENV
+        echo "CPYTHON_BUILDDIR=$(realpath -m ${GITHUB_WORKSPACE}/../cpython-builddir)" >> $GITHUB_ENV
+    - name: Create directories for read-only out-of-tree builds
+      run: mkdir -p $CPYTHON_RO_SRCDIR $CPYTHON_BUILDDIR
+    - name: Bind mount sources read-only
+      run: sudo mount --bind -o ro $GITHUB_WORKSPACE $CPYTHON_RO_SRCDIR
+    - name: Configure CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
+      run: ../cpython-ro-srcdir/configure --with-pydebug --with-openssl=$OPENSSL_DIR
+    - name: Build CPython out-of-tree
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make -j4
     - name: Display build info
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: make pythoninfo
+    - name: Remount sources writable for tests
+      # some tests write to srcdir, lack of pyc files slows down testing
+      run: sudo mount $CPYTHON_RO_SRCDIR -oremount,rw
     - name: Tests
+      working-directory: ${{ env.CPYTHON_BUILDDIR }}
       run: xvfb-run make buildbottest TESTOPTS="-j4 -uall,-cpu"
 
   build_ubuntu_ssltests:

.gitignore

@@ -10,6 +10,7 @@
 *.so.*
 *.dylib
 *.dll
+*.wasm
 *.orig
 *.pyc
 *.pyd
@@ -76,6 +77,7 @@ Misc/python-embed.pc
 Misc/python-config.sh
 Modules/Setup.config
 Modules/Setup.local
+Modules/Setup.stdlib
 Modules/config.c
 Modules/ld_so_aix
 Programs/_freeze_module
@@ -113,6 +115,8 @@ Tools/unicode/data/
 /config.log
 /config.status
 /config.status.lineno
+# hendrikmuhs/ccache-action@v1
+/.ccache
 /platform
 /profile-clean-stamp
 /profile-run-stamp

Doc/Makefile

@@ -117,6 +117,10 @@ suspicious:
 	     "or in build/$(BUILDER)/suspicious.csv.  If all issues are false" \
 	     "positives, append that file to tools/susp-ignored.csv."; \
 	false; }
+	@echo "⚠ make suspicious is deprecated and will be removed soon."
+	@echo "⚠ Use:"
+	@echo "⚠     make check"
+	@echo "⚠ instead."
 
 coverage: BUILDER = coverage
 coverage: build

Doc/c-api/exceptions.rst

@@ -482,7 +482,6 @@ Querying the error indicator
    to an exception that was *already caught*, not to an exception that was
    freshly raised.  This function steals the references of the arguments.
    To clear the exception state, pass ``NULL`` for all three arguments.
-   For general rules about the three arguments, see :c:func:`PyErr_Restore`.
 
    .. note::
 
@@ -493,6 +492,12 @@ Querying the error indicator
 
    .. versionadded:: 3.3
 
+   .. versionchanged:: 3.11
+      The ``type`` and ``traceback`` arguments are no longer used and
+      can be NULL. The interpreter now derives them from the exception
+      instance (the ``value`` argument). The function still steals
+      references of all three arguments.
+
 
 Signal Handling
 ===============

Doc/c-api/init_config.rst

@@ -479,6 +479,9 @@ PyConfig
 
       Fields which are already initialized are left unchanged.
 
+      Fields for :ref:`path configuration <init-path-config>` are no longer
+      calculated or modified when calling this function, as of Python 3.11.
+
       The :c:func:`PyConfig_Read` function only parses
       :c:member:`PyConfig.argv` arguments once: :c:member:`PyConfig.parse_argv`
       is set to ``2`` after arguments are parsed. Since Python arguments are
@@ -493,6 +496,12 @@ PyConfig
          parsed, and arguments are only parsed if
          :c:member:`PyConfig.parse_argv` equals ``1``.
 
+      .. versionchanged:: 3.11
+         :c:func:`PyConfig_Read` no longer calculates all paths, and so fields
+         listed under :ref:`Python Path Configuration <init-path-config>` may
+         no longer be updated until :c:func:`Py_InitializeFromConfig` is
+         called.
+
    .. c:function:: void PyConfig_Clear(PyConfig *config)
 
       Release configuration memory.
@@ -596,13 +605,16 @@ PyConfig
 
       .. versionadded:: 3.10
 
-   .. c:member:: int no_debug_ranges
+   .. c:member:: int code_debug_ranges
 
-      If equals to ``1``, disables the inclusion of the end line and column
+      If equals to ``0``, disables the inclusion of the end line and column
       mappings in code objects. Also disables traceback printing carets to
       specific error locations.
 
-      Default: ``0``.
+      Set to ``0`` by the :envvar:`PYTHONNODEBUGRANGES` environment variable
+      and by the :option:`-X no_debug_ranges <-X>` command line option.
+
+      Default: ``1``.
 
       .. versionadded:: 3.11
 
@@ -845,12 +857,19 @@ PyConfig
 
       Default: value of the ``PLATLIBDIR`` macro which is set by the
       :option:`configure --with-platlibdir option <--with-platlibdir>`
-      (default: ``"lib"``).
+      (default: ``"lib"``, or ``"DLLs"`` on Windows).
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
       .. versionadded:: 3.9
 
+      .. versionchanged:: 3.11
+         This macro is now used on Windows to locate the standard
+         library extension modules, typically under ``DLLs``. However,
+         for compatibility, note that this value is ignored for any
+         non-standard layouts, including in-tree builds and virtual
+         environments.
+
    .. c:member:: wchar_t* pythonpath_env
 
       Module search paths (:data:`sys.path`) as a string separated by ``DELIM``
@@ -867,9 +886,9 @@ PyConfig
 
       Module search paths: :data:`sys.path`.
 
-      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0, the
-      function calculating the :ref:`Python Path Configuration <init-path-config>`
-      overrides the :c:member:`~PyConfig.module_search_paths` and sets
+      If :c:member:`~PyConfig.module_search_paths_set` is equal to 0,
+      :c:func:`Py_InitializeFromConfig` will replace
+      :c:member:`~PyConfig.module_search_paths` and sets
       :c:member:`~PyConfig.module_search_paths_set` to ``1``.
 
       Default: empty list (``module_search_paths``) and ``0``
@@ -941,16 +960,16 @@ PyConfig
 
    .. c:member:: int pathconfig_warnings
 
-      On Unix, if non-zero, calculating the :ref:`Python Path Configuration
-      <init-path-config>` can log warnings into ``stderr``. If equals to 0,
-      suppress these warnings.
-
-      It has no effect on Windows.
+      If non-zero, calculation of path configuration is allowed to log
+      warnings into ``stderr``. If equals to 0, suppress these warnings.
 
       Default: ``1`` in Python mode, ``0`` in isolated mode.
 
       Part of the :ref:`Python Path Configuration <init-path-config>` input.
 
+      .. versionchanged:: 3.11
+         Now also applies on Windows.
+
    .. c:member:: wchar_t* prefix
 
       The site-specific directory prefix where the platform independent Python
@@ -1302,10 +1321,9 @@ variables, command line arguments (:c:member:`PyConfig.argv` is not parsed)
 and user site directory. The C standard streams (ex: ``stdout``) and the
 LC_CTYPE locale are left unchanged. Signal handlers are not installed.
 
-Configuration files are still used with this configuration. Set the
-:ref:`Python Path Configuration <init-path-config>` ("output fields") to ignore these
-configuration files and avoid the function computing the default path
-configuration.
+Configuration files are still used with this configuration to determine
+paths that are unspecified. Ensure :c:member:`PyConfig.home` is specified
+to avoid computing the default path configuration.
 
 
 .. _init-python-config:

Doc/c-api/iter.rst

@@ -9,8 +9,8 @@ There are two functions specifically for working with iterators.
 
 .. c:function:: int PyIter_Check(PyObject *o)
 
-   Return non-zero if the object *o* supports the iterator protocol, and ``0``
-   otherwise.  This function always succeeds.
+   Return non-zero if the object *o* can be safely passed to
+   :c:func:`PyIter_Next`, and ``0`` otherwise.  This function always succeeds.
 
 .. c:function:: int PyAIter_Check(PyObject *o)
 
@@ -21,10 +21,11 @@ There are two functions specifically for working with iterators.
 
 .. c:function:: PyObject* PyIter_Next(PyObject *o)
 
-   Return the next value from the iteration *o*.  The object must be an iterator
-   (it is up to the caller to check this).  If there are no remaining values,
-   returns ``NULL`` with no exception set.  If an error occurs while retrieving
-   the item, returns ``NULL`` and passes along the exception.
+   Return the next value from the iterator *o*.  The object must be an iterator
+   according to :c:func:`PyIter_Check` (it is up to the caller to check this).
+   If there are no remaining values, returns ``NULL`` with no exception set.
+   If an error occurs while retrieving the item, returns ``NULL`` and passes
+   along the exception.
 
 To write a loop which iterates over an iterator, the C code should look
 something like this::

Doc/c-api/typeobj.rst

@@ -1521,9 +1521,9 @@ and :c:type:`PyType_Type` effectively act as defaults.)
 
 .. c:member:: getiterfunc PyTypeObject.tp_iter
 
-   An optional pointer to a function that returns an iterator for the object.  Its
-   presence normally signals that the instances of this type are iterable (although
-   sequences may be iterable without this function).
+   An optional pointer to a function that returns an :term:`iterator` for the
+   object.  Its presence normally signals that the instances of this type are
+   :term:`iterable` (although sequences may be iterable without this function).
 
    This function has the same signature as :c:func:`PyObject_GetIter`::
 
@@ -1536,8 +1536,8 @@ and :c:type:`PyType_Type` effectively act as defaults.)
 
 .. c:member:: iternextfunc PyTypeObject.tp_iternext
 
-   An optional pointer to a function that returns the next item in an iterator.
-   The signature is::
+   An optional pointer to a function that returns the next item in an
+   :term:`iterator`. The signature is::
 
       PyObject *tp_iternext(PyObject *self);
 
@@ -2429,8 +2429,8 @@ Async Object Structures
 
       PyObject *am_await(PyObject *self);
 
-   The returned object must be an iterator, i.e. :c:func:`PyIter_Check` must
-   return ``1`` for it.
+   The returned object must be an :term:`iterator`, i.e. :c:func:`PyIter_Check`
+   must return ``1`` for it.
 
    This slot may be set to ``NULL`` if an object is not an :term:`awaitable`.
 

Doc/faq/library.rst

@@ -240,9 +240,6 @@ Be sure to use the :mod:`threading` module and not the :mod:`_thread` module.
 The :mod:`threading` module builds convenient abstractions on top of the
 low-level primitives provided by the :mod:`_thread` module.
 
-Aahz has a set of slides from his threading tutorial that are helpful; see
-http://www.pythoncraft.com/OSCON2001/.
-
 
 None of my threads seem to run: why?
 ------------------------------------
@@ -614,9 +611,9 @@ use ``p.read(n)``.
 How do I access the serial (RS232) port?
 ----------------------------------------
 
-For Win32, POSIX (Linux, BSD, etc.), Jython:
+For Win32, OSX, Linux, BSD, Jython, IronPython:
 
-   http://pyserial.sourceforge.net
+   https://pypi.org/project/pyserial/
 
 For Unix, see a Usenet post by Mitch Chapman:
 

Doc/faq/programming.rst

@@ -2092,7 +2092,7 @@ Jim Roskind suggests performing steps in the following order in each module:
 * ``import`` statements
 * active code (including globals that are initialized from imported values).
 
-van Rossum doesn't like this approach much because the imports appear in a
+Van Rossum doesn't like this approach much because the imports appear in a
 strange place, but it does work.
 
 Matthias Urlichs recommends restructuring your code so that the recursive import

Doc/glossary.rst

@@ -659,6 +659,11 @@ Glossary
 
       More information can be found in :ref:`typeiter`.
 
+      .. impl-detail::
+
+         CPython does not consistently apply the requirement that an iterator
+         define :meth:`__iter__`.
+
    key function
       A key function or collation function is a callable that returns a value
       used for sorting or ordering.  For example, :func:`locale.strxfrm` is

Doc/howto/annotations.rst

@@ -156,7 +156,7 @@ Manually Un-Stringizing Stringized Annotations
   require annotating with string values that specifically
   *can't* be evaluated.  For example:
 
-  * :pep:`604` union types using `|`, before support for this
+  * :pep:`604` union types using ``|``, before support for this
     was added to Python 3.10.
   * Definitions that aren't needed at runtime, only imported
     when :const:`typing.TYPE_CHECKING` is true.