diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 36b5fbed6d0..f59bf136f3b 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -565,7 +565,6 @@ peps/pep-0681.rst @jellezijlstra peps/pep-0682.rst peps/pep-0683.rst @ericsnowcurrently peps/pep-0684.rst @ericsnowcurrently -# peps/pep-0684.rst peps/pep-0685.rst @brettcannon peps/pep-0686.rst @methane peps/pep-0687.rst @encukou @erlend-aasland @@ -628,21 +627,23 @@ peps/pep-0746.rst @JelleZijlstra peps/pep-0747.rst @JelleZijlstra # ... peps/pep-0749.rst @JelleZijlstra -# ... peps/pep-0750.rst @gvanrossum @lysnikolaou peps/pep-0751.rst @brettcannon peps/pep-0752.rst @warsaw peps/pep-0753.rst @warsaw -# ... # peps/pep-0754.rst -# ... +peps/pep-0755.rst @warsaw peps/pep-0756.rst @vstinner peps/pep-0757.rst @vstinner +peps/pep-0758.rst @pablogsal @brettcannon +peps/pep-0759.rst @warsaw +# ... peps/pep-0789.rst @njsmith # ... peps/pep-0801.rst @warsaw # ... peps/pep-2026.rst @hugovk +# ... peps/pep-3000.rst @gvanrossum peps/pep-3001.rst @birkenfeld # peps/pep-3002.rst diff --git a/.gitignore b/.gitignore index 6004c5e61f7..6e056e9f68d 100644 --- a/.gitignore +++ b/.gitignore @@ -24,4 +24,5 @@ coverage.xml /venv # Builds -/sphinx-warnings.txt \ No newline at end of file +/sphinx-warnings.txt +/peps/numerical.rst diff --git a/pep_sphinx_extensions/pep_theme/static/style.css b/pep_sphinx_extensions/pep_theme/static/style.css index da600ba173d..37323c27794 100644 --- a/pep_sphinx_extensions/pep_theme/static/style.css +++ b/pep_sphinx_extensions/pep_theme/static/style.css @@ -281,6 +281,13 @@ table.pep-zero-table tr td:nth-child(5) { width: 50%; } +/* Numerical index */ +article:has(> section#numerical-index) { + float: unset !important; + width: 90% !important; + max-width: 90% !important; +} + /* Breadcrumbs rules */ section#pep-page-section > header { border-bottom: 1px solid var(--colour-rule-light); diff --git a/pep_sphinx_extensions/pep_theme/templates/page.html b/pep_sphinx_extensions/pep_theme/templates/page.html index 1bc41d89766..f903b787afa 100644 --- a/pep_sphinx_extensions/pep_theme/templates/page.html +++ b/pep_sphinx_extensions/pep_theme/templates/page.html @@ -49,14 +49,16 @@

Python Enhancement Proposals

{{ body }}
+ {%- if not pagename == "numerical" %} + {%- endif %} diff --git a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py index 8fbf5cc7e65..2dc3e7ff52d 100644 --- a/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py +++ b/pep_sphinx_extensions/pep_zero_generator/pep_index_generator.py @@ -63,6 +63,9 @@ def write_peps_json(peps: list[parser.PEP], path: Path) -> None: def create_pep_zero(app: Sphinx, env: BuildEnvironment, docnames: list[str]) -> None: peps = _parse_peps(Path(app.srcdir)) + numerical_index_text = writer.PEPZeroWriter().write_numerical_index(peps) + subindices.update_sphinx("numerical", numerical_index_text, docnames, env) + pep0_text = writer.PEPZeroWriter().write_pep0(peps, builder=env.settings["builder"]) pep0_path = subindices.update_sphinx("pep-0000", pep0_text, docnames, env) peps.append(parser.PEP(pep0_path)) diff --git a/pep_sphinx_extensions/pep_zero_generator/writer.py b/pep_sphinx_extensions/pep_zero_generator/writer.py index e0507cc7f29..030fd6ed65e 100644 --- a/pep_sphinx_extensions/pep_zero_generator/writer.py +++ b/pep_sphinx_extensions/pep_zero_generator/writer.py @@ -132,6 +132,18 @@ def emit_pep_category(self, category: str, peps: list[PEP]) -> None: self.emit_text(" -") self.emit_newline() + def write_numerical_index(self, peps: list[PEP]) -> str: + """Write PEPs by number.""" + self.emit_text(".. _numerical-index:") + self.emit_newline() + + self.emit_title("Numerical Index") + self.emit_table(peps) + self.emit_newline() + + numerical_index_string = "\n".join(self.output) + return numerical_index_string + def write_pep0( self, peps: list[PEP], @@ -139,7 +151,7 @@ def write_pep0( intro: str = INTRO, is_pep0: bool = True, builder: str = None, - ): + ) -> str: if len(peps) == 0: return "" @@ -176,6 +188,15 @@ def write_pep0( ) self.emit_newline() + # PEPs by number + if is_pep0: + self.emit_title("Numerical Index") + self.emit_text( + "The :doc:`numerical index ` contains " + "a table of all PEPs, ordered by number." + ) + self.emit_newline() + # PEPs by category self.emit_title("Index by Category") meta, info, provisional, accepted, open_, finished, historical, deferred, dead = _classify_peps(peps) @@ -188,7 +209,7 @@ def write_pep0( ("Finished PEPs (done, with a stable interface)", finished), ("Historical Meta-PEPs and Informational PEPs", historical), ("Deferred PEPs (postponed pending further research or updates)", deferred), - ("Abandoned, Withdrawn, and Rejected PEPs", dead), + ("Rejected, Superseded, and Withdrawn PEPs", dead), ] for (category, peps_in_category) in pep_categories: # For sub-indices, only emit categories with entries. @@ -203,12 +224,6 @@ def write_pep0( self.emit_newline() - # PEPs by number - self.emit_title("Numerical Index") - self.emit_table(peps) - - self.emit_newline() - # Reserved PEP numbers if is_pep0: self.emit_title("Reserved PEP Numbers") @@ -264,7 +279,7 @@ def write_pep0( self.emit_newline() self.emit_newline() - pep0_string = "\n".join(map(str, self.output)) + pep0_string = "\n".join(self.output) return pep0_string diff --git a/peps/contents.rst b/peps/contents.rst index 5d5c23ee3a6..0dc1f8e5817 100644 --- a/peps/contents.rst +++ b/peps/contents.rst @@ -14,6 +14,7 @@ This is an internal Sphinx page; please go to the :doc:`PEP Index `. :glob: :caption: PEP Table of Contents (needed for Sphinx): - pep-* api/* topic/* + numerical + pep-* diff --git a/peps/pep-0101.rst b/peps/pep-0101.rst index c21e181457b..cb9f8840855 100644 --- a/peps/pep-0101.rst +++ b/peps/pep-0101.rst @@ -539,9 +539,6 @@ the main repo. - Edit all version references in ``README.rst`` - - Move any historical "what's new" entries from ``Misc/NEWS`` to - ``Misc/HISTORY``. - - Edit ``Doc/tutorial/interpreter.rst`` (two references to '[Pp]ython3x', one to 'Python 3.x', also make the date in the banner consistent). @@ -571,6 +568,10 @@ the main repo. ls PC/pyconfig.h.in PCbuild/rt.bat | xargs sed -i 's/python3\(\.\?\)[0-9]\+/python3\19/g' + - Edit the ``bug.yml`` and ``crash.yml`` issue templates in + ``.github/ISSUE_TEMPLATE/`` to add the new branch to the + "versions" dropdown. + - Commit these changes to the main branch:: git status diff --git a/peps/pep-0241.rst b/peps/pep-0241.rst index a33fe482647..289b1b6c4cd 100644 --- a/peps/pep-0241.rst +++ b/peps/pep-0241.rst @@ -9,7 +9,8 @@ Created: 12-Mar-2001 Post-History: `19-Mar-2001 `__ Superseded-By: 314 -.. superseded:: 314 +.. canonical-pypa-spec:: :ref:`packaging:core-metadata` + Introduction ============ diff --git a/peps/pep-0314.rst b/peps/pep-0314.rst index 6817cff3e10..36d3ad709af 100644 --- a/peps/pep-0314.rst +++ b/peps/pep-0314.rst @@ -12,6 +12,8 @@ Post-History: 29-Apr-2003 Replaces: 241 Superseded-By: 345 +.. canonical-pypa-spec:: :ref:`packaging:core-metadata` + Introduction ============ diff --git a/peps/pep-0345.rst b/peps/pep-0345.rst index a9a9f359ba8..b3062e74866 100644 --- a/peps/pep-0345.rst +++ b/peps/pep-0345.rst @@ -13,6 +13,8 @@ Replaces: 314 Superseded-By: 566 Resolution: https://mail.python.org/archives/list/python-dev@python.org/thread/MKHXVV746H7ZDFN62Z72VNAX6KIRXNRO/ +.. canonical-pypa-spec:: :ref:`packaging:core-metadata` + Abstract ======== diff --git a/peps/pep-0426.rst b/peps/pep-0426.rst index f67e7bde1b3..0acb6883cf4 100644 --- a/peps/pep-0426.rst +++ b/peps/pep-0426.rst @@ -14,6 +14,9 @@ Post-History: 14-Nov-2012, 05-Feb-2013, 07-Feb-2013, 09-Feb-2013, 27-May-2013, 20-Jun-2013, 23-Jun-2013, 14-Jul-2013, 21-Dec-2013 Replaces: 345 +Superseded-By: 566 + +.. canonical-pypa-spec:: :ref:`packaging:core-metadata` .. withdrawn:: diff --git a/peps/pep-0569.rst b/peps/pep-0569.rst index 8f1d1a1b2be..fc8d32d73ed 100644 --- a/peps/pep-0569.rst +++ b/peps/pep-0569.rst @@ -1,7 +1,7 @@ PEP: 569 Title: Python 3.8 Release Schedule Author: Łukasz Langa -Status: Active +Status: Final Type: Informational Topic: Release Created: 27-Jan-2018 @@ -34,10 +34,15 @@ Release Manager and Crew 3.8 will receive bugfix updates approximately every 2 months for approximately 18 months. Some time after the release of 3.9.0 final, -the ninth and final 3.8 bugfix update will be released. After that, -it is expected that security updates (source only) will be released -until 5 years after the release of 3.8 final, so until approximately -October 2024. +the ninth and final 3.8 bugfix update was released. After that, +security updates (source only) were released for 5 years until the +release of Python 3.13.0 final. + +As of 2024-10-07, 3.8 has reached the +`end-of-life phase `_ +of its release cycle. 3.8.20 was the final security release. +The codebase for 3.8 is now frozen and no further updates will be +provided nor issues of any kind will be accepted on the bug tracker. Release Schedule @@ -86,7 +91,7 @@ Bugfix releases Source-only security fix releases --------------------------------- -Provided irregularly on an "as-needed" basis until October 2024. +Provided irregularly on an "as-needed" basis until October 7th 2024. - 3.8.11: Monday, 2021-06-28 - 3.8.12: Monday, 2021-08-30 @@ -97,7 +102,7 @@ Provided irregularly on an "as-needed" basis until October 2024. - 3.8.17: Tuesday, 2023-06-06 - 3.8.18: Thursday, 2023-08-24 - 3.8.19: Tuesday, 2024-03-19 -- 3.8.20: Friday, 2024-09-06 +- 3.8.20: Friday, 2024-09-06 (final security release) Features for 3.8 diff --git a/peps/pep-0693.rst b/peps/pep-0693.rst index cb629da1f31..f8122e3d005 100644 --- a/peps/pep-0693.rst +++ b/peps/pep-0693.rst @@ -63,10 +63,10 @@ Actual: - 3.12.4: Thursday, 2024-06-06 - 3.12.5: Tuesday, 2024-08-06 - 3.12.6: Friday, 2024-09-06 +- 3.12.7: Tuesday, 2024-10-01 Expected: -- 3.12.7: Tuesday, 2024-10-01 - 3.12.8: Tuesday, 2024-12-03 - 3.12.9: Tuesday, 2025-02-04 - 3.12.10: Tuesday, 2025-04-08 diff --git a/peps/pep-0703.rst b/peps/pep-0703.rst index 0cc4e84145b..bed80c7842d 100644 --- a/peps/pep-0703.rst +++ b/peps/pep-0703.rst @@ -13,7 +13,7 @@ Post-History: `09-Jan-2023 `__, Resolution: https://discuss.python.org/t/pep-703-making-the-global-interpreter-lock-optional-in-cpython-acceptance/37075 .. note:: - The Steering Council accepts PEP 703, but with clear provisio: that + The Steering Council accepts PEP 703, but with clear proviso: that the rollout be gradual and break as little as possible, and that we can roll back any changes that turn out to be too disruptive – which includes potentially rolling back all of PEP 703 entirely if necessary diff --git a/peps/pep-0719.rst b/peps/pep-0719.rst index 36f939ed05e..420c04422fe 100644 --- a/peps/pep-0719.rst +++ b/peps/pep-0719.rst @@ -49,11 +49,11 @@ Actual: - 3.13.0 beta 4: Thursday, 2024-07-18 - 3.13.0 candidate 1: Thursday, 2024-08-01 - 3.13.0 candidate 2: Friday, 2024-09-06 +- 3.13.0 candidate 3: Tuesday, 2024-10-01 +- 3.13.0 final: Monday, 2024-10-07 Expected: -- 3.13.0 final: Tuesday, 2024-10-01 - Subsequent bugfix releases every two months. diff --git a/peps/pep-0730.rst b/peps/pep-0730.rst index 0d997d87365..64c017a2b7e 100644 --- a/peps/pep-0730.rst +++ b/peps/pep-0730.rst @@ -10,6 +10,9 @@ Created: 09-Oct-2023 Python-Version: 3.13 Resolution: https://discuss.python.org/t/pep-730-adding-ios-as-a-supported-platform/35854/66 +.. canonical-doc:: :ref:`python:using-ios` + + Abstract ======== diff --git a/peps/pep-0735.rst b/peps/pep-0735.rst index bb1ee4c44e0..dd87a5f9aa0 100644 --- a/peps/pep-0735.rst +++ b/peps/pep-0735.rst @@ -51,6 +51,8 @@ which this PEP seeks to support: * non-package projects, such as data science projects +Several motivating use cases are defined in detail in the :ref:`Use Cases Appendix `. + Limitations of ``requirements.txt`` files ----------------------------------------- @@ -95,8 +97,11 @@ package specifiers which are published as part of a package's metadata, and which a user can request under that name, as in ``pip install 'foo[bar]'`` to install ``foo`` with the ``bar`` extra. -Because ``extras`` are package metadata, they are not usable when a project -does not build a distribution (i.e., is not a package). +Because ``extras`` are package metadata, they are not guaranteed to be +statically defined and may require a build system to resolve. +Furthermore, definition of a ``[project.optional-dependencies]`` indicates to +many tools that a project is a package, and may drive tool behaviors such as +validation of the ``[project]`` table. For projects which are packages, ``extras`` are a common solution for defining development dependencies, but even under these circumstances they have @@ -163,24 +168,6 @@ defined in greater detail in the :ref:`Use Cases Appendix `. be used as a location for locked dependency data) * Input data to an environment manager, such as tox, Nox, or Hatch * Configurable IDE discovery of test and linter requirements -* Exposure of package dependencies for install, without the package itself - -Support for Inclusion from ``project.dependencies`` ---------------------------------------------------- - -The ``project.dependencies`` and ``project.optional-dependencies`` tables are -allowed to include Dependency Groups, requiring an update to the specification -of these tables. - -The drivers for such changes are that some usages are well solved by the -addition of such support, and that failing to include support in the initial -Dependency Group PEP, but adding such support later in a subsequent PEP, -would make the support landscape significantly more difficult for tool -maintainers. - -Inclusion of a Dependency Group in ``project.dependencies`` or -``project.optional-dependencies`` takes the form of a Dependency Group Include, -defined in the specification section below. Regarding Poetry and PDM Dependency Groups ------------------------------------------ @@ -255,12 +242,7 @@ Strings in requirement lists must be valid `Dependency Specifiers `__, as defined in :pep:`508`. -Tables in requirement lists must be valid Dependency Object Specifiers, -defined below. - -The ``project`` table in ``pyproject.toml`` is modified such that -``project.dependencies`` and the values of ``project.optional-dependencies`` -may contain Dependency Object Specifiers. +Tables in requirement lists must be valid Dependency Object Specifiers. Dependency Object Specifiers ---------------------------- @@ -310,21 +292,6 @@ Includes, in which case those includes should be expanded as well. Dependency Group Includes MUST NOT include cycles, and tools SHOULD report an error if they detect a cycle. -``project`` Table Changes -------------------------- - -The ``[project]`` table, originally defined in :pep:`621` is extended in two ways. - -``dependencies`` - -In addition to :pep:`508` strings, the array may contain Dependency Object -Specifiers. - -``optional-dependencies`` - -In addition to :pep:`508` strings, the array values in this table may contain -Dependency Object Specifiers. - Example Dependency Groups Table ------------------------------- @@ -404,11 +371,10 @@ Tools MAY choose to provide the same interfaces for installing Dependency Groups as they do for installing extras. Note that this specification does not forbid having an extra whose name matches -a Dependency Group. In such cases, tools must define their own semantics for -precedence order or disambiguation. +a Dependency Group. Users are advised to avoid creating Dependency Groups whose names match extras. -Tools SHOULD NOT treat such matching as an error. +Tools MAY treat such matching as an error. Validation and Compatibility ---------------------------- @@ -482,7 +448,7 @@ The output is therefore valid ``requirements.txt`` data. def _resolve_dependency_group( - dependency_groups: dict, group: str, past_groups: tuple[str] = () + dependency_groups: dict, group: str, past_groups: tuple[str, ...] = () ) -> list[str]: if group in past_groups: raise ValueError(f"Cyclic dependency group include: {group} -> {past_groups}") @@ -542,16 +508,15 @@ At time of writing, the ``dependency-groups`` namespace within a reserved for use only by standards specified at packaging.python.org, there are no direct backwards compatibility concerns. -However, the introduction of the feature as a potential component of -``project`` data has implications for a number of ecosystem tools. +However, the introduction of the feature has implications for a +number of ecosystem tools, especially those which attempt to support +examination of data in ``setup.py`` and ``requirements.txt``. Audit and Update Tools ---------------------- A wide range of tools understand Python dependency data as expressed in -``project.dependencies`` and ``project.optional-dependencies`` and may -additionally support ``setup.cfg``, ``requirements.txt``, and even -``setup.py``. (e.g., Dependabot, Tidelift, etc) +``requirements.txt`` files. (e.g., Dependabot, Tidelift, etc) Such tools inspect dependency data and, in some cases, offer tool-assisted or fully automated updates. @@ -563,31 +528,6 @@ As a result, users of Dependency Groups would experience a degradation in their workflows and tool support at the time that they start using Dependency Groups. This is true of any new standard for where and how dependency data are encoded. -Repackaging ------------ - -Repackaging, and in particular tool-assisted repackaging such as Grayskull or -PyInstaller, will need to contend with the change to package metadata -definitions as well. - -Because repackagers for alternate ecosystems such as conda and linux distros -are often distinct persons vs the package publishers, this compatibility -concern is more difficult to address than cases in which the package -maintainers experience the impact. Package maintainers may be unaware of the -impact of beginning to use Dependency Groups, and may unknowingly make changes -which harm downstream repackaging workflows. - -There are two primary ways in which this issue can be addressed: - -* via education and "How to Teach This" -- users whose packages are repackaged - should be made aware that using new standards may cause issues for downstream - package consumers - -* by driving more behavior through build-backends -- so long as dependency - metadata are gathered via :pep:`517` interfaces, downstream repackagers can - remain ignorant of which build system is being used and whether or not it - supports Dependency Groups - Security Implications ===================== @@ -632,22 +572,6 @@ defined dynamically. Requirements loaded from ``requirements.txt`` files and definitions of static lists prior to ``setup()`` invocation readily analogize with Dependency Groups. -Notes for Packages which are Repackaged ---------------------------------------- - -A special note should be given to package maintainers whose packages are -repackaged by linux distros, homebrew, conda, etc. - -Use of Dependency Group Includes in core project metadata, -``project.dependencies`` and ``project.optional-dependencies``, may break these -consumers' uses of your package. Because they may be consuming and directly -interact with the source for your repository, their toolchains may not support -Dependency Groups at the same time that the package maintainers' tools are -updated. - -Ensure that repackaging consumers can contact you if there is an issue, and -make sure to note transitions to use Dependency Groups in your changelogs. - Interfaces for Use of Dependency Groups --------------------------------------- @@ -779,18 +703,124 @@ has impacts on high-level tools like IDEs and Dependabot, which cannot support deep integration with these Dependency Groups. (For example, at time of writing Dependabot will not flag dependencies which are pinned in ``tox.ini`` files.) +Deferred Ideas +============== + +Why not support Dependency Group Includes in ``[project.dependencies]`` or ``[project.optional-dependencies]``? +--------------------------------------------------------------------------------------------------------------- + +Earlier drafts of this specification allowed Dependency Group Includes to be +used in the ``[project]`` table. +However, there were several issues raised during community feedback which led +to its removal. + +Only a small number of additional use cases would be addressed by the inclusion +of Dependency Groups, and it increased the scope of the specification +significantly. In particular, this inclusion would increase the number of parties +impacted by the addition. Many readers of the ``[project]`` table, including build +backends, SBOM generators, and dependency analyzers are implicated by a change to +``[project]`` but may continue to operate as-is in the presence of a new (but +unconnected) ``[dependency-groups]`` table. + +Separately from the above concern, allowing inclusion of dependency groups from the +``[project]`` table encourages package maintainers to move dependency metadata out +of the current standard location. +This complicates static ``pyproject.toml`` metadata and conflicts with the goal of +:pep:`621` to store dependency metadata in a single location. + +Finally, exclusion of ``[project]`` support from this PEP is not final. The +use of includes from that table, or an inclusion syntax from +``[dependency-groups]`` into ``[project]``, could be introduced by a future +PEP and considered on its own merits. + +Use Cases for Dependency Group Includes From ``[project]`` +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +Although deferred in this PEP, allowing includes from the ``[project]`` +table would address several use cases. + +In particular, there are cases in which package developers would like to +install only the dependencies of a package, without the package itself. + +For example: + +* Specify different environment variables or options when building dependencies + vs when building the package itself + +* Creating layered container images in which the dependencies are isolated from + the package being installed + +* Providing the dependencies to analysis environments (e.g., type checking) + without having to build and install the package itself + +For an example of the last case, consider the following sample +``pyproject.toml``: + +.. code-block:: toml + + [project] + dependencies = [{include = "runtime"}] + [optional-dependencies] + foo = [{include = "foo"}] + [dependency-groups] + runtime = ["a", "b"] + foo = ["c", "d"] + typing = ["mypy", {include = "runtime"}, {include = "foo"}] + +In this case, a ``typing`` group can be defined, with all of the package's +runtime dependencies, but without the package itself. This allows uses of the +``typing`` Dependency Group to skip installation of the package -- not only is +this more efficient, but it may reduce the requirements for testing systems. + Why not support Dependency Group Includes in ``[build-system.requires]``? ------------------------------------------------------------------------- -Although it may be interesting to allow this in the future, incorporating the -proposal into the build-system table reduces the ability of users to rely on -this to bootstrap support. :pep:`517` frontends would be required to support -Dependency Groups in order to achieve this, and the versions of these frontends -are not easily controlled by packages. +Given that we will not allow for ``[project]`` usage of Dependency Groups, +``[build-system.requires]`` can be considered in comparison with +``[project.dependencies]``. + +There are fewer theoretical usages for build requirements specified in a group +than package requirements. Additionally, the impact of such a change implicates +:pep:`517` frontend, which would need to support Dependency Groups in order to +prepare a build environment. + +Compared with changes to ``[project.dependencies]`` and +``[project.optional-dependencies]``, changing the behaviors of +``[build-system.requires]`` is higher impact and has fewer potential uses. +Therefore, given that this PEP declines to make changes to the ``[project]`` +table, changing ``[build-system]`` is also deferred. + + +Why not support a Dependency Group which includes the current project? +---------------------------------------------------------------------- + +Several usage scenarios for dependency groups revolve around installing a +dependency group alongside a package defined in the ``[project]`` table. +For example, testing a package involves installing testing dependencies and the +package itself. Additionally, the compatibility of a dependency group with the +main package is a valuable input to lockfile generators. + +In such cases, it is desirable for a Dependency Group to declare that it +depends upon the project itself. Example syntaxes from discussions included +``{include-project = true}`` and ``{include-group = ":project:"}``. + +However, if a specification is established to extend :pep:`508` with Path +Dependencies, this would result in Dependency Groups having two ways of +specifying the main package. For example, if ``.`` becomes formally supported, +and ``{include-project = true}`` is included in this PEP, then dependency +groups may specify any of the following groups + +.. code-block:: toml + + [dependency-groups] + case1 = [{include-project = true}] + case2 = ["."] + case3 = [{include-project = true}, "."] + case4 = [{include-project = false}, "."] -By only defining build backend support, it is possible for packages to start -leveraging the new syntax and capabilities without being concerned about -controlling the environment in which the package is built and installed. +In order to avoid a confusing future in which multiple different options +specify the package defined in ``pyproject.toml``, any syntax for declaring +this relationship is omitted from this PEP. .. _prior_art: @@ -1411,42 +1441,6 @@ for various purposes. This declaration allows the project author's knowledge of the appropriate tools for the project to be shared with all editors of that project. -Exposure of package dependencies without the package itself ------------------------------------------------------------ - -There are a variety of use-cases in which package developers would like to -install only the dependencies of a package, without the package itself. - -For example: - -* Specify different environment variables or options when building dependencies - vs when building the package itself - -* Creating layered container images in which the dependencies are isolated from - the package being installed - -* Providing the dependencies to analysis environments (e.g., type checking) - without having to build and install the package itself - -For an example of the last case, consider the following sample -``pyproject.toml``: - -.. code-block:: toml - - [project] - dependencies = [{include = "runtime"}] - [optional-dependencies] - foo = [{include = "foo"}] - [dependency-groups] - runtime = ["a", "b"] - foo = ["c", "d"] - typing = ["mypy", {include = "runtime"}, {include = "foo"}] - -In this case, a ``typing`` group can be defined, with all of the package's -runtime dependencies, but without the package itself. This allows uses of the -``typing`` Dependency Group to skip installation of the package -- not only is -this more efficient, but it may reduce the requirements for testing systems. - Copyright ========= diff --git a/peps/pep-0738.rst b/peps/pep-0738.rst index aa47c47b1fb..ede9aeeb1d6 100644 --- a/peps/pep-0738.rst +++ b/peps/pep-0738.rst @@ -3,12 +3,14 @@ Title: Adding Android as a supported platform Author: Malcolm Smith Sponsor: Petr Viktorin Discussions-To: https://discuss.python.org/t/pep-738-adding-android-as-a-supported-platform/40975 -Status: Accepted +Status: Final Type: Standards Track Created: 12-Dec-2023 Python-Version: 3.13 Resolution: https://discuss.python.org/t/pep-738-adding-android-as-a-supported-platform/40975/23 +.. canonical-doc:: :ref:`python:using-android` + Abstract ======== @@ -289,8 +291,6 @@ namedtuple containing the following: * ``release`` - Android version of the device, as a string (e.g. ``"14"``) * ``api_level`` - :ref:`API level <738-os-versions>` of the device, as an integer (e.g. ``34``) -* ``min_api_level`` - Minimum API level this build of Python can run on, as - an integer (e.g. ``23``). This is the same as ``sys.getandroidapilevel``. * ``manufacturer`` - `manufacturer `__ of the device, as a string (e.g. ``"Google"``) @@ -300,6 +300,8 @@ namedtuple containing the following: * ``device`` - `device name `__ of the device, as a string (e.g. ``"panther"``) +* ``is_emulator`` - ``True`` if the device is an emulator; ``False`` if it’s a + physical device. Which one of ``model`` and ``device`` is more likely to be unique, and which one is more likely to resemble the marketing name, varies between different @@ -437,6 +439,20 @@ example of a test suite that is executed on the Android emulator using GitHub Actions. +Rejected Ideas +============== + +The following changes were made to the original specification of +``platform.android_ver()``: + +* The ``min_api_level`` field was removed, because unlike all the other fields, + it isn't a property of the current device. This information is still available + from the pre-existing function ``sys.getandroidapilevel()``. + +* The ``is_emulator`` field was added, since experience during testing showed + that some issues were emulator-specific. + + Copyright ========= diff --git a/peps/pep-0740.rst b/peps/pep-0740.rst index 7679158b238..9493cb8a215 100644 --- a/peps/pep-0740.rst +++ b/peps/pep-0740.rst @@ -255,7 +255,7 @@ The following changes are made to the These changes require a version change to the JSON API: -* The ``api-version`` **SHALL** specify version 1.2 or later. +* The ``api-version`` **SHALL** specify version 1.3 or later. .. _attestation-object: diff --git a/peps/pep-0746.rst b/peps/pep-0746.rst index f2e0f1f8229..342b730aa69 100644 --- a/peps/pep-0746.rst +++ b/peps/pep-0746.rst @@ -15,7 +15,7 @@ Abstract This PEP proposes a mechanism for type checking metadata that uses the :py:data:`typing.Annotated` type. Metadata objects that implement -the new ``__supports_type__`` protocol will be type checked by static +the new ``__supports_annotated_base__`` protocol will be type checked by static type checkers to ensure that the metadata is valid for the given type. Motivation @@ -47,11 +47,11 @@ Specification ============= This PEP introduces a protocol that can be used by static and runtime type checkers to validate the consistency between ``Annotated`` metadata and a given type. -Objects that implement this protocol have an attribute called ``__supports_type__`` +Objects that implement this protocol have an attribute called ``__supports_annotated_base__`` that specifies whether the metadata is valid for a given type:: class Int64: - __supports_type__: int + __supports_annotated_base__: int The attribute may also be marked as a ``ClassVar`` to avoid interaction with dataclasses:: @@ -61,14 +61,14 @@ The attribute may also be marked as a ``ClassVar`` to avoid interaction with dat @dataclass class Gt: value: int - __supports_type__: ClassVar[int] + __supports_annotated_base__: ClassVar[int] When a static type checker encounters a type expression of the form ``Annotated[T, M1, M2, ...]``, it should enforce that for each metadata element in ``M1, M2, ...``, one of the following is true: -* The metadata element evaluates to an object that does not have a ``__supports_type__`` attribute; or -* The metadata element evaluates to an object ``M`` that has a ``__supports_type__`` attribute; - and ``T`` is assignable to the type of ``M.__supports_type__``. +* The metadata element evaluates to an object that does not have a ``__supports_annotated_base__`` attribute; or +* The metadata element evaluates to an object ``M`` that has a ``__supports_annotated_base__`` attribute; + and ``T`` is assignable to the type of ``M.__supports_annotated_base__``. To support generic ``Gt`` metadata, one might write:: @@ -79,7 +79,7 @@ To support generic ``Gt`` metadata, one might write:: ... class Gt[T]: - __supports_type__: ClassVar[SupportsGt[T]] + __supports_annotated_base__: ClassVar[SupportsGt[T]] def __init__(self, value: T) -> None: self.value = value @@ -137,8 +137,8 @@ does not generally use marker base classes. In addition, it provides less flexib the current proposal: it would not allow overloads, and it would require metadata objects to add a new base class, which may make their runtime implementation more complex. -Using a method instead of an attribute for ``__supports_type__`` -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ +Using a method instead of an attribute for ``__supports_annotated_base__`` +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ We considered using a method instead of an attribute for the protocol, so that this method can be used at runtime to check the validity of the metadata and to support overloads or returning boolean literals. diff --git a/peps/pep-0747.rst b/peps/pep-0747.rst index dd3a053ae69..1d5483ad77d 100644 --- a/peps/pep-0747.rst +++ b/peps/pep-0747.rst @@ -26,7 +26,7 @@ an overly-wide type like ``object``, which makes some use cases impossible and generally reduces type safety. This PEP addresses this limitation by introducing a new special form ``typing.TypeForm``. -This PEP makes no changes to the Python grammar. ``TypeForm`` is +This PEP makes no changes to the Python grammar. Correct usage of ``TypeForm`` is intended to be enforced only by type checkers, not by the Python runtime. @@ -95,13 +95,10 @@ would benefit from ``TypeForm``: - Assignability checkers: - Determines whether a value is assignable to a specified type - - Pattern 1: - - ``def is_assignable[T](value: object, typx: TypeForm[T]) -> TypeIs[T]`` - - Pattern 2: + - Pattern 1: ``def is_assignable[T](value: object, typx: TypeForm[T]) -> TypeIs[T]`` - ``def is_match[T](value: object, typx: TypeForm[T]) -> TypeGuard[T]`` + - Pattern 2: ``def is_match[T](value: object, typx: TypeForm[T]) -> TypeGuard[T]`` - Examples: beartype.\ `is_bearable`_, trycast.\ `isassignable`_, typeguard.\ `check_type`_, xdsl.\ `isa`_ @@ -119,7 +116,9 @@ would benefit from ``TypeForm``: - Pattern 1: - ``def convert[T](value: object, typx: TypeForm[T]) -> T`` + :: + + def convert[T](value: object, typx: TypeForm[T]) -> T - Examples: cattrs.BaseConverter.\ `structure`_, trycast.\ `checkcast`_, typedload.\ `load`_ @@ -223,7 +222,7 @@ For example, if a static type checker encounters the expression ``str | None``, it may normally evaluate its type as ``UnionType`` because it produces a runtime value that is an instance of ``types.UnionType``. However, because this expression is a valid type expression, it is also assignable to the -type ``TypeForm[str | None]``: +type ``TypeForm[str | None]``:: v1_actual: UnionType = str | None # OK v1_type_form: TypeForm[str | None] = str | None # OK diff --git a/peps/pep-0749.rst b/peps/pep-0749.rst index 6503ef832f1..5afd763a2d2 100644 --- a/peps/pep-0749.rst +++ b/peps/pep-0749.rst @@ -27,10 +27,14 @@ specification: * We specify the behavior of wrapper objects that provide annotations, such as :py:func:`classmethod` and code that uses :py:func:`functools.wraps`. * There will not be a code flag for marking ``__annotate__`` functions - that can be run in a "fake globals" environment. -* Setting the ``__annotations__`` attribute directly will not affect the ``__annotate__`` attribute. + that can be run in a "fake globals" environment. Instead, we add a fourth format, + ``VALUE_WITH_FAKE_GLOBALS``, to allow third-party implementors of annotate functions to + indicate what formats they support. +* Deleting the ``__annotations__`` attribute directly will also clear ``__annotate__``. * We add functionality to allow evaluating type alias values and type parameter bounds and defaults (which were added by :pep:`695` and :pep:`696`) using PEP 649-like semantics. +* The ``SOURCE`` format is renamed to ``STRING`` to improve clarity and reduce the risk of + user confusion. Motivation ========== @@ -74,6 +78,11 @@ We suggest the following deprecation plan: - In Python 3.14, ``from __future__ import annotations`` will continue to work as it did before, converting annotations into strings. + + - If the future import is active, the ``__annotate__`` function of objects with + annotations will return the annotations as strings when called with the ``VALUE`` + format, reflecting the behavior of ``__annotations__``. + - Sometime after the last release that did not support :pep:`649` semantics (expected to be 3.13) reaches its end-of-life, ``from __future__ import annotations`` is deprecated. Compiling any code that uses the future import will emit a :py:exc:`DeprecationWarning`. This will @@ -194,7 +203,8 @@ The module will contain the following functionality: * ``Format``: an enum that contains the possible formats of annotations. This will replace the ``VALUE``, ``FORWARDREF``, and ``SOURCE`` formats in :pep:`649`. PEP 649 proposed to make these values global members of the :py:mod:`inspect` - module; we prefer to place them within an enum. + module; we prefer to place them within an enum. We propose to add a fourth format, + ``VALUE_WITH_FAKE_GLOBALS`` (see below). * ``ForwardRef``: a class representing a forward reference; it may be returned by ``get_annotations()`` when the format is ``FORWARDREF``. The existing class :py:class:`typing.ForwardRef` will become an alias of this class. Its members include: @@ -221,6 +231,16 @@ The module will contain the following functionality: dictionary. This is intended to be used for evaluating deferred attributes introduced by :pep:`695` and :pep:`696`; see below for details. *func* may be ``None`` for convenience; if ``None`` is passed, the function also returns ``None``. +* ``annotations_to_string(annotations: dict[str, object]) -> dict[str, str]``: a function that + converts each value in an annotations dictionary to a string representation. + This is useful for + implementing the ``SOURCE`` format in cases where the original source is not available, + such as in the functional syntax for :py:class:`typing.TypedDict`. +* ``value_to_string(value: object) -> str``: a function that converts a single value to a + string representation. This is used by ``annotations_to_string``. + It uses ``repr()`` for most values, but for types it returns the fully qualified name. + It is also useful as a helper for the ``repr()`` of a number of objects in the + :py:mod:`typing` and :py:mod:`collections.abc` modules. A new function is also added to the :py:mod:`!typing` module, ``typing.evaluate_forward_ref``. This function is a wrapper around the ``ForwardRef.evaluate`` method, but it performs @@ -241,6 +261,9 @@ What should this module be called? Some ideas: and ``from __future__ import annotations`` in the same module. The use of a common word as the name will make the module harder to search for. There is a PyPI package :pypi:`annotations`, but it had only a single release in 2015 and looks abandoned. +- ``annotation`` (in the singular): Similar, but does not cause confusion with the future + import. There is an abandoned PyPI package :pypi:`annotation`, but it apparently never + released any artifacts. - ``annotools``: Analogous to :py:mod:`itertools` and :py:mod:`functools`, but "anno" is a less obvious abbreviation than "iter" or "func". As of this writing, there is no PyPI package with this name. @@ -551,8 +574,8 @@ This approach would also mean that accessing ``.__annotations__`` on an instance of an annotated class no longer works. While this behavior is not documented, it is a long-standing feature of Python and is relied upon by some users. -Remove code flag for marking ``__annotate__`` functions -======================================================= +Adding the ``VALUE_WITH_FAKE_GLOBALS`` format +============================================= :pep:`649` specifies: @@ -567,57 +590,64 @@ Remove code flag for marking ``__annotate__`` functions it's expected that only ``__annotate__`` methods generated by the Python compiler will set it. -We have not found a need for this mechanism during our work to -add :pep:`649` support to the standard library. While it is true -that custom ``__annotate__`` functions may not work well with the -"fake globals" environment, this technique is used only when the -``__annotate__`` function raises :py:exc:`NotImplementedError` to -signal that it does not support the requested format. However, -manually implemented ``__annotate__`` functions are likely to support -all three annotation formats; often, they will consist of a call to -``annotationlib.call_annotate_function`` plus some transformation of the -result. - -In addition, the proposed mechanism couples the implementation with +However, this mechanism couples the implementation with low-level details of the code object. The code object flags are CPython-specific and the documentation :py:ref:`explicitly warns ` against relying on the values. +Larry Hastings suggested an alternative approach that does not +rely on code flags: a fourth format, ``VALUE_WITH_FAKE_GLOBALS``. +Compiler-generated annotate functions would support only the +``VALUE`` and ``VALUE_WITH_FAKE_GLOBALS`` formats, both of which are +implemented identically. The standard library would use the +``VALUE_WITH_FAKE_GLOBALS`` format when invoking an annotate function +in one of the special "fake globals" environments. + +This approach is useful as a forward-compatible mechanism for +adding new annotation formats in the future. Users who manually +write annotate functions should raise ``NotImplementedError`` if +the ``VALUE_WITH_FAKE_GLOBALS`` format is requested, so the standard +library will not call the manually written annotate function with +"fake globals", which could have unpredictable results. + Specification ------------- -The standard library will use the "fake globals" technique on any -``__annotate__`` function that raises :py:exc:`NotImplementedError` -when the requested format is not supported. +An additional format, ``FAKE_GLOBALS_VALUE``, is added to the ``Format`` enum in the +``annotationlib`` module, with value equal to 2. (As a result, the values of the +other formats will shift relative to PEP 649: ``FORWARDREF`` will be 3 and ``SOURCE`` +will be 4.) + +Compiler-generated +annotate functions will support this format and return the same value as +they would return for the ``VALUE`` format. The standard library will pass +this format to the ``__annotate__`` function when it is called in a "fake globals" +environment, as used to implement the ``FORWARDREF`` and ``SOURCE`` formats. +All public functions in the ``annotationlib`` module that accept a format +argument will raise :py:exc:`NotImplementedError` if the format is ``FAKE_GLOBALS_VALUE``. -Third-party code that implements ``__annotate__`` functions should either -support all three annotation formats, or be prepared to handle the -"fake globals" environment. This should be mentioned in the data model -documentation for ``__annotate__``. +Third-party code that implements ``__annotate__`` functions should raise +:py:exc:`NotImplementedError` if the ``FAKE_GLOBALS_VALUE`` format is passed +and the function is not prepared to be run in a "fake globals" environment. +This should be mentioned in the data model documentation for ``__annotate__``. -Effect of setting ``__annotations__`` -===================================== +Effect of deleting ``__annotations__`` +====================================== :pep:`649` specifies: Setting ``o.__annotations__`` to a legal value automatically sets ``o.__annotate__`` to ``None``. -We would prefer to keep ``__annotate__`` unchanged when ``__annotations__`` -is written to. Conceptually, ``__annotate__`` provides the ground truth -and ``__annotations__`` is merely a cache, and we shouldn't throw away the -ground truth if the cache is modified. - -The motivation for :pep:`649`'s behavior is to keep the two attributes in sync. -However, this is impossible in general; if the ``__annotations__`` dictionary -is modified in place, this will not be reflected in the ``__annotate__`` attribute. -The overall mental model for this area will be simpler if setting ``__annotations__`` -has no effect on ``__annotate__``. +However, the PEP does not say what happens if the ``__annotations__`` attribute +is deleted (using ``del``). It seems most consistent that deleting the attribute +will also delete ``__annotate__``. Specification ------------- -The value of ``__annotate__`` is not changed when ``__annotations__`` is set. +Deleting the ``__annotations__`` attribute on functions, modules, and classes +results in setting ``__annotate__`` to None. Deferred evaluation of PEP 695 and 696 objects ============================================== @@ -661,6 +691,69 @@ Usually, users would use these attributes in combinations with in SOURCE format, one could write ``annotationlib.call_evaluate_function(T.evaluate_bound, annotationlib.Format.SOURCE)``. +Behavior of dataclass field types +================================= + +One consequence of the deferred evaluation of annotations is that +dataclasses can use forward references in their annotations: + +.. code:: pycon + + >>> from dataclasses import dataclass + >>> @dataclass + ... class D: + ... x: undefined + ... + +However, the ``FORWARDREF`` format leaks into the field types of the dataclass: + +.. code:: pycon + + >>> fields(D)[0].type + ForwardRef('undefined') + +We considered a change where the ``.type`` attribute of a field object would +trigger evaluation of annotations, so that the field type could contain actual +values in the case of forward references that were defined after the dataclass +itself was created, but before the field type is accessed. +However, this would also mean that accessing ``.type`` could now run arbitrary +code in the annotation, and potentially throws errors such as :py:exc:`NameError`. + +Therefore, we consider it more user-friendly to keep the ``ForwardRef`` object +in the type, and document that users who want to resolve forward references +can use the ``ForwardRef.evaluate`` method. + +If use cases come up in the future, we could add additional functionality, +such as a new method that re-evaluates the annotation from scratch. + +Renaming ``SOURCE`` to ``STRING`` +================================= + +The ``SOURCE`` format is meant for tools that need to show a human-readable +format that is close to the original source code. However, we cannot retrieve +the original source in ``__annotate__`` functions, and in some cases, we have +``__annotate__`` functions in Python code that do not have access to the original +code. For example, this applies to :py:func:`dataclasses.make_dataclass` +and the call-based syntax for :py:class:`typing.TypedDict`. + +This makes the name ``SOURCE`` a bit of a misnomer. The goal of the format +should indeed be to recreate the source, but the name is likely to mislead +users in practice. A more neutral name would emphasize that the format returns +an annotation dictionary with only strings. We suggest ``STRING``. + +Specification +------------- + +The ``SOURCE`` format is renamed to ``STRING``. To reiterate the changes in this +PEP, the four supported formats are now: + +- ``VALUE``: the default format, which evaluates the annotations and returns the + resulting values. +- ``VALUE_WITH_FAKE_GLOBALS``: for internal use; should be handled like ``VALUE`` + by annotate functions that support execution with fake globals. +- ``FORWARDREF``: replaces undefined names with ``ForwardRef`` objects. +- ``STRING``: returns strings, attempts to recreate code close to the original source. + Miscellaneous implementation details ==================================== @@ -709,7 +802,7 @@ Signature of ``__annotate__`` functions ``__annotate__(format: int) -> dict`` However, using ``format`` as a parameter name could lead to collisions -if an annotation uses a class named ``format``. The parameter should be +if an annotation uses a symbol named ``format``. The parameter should be positional-only and have a name that cannot be a legal identifier in order to avoid this problem. @@ -793,63 +886,6 @@ as we make progress on implementing it. Readers are encouraged to follow the implementation of the PEP and try out the draft implementation. Any feedback may be incorporated into future versions of this PEP. -Should dataclass field types use deferred evaluation? ------------------------------------------------------ - -The current draft implementation already supports deferred evaluation in dataclasses, -so this works: - -.. code:: pycon - - >>> from dataclasses import dataclass - >>> @dataclass - ... class D: - ... x: undefined - ... - -However, the ``FORWARDREF`` format leaks into the field types of the dataclass: - -.. code:: pycon - - >>> fields(D)[0].type - ForwardRef('undefined') - -We could instead add deferred evaluation for the field type, similar to that outlined -above for type alias values. - -Accessing ``.type`` might throw an error: - -.. code:: pycon - - >>> @dataclass - ... class D: - ... x: undefined - ... - >>> field = fields(D)[0] - >>> field.type - Traceback (most recent call last): - File "", line 1, in - field.type - File ".../dataclasses.py", line 308, in type - annos = self._annotate(annotationlib.Format.VALUE) - File "", line 3, in __annotate__ - x: undefined - ^^^^^^^^^ - NameError: name 'undefined' is not defined - -But users could use ``annotationlib.call_evaluate_function`` to get the type in other formats: - -.. code:: pycon - - >>> annotationlib.call_evaluate_function(field.evaluate_type, annotationlib.Format.SOURCE) - 'undefined' - >>> annotationlib.call_evaluate_function(field.evaluate_type, annotationlib.Format.FORWARDREF) - ForwardRef('undefined') - -Other variations are possible. For example, we could leave the ``type`` attribute unchanged, -and only add the ``evaluate_type`` method. This avoids unpleasant surprises where accessing -``.type`` may throw an exception. - Acknowledgments =============== @@ -858,7 +894,8 @@ initial decisions, but the overall design is still his. I thank Carl Meyer and Alex Waygood for feedback on early drafts of this PEP. Alex Waygood, Alyssa Coghlan, and David Ellis provided insightful feedback and suggestions on the -interaction between metaclasses and ``__annotations__``. +interaction between metaclasses and ``__annotations__``. Larry Hastings also provided useful +feedback on this PEP. Appendix ======== diff --git a/peps/pep-0753.rst b/peps/pep-0753.rst index accd2490fb4..1222387b393 100644 --- a/peps/pep-0753.rst +++ b/peps/pep-0753.rst @@ -15,13 +15,14 @@ Post-History: `26-Aug-2024 ` for normalizing and + assigning semantics to ``Project-URL`` labels during consumer-side metadata + processing. Rationale and Motivation ======================== @@ -35,21 +36,21 @@ for expressing a package's relationship to external resources, via URLs: 1. Metadata 1.0 introduced ``Home-page``, a single-use field containing a URL to the distribution's home page. - .. code-block:: + .. code-block:: email Home-page: https://example.com/sampleproject 2. Metadata 1.1 introduced ``Download-URL``, a complementary single-use field containing a URL suitable for downloading the current distribution. - .. code-block:: + .. code-block:: email Download-URL: https://example.com/sampleproject/sampleproject-1.2.3.tar.gz 3. Metadata 1.2 introduced ``Project-URL``, a **multiple-use** field containing a label-and-URL pair. Each label is free text conveying the URL's semantics. - .. code-block:: + .. code-block:: email Project-URL: Homepage, https://example.com/sampleproject Project-URL: Download, https://example.com/sampleproject/sampleproject-1.2.3.tar.gz @@ -93,9 +94,6 @@ This PEP stipulates the following for metadata producers: * When generating metadata 1.2 or later, producers **SHOULD** emit only ``Project-URL``, and **SHOULD NOT** emit ``Home-page`` or ``Download-URL`` fields. -* When generating ``Project-URL`` equivalents for ``Home-page`` and - ``Download-URL``, producers **SHOULD** use the - :ref:`label conventions ` described below. These stipulations do not change the optionality of URL fields in core metadata. In other words, producers **MAY** choose to omit ``Project-URL`` entirely @@ -106,6 +104,11 @@ or ``Download-URL``. However, see :ref:`future-considerations` for thoughts on how a new (as of yet unspecified) major core metadata version could complete the deprecation cycle via removal of these deprecated fields. +Similarly, this PEP does **not** propose that metadata producers emit +:ref:`normalized labels `. Label normalization is performed +solely on the processing and consumption side (i.e. within indices and other +consumers of distribution metadata). + .. _package-indices: Package indices @@ -153,10 +156,13 @@ informal relationship between ``Home-page``, ``Download-URL``, and their This formalization has two parts: -1. A set of rules for normalizing ``Project-URL`` labels; +1. A set of rules for normalizing ``Project-URL`` labels during index-side + processing; 2. A set of "well-known" normalized label values that indices may specialize URL presentation for. +.. _label-normalization: + Label normalization ------------------- @@ -190,14 +196,22 @@ normalization: "``Change_Log``", "``changelog``" "``What's New?``", "``whatsnew``" -Metadata producers **SHOULD** emit the normalized form of a user -specified label, but **MAY** choose to emit the un-normalized form so -long as it adheres to the existing 32 character constraint. +When processing distribution metadata, package indices **SHOULD** perform +label normalization to determine if a given label is +:ref:`well known ` for subsequent special processing. +Labels that are not well-known **MUST** be processed in their un-normalized +form. + +Normalization does **not** change pre-existing semantics around duplicated +``Project-URL`` labels. In other words, normalization may result in duplicate +labels in the project's metadata, but only in the same manner that was already permitted +(since the :ref:`core metadata specification ` does +not stipulate that labels are unique). -Package indices **SHOULD NOT** use the normalized labels belonging to the set -of well-known labels directly as UI elements (instead replacing them with -appropriately capitalized text labels). Labels not belonging to the well-known -set **MAY** be used directly as UI elements. +Excerpted examples of normalized metadata fields are provided in +:ref:`Appendix A `. + +.. _well-known: Well-known labels ----------------- @@ -243,15 +257,9 @@ Indices **MAY** choose to use the human-readable equivalents suggested above in their UI elements, if appropriate. Alternatively, indices **MAY** choose their own appropriate human-readable equivalents for UI elements. -Packagers and metadata producers **MAY** choose to use these well-known -labels or their aliases to communicate specific URL intents to package indices -and downstreams. - -Packagers and metadata producers **SHOULD** produce the normalized version -of the well-known labels or their aliases in package metadata. Packaging tools -**MUST NOT** transform between equivalent aliases, i.e.. **SHOULD** -normalize ``GitHub`` to ``github`` but **MUST NOT** transform -``github`` to ``source``. +Packagers and metadata producers **MAY** choose to use any label that normalizes +to these labels (or their aliases) to communicate specific URL intents to +package indices and downstreams. Similarly, indices **MAY** choose to specialize their rendering or presentation of URLs with these labels, e.g. by presenting an appropriate icon or tooltip @@ -264,12 +272,9 @@ for documentation hosting or issue tracking). This PEP recognizes that the list of well-known labels is unlikely to remain static, and that subsequent additions to it should not require the overhead -associated with a formal PEP process or new metadata version. As the primary -expected use case for this information is to control the way project URLs are -displayed on the Python Package Index, this PEP proposes that the list above -become a "living" list within PyPI's documentation (at time of writing, the -documentation for influencing PyPI's URL display can be found -`here `__). +associated with a formal PEP process or new metadata version. As such, +this PEP proposes that the list above become a "living" list within +the :ref:`PyPA specifications `. Backwards Compatibility ======================= @@ -349,6 +354,44 @@ already contains an informal description of PyPI's URL handling behavior. If this PEP is accepted, the authors of this PEP will coordinate to update and cross-link the resources mentioned above. +.. _appendix-a: + +Appendix A: Label normalization examples +======================================== + +This appendix provides an illustrative excerpt of a distribution's +metadata, before and after index-side processing: + +Before: + +.. code-block:: email + + Project-URL: Home-page, https://example.com + Project-URL: Homepage, https://another.example.com + Project-URL: Source, https://github.com/example/example + Project-URL: GitHub, https://github.com/example/example + Project-URL: Another Service, https://custom.example.com + + +After: + +.. code-block:: email + + Project-URL: homepage, https://example.com + Project-URL: homepage, https://another.example.com + Project-URL: source, https://github.com/example/example + Project-URL: github, https://github.com/example/example + Project-URL: Another Service, https://custom.example.com + +In particular, observe: + +* Normalized duplicates are preserved (both ``Home-page`` and ``Homepage`` + become ``homepage``); +* ``Source`` and ``GitHub`` are both normalized into their respective forms, + but ``github`` is **not** transformed into ``source``. +* ``Another Service`` is **not** normalized, since its normal form + (``anotherservice``) is not a :ref:`well-known label `. + Copyright ========= diff --git a/peps/pep-0756.rst b/peps/pep-0756.rst index 926f918ac65..fd5447e860e 100644 --- a/peps/pep-0756.rst +++ b/peps/pep-0756.rst @@ -21,9 +21,8 @@ Add functions to the limited C API version 3.14: view. * ``PyUnicode_Import()``: import a Python str object. -In general, ``PyUnicode_Export()`` has an *O*\ (1) complexity: no memory -copy is needed. See the :ref:`specification ` for -cases when a copy is needed. +On CPython, ``PyUnicode_Export()`` has an *O*\ (1) complexity: no memory +is copied and no conversion is done. Rationale @@ -67,9 +66,10 @@ possible to write code specialized for UCS formats. A C extension using the limited C API can only use less efficient code paths and string formats. -For example, the MarkupSafe project has a C extension specialized for -UCS formats for best performance, and so cannot use the limited C -API. +For example, the `MarkupSafe project +`_ has a C extension +specialized for UCS formats for best performance, and so cannot use the +limited C API. Specification @@ -146,22 +146,21 @@ UCS-2 and UCS-4 use the native byte order. *requested_formats* can be a single format or a bitwise combination of the formats in the table above. On success, the returned format will be set to a single one of the requested -flags. +formats. Note that future versions of Python may introduce additional formats. +No memory is copied and no conversion is done. + + + .. _export-complexity: Export complexity ----------------- -In general, an export has a complexity of *O*\ (1): no memory copy is -needed. There are cases when a copy is needed, *O*\ (*n*) complexity: - -* If only UCS-2 is requested and the native format is UCS-1. -* If only UCS-4 is requested and the native format is UCS-1 or UCS-2. -* If only UTF-8 is requested: the string is encoded to UTF-8 at the - first call, and then the encoded UTF-8 string is cached. +On CPython, an export has a complexity of *O*\ (1): no memory is copied +and no conversion is done. To get the best performance on CPython and PyPy, it's recommended to support these 4 formats:: @@ -216,31 +215,29 @@ See ``PyUnicode_Export()`` for the available formats. UTF-8 format ------------ -CPython 3.14 doesn't use the UTF-8 format internally. The format is -provided for compatibility with PyPy which uses UTF-8 natively for -strings. However, in CPython, the encoded UTF-8 string is cached which -makes it convenient to be exported. +CPython 3.14 doesn't use the UTF-8 format internally and doesn't support +exporting a string as UTF-8. The ``PyUnicode_AsUTF8AndSize()`` function +can be used instead. + +The ``PyUnicode_FORMAT_UTF8`` format is provided for compatibility with +alternate implementations which may use UTF-8 natively for strings. -On CPython, the UTF-8 format has the lowest priority: ASCII and UCS -formats are preferred. ASCII format ------------ When the ``PyUnicode_FORMAT_ASCII`` format is request for export, the -``PyUnicode_FORMAT_UCS1`` export format is used for ASCII and Latin-1 -strings. +``PyUnicode_FORMAT_UCS1`` export format is used for ASCII strings. The ``PyUnicode_FORMAT_ASCII`` format is mostly useful for -``PyUnicode_Import()`` to validate that the string only contains ASCII +``PyUnicode_Import()`` to validate that a string only contains ASCII characters. -Surrogate characters and NUL characters ---------------------------------------- +Surrogate characters and embedded NUL characters +------------------------------------------------ -Surrogate characters are allowed: they can be imported and exported. For -example, the UTF-8 format uses the ``surrogatepass`` error handler. +Surrogate characters are allowed: they can be imported and exported. Embedded NUL characters are allowed: they can be imported and exported. @@ -347,6 +344,7 @@ to return NULL on embedded null characters Rejecting embedded NUL characters require to scan the string which has an *O*\ (*n*) complexity. + Reject surrogate characters --------------------------- @@ -365,6 +363,34 @@ this issue. For example, the UTF-8 codec can be used with the characters. +Conversions on demand +--------------------- + +It would be convenient to convert formats on demand. For example, +convert UCS-1 and UCS-2 to UCS-4 if an export to only UCS-4 is +requested. + +The problem is that most users expect an export to require no memory +copy and no conversion: an *O*\ (1) complexity. It is better to have an +API where all operations have an *O*\ (1) complexity. + +Export to UTF-8 +--------------- + +CPython 3.14 has a cache to encode a string to UTF-8. It is tempting to +allow exporting to UTF-8. + +The problem is that the UTF-8 cache doesn't support surrogate +characters. An export is expected to provide the whole string content, +including embedded NUL characters and surrogate characters. To export +surrogate characters, a different code path using the ``surrogatepass`` +error handler is needed and each export operation has to allocate a +temporary buffer: *O*\ (n) complexity. + +An export is expected to have an *O*\ (1) complexity, so the idea to +export UTF-8 in CPython was abadonned. + + Discussions =========== diff --git a/peps/pep-0757.rst b/peps/pep-0757.rst index 5fe1ba52dc6..12b967c1bbf 100644 --- a/peps/pep-0757.rst +++ b/peps/pep-0757.rst @@ -388,23 +388,6 @@ There is no impact on the backward compatibility, only new APIs are added. -Open Questions -============== - -* Should we add *digits_order* and *endianness* members to :data:`sys.int_info` - and remove :c:func:`PyLong_GetNativeLayout()`? The - :c:func:`PyLong_GetNativeLayout()` function returns a C structure - which is more convenient to use in C than :data:`sys.int_info` which uses - Python objects. -* Currently, all required information for :class:`int` import/export is - already available via :c:func:`PyLong_GetInfo()` or :data:`sys.int_info`. - Native endianness of "digits" and current order of digits (least - significant digit first) --- is a common denominator of all libraries - for arbitrary precision integer arithmetic. So, shouldn't we just remove - from API both :c:struct:`PyLongLayout` and :c:func:`PyLong_GetNativeLayout()` (which - is actually just a minor convenience)? - - Rejected Ideas ============== @@ -426,6 +409,46 @@ If later there are use cases for arbitrary layouts, new APIs can be added. +Don't add :c:func:`PyLong_GetNativeLayout` function +--------------------------------------------------- + +Currently, most required information for :class:`int` import/export is already +available via :c:func:`PyLong_GetInfo()` (and :data:`sys.int_info`). We also +can add more (like order of digits), this interface doesn't poses any +constraints on future evolution of the :c:type:`PyLongObject`. + +The problem is that the :c:func:`PyLong_GetInfo()` returns a Python object, +:term:`named tuple`, not a convenient C structure and that might distract +people from using it in favor e.g. of current semi-private macros like +:c:macro:`!PyLong_SHIFT` and :c:macro:`!PyLong_BASE`. + + +Provide mpz_import/export-like API instead +------------------------------------------ + +The other approach to import/export data from :class:`int` objects might be +following: expect, that C extensions provide contiguous buffers that CPython +then exports (or imports) the *absolute* value of an integer. + +API example:: + + struct PyLongLayout { + uint8_t bits_per_digit; + uint8_t digit_size; + int8_t digits_order; + }; + + size_t PyLong_GetDigitsNeeded(PyLongObject *obj, PyLongLayout layout); + int PyLong_Export(PyLongObject *obj, PyLongLayout layout, void *buffer); + PyLongObject *PyLong_Import(PyLongLayout layout, void *buffer); + +This might work for the GMP, as this it has :c:func:`!mpz_limbs_read()` and +:c:func:`!mpz_limbs_write()` functions, that can provide required "buffers". + +The major drawback of this approach is that it's much more complex on the +CPython side (i.e. actual conversion between different layouts). + + Discussions =========== diff --git a/peps/pep-0758.rst b/peps/pep-0758.rst new file mode 100644 index 00000000000..d0c7f405027 --- /dev/null +++ b/peps/pep-0758.rst @@ -0,0 +1,203 @@ +PEP: 758 +Title: Allow ``except`` and ``except*`` expressions without parentheses +Author: Pablo Galindo , Brett Cannon +PEP-Delegate: TBD +Status: Draft +Type: Standards Track +Created: 30-Sep-2024 +Python-Version: 3.14 +Post-History: `02-Oct-2024 `__ + +Abstract +======== + +This PEP [1]_ proposes to allow unparenthesized ``except`` and ``except*`` +blocks in Python's exception handling syntax. Currently, when catching multiple +exceptions, parentheses are required around the exception types. This was a +Python 2 remnant. This PEP suggests allowing the omission of these parentheses, +simplifying the syntax, making it more consistent with other parts of the syntax +that make parentheses optional, and improving readability in certain cases. + + +Motivation +========== + +The current syntax for catching multiple exceptions requires parentheses in the +``except`` expression (equivalently for the ``except*`` expression). For +example: + +.. code-block:: python + + try: + ... + except (ExceptionA, ExceptionB, ExceptionC): + ... + +While this syntax is clear and unambiguous, it can be seen as unnecessarily +verbose in some cases, especially when catching a large number of exceptions. By +allowing the omission of parentheses, we can simplify the syntax: + +.. code-block:: python + + try: + ... + except ExceptionA, ExceptionB, ExceptionC: + ... + +This change would bring the syntax more in line with other comma-separated lists +in Python, such as function arguments, generator expressions inside of a +function call, and tuple literals, where parentheses are optional. + +The same change would apply to ``except*`` expressions. For example: + +.. code-block:: python + + try: + ... + except* ExceptionA, ExceptionB, ExceptionC: + ... + +Both forms will also allow the use of the ``as`` clause to capture the exception +instance as before: + +.. code-block:: python + + try: + ... + except ExceptionA, ExceptionB, ExceptionC as e: + ... + + +Rationale +========= + +The decision to allow unparenthesized ``except`` blocks is based on the +following considerations: + +1. Simplicity: Removing the requirement for parentheses simplifies the syntax, + making it more consistent with other parts of the language. + +2. Readability: In cases where many exceptions are being caught, the removal of + parentheses can improve readability by reducing visual clutter. + +3. Consistency: This change makes the ``except`` clause more consistent with + other parts of Python where unambiguous, comma-separated lists don't require + parentheses. + +Specification +============= + +The syntax for the except clause will be modified to allow an unparenthesized +list of exception types. The grammar will be updated as follows: + +.. code-block:: peg + + except_block: + | 'except' expressions ['as' NAME] ':' block + | 'except' ':' block + + except_star_block + | 'except' '*' expressions ['as' NAME] ':' block + +This allows both the current parenthesized syntax and the new unparenthesized +syntax: + +.. code-block:: python + + try: + ... + except (ExceptionA, ExceptionB): # Still valid + ... + except ExceptionC, ExceptionD: # New syntax + ... + +The semantics of exception handling remain unchanged. The interpreter will catch +any of the listed exceptions, regardless of whether they are parenthesized or +not. + + +Backwards Compatibility +======================= + +This change is fully backwards compatible. All existing code using parenthesized +``except`` and ``except*`` blocks will continue to work without modification. +The new syntax is purely additive and does not break any existing code. + +It's worth noting that in Python 2 the unparenthesized syntax was allowed with +two elements, but had different semantics, in which the first element of the +list was used as the exception type and the second element as the capture +variable. This change does not reintroduce the Python 2 semantics, and the +unparenthesized syntax will behave identically to the parenthesized version. + + +Security Implications +===================== + +There are no known security implications for this change. The semantics of +exception handling remain the same, and this is purely a syntactic change. + + +How to Teach This +================= + +For new Python users, the unparenthesized syntax can be taught as the standard +way to catch multiple exceptions: + +.. code-block:: python + + try: + risky_operation() + except ValueError, TypeError, OSError: + handle_errors() + +For experienced users, it can be introduced as a new, optional syntax that can +be used interchangeably with the parenthesized version. Documentation should +note that both forms are equivalent: + +.. code-block:: python + + # These are equivalent: + except (ValueError, TypeError): + ... + + except ValueError, TypeError: + ... + +It should be emphasized that this is purely a syntactic change and does not +affect the behaviour of exception handling. + + +Reference Implementation +======================== + +A proof-of-concept implementation is available at +https://github.com/pablogsal/cpython/commits/notuples/. This implementation +modifies the Python parser to accept the new syntax and ensures that it behaves +identically to the parenthesized version. + + +Rejected Ideas +============== + +1. Allowing mixed parenthesized and unparenthesized syntax: + + .. code-block:: python + + try: + ... + except (ValueError, TypeError), OSError: + ... + + This was rejected due to the potential for confusion and to maintain a clear + distinction between the two styles. + +Footnotes +========= + +.. [1] Originally named "Parenthetically Speaking, We Don't Need 'Em" + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. diff --git a/peps/pep-0759.rst b/peps/pep-0759.rst new file mode 100644 index 00000000000..810f4d19247 --- /dev/null +++ b/peps/pep-0759.rst @@ -0,0 +1,472 @@ +PEP: 759 +Title: External Wheel Hosting +Author: Barry Warsaw , + Ethan Smith +PEP-Delegate: Donald Stufft +Status: Draft +Type: Standards Track +Topic: Packaging +Created: 01-Oct-2024 +Post-History: + +Abstract +======== + +This PEP proposes a mechanism by which projects hosted on `pypi.org +`__ can safely host wheel artifacts on external sites other +than PyPI. This PEP explicitly does *not* propose external hosting of +projects, packages, or their metadata. That functionality is already available +by externally hosting independent package indexes. Because this PEP only +provides a mechanism for projects to customize the download URL for specific +released wheel artifacts, dependency resolution as already implemented by +common installer tools such as `pip `__ and +`uv `__ does not need to change. + +This PEP defines what it means to be "safe" in this context, along with a new +package upload file format called a ``.rim`` file. It defines how ``.rim`` +files affect the metadata returned for a package's :ref:`Simple Repository API +` +in both HTML and JSON formats, and how traditional wheels can easily be turned +into ``.rim`` files. + +Rationale +========= + +The Python Package Index, hosted at https://pypi.org, imposes `default limits +`__ on upload artifact file size (100 MiB) and total project size +(10 GiB). Most projects can comfortably fit within these limits during the lifetime of the +project, through years of uploads. A few projects have encountered these limits, and have +been granted both file size and project size exceptions, allowing them to continue +uploading new releases without having to take more drastic measures, such as removing +files which may potentially still be in use by consumers (e.g. through version pins). + +A related workaround is the `"wheel stub" `__ +approach, which provides an indirect link between PyPI and an external third party package +index, where such limitations can be avoided. Wheel stubs are :ref:`source distributions +` (a.k.a. "sdists") which utilize a :pep:`517` build +backend that, instead of turning source code into a binary wheel, performs some logic to +calculate the URL for an existing, externally hosted wheel to download and install. This +approach works, but it obscures the connection between PyPI, the sdist, and the externally +hosted wheel, since there is no way to present this information to ``pip`` or other such +tools. + +Historical context +------------------ + +In 2013, :pep:`438` proposed a "backward-compatible two-phase transition +process" to modify several aspects of release file hosting on PyPI. As this +PEP describes, PyPI originally supported only project and release +*registration* without also allowing for artifact file hosting. As such, most +projects hosted release file artifacts elsewhere. Artifact hosting was later +added, but the mix of externally and PyPI-hosted files led to a wide range of +usability and potential security-related problems. PEP 438 was an attempt to +provide several facilities to allow external hosting while promoting a +PyPI-first hosting preference. + +PEP 438 was complex, with three different "hosting modes", ``rel`` metadata in +the simple HTML index pages to signify hosting locations, and a two-phase +transition plan affecting PyPI and installer tools. PEP 438 was ultimately +retracted in 2015 by :pep:`470`, which acknowledges that PEP 438 did succeed +in... + + bringing about more people to utilize PyPI's repository features, an + altogether good thing given the global CDN powering PyPI providing speed + ups for a lot of people[...] + +Instead of external hosting, PEP 470 promoted the use of explicit multiple +repositories, providing full package indexing and artifact hosting, and +enabled through installer tool support, such as ``pip install +--extra-index-url`` allowing ``pip`` to essentially treat multiple +repositories as `one single global repository +`__ +for package installation resolution. Because this has been the blessed norm +for so many years, all Python package installation tools support querying +multiple indexes for dependency resolution. + +The problem with multiple indexes +--------------------------------- + +Why then does this PEP propose to allow a more limited form of external +hosting, and how does this proposal avoid the problems documented in PEP 470? + +One well-known problem that consolidating multiple indexes enables is +`dependency confusion attacks +`__, to +which Python *can* be particularly vulnerable, due to the algorithm that ``pip +install`` uses for resolving package dependencies and preferred versions. The +``uv`` tool addresses this by supporting an additional `index strategy +`__ option, +whereby users can select between, e.g. a ``pip``-compatible strategy, and a +more limited strategy that prevents such dependency confusion attacks. + +:pep:`708` provides additional background about dependency confusion attacks, +and takes a different approach to preventing them. At its core, PEP 708 allows +repository owners to indicate that projects track across different +repositories, which allows installers to determine how to treat the global +package namespace when combined across multiple repositories. PEP 708 has been +provisionally accepted, pending several required conditions as outlined in PEP +708, some of which may have an indeterminate future. As PEP 708 itself says, +this won't by itself solve dependency confusion attacks, but is one way to +provide enough information to installers to help minimize these attacks. + +While there can still be valid use cases for standing up a totally independent +package index (such as providing richer platform support for GPUs until a +fully formed `variant proposal +`__ +is accepted), this PEP takes a different, simpler approach and doesn't replace +any of the existing, proposed, or approved package index cooperation +specifications. + +This PEP also preserves the core purpose of PyPI, and allows it to +remain the traditional, canonical, centralized index of all Python +packages. + +Addressing PyPI limits +---------------------- + +This proposal also addresses the problem of size limits imposed by PyPI, where there is a +`default artifact size limit `__ of 100 MiB and a +default overall `project size limit `__ of 10 +GiB. Most packages and artifacts can easily fit in these limits, even for packages +containing binary extension modules for a variety of platforms. A small, but important +class of packages routinely exceed these limits, requiring them to submit PyPI `exception +request support tickets`_. It's not necessarily difficult to get resolution on such +exceptions, but it is a special process that can take some time to resolve, and the +criteria for granting such exceptions aren't well documented. + +Reducing operational complexity +------------------------------- + +Setting up and maintaining an entire package index can be a complex +operational solution, both time and resource intensive. This is especially +true if the main purpose of such an index is just to avoid file size +limitations. The external index approach also imposes a tricky UX on consumers +of projects on the external index, requiring them to understand how CLI +options such as ``--external-index-url`` work, along with the security +implications of such flags. It would be much easier for both producers and +consumers of large wheel packages to just set up and maintain a simple web +server, capable of serving individual files with no more complex API than +``HTTP GET``. Such an interface is also easily cacheable or placed behind a +`CDN `__. Simple HTTP +servers are also much easier to audit for security purposes, easier to proxy, +and usually take much less resources to run, support, and maintain. Even +something like `Amazon S3 `__ could be used to +host external wheels. + +This PEP proposes an approach that favors such operational simplicity. + +Specification +============= + +A new type of uploadable file is defined, called a "RIM" (i.e. ``.rim``), or "Remote +Installable Metadata" file. The name evokes the image of a wheel with the tire removed, +and emphasizes that ``.rim`` files are easily derived from ``.whl`` files. The process of +turning a ``.whl`` into a ``.rim`` is :ref:`outlined below `. The file name +format exactly matches the :ref:`wheel file naming format +` specification, except that RIM files use the suffix +``.rim``. This means that all the tags used to discriminate ``.whl`` files also +distinguish between different ``.rim`` files, and thus can be used during dependency +resolution steps, exactly as ``.whl`` files are today. In this respect, ``.whl`` and +``.rim`` files are interchangeable. + +The content of a ``.rim`` file is *nearly* identical to ``.whl`` files, however ``.rim`` +files **MUST** contain only the ``.dist-info`` directory from a wheel. No other top-level +file or directory is allowed in the ``.rim`` zip file. The ``.dist-info`` directory +**MUST** contain a single additional file in addition to those `allowed`_ in a ``.whl`` +file's ``.dist-info`` directory: a file called ``EXTERNAL-HOSTING.json``. + +.. _file-format: + +This is a JSON file contains containing the following keys: + +``version`` + This is the file format version, which for this PEP **MUST** be ``1.0``. +``owner`` + This **MUST** name the PyPI organization owner of this externally hosted file, for + reasons which will be described in :ref:`detail below `. +``uri`` + This is a single URL naming the location of the physical ``.whl`` file hosted on an + external site. This URL **MUST** use the ``https`` scheme. +``size`` + This is an integer value describing the size in bytes of the physical ``.whl`` file on + the remote host. +``hashes`` + This is a dictionary of the format described in :pep:`694`, used to capture both the + :pep:`694#upload-each-file` of the physical ``.whl`` file, with the same + constraints as proposed in that PEP. Since these hashes are immutable once uploaded + to PyPI, they serve as a critical validation that the externally hosted wheel hasn't + been corrupted or compromised. + +Effects of the RIM file +----------------------- + +The only effect of a ``.rim`` file is to change the download URL for the wheel artifact in +both the HTML and JSON interfaces in the `simple repository API`_. In the HTML page for a +package release, the ``href`` attribute **MUST** be the value of the ``uri`` key, +including a ``#=`` fragment. this hash fragment **MUST** be in +exactly the same format as described the :pep:`376` originated `signed wheel file format`_ +in the ``.dist-info/RECORD`` file. The exact same rules for selection of hash algorithm +and encoding is used here. + +Similarly in the `JSON response`_ the ``url`` key pointing to the download file must be +the value of the :ref:`uri ` key, and the ``hashes`` dictionary **MUST** be +included with values populated from the ``hashes`` dictionary provided above. + +In all other respects, a compliant package index should treat ``.rim`` files the same as +``.whl`` files, with some other minor exceptions as outlined below. For example, ``.rim`` +files can be `deleted `__ and yanked (:pep:`592`) just +like any ``.whl`` file, with the exact same semantics (i.e. deletions are permanent). When +a ``.rim`` is deleted, an index **MUST NOT** allow a matching ``.whl`` or ``.rim`` file to +be (re-)uploaded. + +Availability order +------------------ + +Externally hosted wheels **MUST** be available before the corresponding ``.rim`` file is +uploaded to PyPI, otherwise a publishing race condition is introduced, although this +requirement **MAY** be relaxed for ``.rim`` files uploaded to a :pep:`694` staged release. + +Wheels can override RIMs +------------------------ + +Indexes **MUST** reject ``.rim`` files if a matching ``.whl`` file already exists with the +exact same file name tags. However, indexes **MAY** accept a ``.whl`` file if a matching +``.rim`` file exists, as long as that ``.rim`` file hasn't been deleted or yanked. This +allows uploaders to replace an externally hosted wheel file with an index hosted wheel +file, but the converse is prohibited. Since the default is to host wheels on the same +package index that contains the package metadata, it is not allowed to "downgrade" an +existing wheel file once uploaded. When a ``.whl`` replaces a ``.rim``, the index **MUST** +provide download URLs for the package using its own hosted file service. When uploading +the overriding ``.whl`` file, the package index **MUST** validate the hash from the +existing ``.rim`` file, and these hashes must match or the overriding upload **MUST** be +rejected. + +PyPI API bump unnecessary +------------------------- + +It's likely that the changes are backward compatible enough that a bump in the `PyPI +repository version`_ is not necessary. Since ``.rim`` files are essentially changes only +to the upload API, package resolvers and package installers can continue to function with +the APIs they've always supported. + +.. _resiliency: + +External hosting resiliency +=========================== + +One of the key concerns leading to PEP 438's revocation in PEP 470 was +potential user confusion when an external index disappeared. From PEP 470: + + This confusion comes down to end users of projects not realizing if a + project is hosted on PyPI or if it relies on an external service. This + often manifests itself when the external service is down but PyPI is + not. People will see that PyPI works, and other projects works, but this + one specific one does not. They oftentimes do not realize who they need to + contact in order to get this fixed or what their remediation steps are. + +While the problem of external wheel hosting service going down is not directly +solved by this PEP, several safeguards are in place to greatly reduce the +potential burden on PyPI administrators. + +This PEP thus proposes that: + +- External wheel hosting is only allowed for packages which are owned by + `organization accounts `__. + External hosting is an organization-wide setting. +- Organization accounts do not automatically gain the ability to externally + host wheels; this feature MUST be explicitly enabled by PyPI admins at their discretion. Since + this will not be a common request, we don't expect the overhead to be nearly + as burdensome as :pep:`541` resolutions, account recovery requests, or even + file/project size increase requests. External hosting requests would be + handled in the same manner as those requests, i.e. via the `PyPI GitHub + support tracker `__. +- Organization accounts requesting external wheel hosting **MUST** register their own + support contact URI, be it a ``mailto`` URI for a contact email address, or the URL to + the organization's support tracker. Such a contact URI is optional for organizations + which do not avail themselves of external wheel file hosting. + +Combined with the ``EXTERNAL-HOSTING.json`` file's ``owner`` key, this allows for +installer tools to unambiguously redirect any download errors away from the PyPI support +admins and squarely to the organization's support admins. + +While the exact mechanics of storing and retrieving this organization support +URL will be defined separately, for the sake of example, let's say a package +``foo`` externally hosts wheel files on ```https://foo.example.com`` +`__ and that host becomes unreachable. When an +installer tool tries to download and install the package ``foo`` wheel, the +download step will fail. The installer would then be able to query PyPI to +provide a useful error message to the end user: + +- The installer downloads the ``.rim`` file and reads the ``owner`` key from the + ``EXTERNAL-HOSTING.json`` file inside the ``.rim`` zip file. +- The installer queries PyPI for the support URI for the organization + owner of the externally hosted wheel. +- An informative error message would then be displayed, e.g.: + + The externally hosted wheel file ``foo-....whl`` could not be + downloaded. Please contact support@foo.example.com for help. Do not report + this to the PyPI administrators. + +.. _dismounting: + +Dismounting wheels +================== + +It is generally very easy to produce a ``.rim`` file from an existing ``.whl`` +file. This could be done efficiently by a :pep:`518` build backend with an +additional command line option, or a separate tool which takes a ``.whl`` file +as input and creates the associated ``.rim`` file. To complete the analogy, +the act of turning a ``.whl`` into a ``.rim`` is called "dismounting". The +steps such a tool would take are: + +- Accept as input the source ``.whl`` file, the organization owner of the + package, and URL at which the ``.whl`` will be hosted, and the support URI + to report download problems from. These could in fact be captured in the + ``pyproject.toml`` file, but that specification is out of scope for this + PEP. +- Unzip the ``.whl`` and create the ``.rim`` zip archive. +- Omit from the ``.rim`` file any path in the ``.whl`` that **isn't** rooted + at the ``.dist-info`` directory. +- Calculate the hash of the source ``.whl`` file. +- Add the ``EXTERNAL-HOSTING.json`` file containing the JSON keys and values as described + above, to the ``.rim`` archive. + +Changes to tools +================ + +Theoretically, installer tools shouldn't need any changes, since when they +have identified the wheel to download and install, they simply consult the +download URLs returned by PyPI's Simple API. In practice though, tools such as +``pip`` and ``uv`` may have constrained lists of hosts they will allow +downloads from, such as PyPI's own ``pythonhosted.org`` domain. + +In this case, such tools will need to relax those constraints, but the exact policy for +this is left to the installer tools themselves. Any number of approaches could be +implemented, such as downloading the ``.rim`` file and verifying the +``EXTERNAL-HOSTING.json`` metadata, or simply trusting the external downloads for any +wheel with a matching checksum. They could also query PyPI for the project's organization +owner and support URI before trusting the download. They could warn the user when +externally hosted wheel files are encountered, and/or require the use of a command line +option to enable additional download hosts. Any of these verification policies could be +chosen in configuration files. + +Installer tools should also probably provide better error messages when +externally hosted wheels cannot be downloaded, e.g. because a host is +unreachable. As described above, such tools could query enough metadata from +PyPI to provide clear and distinct error messages pointing users to the +package's external hosting support email or issue tracker. + +Constraints for external hosting services +========================================= + +The following constraints lead to reliable and compatible external wheel hosting services: + +- External wheels **MUST** be served over HTTPS, with a certificate signed by + `Mozilla's root certificate store `__. This ensures + compatibility with `pip `__ + and `uv + `__. At + the time of this writing, ``pip`` 24.2 on Python 3.10 or newer uses the system + certificate store in addition to the Mozilla store provided by the third party `certifi + `__ Python package. ``uv`` uses the Mozilla store + provided by the `webpki-roots `__ crate, but not + the system store unless the ``--native-tls`` flag is given [#fn1]_. *The PyPI + administrators may modify this requirement in the future, but compatibility with popular + installers will not be compromised.* +- External wheel hosts **SHOULD** use a content delivery network (`CDN + `__), just as PyPI does. +- External wheel hosts **MUST** commit to a stable URL for all wheels they host. +- Externally hosted wheels **MUST NOT** be removed from an external wheel host unless the + corresponding ``.rim`` file is deleted from PyPI first, and **MUST NOT** remove external + wheels for yanked releases. +- External wheel hosts **MUST** support `HTTP range requests`_. +- External wheel hosts **SHOULD** support the `HTTP/2`_ protocol. + +Security +======== + +Several factors as described in this proposal should mitigate security +concerns with externally hosted wheels, such as: + +- Wheel file checksums **MUST** be included in ``.rim`` files, and once uploaded cannot be + changed. Since the checksum stored on PyPI is immutable and required, it is not possible + to spoof an external wheel file, even if the owning organization lost control of their + hosting domain. +- Externally hosted wheels **MUST** be served over HTTPS. +- In order to serve externally hosted wheels, organizations **MUST** be approved by the + PyPI admins. + +When users identify malware or vulnerabilities in PyPI-hosted projects, they can now +report this using the `malware reporting facilities `__ on +PyPI, as also described in this `blog post`_. The same process can be used to report +security issues in externally hosted wheels, and the same remediation process should be +used. In addition, since organizations with external hosting enabled MUST provide a +support contact URI, that URI can be used in some cases to report the security issue to +the hosting organization. Such organization reporting won't make sense for malware, but +could indeed be a very useful way to report security vulnerabilities in externally hosted +wheels. + +Rejected ideas +============== + +Several ideas were considered and rejected. + +- Requiring digital signatures on externally hosted wheel files, either in + addition to or other than hashes. We deem this unnecessary since the + checksum requirement should be enough to validate that the metadata on PyPI + for a wheel exactly matches the downloaded wheel. The added complexity of + key management outweighs any additional benefit such digital signatures + might convey. + +- Hash verification on ``.rim`` file uploads. PyPI *could* verify that the hash in the + uploaded ``.rim`` file matches the externally hosted wheel before it accepts the upload, + but this requires downloading the external wheel and performing the checksum, which also + implies that the upload of the ``.rim`` file cannot be accepted until this external + ``.whl`` file is downloaded and verified. This increases PyPI bandwidth and slows down + the upload query, although :pep:`694` draft uploads could potentially mitigate these + concerns. Still, the benefit is not likely worth the additional complexity. + +- Periodic verification of the download URLs by the index. PyPI could try to periodically + ensure that the external wheel host or the external ``.whl`` file itself is still + available, e.g. via an :rfc:`HTTP HEAD <9110#section-9.3.2>` request. This is likely overkill and without also + providing the file's checksum in the response [#fn2]_, may not provide much additional + benefit. + +- This PEP could allow for an organization to provide fallback download hosts, + such that a secondary is available if the primary goes down. We believe + that DNS-based replication is a much better, well-known technique, and + probably much more resilient anyway. + +- ``.rim`` file replacement. While it is allowed for ``.whl`` files to replace + existing ``.rim`` files, as long as a) the ``.rim`` file hasn't been deleted + or yanked, b) the checksums match, we do not allow replacing ``.whl`` files + with ``.rim`` files, nor do we allow a ``.rim`` file to overwrite an + existing ``.rim`` file. This latter could be a technique to change the + hosting URL for an externally hosted ``.whl``; however, we do not think this + is a good idea. There are other ways to "fix" an external host URL as + described above, and we do not want to encourage mass re-uploads of existing + ``.rim`` files. + +Footnotes +========= +.. [#fn1] The ``uv --native-tls`` flag `replaces + `__ + the ``webpki-roots`` store. +.. [#fn2] There being no standard way to return the file's checksum in response to an + :rfc:`HTTP HEAD <9110#section-9.3.2>` request. + +Copyright +========= + +This document is placed in the public domain or under the +CC0-1.0-Universal license, whichever is more permissive. + +.. _`exception request support tickets`: https://github.com/pypi/support/issues?q=is%3Aissue+is%3Aclosed+file+limit+request +.. _`allowed`: https://packaging.python.org/en/latest/specifications/binary-distribution-format/#the-dist-info-directory +.. _`signed wheel file format`: https://packaging.python.org/en/latest/specifications/binary-distribution-format/#signed-wheel-files +.. _`simple repository API`: https://packaging.python.org/en/latest/specifications/simple-repository-api/# +.. _`JSON response`: https://packaging.python.org/en/latest/specifications/simple-repository-api/#json-based-simple-api-for-python-package-indexes +.. _`PyPI repository version`: https://packaging.python.org/en/latest/specifications/simple-repository-api/#versioning-pypi-s-simple-api +.. _`blog post`: https://blog.pypi.org/posts/2024-03-06-malware-reporting-evolved/ +.. _`HTTP range requests`: https://http.dev/range-request +.. _`HTTP/2`: https://http.dev/2 diff --git a/peps/pep-2026.rst b/peps/pep-2026.rst index 63626d75d8c..d14fbd27e64 100644 --- a/peps/pep-2026.rst +++ b/peps/pep-2026.rst @@ -532,8 +532,24 @@ However, YY.MM versioning is rejected for many of the same reasons as YY.0 versi For example, Python 3.2026 would be released in 2026. It's clearer that the minor version is a year when using a four digits, and -avoids confusion with Ubuntu versions which use YY.MM. However, this is -rejected as changing from two to four digits would break more code than 3.YY versioning. +avoids confusion with Ubuntu versions which use YY.MM. + +``PY_VERSION_HEX`` +'''''''''''''''''' + +CPython's C API :external+python:c:macro:`PY_VERSION_HEX` macro currently uses +eight bits to encode the minor version, accommodating a maximum minor version of +255. To hold a four-digit year, it would need to be expanded to 11 bits to fit +2047 or rather 12 bits for 4095. + +This looks feasible, as it's intended for numeric comparisons, such as +``#if PY_VERSION_HEX >= ...``. In the `top 8,000 PyPI projects +`__ +only one instance was found of bit shifting +(``hexversion >> 16 != PY_VERSION_HEX >> 16``). + +However, 3.YYYY is rejected as changing from two to four digits would +nevertheless need more work and break more code than simpler 3.YY versioning. Editions --------