Switch to the hatchling build backend.

This should elminate flaky CI due to ephemeral dirs in the source root
created by the setuptools build backend which were not multi-process
safe.

The `hatchling` build backend supports metadata plugins and it performs
isolated builds making it a good replacement for the prior setuptools
hack for effecting dynamic `Requires-Python` to enable Python 3.13
shadow-support. Unfortunately, there is no nice way to specify an
in-tree plugin; so we actually use a very thin in-tree build backend
that wraps `hatchling.build`, to allow discovery of our in-tree plugin.

build-backend/pex_build.dist-info/entry_points.txt

@@ -0,0 +1,2 @@
+[hatch]
+pex-dynamic-requires-python = pex_build.hatchling.hooks

build-backend/pex_build/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).

build-backend/pex_build/hatchling/__init__.py

@@ -0,0 +1,2 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).

build-backend/pex_build/hatchling/build.py

@@ -0,0 +1,7 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+# We re-export all hatchling's PEP-517 build backend hooks here for the build frontend to call.
+from hatchling.build import *  # NOQA

build-backend/pex_build/hatchling/dynamic_requires_python.py

@@ -0,0 +1,27 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import, print_function
+
+import os
+import sys
+from typing import Any, Dict
+
+from hatchling.metadata.plugin.interface import MetadataHookInterface
+
+
+class DynamicRequiresPythonHook(MetadataHookInterface):
+    """Allows dynamically specifying requires-python metadata via _PEX_REQUIRES_PYTHON env var."""
+
+    PLUGIN_NAME = "pex-dynamic-requires-python"
+
+    def update(self, metadata):
+        # type: (Dict[str, Any]) -> None
+        requires_python = os.environ.get("_PEX_REQUIRES_PYTHON")
+        if requires_python:
+            print(
+                "pex_build: Dynamically modifying pyproject.toml requires-python of {original} to "
+                "{dynamic}".format(original=metadata["requires-python"], dynamic=requires_python),
+                file=sys.stderr,
+            )
+            metadata["requires-python"] = requires_python

build-backend/pex_build/hatchling/hooks.py

@@ -0,0 +1,15 @@
+# Copyright 2024 Pants project contributors (see CONTRIBUTORS.md).
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+from typing import Type
+
+from hatchling.plugin import hookimpl
+from pex_build.hatchling.dynamic_requires_python import DynamicRequiresPythonHook
+
+
+@hookimpl
+def hatch_register_metadata_hook():
+    # type: () -> Type[DynamicRequiresPythonHook]
+    return DynamicRequiresPythonHook

mypy.ini

@@ -45,3 +45,10 @@ ignore_missing_imports = True
 
 [mypy-pkg_resources]
 ignore_missing_imports = True
+
+[mypy-hatchling.*]
+# Currently we use one venv for MyPy checks and the hatchling requirements are not compatible with
+# the rest; so we ignore. Ideally pex.hatchling build backend type-checking would be segregated as
+# scripts are today and each with its own venv.
+ignore_missing_imports = True
+

pex/pip/tool.py

@@ -436,6 +436,7 @@ def spawn_download_distributions(
 
         download_cmd = ["download", "--dest", download_dir]
         extra_env = {}  # type: Dict[str, str]
+        pex_extra_sys_path = []  # type: List[str]
 
         if not build:
             download_cmd.extend(["--only-binary", ":all:"])
@@ -451,7 +452,7 @@ def spawn_download_distributions(
 
         if not build_isolation:
             download_cmd.append("--no-build-isolation")
-            extra_env.update(PEP517_BACKEND_PATH=os.pathsep.join(sys.path))
+            pex_extra_sys_path.extend(sys.path)
 
         if allow_prereleases:
             download_cmd.append("--pre")
@@ -483,7 +484,6 @@ def spawn_download_distributions(
             )
 
         log_analyzers = []  # type: List[LogAnalyzer]
-        pex_extra_sys_path = []  # type: List[str]
         for obs in (foreign_platform_observer, observer):
             if obs:
                 if obs.analyzer:
@@ -492,10 +492,10 @@ def spawn_download_distributions(
                 extra_sys_path = obs.patch_set.emit_patches(package=self._PATCHES_PACKAGE_NAME)
                 if extra_sys_path:
                     pex_extra_sys_path.append(extra_sys_path)
+                    extra_env[self._PATCHES_PACKAGE_ENV_VAR_NAME] = self._PATCHES_PACKAGE_NAME
 
         if pex_extra_sys_path:
             extra_env["PEX_EXTRA_SYS_PATH"] = os.pathsep.join(pex_extra_sys_path)
-            extra_env[self._PATCHES_PACKAGE_ENV_VAR_NAME] = self._PATCHES_PACKAGE_NAME
 
         # The Pip 2020 resolver hides useful dependency conflict information in stdout interspersed
         # with other information we want to suppress. We jump though some hoops here to get at that
@@ -624,7 +624,7 @@ def spawn_build_wheels(
         if not build_isolation:
             wheel_cmd.append("--no-build-isolation")
             interpreter = interpreter or PythonInterpreter.get()
-            extra_env.update(PEP517_BACKEND_PATH=os.pathsep.join(interpreter.sys_path))
+            extra_env.update(PEX_EXTRA_SYS_PATH=os.pathsep.join(interpreter.sys_path))
 
         if not verify:
             wheel_cmd.append("--no-verify")

pex/vendor/__init__.py

@@ -151,14 +151,7 @@ def create_packages(self):
         if not self.rewrite:
             # The extra package structure is only required by Pex for vendored code used via import
             # rewrites.
-
-            # N.B.: Although we've historically early-returned here, the switch from flit to
-            # setuptools for our build backend necessitates all vendored dists are seen as part of
-            # the `pex` package tree by setuptools to get all vendored code properly included in
-            # our distribution.
-            # TODO(John Sirois): re-introduce early return once it is no longer foils our build
-            #  process.
-            pass
+            return
 
         for index, _ in enumerate(self._subpath_components):
             relpath = _PACKAGE_COMPONENTS + self._subpath_components[: index + 1] + ["__init__.py"]

pex/vendor/_vendored/pip/.layout.json

@@ -1 +1 @@
-{"fingerprint": "69c1ee1f3c238f9f7a88645a63f68c3c5802b6e3d349612fde5fe2b8c692a668", "record_relpath": "pip-20.3.4.dist-info/RECORD", "stash_dir": ".prefix"}
+{"fingerprint": "120267325b80f5c4b4adac019eb6617ab3319395c043d2871eedf70dd6ae2954", "record_relpath": "pip-20.3.4.dist-info/RECORD", "stash_dir": ".prefix"}

pyproject.toml

@@ -1,9 +1,76 @@
 [build-system]
-# N.B.: We use `setup.cfg` for declarative metadata instead of `[project]` here since support for
-# pyproject.toml projects was introduced in setuptools 61.0.0 which only supports Python>=3.7. We
-# need to also support 2.7, 3.5 and 3.6.
-requires = ["setuptools"]
-build-backend = "setuptools.build_meta"
+# N.B.: We use an in-tree backend just to get our hatchling in-tree plugins visible without having
+# to publish a plugin distribution. The real backend is `hatchling.build` and the
+# `pex_build.hatchling.build` backend is a very thin wrapper that just serves to install our hooks
+# just in time.
+backend-path = ["build-backend"]
+build-backend = "pex_build.hatchling.build"
+requires = ["hatchling"]
+
+[tool.hatch.metadata.hooks.pex-dynamic-requires-python]
+# We need this empty table to enable our hook.
+
+[project]
+name = "pex"
+dynamic = ["version"]
+requires-python = ">=2.7,<3.13,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*"
+authors = [
+  {name = "The PEX developers", email="pantsbuild@gmail.com"}
+]
+description = "The PEX packaging toolchain."
+readme = "README.rst"
+license-files = { paths = ["LICENSE"] }
+keywords = ["package", "executable", "virtualenv", "lock", "freeze"]
+classifiers = [
+  "Development Status :: 5 - Production/Stable",
+  "Intended Audience :: Developers",
+  "License :: OSI Approved :: Apache Software License",
+  "Operating System :: Unix",
+  "Operating System :: POSIX :: Linux",
+  "Operating System :: MacOS :: MacOS X",
+  "Programming Language :: Python",
+  "Programming Language :: Python :: 2",
+  "Programming Language :: Python :: 2.7",
+  "Programming Language :: Python :: 3",
+  "Programming Language :: Python :: 3.5",
+  "Programming Language :: Python :: 3.6",
+  "Programming Language :: Python :: 3.7",
+  "Programming Language :: Python :: 3.8",
+  "Programming Language :: Python :: 3.9",
+  "Programming Language :: Python :: 3.10",
+  "Programming Language :: Python :: 3.11",
+  "Programming Language :: Python :: 3.12",
+  "Programming Language :: Python :: Implementation :: CPython",
+  "Programming Language :: Python :: Implementation :: PyPy",
+  "Topic :: Software Development :: Build Tools",
+  "Topic :: System :: Archiving :: Packaging",
+  "Topic :: System :: Software Distribution",
+  "Topic :: Utilities",
+]
+
+[project.optional-dependencies]
+subprocess = [
+  "subprocess32>=3.2.7; python_version < '3'"
+]
+
+[project.scripts]
+pex = "pex.bin.pex:main"
+pex3 = "pex.cli.pex:main"
+pex-tools = "pex.tools.main:main"
+
+[project.entry-points."distutils.commands"]
+bdist_pex = "pex.distutils.commands.bdist_pex:bdist_pex"
+
+[project.urls]
+Homepage = "https://github.com/pantsbuild/pex"
+Download = "https://github.com/pantsbuild/pex/releases/latest/download/pex"
+Changelog = "https://github.com/pantsbuild/pex/blob/main/CHANGES.md"
+Documentation = "https://pex.readthedocs.io/en/latest/"
+Source = "https://github.com/pantsbuild/pex"
+
+[tool.hatch.version]
+path = "pex/version.py"
+pattern = '__version__ = "(?P<version>[^"]+)"'
 
 [tool.black]
 line-length = 100

scripts/format.py

@@ -35,7 +35,7 @@ def run_black(*args: str) -> None:
         dest=sys.stdout,
     ) as out_fd:
         subprocess.run(
-            args=["black", "--color", *args, "setup.py", "pex", "scripts", "testing", "tests"],
+            args=["black", "--color", *args, "build-backend", "pex", "scripts", "testing", "tests"],
             stdout=out_fd,
             stderr=subprocess.STDOUT,
             check=True,
@@ -44,7 +44,7 @@ def run_black(*args: str) -> None:
 
 def run_isort(*args: str) -> None:
     subprocess.run(
-        args=["isort", *args, "setup.py", "pex", "scripts", "testing", "tests"], check=True
+        args=["isort", *args, "build-backend", "pex", "scripts", "testing", "tests"], check=True
     )
 
 

scripts/lint.py

@@ -24,7 +24,7 @@ def run_autoflake(*args: str) -> None:
             "--exclude",
             ",".join(excludes),
             "--recursive",
-            "setup.py",
+            "build-backend",
             "pex",
             "scripts",
             "testing",

scripts/package.py

@@ -6,6 +6,7 @@
 import subprocess
 import sys
 from argparse import ArgumentDefaultsHelpFormatter, ArgumentParser
+from email.parser import Parser
 from enum import Enum, unique
 from http.server import HTTPServer, SimpleHTTPRequestHandler
 from pathlib import Path, PurePath
@@ -46,7 +47,12 @@ def build_pex_pex(output_file: PurePath, verbosity: int = 0) -> None:
     subprocess.run(args, check=True)
 
 
-def describe_git_rev() -> str:
+def describe_rev() -> str:
+    if not os.path.isdir(".git") and os.path.isfile("PKG-INFO"):
+        # We're being build from an unpacked sdist.
+        with open("PKG-INFO") as fp:
+            return Parser().parse(fp).get("Version", "Unknown Version")
+
     git_describe = subprocess.run(
         ["git", "describe"], check=True, stdout=subprocess.PIPE, encoding="utf-8"
     )
@@ -103,9 +109,9 @@ def main(
         print(f"Building Pex PEX to `{pex_output_file}` ...")
         build_pex_pex(pex_output_file, verbosity)
 
-        git_rev = describe_git_rev()
+        rev = describe_rev()
         sha256, size = describe_file(pex_output_file)
-        print(f"Built Pex PEX @ {git_rev}:")
+        print(f"Built Pex PEX @ {rev}:")
         print(f"sha256: {sha256}")
         print(f"  size: {size}")
 

scripts/typecheck.py

@@ -38,6 +38,9 @@ def run_mypy(python_version: str, files: Sequence[str], subject: str = "files")
 
 
 def main() -> None:
+    run_mypy(
+        "2.7", files=sorted(find_files_to_check(include=["build-backend"])), subject="build-backend"
+    )
     run_mypy("3.8", files=sorted(find_files_to_check(include=["scripts"])), subject="scripts")
 
     source_and_tests = sorted(