infer the PBS release tag from the user-supplied URLs

docs/notes/2.25.x.md

@@ -64,6 +64,8 @@ The AWS Lambda backend now provides built-in complete platforms for the Python 3
 
 The Python Build Standalone backend (`pants.backend.python.providers.experimental.python_build_standalone`) now supports filtering PBS releases via their "release tag" via [the new `--python-build-standalone-release-constraints` option](https://www.pantsbuild.org/2.25/reference/subsystems/python-build-standalone-python-provider#release_constraints).
 
+Also, the PBS "release tag" will be inferred for PBS releases supplied via the `--python-build-standalone-known-python-versions` option from the given URLs if those URLs conform to the naming convention used by the PBS project. The new advanced option `--python-build-standalone-require-inferrable-release-tag` controls whether Pants requires the tag to be inferrable. This option currently defaults to `False`, but will be migrated to `True` in a future Pants release. (The release tag cannot currently be specified via `--python-build-standalone-known-python-versions` since changing that option would not be a backward compatible change.)
+
 The default version of the [Pex](https://docs.pex-tool.org/) tool has been updated from 2.20.3 to [2.27.1](https://github.com/pex-tool/pex/releases/tag/v2.24.3).  Among many improvements and bug fixes, this unlocks support for pip [24.3.1](https://pip.pypa.io/en/stable/news/#v24-3-1).
 
 #### Shell

src/python/pants/backend/python/providers/python_build_standalone/rules.py

@@ -4,7 +4,10 @@
 
 import functools
 import json
+import posixpath
+import re
 import textwrap
+import urllib
 import uuid
 from pathlib import PurePath
 from typing import Iterable, Mapping, TypedDict, cast
@@ -42,7 +45,7 @@
 from pants.engine.unions import UnionRule
 from pants.option.errors import OptionsError
 from pants.option.global_options import NamedCachesDirOption
-from pants.option.option_types import StrListOption, StrOption
+from pants.option.option_types import BoolOption, StrListOption, StrOption
 from pants.option.subsystem import Subsystem
 from pants.util.docutil import bin_name
 from pants.util.frozendict import FrozenDict
@@ -150,6 +153,18 @@ class PBSPythonProviderSubsystem(Subsystem):
         ),
     )
 
+    require_inferrable_release_tag = BoolOption(
+        default=False,
+        help=textwrap.dedent(
+            """
+            Normally, Pants will try to infer the PBS release "tag" from URLs supplied to the
+            `--python-build-standalone-known-python-versions` option. If this option is True,
+            then it is an error if Pants cannot infer the tag from the URL.
+            """
+        ),
+        advanced=True,
+    )
+
     @memoized_property
     def release_constraints(self) -> ConstraintsList:
         rcs = self._release_constraints
@@ -163,27 +178,75 @@ def release_constraints(self) -> ConstraintsList:
                 f"The `[{PBSPythonProviderSubsystem.options_scope}].release_constraints option` is not valid: {e}"
             ) from None
 
-    def get_all_pbs_pythons(self) -> dict[str, dict[str, PBSPythonInfo]]:
-        all_pythons = load_pbs_pythons().copy()
+    def get_user_supplied_pbs_pythons(
+        self, require_tag: bool
+    ) -> dict[str, dict[str, PBSPythonInfo]]:
+        extract_re = re.compile(r"^cpython-([0-9.]+)\+([0-9]+)-.*\.tar\.\w+$")
+
+        def extract_version_and_tag(url: str) -> tuple[str, str] | None:
+            parsed_url = urllib.parse.urlparse(urllib.parse.unquote(url))
+            base_path = posixpath.basename(parsed_url.path)
+
+            nonlocal extract_re
+            if m := extract_re.fullmatch(base_path):
+                return (m.group(1), m.group(2))
+
+            return None
+
+        user_supplied_pythons: dict[str, dict[str, PBSPythonInfo]] = {}
 
         for version_info in self.known_python_versions or []:
-            try:
-                pyversion, platform, sha256, filesize, url = (
-                    x.strip() for x in version_info.split("|")
+            version_parts = version_info.split("|")
+            if len(version_parts) != 5:
+                raise ExternalToolError(
+                    f"Each value for the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option "
+                    "must be five values separated by a `|` character as follows: PYTHON_VERSION|PLATFORM|SHA256|FILE_SIZE|URL "
+                    f"\n\nInstead, the following value was provided: {version_info}"
                 )
-            except ValueError:
+
+            py_version, platform, sha256, filesize, url = (x.strip() for x in version_parts)
+
+            tag: str | None = None
+            maybe_inferred_py_version_and_tag = extract_version_and_tag(url)
+            if maybe_inferred_py_version_and_tag:
+                inferred_py_version, inferred_tag = maybe_inferred_py_version_and_tag
+                if inferred_py_version != py_version:
+                    raise ExternalToolError(
+                        f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "
+                        f"the value `{version_info}` declares Python version `{py_version}` in the first field, but the URL"
+                        f"provided references Python version `{inferred_py_version}`. These must be the same."
+                    )
+                tag = inferred_tag
+
+            if require_tag and tag is None:
                 raise ExternalToolError(
-                    f"Bad value for [{PBSPythonProviderSubsystem.options_scope}].known_python_versions: {version_info}"
+                    f"While parsing the `[{PBSPythonProviderSubsystem.options_scope}].known_python_versions` option, "
+                    f'the PBS release "tag" could not be inferred from the supplied URL: {url}'
+                    "\n\nThis is an error because the option"
+                    f"`[{PBSPythonProviderSubsystem.options_scope}].require_inferrable_release_tag` is set to True."
                 )
 
-            if pyversion not in all_pythons:
-                all_pythons[pyversion] = {}
+            if py_version not in user_supplied_pythons:
+                user_supplied_pythons[py_version] = {}
 
-            # Note: Tag is set "" which means this version will always match the release constraint.
-            # TODO: Infer the release tag from the URL.
-            all_pythons[pyversion][platform] = PBSPythonInfo(
-                url=url, sha256=sha256, size=int(filesize), tag=""
+            pbs_python_info = PBSPythonInfo(
+                url=url, sha256=sha256, size=int(filesize), tag=tag or ""
             )
+            user_supplied_pythons[py_version][platform] = pbs_python_info
+
+        return user_supplied_pythons
+
+    def get_all_pbs_pythons(self) -> dict[str, dict[str, PBSPythonInfo]]:
+        all_pythons = load_pbs_pythons().copy()
+
+        user_supplied_pythons = self.get_user_supplied_pbs_pythons(
+            require_tag=self.require_inferrable_release_tag
+        )
+        for py_version, platform_metadatas_for_py_version in user_supplied_pythons.items():
+            for platform_name, platform_metadata in platform_metadatas_for_py_version.items():
+                if py_version not in all_pythons:
+                    all_pythons[py_version] = {}
+                all_pythons[py_version][platform_name] = platform_metadata
 
         return all_pythons
 

src/python/pants/backend/python/providers/python_build_standalone/rules_integration_test.py

@@ -19,10 +19,12 @@
 from pants.backend.python.util_rules.interpreter_constraints import InterpreterConstraints
 from pants.build_graph.address import Address
 from pants.core.goals.run import RunRequest
+from pants.core.util_rules.external_tool import ExternalToolError
 from pants.engine.platform import Platform
 from pants.engine.process import InteractiveProcess
 from pants.engine.rules import QueryRule
 from pants.engine.target import Target
+from pants.testutil.option_util import create_subsystem
 from pants.testutil.rule_runner import RuleRunner, mock_console
 
 
@@ -36,6 +38,7 @@ def rule_runner() -> RuleRunner:
             *target_types_rules.rules(),
             QueryRule(RunRequest, (PythonSourceFieldSet,)),
             QueryRule(Platform, ()),
+            QueryRule(pbs.PBSPythonProviderSubsystem, ()),
         ],
         target_types=[
             PythonSourcesGeneratorTarget,
@@ -148,6 +151,36 @@ def test_additional_versions(rule_runner, mock_empty_versions_resource):
     assert version.startswith("3.9.16")
 
 
+# Confirm whether the PBS tag data can be inferred from a URL.
+def test_tag_inference_from_url() -> None:
+    subsystem = create_subsystem(
+        pbs.PBSPythonProviderSubsystem,
+        known_python_versions=[
+            "3.10.13|linux_arm|abc123|123|https://github.com/indygreg/python-build-standalone/releases/download/20240224/cpython-3.10.13%2B20240224-aarch64-unknown-linux-gnu-install_only.tar.gz",
+        ],
+    )
+
+    user_supplied_pbs_versions = subsystem.get_user_supplied_pbs_pythons(require_tag=False)
+    assert user_supplied_pbs_versions["3.10.13"]["linux_arm"] == pbs.PBSPythonInfo(
+        url="https://github.com/indygreg/python-build-standalone/releases/download/20240224/cpython-3.10.13%2B20240224-aarch64-unknown-linux-gnu-install_only.tar.gz",
+        sha256="abc123",
+        size=123,
+        tag="20240224",
+    )
+
+    # Confirm whether requiring tag inference results in an error.
+    subsystem = create_subsystem(
+        pbs.PBSPythonProviderSubsystem,
+        known_python_versions=[
+            "3.10.13|linux_arm|abc123|123|file:///releases/20240224/cpython.tar.gz",
+        ],
+    )
+    with pytest.raises(
+        ExternalToolError, match='the PBS release "tag" could not be inferred from the supplied URL'
+    ):
+        _ = subsystem.get_user_supplied_pbs_pythons(require_tag=True)
+
+
 def test_venv_pex_reconstruction(rule_runner):
     """A VenvPex refers to the location of the venv so it doesn't have to re-construct if it exists.