feat(bzlmod): cross-platform builds without experimental_index_url

CHANGELOG.md

@@ -31,7 +31,11 @@ A brief description of the categories of changes:
 
 {#v0-0-0-fixed}
 ### Fixed
-- Nothing yet
+- The extension evaluation has been adjusted to always generate the same lock
+  file irrespective if `experimental_index_url` is set by any module or not.
+  Fixes [#2268](https://github.com/bazelbuild/rules_python/issues/2268). A known
+  issue is that it may break `bazel query` and in these usecases it is advisable
+  to use `cquery` until we have `sdist` cross-building from source fully working.
 
 {#v0-0-0-added}
 ### Added

python/private/pypi/config_settings.bzl

@@ -131,6 +131,17 @@ def config_settings(
                 constraint_values.append("@platforms//cpu:" + cpu)
                 suffix += "_" + cpu
 
+            if suffix:
+                # Add python version + platform config settings
+                _dist_config_setting(
+                    name = suffix.strip("_"),
+                    flag_values = {},
+                    python_version = python_version,
+                    is_python = is_python,
+                    visibility = visibility,
+                    native = native,
+                )
+
             _dist_config_settings(
                 suffix = suffix,
                 plat_flag_values = _plat_flag_values(
@@ -277,7 +288,7 @@ def _plat_flag_values(os, cpu, osx_versions, glibc_versions, muslc_versions):
 
     return ret
 
-def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native = native, **kwargs):
+def _dist_config_setting(*, name, is_python, python_version, is_pip_whl = None, native = native, **kwargs):
     """A macro to create a target that matches is_pip_whl_auto and one more value.
 
     Args:
@@ -310,6 +321,10 @@ def _dist_config_setting(*, name, is_pip_whl, is_python, python_version, native
         # `python_version` setting.
         return
 
+    if not is_pip_whl:
+        native.config_setting(name = _name, **kwargs)
+        return
+
     config_setting_name = _name + "_setting"
     native.config_setting(name = config_setting_name, **kwargs)
 

python/private/pypi/extension.bzl

@@ -24,7 +24,7 @@ load("//python/private:version_label.bzl", "version_label")
 load(":attrs.bzl", "use_isolated")
 load(":evaluate_markers.bzl", "evaluate_markers", EVALUATE_MARKERS_SRCS = "SRCS")
 load(":hub_repository.bzl", "hub_repository")
-load(":parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement")
+load(":parse_requirements.bzl", "parse_requirements")
 load(":parse_whl_name.bzl", "parse_whl_name")
 load(":pip_repository_attrs.bzl", "ATTRS")
 load(":render_pkg_aliases.bzl", "whl_alias")
@@ -212,7 +212,6 @@ def _create_whl_repos(
         logger = logger,
     )
 
-    repository_platform = host_platform(module_ctx)
     for whl_name, requirements in requirements_by_platform.items():
         # We are not using the "sanitized name" because the user
         # would need to guess what name we modified the whl name
@@ -260,10 +259,10 @@ def _create_whl_repos(
             if v != default
         })
 
+        is_exposed = False
         if get_index_urls:
             # TODO @aignas 2024-05-26: move to a separate function
             found_something = False
-            is_exposed = False
             for requirement in requirements:
                 is_exposed = is_exposed or requirement.is_exposed
                 dists = requirement.whls
@@ -319,35 +318,32 @@ def _create_whl_repos(
                     exposed_packages[whl_name] = None
                 continue
 
-        requirement = select_requirement(
-            requirements,
-            platform = None if pip_attr.download_only else repository_platform,
-        )
-        if not requirement:
-            # Sometimes the package is not present for host platform if there
-            # are whls specified only in particular requirements files, in that
-            # case just continue, however, if the download_only flag is set up,
-            # then the user can also specify the target platform of the wheel
-            # packages they want to download, in that case there will be always
-            # a requirement here, so we will not be in this code branch.
-            continue
-        elif get_index_urls:
-            logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
-
-        whl_library_args["requirement"] = requirement.requirement_line
-        if requirement.extra_pip_args:
-            whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+        for i, requirement in enumerate(requirements):
+            is_exposed = is_exposed or requirement.is_exposed
+            if get_index_urls:
+                logger.warn(lambda: "falling back to pip for installing the right file for {}".format(requirement.requirement_line))
+
+            whl_library_args["requirement"] = requirement.requirement_line
+            if requirement.extra_pip_args:
+                whl_library_args["extra_pip_args"] = requirement.extra_pip_args
+
+            # We sort so that the lock-file remains the same no matter the order of how the
+            # args are manipulated in the code going before.
+            repo_name = "{}_{}".format(pip_name, whl_name)
+            if len(requirements) > 1:
+                repo_name = "{}__{}".format(repo_name, i)
+
+            whl_libraries[repo_name] = dict(whl_library_args.items())
+            whl_map.setdefault(whl_name, []).append(
+                whl_alias(
+                    repo = repo_name,
+                    version = major_minor,
+                    target_platforms = requirement.target_platforms if len(requirements) > 1 else None,
+                ),
+            )
 
-        # We sort so that the lock-file remains the same no matter the order of how the
-        # args are manipulated in the code going before.
-        repo_name = "{}_{}".format(pip_name, whl_name)
-        whl_libraries[repo_name] = dict(whl_library_args.items())
-        whl_map.setdefault(whl_name, []).append(
-            whl_alias(
-                repo = repo_name,
-                version = major_minor,
-            ),
-        )
+        if is_exposed:
+            exposed_packages[whl_name] = None
 
     return struct(
         is_reproducible = is_reproducible,

python/private/pypi/parse_requirements.bzl

@@ -212,6 +212,8 @@ def parse_requirements(
 def select_requirement(requirements, *, platform):
     """A simple function to get a requirement for a particular platform.
 
+    Only used in WORKSPACE.
+
     Args:
         requirements (list[struct]): The list of requirements as returned by
             the `parse_requirements` function above.
@@ -243,6 +245,8 @@ def select_requirement(requirements, *, platform):
 def host_platform(ctx):
     """Return a string representation of the repository OS.
 
+    Only used in WORKSPACE.
+
     Args:
         ctx (struct): The `module_ctx` or `repository_ctx` attribute.
 

python/private/pypi/render_pkg_aliases.bzl

@@ -318,16 +318,19 @@ def multiplatform_whl_aliases(*, aliases, **kwargs):
     ret = []
     versioned_additions = {}
     for alias in aliases:
-        if not alias.filename:
+        if not alias.filename and not alias.target_platforms:
             ret.append(alias)
             continue
 
         config_settings, all_versioned_settings = get_filename_config_settings(
             # TODO @aignas 2024-05-27: pass the parsed whl to reduce the
             # number of duplicate operations.
-            filename = alias.filename,
+            filename = alias.filename or "",
             target_platforms = alias.target_platforms,
             python_version = alias.version,
+            # If we have multiple platforms but no wheel filename, lets use different
+            # config settings.
+            non_whl_prefix = "sdist" if alias.filename else "",
             **kwargs
         )
 
@@ -429,10 +432,7 @@ def get_whl_flag_versions(aliases):
         if a.version:
             python_versions[a.version] = None
 
-        if not a.filename:
-            continue
-
-        if a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"):
+        if a.filename and a.filename.endswith(".whl") and not a.filename.endswith("-any.whl"):
             parsed = parse_whl_name(a.filename)
         else:
             for plat in a.target_platforms or []:
@@ -491,10 +491,11 @@ def get_filename_config_settings(
         *,
         filename,
         target_platforms,
-        glibc_versions,
-        muslc_versions,
-        osx_versions,
-        python_version):
+        python_version,
+        glibc_versions = None,
+        muslc_versions = None,
+        osx_versions = None,
+        non_whl_prefix = "sdist"):
     """Get the filename config settings.
 
     Args:
@@ -504,6 +505,8 @@ def get_filename_config_settings(
         muslc_versions: list[tuple[int, int]], list of versions.
         osx_versions: list[tuple[int, int]], list of versions.
         python_version: the python version to generate the config_settings for.
+        non_whl_prefix: the prefix of the config setting when the whl we don't have
+            a filename ending with ".whl".
 
     Returns:
         A tuple:
@@ -512,19 +515,20 @@ def get_filename_config_settings(
     """
     prefixes = []
     suffixes = []
-    if (0, 0) in glibc_versions:
-        fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value")
-    if (0, 0) in muslc_versions:
-        fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value")
-    if (0, 0) in osx_versions:
-        fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value")
-
-    glibc_versions = sorted(glibc_versions)
-    muslc_versions = sorted(muslc_versions)
-    osx_versions = sorted(osx_versions)
     setting_supported_versions = {}
 
     if filename.endswith(".whl"):
+        if (0, 0) in glibc_versions:
+            fail("Invalid version in 'glibc_versions': cannot specify (0, 0) as a value")
+        if (0, 0) in muslc_versions:
+            fail("Invalid version in 'muslc_versions': cannot specify (0, 0) as a value")
+        if (0, 0) in osx_versions:
+            fail("Invalid version in 'osx_versions': cannot specify (0, 0) as a value")
+
+        glibc_versions = sorted(glibc_versions)
+        muslc_versions = sorted(muslc_versions)
+        osx_versions = sorted(osx_versions)
+
         parsed = parse_whl_name(filename)
         if parsed.python_tag == "py2.py3":
             py = "py"
@@ -539,10 +543,10 @@ def get_filename_config_settings(
             abi = parsed.abi_tag
 
         if parsed.platform_tag == "any":
-            prefixes = ["{}_{}_any".format(py, abi)]
+            prefixes = ["_{}_{}_any".format(py, abi)]
             suffixes = [_non_versioned_platform(p) for p in target_platforms or []]
         else:
-            prefixes = ["{}_{}".format(py, abi)]
+            prefixes = ["_{}_{}".format(py, abi)]
             suffixes = _whl_config_setting_suffixes(
                 platform_tag = parsed.platform_tag,
                 glibc_versions = glibc_versions,
@@ -551,22 +555,22 @@ def get_filename_config_settings(
                 setting_supported_versions = setting_supported_versions,
             )
     else:
-        prefixes = ["sdist"]
+        prefixes = [""] if not non_whl_prefix else ["_" + non_whl_prefix]
         suffixes = [_non_versioned_platform(p) for p in target_platforms or []]
 
     versioned = {
-        ":is_cp{}_{}_{}".format(python_version, p, suffix): {
-            version: ":is_cp{}_{}_{}".format(python_version, p, setting)
+        ":is_cp{}{}_{}".format(python_version, p, suffix): {
+            version: ":is_cp{}{}_{}".format(python_version, p, setting)
             for version, setting in versions.items()
         }
         for p in prefixes
         for suffix, versions in setting_supported_versions.items()
     }
 
     if suffixes or versioned:
-        return [":is_cp{}_{}_{}".format(python_version, p, s) for p in prefixes for s in suffixes], versioned
+        return [":is_cp{}{}_{}".format(python_version, p, s) for p in prefixes for s in suffixes], versioned
     else:
-        return [":is_cp{}_{}".format(python_version, p) for p in prefixes], setting_supported_versions
+        return [":is_cp{}{}".format(python_version, p) for p in prefixes], setting_supported_versions
 
 def _whl_config_setting_suffixes(
         platform_tag,

tests/pypi/extension/extension_tests.bzl

@@ -150,14 +150,100 @@ def _test_simple(env):
     )
 
     pypi.is_reproducible().equals(True)
-    pypi.exposed_packages().contains_exactly({"pypi": []})
+    pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
     pypi.hub_group_map().contains_exactly({"pypi": {}})
-    pypi.hub_whl_map().contains_exactly({"pypi": {}})
-    pypi.whl_libraries().contains_exactly({})
+    pypi.hub_whl_map().contains_exactly({"pypi": {
+        "simple": [
+            struct(
+                config_setting = "//_config:is_python_3.15",
+                filename = None,
+                repo = "pypi_315_simple",
+                target_platforms = None,
+                version = "3.15",
+            ),
+        ],
+    }})
+    pypi.whl_libraries().contains_exactly({
+        "pypi_315_simple": {
+            "dep_template": "@pypi//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "repo": "pypi_315",
+            "requirement": "simple==0.0.1 --hash=sha256:deadbeef",
+        },
+    })
     pypi.whl_mods().contains_exactly({})
 
 _tests.append(_test_simple)
 
+def _test_simple_multiple_requirements(env):
+    pypi = _parse_modules(
+        env,
+        module_ctx = _mock_mctx(
+            _mod(
+                name = "rules_python",
+                parse = [
+                    _parse(
+                        hub_name = "pypi",
+                        python_version = "3.15",
+                        requirements_darwin = "darwin.txt",
+                        requirements_windows = "win.txt",
+                    ),
+                ],
+            ),
+            read = lambda x: {
+                "darwin.txt": "simple==0.0.2 --hash=sha256:deadb00f",
+                "win.txt": "simple==0.0.1 --hash=sha256:deadbeef",
+            }[x],
+        ),
+        available_interpreters = {
+            "python_3_15_host": "unit_test_interpreter_target",
+        },
+    )
+
+    pypi.is_reproducible().equals(True)
+    pypi.exposed_packages().contains_exactly({"pypi": ["simple"]})
+    pypi.hub_group_map().contains_exactly({"pypi": {}})
+    pypi.hub_whl_map().contains_exactly({"pypi": {
+        "simple": [
+            struct(
+                config_setting = "//_config:is_python_3.15",
+                filename = None,
+                repo = "pypi_315_simple__0",
+                target_platforms = [
+                    "cp315_windows_x86_64",
+                ],
+                version = "3.15",
+            ),
+            struct(
+                config_setting = "//_config:is_python_3.15",
+                filename = None,
+                repo = "pypi_315_simple__1",
+                target_platforms = [
+                    "cp315_osx_aarch64",
+                    "cp315_osx_x86_64",
+                ],
+                version = "3.15",
+            ),
+        ],
+    }})
+    pypi.whl_libraries().contains_exactly({
+        "pypi_315_simple__0": {
+            "dep_template": "@pypi//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "repo": "pypi_315",
+            "requirement": "simple==0.0.1 --hash=sha256:deadbeef",
+        },
+        "pypi_315_simple__1": {
+            "dep_template": "@pypi//{name}:{target}",
+            "python_interpreter_target": "unit_test_interpreter_target",
+            "repo": "pypi_315",
+            "requirement": "simple==0.0.2 --hash=sha256:deadb00f",
+        },
+    })
+    pypi.whl_mods().contains_exactly({})
+
+_tests.append(_test_simple_multiple_requirements)
+
 def _test_simple_get_index(env):
     got_simpleapi_download_args = []
     got_simpleapi_download_kwargs = {}