diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl index 6462880634..678a96ba70 100644 --- a/python/private/whl_target_platforms.bzl +++ b/python/private/whl_target_platforms.bzl @@ -103,12 +103,12 @@ def _whl_priority(value): # Windows does not have multiple wheels for the same target platform return (False, False, 0, 0) -def select_whls(*, whls, want_version = None, want_abis = [], want_platforms = []): +def select_whls(*, whls, want_version = "3.0", want_abis = [], want_platforms = []): """Select a subset of wheels suitable for target platforms from a list. Args: whls(list[struct]): A list of candidates. - want_version(str, optional): An optional parameter to filter whls by version. + want_version(str): An optional parameter to filter whls by version. Defaults to '3.0'. want_abis(list[str]): A list of ABIs that are supported. want_platforms(str): The platforms @@ -124,55 +124,70 @@ def select_whls(*, whls, want_version = None, want_abis = [], want_platforms = [ version_limit = int(want_version.split(".")[1]) candidates = {} - any_whls = [] for whl in whls: parsed = parse_whl_name(whl.filename) - is_python_implementation_compatible = True + supported_implementations = {} + whl_version_min = 0 for tag in parsed.python_tag.split("."): - if not tag[:2] in ["py", "cp"]: - is_python_implementation_compatible = False - break + supported_implementations[tag[:2]] = None - if not is_python_implementation_compatible: + if tag.startswith("cp3") or tag.startswith("py3"): + version = int(tag[len("..3"):] or 0) + else: + # tag.startswith("cp2") or tag.startswith("py2") + continue + + if whl_version_min == 0 or version < whl_version_min: + whl_version_min = version + + if not ("cp" in supported_implementations or "py" in supported_implementations): continue if want_abis and parsed.abi_tag not in want_abis: # Filter out incompatible ABIs continue - if parsed.platform_tag == "any" or not want_platforms: - whl_version_min = 0 - if parsed.python_tag.startswith("cp3") or parsed.python_tag.startswith("py3"): - whl_version_min = int(parsed.python_tag[len("xx3"):] or 0) - - if version_limit != -1 and whl_version_min > version_limit: - continue - elif version_limit != -1 and parsed.abi_tag in ["none", "abi3"]: - # We want to prefer by version and then we want to prefer abi3 over none - any_whls.append((whl_version_min, parsed.abi_tag == "abi3", whl)) - continue - - candidates["any"] = whl + if version_limit != -1 and whl_version_min > version_limit: continue compatible = False - for p in whl_target_platforms(parsed.platform_tag): - if p.target_platform in want_platforms: - compatible = True - break - - if compatible: - candidates[parsed.platform_tag] = whl + if parsed.platform_tag == "any": + compatible = True + else: + for p in whl_target_platforms(parsed.platform_tag): + if p.target_platform in want_platforms: + compatible = True + break - if any_whls: - # Overwrite the previously select any whl with the best fitting one. - # The any_whls will be only populated if we pass the want_version - # parameter. - _, _, whl = sorted(any_whls)[-1] - candidates["any"] = whl + if not compatible: + continue - return candidates.values() + for implementation in supported_implementations: + candidates.setdefault( + ( + parsed.abi_tag, + parsed.platform_tag, + ), + {}, + ).setdefault( + ( + # prefer cp implementation + implementation == "cp", + # prefer higher versions + whl_version_min, + # prefer abi3 over none + parsed.abi_tag != "none", + # prefer cpx abi over abi3 + parsed.abi_tag != "abi3", + ), + [], + ).append(whl) + + return [ + candidates[key][sorted(v)[-1]][-1] + for key, v in candidates.items() + ] def select_whl(*, whls, want_platform): """Select a suitable wheel from a list. @@ -193,6 +208,7 @@ def select_whl(*, whls, want_platform): # the repository context instead of `select_whl`. whls = select_whls( whls = whls, + want_version = "", want_platforms = [want_platform], ) diff --git a/tests/private/whl_target_platforms/select_whl_tests.bzl b/tests/private/whl_target_platforms/select_whl_tests.bzl index aafcc6d2ad..e099bd57a3 100644 --- a/tests/private/whl_target_platforms/select_whl_tests.bzl +++ b/tests/private/whl_target_platforms/select_whl_tests.bzl @@ -18,62 +18,53 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("//python/private:whl_target_platforms.bzl", "select_whl", "select_whls") # buildifier: disable=bzl-visibility WHL_LIST = [ - struct( - filename = f, - url = "https://" + f, - sha256 = "sha256://" + f, - ) - for f in [ - "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp311-cp311-win32.whl", - "pkg-0.0.1-cp311-cp311-win_amd64.whl", - "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp37-cp37m-win32.whl", - "pkg-0.0.1-cp37-cp37m-win_amd64.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl", - "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl", - "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", - "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl", - "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", - "pkg-0.0.1-cp39-cp39-win32.whl", - "pkg-0.0.1-cp39-cp39-win_amd64.whl", - "pkg-0.0.1-cp39-abi3-any.whl", - "pkg-0.0.1-py310-abi3-any.whl", - "pkg-0.0.1-py3-abi3-any.whl", - "pkg-0.0.1-py3-none-any.whl", - # Also add other interpreters to ensure that we don't get them - "pkg-0.0.1-pp37.pp38.pp39-none-any.whl", - ] + "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl", + "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp311-cp311-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp311-cp311-win32.whl", + "pkg-0.0.1-cp311-cp311-win_amd64.whl", + "pkg-0.0.1-cp37-cp37m-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp37-cp37m-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp37-cp37m-win32.whl", + "pkg-0.0.1-cp37-cp37m-win_amd64.whl", + "pkg-0.0.1-cp39-cp39-macosx_10_9_universal2.whl", + "pkg-0.0.1-cp39-cp39-macosx_10_9_x86_64.whl", + "pkg-0.0.1-cp39-cp39-macosx_11_0_arm64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", + "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_aarch64.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_i686.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_ppc64le.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_s390x.whl", + "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp39-cp39-win32.whl", + "pkg-0.0.1-cp39-cp39-win_amd64.whl", + "pkg-0.0.1-cp39-abi3-any.whl", + "pkg-0.0.1-py310-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", ] def _match(env, got, *want_filenames): @@ -85,18 +76,38 @@ def _match(env, got, *want_filenames): env.expect.that_collection(got_filenames).contains_exactly(want_filenames) if got: - env.expect.that_str(got[0].sha256).equals("sha256://" + want_filenames[0]) - env.expect.that_str(got[0].url).equals("https://" + want_filenames[0]) + # Check that we pass the original structs + env.expect.that_str(got[0].other).equals("dummy") def _select_whl(**kwargs): """A small wrapper to make the tests more DRY.""" got_single = select_whl(**kwargs) return [got_single] if got_single else [] +def _select_whls(whls, **kwargs): + return select_whls( + whls = [ + struct( + filename = f, + other = "dummy", + ) + for f in whls + ], + **kwargs + ) + _tests = [] def _test_simplest(env): - got = select_whls(whls = WHL_LIST, want_abis = ["none"], want_platforms = ["ignored"]) + got = _select_whls( + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + ], + want_abis = ["none"], + want_platforms = ["ignored"], + ) _match( env, got, @@ -106,7 +117,15 @@ def _test_simplest(env): _tests.append(_test_simplest) def _test_select_abi3(env): - got = select_whls(whls = WHL_LIST, want_abis = ["abi3"], want_platforms = ["ignored"]) + got = _select_whls( + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py3-none-any.whl", + ], + want_abis = ["abi3"], + want_platforms = ["ignored"], + ) _match( env, got, @@ -116,32 +135,79 @@ def _test_select_abi3(env): _tests.append(_test_select_abi3) def _test_select_by_supported_py_version(env): - got = select_whls(whls = WHL_LIST, want_abis = ["abi3"], want_platforms = ["ignored"], want_version = "3.8") - _match( - env, - got, - "pkg-0.0.1-py3-abi3-any.whl", - ) + for want_version, match in { + "3.11": "pkg-0.0.1-py311-abi3-any.whl", + "3.8": "pkg-0.0.1-py3-abi3-any.whl", + }.items(): + got = _select_whls( + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + ], + want_abis = ["abi3"], + want_platforms = ["ignored"], + want_version = want_version, + ) + _match(env, got, match) - got = select_whls(whls = WHL_LIST, want_abis = ["abi3"], want_platforms = ["ignored"], want_version = "3.9") - _match( - env, - got, - "pkg-0.0.1-cp39-abi3-any.whl", - ) +_tests.append(_test_select_by_supported_py_version) - got = select_whls(whls = WHL_LIST, want_abis = ["abi3"], want_platforms = ["ignored"], want_version = "3.10") - _match( - env, - got, - "pkg-0.0.1-py310-abi3-any.whl", +def _test_select_by_supported_cp_version(env): + for want_version, match in { + "3.11": "pkg-0.0.1-cp311-abi3-any.whl", + "3.8": "pkg-0.0.1-py3-abi3-any.whl", + }.items(): + got = _select_whls( + whls = [ + "pkg-0.0.1-py2.py3-abi3-any.whl", + "pkg-0.0.1-py3-abi3-any.whl", + "pkg-0.0.1-py311-abi3-any.whl", + "pkg-0.0.1-cp311-abi3-any.whl", + ], + want_abis = ["abi3"], + want_platforms = ["ignored"], + want_version = want_version, + ) + _match(env, got, match) + +_tests.append(_test_select_by_supported_cp_version) + +def _test_supported_cp_version_manylinux(env): + for want_version, match in { + "3.11": "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", + "3.8": "pkg-0.0.1-py3-none-manylinux_x86_64.whl", + }.items(): + got = _select_whls( + whls = [ + "pkg-0.0.1-py2.py3-none-manylinux_x86_64.whl", + "pkg-0.0.1-py3-none-manylinux_x86_64.whl", + "pkg-0.0.1-py311-none-manylinux_x86_64.whl", + "pkg-0.0.1-cp311-none-manylinux_x86_64.whl", + ], + want_abis = ["none"], + want_platforms = ["linux_x86_64"], + want_version = want_version, + ) + _match(env, got, match) + +_tests.append(_test_supported_cp_version_manylinux) + +def _test_ignore_unsupported(env): + got = _select_whls( + whls = [ + "pkg-0.0.1-xx3-abi3-any.whl", + ], + want_abis = ["abi3"], + want_platforms = ["ignored"], ) + _match(env, got) -_tests.append(_test_select_by_supported_py_version) +_tests.append(_test_ignore_unsupported) def _test_match_abi_and_not_py_version(env): # Check we match the ABI and not the py version - got = select_whls(whls = WHL_LIST, want_abis = ["cp37m"], want_platforms = ["linux_x86_64"]) + got = _select_whls(whls = WHL_LIST, want_abis = ["cp37m"], want_platforms = ["linux_x86_64"], want_version = "3.7") _match( env, got, @@ -155,7 +221,7 @@ _tests.append(_test_match_abi_and_not_py_version) def _test_select_filename_with_many_tags(env): # Check we can select a filename with many platform tags - got = select_whls(whls = WHL_LIST, want_abis = ["cp39"], want_platforms = ["linux_x86_32"]) + got = _select_whls(whls = WHL_LIST, want_abis = ["cp39"], want_platforms = ["linux_x86_32"], want_version = "3.9") _match( env, got, @@ -169,7 +235,12 @@ _tests.append(_test_select_filename_with_many_tags) def _test_osx_prefer_arch_specific(env): # Check that we prefer the specific wheel - got = select_whls(whls = WHL_LIST, want_abis = ["cp311"], want_platforms = ["osx_x86_64", "osx_x86_32"]) + got = _select_whls( + whls = WHL_LIST, + want_abis = ["cp311"], + want_platforms = ["osx_x86_64", "osx_x86_32"], + want_version = "3.11", + ) _match( env, got, @@ -179,7 +250,7 @@ def _test_osx_prefer_arch_specific(env): got = _select_whl(whls = got, want_platform = "osx_x86_64") _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl") - got = select_whls(whls = WHL_LIST, want_abis = ["cp311"], want_platforms = ["osx_aarch64"]) + got = _select_whls(whls = WHL_LIST, want_abis = ["cp311"], want_platforms = ["osx_aarch64"], want_version = "3.11") _match( env, got, @@ -193,7 +264,7 @@ _tests.append(_test_osx_prefer_arch_specific) def _test_osx_fallback_to_universal2(env): # Check that we can use the universal2 if the arm wheel is not available - got = select_whls(whls = [w for w in WHL_LIST if "arm64" not in w.filename], want_abis = ["cp311"], want_platforms = ["osx_aarch64"]) + got = _select_whls(whls = [w for w in WHL_LIST if "arm64" not in w], want_abis = ["cp311"], want_platforms = ["osx_aarch64"], want_version = "3.11") _match( env, got, @@ -206,12 +277,13 @@ _tests.append(_test_osx_fallback_to_universal2) def _test_prefer_manylinux_wheels(env): # Check we prefer platform specific wheels - got = select_whls(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_platforms = ["linux_x86_64"]) + got = _select_whls(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_platforms = ["linux_x86_64"], want_version = "3.9") _match( env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", "pkg-0.0.1-cp39-cp39-musllinux_1_1_x86_64.whl", + "pkg-0.0.1-cp39-abi3-any.whl", "pkg-0.0.1-py3-none-any.whl", ) got = _select_whl(whls = got, want_platform = "linux_x86_64")