diff --git a/CHANGELOG.md b/CHANGELOG.md index 1040dedd0d..ed39bdd0cd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,14 +25,12 @@ A brief description of the categories of changes: * (toolchains) Optional toolchain dependency: `py_binary`, `py_test`, and `py_library` now depend on the `//python:exec_tools_toolchain_type` for build tools. - * (deps): Bumped `bazel_skylib` to 1.6.1. * (bzlmod): The `python` and internal `rules_python` extensions have been marked as `reproducible` and will not include any lock file entries from now on. ### Fixed - * (gazelle) Remove `visibility` from `NonEmptyAttr`. Now empty(have no `deps/main/srcs/imports` attr) `py_library/test/binary` rules will be automatically deleted correctly. For example, if `python_generation_mode` @@ -64,9 +62,16 @@ A brief description of the categories of changes: `transitive_pyc_files`, which tell the pyc files a target makes available directly and transitively, respectively. * `//python:features.bzl` added to allow easy feature-detection in the future. +* (pip) Allow specifying the requirements by (os, arch) and add extra + validations when parsing the inputs. This is a non-breaking change for most + users unless they have been passing multiple `requirements_*` files together + with `extra_pip_args = ["--platform=manylinux_2_4_x86_64"]`, that was an + invalid usage previously but we were not failing the build. From now on this + is explicitly disallowed. [precompile-docs]: /precompiling + ## [0.32.2] - 2024-05-14 [0.32.2]: https://github.com/bazelbuild/rules_python/releases/tag/0.32.2 diff --git a/MODULE.bazel b/MODULE.bazel index 8acde16c33..7064dfc84f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -61,9 +61,11 @@ pip.parse( experimental_index_url = "https://pypi.org/simple", hub_name = "rules_python_publish_deps", python_version = "3.11", - requirements_darwin = "//tools/publish:requirements_darwin.txt", - requirements_lock = "//tools/publish:requirements.txt", - requirements_windows = "//tools/publish:requirements_windows.txt", + requirements_by_platform = { + "//tools/publish:requirements.txt": "linux_*", + "//tools/publish:requirements_darwin.txt": "osx_*", + "//tools/publish:requirements_windows.txt": "windows_*", + }, ) use_repo(pip, "rules_python_publish_deps") diff --git a/docs/sphinx/pip.md b/docs/sphinx/pip.md index e73c0c6a56..e1c8e343f0 100644 --- a/docs/sphinx/pip.md +++ b/docs/sphinx/pip.md @@ -19,11 +19,41 @@ load("@pip_deps//:requirements.bzl", "install_deps") install_deps() ``` +For `bzlmod` an equivalent `MODULE.bazel` would look like: +```starlark +pip = use_extension("//python/extensions:pip.bzl", "pip") +pip.parse( + hub_name = "pip_deps", + requirements_lock = ":requirements.txt", +) +use_repo(pip, "pip_deps") +``` + You can then reference installed dependencies from a `BUILD` file with: ```starlark load("@pip_deps//:requirements.bzl", "requirement") +py_library( + name = "bar", + ... + deps = [ + "//my/other:dep", + "@pip_deps//requests", + "@pip_deps//numpy", + ], +) +``` + +The rules also provide a convenience macro for translating the entries in the +`requirements.txt` file (e.g. `opencv-python`) to the right bazel label (e.g. +`@pip_deps//opencv_python`). The convention of bazel labels is lowercase +`snake_case`, but you can use the helper to avoid depending on this convention +as follows: + +```starlark +load("@pip_deps//:requirements.bzl", "requirement") + py_library( name = "bar", ... @@ -35,33 +65,39 @@ py_library( ) ``` -In addition to the `requirement` macro, which is used to access the generated `py_library` -target generated from a package's wheel, The generated `requirements.bzl` file contains -functionality for exposing [entry points][whl_ep] as `py_binary` targets as well. +If you would like to access [entry points][whl_ep], see the `py_console_script_binary` rule documentation. [whl_ep]: https://packaging.python.org/specifications/entry-points/ +(per-os-arch-requirements)= +## Requirements for a specific OS/Architecture + +In some cases you may need to use different requirements files for different OS, Arch combinations. This is enabled via the `requirements_by_platform` attribute in `pip.parse` extension and the `pip_parse` repository rule. The keys of the dictionary are labels to the file and the values are a list of comma separated target (os, arch) tuples. + +For example: ```starlark -load("@pip_deps//:requirements.bzl", "entry_point") - -alias( - name = "pip-compile", - actual = entry_point( - pkg = "pip-tools", - script = "pip-compile", - ), -) + # ... + requirements_by_platform = { + "requirements_linux_x86_64.txt": "linux_x86_64", + "requirements_osx.txt": "osx_*", + "requirements_linux_exotic.txt": "linux_exotic", + "requirements_some_platforms.txt": "linux_aarch64,windows_*", + }, + # For the list of standard platforms that the rules_python has toolchains for, default to + # the following requirements file. + requirements_lock = "requirements_lock.txt", ``` -Note that for packages whose name and script are the same, only the name of the package -is needed when calling the `entry_point` macro. +In case of duplicate platforms, `rules_python` will raise an error as there has +to be unambiguous mapping of the requirement files to the (os, arch) tuples. +An alternative way is to use per-OS requirement attributes. ```starlark -load("@pip_deps//:requirements.bzl", "entry_point") - -alias( - name = "flake8", - actual = entry_point("flake8"), + # ... + requirements_windows = "requirements_windows.txt", + requirements_darwin = "requirements_darwin.txt", + # For the remaining platforms (which is basically only linux OS), use this file. + requirements_lock = "requirements_lock.txt", ) ``` diff --git a/examples/bzlmod/MODULE.bazel b/examples/bzlmod/MODULE.bazel index 1134487145..0d30161147 100644 --- a/examples/bzlmod/MODULE.bazel +++ b/examples/bzlmod/MODULE.bazel @@ -128,8 +128,17 @@ pip.parse( ], hub_name = "pip", python_version = "3.9", - requirements_lock = "//:requirements_lock_3_9.txt", - requirements_windows = "//:requirements_windows_3_9.txt", + # The requirements files for each platform that we want to support. + requirements_by_platform = { + # Default requirements file for needs to explicitly provide the platforms + "//:requirements_lock_3_9.txt": "linux_*,osx_*", + # This API allows one to specify additional platforms that the users + # configure the toolchains for themselves. In this example we add + # `windows_aarch64` to illustrate that `rules_python` won't fail to + # process the value, but it does not mean that this example will work + # on Windows ARM. + "//:requirements_windows_3_9.txt": "windows_x86_64,windows_aarch64", + }, # These modifications were created above and we # are providing pip.parse with the label of the mod # and the name of the wheel. @@ -193,14 +202,3 @@ local_path_override( module_name = "other_module", path = "other_module", ) - -# ===== -# Config for testing duplicate packages in requirements -# ===== -# -pip.parse( - hub_name = "dupe_requirements", - python_version = "3.9", # Must match whatever is marked is_default=True - requirements_lock = "//tests/dupe_requirements:requirements.txt", -) -use_repo(pip, "dupe_requirements") diff --git a/examples/bzlmod/tests/dupe_requirements/BUILD.bazel b/examples/bzlmod/tests/dupe_requirements/BUILD.bazel deleted file mode 100644 index 47eb7ca0fb..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/BUILD.bazel +++ /dev/null @@ -1,19 +0,0 @@ -load("@rules_python//python:pip.bzl", "compile_pip_requirements") -load("@rules_python//python:py_test.bzl", "py_test") - -py_test( - name = "dupe_requirements_test", - srcs = ["dupe_requirements_test.py"], - deps = [ - "@dupe_requirements//pyjwt", - ], -) - -compile_pip_requirements( - name = "requirements", - src = "requirements.in", - requirements_txt = "requirements.txt", - # This is to make the requirements diff test not run on CI. The content we - # need in requirements.txt isn't exactly what will be generated. - tags = ["manual"], -) diff --git a/examples/bzlmod/tests/dupe_requirements/dupe_requirements_test.py b/examples/bzlmod/tests/dupe_requirements/dupe_requirements_test.py deleted file mode 100644 index 1139dc5252..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/dupe_requirements_test.py +++ /dev/null @@ -1,4 +0,0 @@ -# There's nothing to test at runtime. Building indicates success. -# Just import the relevant modules as a basic check. -import cryptography -import jwt diff --git a/examples/bzlmod/tests/dupe_requirements/requirements.in b/examples/bzlmod/tests/dupe_requirements/requirements.in deleted file mode 100644 index b1f623395a..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/requirements.in +++ /dev/null @@ -1,2 +0,0 @@ -pyjwt -pyjwt[crypto] diff --git a/examples/bzlmod/tests/dupe_requirements/requirements.txt b/examples/bzlmod/tests/dupe_requirements/requirements.txt deleted file mode 100644 index 785f556624..0000000000 --- a/examples/bzlmod/tests/dupe_requirements/requirements.txt +++ /dev/null @@ -1,97 +0,0 @@ -# -# This file is manually tweaked output from the automatic generation. -# To generate: -# 1. bazel run //tests/dupe_requirements:requirements.update -# 2. Then copy/paste the pyjtw lines so there are duplicates -# -pyjwt==2.8.0 \ - --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ - --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 - # via -r tests/dupe_requirements/requirements.in -pyjwt[crypto]==2.8.0 \ - --hash=sha256:57e28d156e3d5c10088e0c68abb90bfac3df82b40a71bd0daa20c65ccd5c23de \ - --hash=sha256:59127c392cc44c2da5bb3192169a91f429924e17aff6534d70fdc02ab3e04320 - # via -r tests/dupe_requirements/requirements.in -cffi==1.16.0 \ - --hash=sha256:0c9ef6ff37e974b73c25eecc13952c55bceed9112be2d9d938ded8e856138bcc \ - --hash=sha256:131fd094d1065b19540c3d72594260f118b231090295d8c34e19a7bbcf2e860a \ - --hash=sha256:1b8ebc27c014c59692bb2664c7d13ce7a6e9a629be20e54e7271fa696ff2b417 \ - --hash=sha256:2c56b361916f390cd758a57f2e16233eb4f64bcbeee88a4881ea90fca14dc6ab \ - --hash=sha256:2d92b25dbf6cae33f65005baf472d2c245c050b1ce709cc4588cdcdd5495b520 \ - --hash=sha256:31d13b0f99e0836b7ff893d37af07366ebc90b678b6664c955b54561fc36ef36 \ - --hash=sha256:32c68ef735dbe5857c810328cb2481e24722a59a2003018885514d4c09af9743 \ - --hash=sha256:3686dffb02459559c74dd3d81748269ffb0eb027c39a6fc99502de37d501faa8 \ - --hash=sha256:582215a0e9adbe0e379761260553ba11c58943e4bbe9c36430c4ca6ac74b15ed \ - --hash=sha256:5b50bf3f55561dac5438f8e70bfcdfd74543fd60df5fa5f62d94e5867deca684 \ - --hash=sha256:5bf44d66cdf9e893637896c7faa22298baebcd18d1ddb6d2626a6e39793a1d56 \ - --hash=sha256:6602bc8dc6f3a9e02b6c22c4fc1e47aa50f8f8e6d3f78a5e16ac33ef5fefa324 \ - --hash=sha256:673739cb539f8cdaa07d92d02efa93c9ccf87e345b9a0b556e3ecc666718468d \ - --hash=sha256:68678abf380b42ce21a5f2abde8efee05c114c2fdb2e9eef2efdb0257fba1235 \ - --hash=sha256:68e7c44931cc171c54ccb702482e9fc723192e88d25a0e133edd7aff8fcd1f6e \ - --hash=sha256:6b3d6606d369fc1da4fd8c357d026317fbb9c9b75d36dc16e90e84c26854b088 \ - --hash=sha256:748dcd1e3d3d7cd5443ef03ce8685043294ad6bd7c02a38d1bd367cfd968e000 \ - --hash=sha256:7651c50c8c5ef7bdb41108b7b8c5a83013bfaa8a935590c5d74627c047a583c7 \ - --hash=sha256:7b78010e7b97fef4bee1e896df8a4bbb6712b7f05b7ef630f9d1da00f6444d2e \ - --hash=sha256:7e61e3e4fa664a8588aa25c883eab612a188c725755afff6289454d6362b9673 \ - --hash=sha256:80876338e19c951fdfed6198e70bc88f1c9758b94578d5a7c4c91a87af3cf31c \ - --hash=sha256:8895613bcc094d4a1b2dbe179d88d7fb4a15cee43c052e8885783fac397d91fe \ - --hash=sha256:88e2b3c14bdb32e440be531ade29d3c50a1a59cd4e51b1dd8b0865c54ea5d2e2 \ - --hash=sha256:8f8e709127c6c77446a8c0a8c8bf3c8ee706a06cd44b1e827c3e6a2ee6b8c098 \ - --hash=sha256:9cb4a35b3642fc5c005a6755a5d17c6c8b6bcb6981baf81cea8bfbc8903e8ba8 \ - --hash=sha256:9f90389693731ff1f659e55c7d1640e2ec43ff725cc61b04b2f9c6d8d017df6a \ - --hash=sha256:a09582f178759ee8128d9270cd1344154fd473bb77d94ce0aeb2a93ebf0feaf0 \ - --hash=sha256:a6a14b17d7e17fa0d207ac08642c8820f84f25ce17a442fd15e27ea18d67c59b \ - --hash=sha256:a72e8961a86d19bdb45851d8f1f08b041ea37d2bd8d4fd19903bc3083d80c896 \ - --hash=sha256:abd808f9c129ba2beda4cfc53bde801e5bcf9d6e0f22f095e45327c038bfe68e \ - --hash=sha256:ac0f5edd2360eea2f1daa9e26a41db02dd4b0451b48f7c318e217ee092a213e9 \ - --hash=sha256:b29ebffcf550f9da55bec9e02ad430c992a87e5f512cd63388abb76f1036d8d2 \ - --hash=sha256:b2ca4e77f9f47c55c194982e10f058db063937845bb2b7a86c84a6cfe0aefa8b \ - --hash=sha256:b7be2d771cdba2942e13215c4e340bfd76398e9227ad10402a8767ab1865d2e6 \ - --hash=sha256:b84834d0cf97e7d27dd5b7f3aca7b6e9263c56308ab9dc8aae9784abb774d404 \ - --hash=sha256:b86851a328eedc692acf81fb05444bdf1891747c25af7529e39ddafaf68a4f3f \ - --hash=sha256:bcb3ef43e58665bbda2fb198698fcae6776483e0c4a631aa5647806c25e02cc0 \ - --hash=sha256:c0f31130ebc2d37cdd8e44605fb5fa7ad59049298b3f745c74fa74c62fbfcfc4 \ - --hash=sha256:c6a164aa47843fb1b01e941d385aab7215563bb8816d80ff3a363a9f8448a8dc \ - --hash=sha256:d8a9d3ebe49f084ad71f9269834ceccbf398253c9fac910c4fd7053ff1386936 \ - --hash=sha256:db8e577c19c0fda0beb7e0d4e09e0ba74b1e4c092e0e40bfa12fe05b6f6d75ba \ - --hash=sha256:dc9b18bf40cc75f66f40a7379f6a9513244fe33c0e8aa72e2d56b0196a7ef872 \ - --hash=sha256:e09f3ff613345df5e8c3667da1d918f9149bd623cd9070c983c013792a9a62eb \ - --hash=sha256:e4108df7fe9b707191e55f33efbcb2d81928e10cea45527879a4749cbe472614 \ - --hash=sha256:e6024675e67af929088fda399b2094574609396b1decb609c55fa58b028a32a1 \ - --hash=sha256:e70f54f1796669ef691ca07d046cd81a29cb4deb1e5f942003f401c0c4a2695d \ - --hash=sha256:e715596e683d2ce000574bae5d07bd522c781a822866c20495e52520564f0969 \ - --hash=sha256:e760191dd42581e023a68b758769e2da259b5d52e3103c6060ddc02c9edb8d7b \ - --hash=sha256:ed86a35631f7bfbb28e108dd96773b9d5a6ce4811cf6ea468bb6a359b256b1e4 \ - --hash=sha256:ee07e47c12890ef248766a6e55bd38ebfb2bb8edd4142d56db91b21ea68b7627 \ - --hash=sha256:fa3a0128b152627161ce47201262d3140edb5a5c3da88d73a1b790a959126956 \ - --hash=sha256:fcc8eb6d5902bb1cf6dc4f187ee3ea80a1eba0a89aba40a5cb20a5087d961357 - # via cryptography -cryptography==41.0.7 \ - --hash=sha256:079b85658ea2f59c4f43b70f8119a52414cdb7be34da5d019a77bf96d473b960 \ - --hash=sha256:09616eeaef406f99046553b8a40fbf8b1e70795a91885ba4c96a70793de5504a \ - --hash=sha256:13f93ce9bea8016c253b34afc6bd6a75993e5c40672ed5405a9c832f0d4a00bc \ - --hash=sha256:37a138589b12069efb424220bf78eac59ca68b95696fc622b6ccc1c0a197204a \ - --hash=sha256:3c78451b78313fa81607fa1b3f1ae0a5ddd8014c38a02d9db0616133987b9cdf \ - --hash=sha256:43f2552a2378b44869fe8827aa19e69512e3245a219104438692385b0ee119d1 \ - --hash=sha256:48a0476626da912a44cc078f9893f292f0b3e4c739caf289268168d8f4702a39 \ - --hash=sha256:49f0805fc0b2ac8d4882dd52f4a3b935b210935d500b6b805f321addc8177406 \ - --hash=sha256:5429ec739a29df2e29e15d082f1d9ad683701f0ec7709ca479b3ff2708dae65a \ - --hash=sha256:5a1b41bc97f1ad230a41657d9155113c7521953869ae57ac39ac7f1bb471469a \ - --hash=sha256:68a2dec79deebc5d26d617bfdf6e8aab065a4f34934b22d3b5010df3ba36612c \ - --hash=sha256:7a698cb1dac82c35fcf8fe3417a3aaba97de16a01ac914b89a0889d364d2f6be \ - --hash=sha256:841df4caa01008bad253bce2a6f7b47f86dc9f08df4b433c404def869f590a15 \ - --hash=sha256:90452ba79b8788fa380dfb587cca692976ef4e757b194b093d845e8d99f612f2 \ - --hash=sha256:928258ba5d6f8ae644e764d0f996d61a8777559f72dfeb2eea7e2fe0ad6e782d \ - --hash=sha256:af03b32695b24d85a75d40e1ba39ffe7db7ffcb099fe507b39fd41a565f1b157 \ - --hash=sha256:b640981bf64a3e978a56167594a0e97db71c89a479da8e175d8bb5be5178c003 \ - --hash=sha256:c5ca78485a255e03c32b513f8c2bc39fedb7f5c5f8535545bdc223a03b24f248 \ - --hash=sha256:c7f3201ec47d5207841402594f1d7950879ef890c0c495052fa62f58283fde1a \ - --hash=sha256:d5ec85080cce7b0513cfd233914eb8b7bbd0633f1d1703aa28d1dd5a72f678ec \ - --hash=sha256:d6c391c021ab1f7a82da5d8d0b3cee2f4b2c455ec86c8aebbc84837a631ff309 \ - --hash=sha256:e3114da6d7f95d2dee7d3f4eec16dacff819740bbab931aff8648cb13c5ff5e7 \ - --hash=sha256:f983596065a18a2183e7f79ab3fd4c475205b839e02cbc0efbbf9666c4b3083d - # via pyjwt -pycparser==2.21 \ - --hash=sha256:8ee45429555515e1f6b185e78100aea234072576aa43ab53aefcae078162fca9 \ - --hash=sha256:e644fdec12f7872f86c58ff790da456218b10f863970249516d60a5eaca77206 - # via cffi diff --git a/examples/pip_parse/MODULE.bazel b/examples/pip_parse/MODULE.bazel index b0e38f2218..f9ca90833f 100644 --- a/examples/pip_parse/MODULE.bazel +++ b/examples/pip_parse/MODULE.bazel @@ -21,6 +21,7 @@ use_repo( pip = use_extension("@rules_python//python/extensions:pip.bzl", "pip") pip.parse( + download_only = True, experimental_requirement_cycles = { "sphinx": [ "sphinx", diff --git a/examples/pip_parse_vendored/requirements.bzl b/examples/pip_parse_vendored/requirements.bzl index de5d187262..5c2391bd4c 100644 --- a/examples/pip_parse_vendored/requirements.bzl +++ b/examples/pip_parse_vendored/requirements.bzl @@ -1,7 +1,6 @@ """Starlark representation of locked requirements. -@generated by rules_python pip_parse repository rule -from @//:requirements.txt +@generated by rules_python pip_parse repository rule. """ load("@rules_python//python:pip.bzl", "pip_utils") diff --git a/python/pip_install/BUILD.bazel b/python/pip_install/BUILD.bazel index e794075af0..91f2ec7b59 100644 --- a/python/pip_install/BUILD.bazel +++ b/python/pip_install/BUILD.bazel @@ -23,7 +23,6 @@ bzl_library( srcs = ["pip_repository.bzl"], deps = [ ":repositories_bzl", - ":requirements_parser_bzl", "//python:repositories_bzl", "//python:versions_bzl", "//python/pip_install/private:generate_group_library_build_bazel_bzl", @@ -32,6 +31,7 @@ bzl_library( "//python/private:bzlmod_enabled_bzl", "//python/private:envsubst_bzl", "//python/private:normalize_name_bzl", + "//python/private:parse_requirements_bzl", "//python/private:parse_whl_name_bzl", "//python/private:patch_whl_bzl", "//python/private:render_pkg_aliases_bzl", diff --git a/python/pip_install/pip_repository.bzl b/python/pip_install/pip_repository.bzl index db6736836f..17d80838e0 100644 --- a/python/pip_install/pip_repository.bzl +++ b/python/pip_install/pip_repository.bzl @@ -18,13 +18,13 @@ load("@bazel_skylib//lib:sets.bzl", "sets") load("//python:repositories.bzl", "is_standalone_interpreter") load("//python:versions.bzl", "WINDOWS_NAME") load("//python/pip_install:repositories.bzl", "all_requirements") -load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load("//python/pip_install/private:generate_group_library_build_bazel.bzl", "generate_group_library_build_bazel") load("//python/pip_install/private:generate_whl_library_build_bazel.bzl", "generate_whl_library_build_bazel") load("//python/pip_install/private:srcs.bzl", "PIP_INSTALL_PY_SRCS") load("//python/private:auth.bzl", "AUTH_ATTRS", "get_auth") load("//python/private:envsubst.bzl", "envsubst") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load("//python/private:parse_whl_name.bzl", "parse_whl_name") load("//python/private:patch_whl.bzl", "patch_whl") load("//python/private:render_pkg_aliases.bzl", "render_pkg_aliases", "whl_alias") @@ -272,38 +272,30 @@ package(default_visibility = ["//visibility:public"]) exports_files(["requirements.bzl"]) """ -def locked_requirements_label(ctx, attr): - """Get the preferred label for a locked requirements file based on platform. - - Args: - ctx: repository or module context - attr: attributes for the repo rule or tag extension - - Returns: - Label - """ - os = ctx.os.name.lower() - requirements_txt = attr.requirements_lock - if os.startswith("mac os") and attr.requirements_darwin != None: - requirements_txt = attr.requirements_darwin - elif os.startswith("linux") and attr.requirements_linux != None: - requirements_txt = attr.requirements_linux - elif "win" in os and attr.requirements_windows != None: - requirements_txt = attr.requirements_windows - if not requirements_txt: - fail("""\ -A requirements_lock attribute must be specified, or a platform-specific lockfile using one of the requirements_* attributes. -""") - return requirements_txt - def _pip_repository_impl(rctx): - requirements_txt = locked_requirements_label(rctx, rctx.attr) - content = rctx.read(requirements_txt) - parsed_requirements_txt = parse_requirements(content) - - packages = [(normalize_name(name), requirement) for name, requirement in parsed_requirements_txt.requirements] + requirements_by_platform = parse_requirements( + rctx, + requirements_by_platform = rctx.attr.requirements_by_platform, + requirements_linux = rctx.attr.requirements_linux, + requirements_lock = rctx.attr.requirements_lock, + requirements_osx = rctx.attr.requirements_darwin, + requirements_windows = rctx.attr.requirements_windows, + extra_pip_args = rctx.attr.extra_pip_args, + ) + selected_requirements = {} + options = None + repository_platform = host_platform(rctx.os) + for name, requirements in requirements_by_platform.items(): + r = select_requirement( + requirements, + platform = repository_platform, + ) + if not r: + continue + options = options or r.extra_pip_args + selected_requirements[name] = r.requirement_line - bzl_packages = sorted([normalize_name(name) for name, _ in parsed_requirements_txt.requirements]) + bzl_packages = sorted(selected_requirements.keys()) # Normalize cycles first requirement_cycles = { @@ -347,13 +339,6 @@ def _pip_repository_impl(rctx): rctx.file(filename, json.encode_indent(json.decode(annotation))) annotations[pkg] = "@{name}//:{filename}".format(name = rctx.attr.name, filename = filename) - tokenized_options = [] - for opt in parsed_requirements_txt.options: - for p in opt.split(" "): - tokenized_options.append(p) - - options = tokenized_options + rctx.attr.extra_pip_args - config = { "download_only": rctx.attr.download_only, "enable_implicit_namespace_pkgs": rctx.attr.enable_implicit_namespace_pkgs, @@ -419,10 +404,9 @@ def _pip_repository_impl(rctx): "%%PACKAGES%%": _format_repr_list( [ ("{}_{}".format(rctx.attr.name, p), r) - for p, r in packages + for p, r in sorted(selected_requirements.items()) ], ), - "%%REQUIREMENTS_LOCK%%": str(requirements_txt), }) return @@ -625,6 +609,15 @@ pip_repository_attrs = { "annotations": attr.string_dict( doc = "Optional annotations to apply to packages", ), + "requirements_by_platform": attr.label_keyed_string_dict( + doc = """\ +The requirements files and the comma delimited list of target platforms as values. + +The keys are the requirement files and the values are comma-separated platform +identifiers. For now we only support `_` values that are present in +`@platforms//os` and `@platforms//cpu` packages respectively. +""", + ), "requirements_darwin": attr.label( allow_single_file = True, doc = "Override the requirements_lock attribute when the host platform is Mac OS", @@ -643,6 +636,11 @@ individual repositories for each of your dependencies so that wheels are fetched/built only for the targets specified by 'build/run/test'. Note that if your lockfile is platform-dependent, you can use the `requirements_[platform]` attributes. + +Note, that in general requirements files are compiled for a specific platform, +but sometimes they can work for multiple platforms. `rules_python` right now +supports requirements files that are created for a particular platform without +platform markers. """, ), "requirements_windows": attr.label( diff --git a/python/pip_install/pip_repository_requirements.bzl.tmpl b/python/pip_install/pip_repository_requirements.bzl.tmpl index 8e17720374..07b4b08148 100644 --- a/python/pip_install/pip_repository_requirements.bzl.tmpl +++ b/python/pip_install/pip_repository_requirements.bzl.tmpl @@ -1,7 +1,6 @@ """Starlark representation of locked requirements. -@generated by rules_python pip_parse repository rule -from %%REQUIREMENTS_LOCK%% +@generated by rules_python pip_parse repository rule. """ %%IMPORTS%% diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 181175679a..45f50effb0 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -128,6 +128,17 @@ bzl_library( deps = [":parse_whl_name_bzl"], ) +bzl_library( + name = "parse_requirements_bzl", + srcs = ["parse_requirements.bzl"], + deps = [ + ":normalize_name_bzl", + ":pypi_index_sources_bzl", + ":whl_target_platforms_bzl", + "//python/pip_install:requirements_parser_bzl", + ], +) + bzl_library( name = "parse_whl_name_bzl", srcs = ["parse_whl_name.bzl"], @@ -145,6 +156,11 @@ bzl_library( ], ) +bzl_library( + name = "pypi_index_sources_bzl", + srcs = ["pypi_index_sources.bzl"], +) + bzl_library( name = "py_cc_toolchain_bzl", srcs = [ diff --git a/python/private/bzlmod/BUILD.bazel b/python/private/bzlmod/BUILD.bazel index 9edd3380bb..2eab575726 100644 --- a/python/private/bzlmod/BUILD.bazel +++ b/python/private/bzlmod/BUILD.bazel @@ -31,10 +31,10 @@ bzl_library( deps = [ ":pip_repository_bzl", "//python/pip_install:pip_repository_bzl", - "//python/pip_install:requirements_parser_bzl", "//python/private:pypi_index_bzl", "//python/private:full_version_bzl", "//python/private:normalize_name_bzl", + "//python/private:parse_requirements_bzl", "//python/private:parse_whl_name_bzl", "//python/private:version_label_bzl", ":bazel_features_bzl", diff --git a/python/private/bzlmod/pip.bzl b/python/private/bzlmod/pip.bzl index ce681259ed..80ee852573 100644 --- a/python/private/bzlmod/pip.bzl +++ b/python/private/bzlmod/pip.bzl @@ -18,16 +18,15 @@ load("@bazel_features//:features.bzl", "bazel_features") load("@pythons_hub//:interpreters.bzl", "DEFAULT_PYTHON_VERSION", "INTERPRETER_LABELS") load( "//python/pip_install:pip_repository.bzl", - "locked_requirements_label", "pip_repository_attrs", "use_isolated", "whl_library", ) -load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load("//python/private:auth.bzl", "AUTH_ATTRS") load("//python/private:normalize_name.bzl", "normalize_name") +load("//python/private:parse_requirements.bzl", "host_platform", "parse_requirements", "select_requirement") load("//python/private:parse_whl_name.bzl", "parse_whl_name") -load("//python/private:pypi_index.bzl", "get_simpleapi_sources", "simpleapi_download") +load("//python/private:pypi_index.bzl", "simpleapi_download") load("//python/private:render_pkg_aliases.bzl", "whl_alias") load("//python/private:version_label.bzl", "version_label") load("//python/private:whl_target_platforms.bzl", "select_whl") @@ -130,27 +129,6 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s ) major_minor = _major_minor_version(pip_attr.python_version) - requirements_lock = locked_requirements_label(module_ctx, pip_attr) - - # Parse the requirements file directly in starlark to get the information - # needed for the whl_libary declarations below. - requirements_lock_content = module_ctx.read(requirements_lock) - parse_result = parse_requirements(requirements_lock_content) - - # Replicate a surprising behavior that WORKSPACE builds allowed: - # Defining a repo with the same name multiple times, but only the last - # definition is respected. - # The requirement lines might have duplicate names because lines for extras - # are returned as just the base package name. e.g., `foo[bar]` results - # in an entry like `("foo", "foo[bar] == 1.0 ...")`. - requirements = { - normalize_name(entry[0]): entry - # The WORKSPACE pip_parse sorted entries, so mimic that ordering. - for entry in sorted(parse_result.requirements) - }.values() - - extra_pip_args = pip_attr.extra_pip_args + parse_result.options - if hub_name not in whl_map: whl_map[hub_name] = {} @@ -180,6 +158,18 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s whl_group_mapping = {} requirement_cycles = {} + # Create a new wheel library for each of the different whls + + requirements_by_platform = parse_requirements( + module_ctx, + requirements_by_platform = pip_attr.requirements_by_platform, + requirements_linux = pip_attr.requirements_linux, + requirements_lock = pip_attr.requirements_lock, + requirements_osx = pip_attr.requirements_darwin, + requirements_windows = pip_attr.requirements_windows, + extra_pip_args = pip_attr.extra_pip_args, + ) + index_urls = {} if pip_attr.experimental_index_url: if pip_attr.download_only: @@ -191,7 +181,11 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s index_url = pip_attr.experimental_index_url, extra_index_urls = pip_attr.experimental_extra_index_urls or [], index_url_overrides = pip_attr.experimental_index_url_overrides or {}, - sources = [requirements_lock_content], + sources = list({ + req.distribution: None + for reqs in requirements_by_platform.values() + for req in reqs + }), envsubst = pip_attr.envsubst, # Auth related info netrc = pip_attr.netrc, @@ -201,8 +195,21 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s parallel_download = pip_attr.parallel_download, ) - # Create a new wheel library for each of the different whls - for whl_name, requirement_line in requirements: + repository_platform = host_platform(module_ctx.os) + for whl_name, requirements in requirements_by_platform.items(): + requirement = select_requirement( + requirements, + platform = repository_platform, + ) + if not requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + continue + # We are not using the "sanitized name" because the user # would need to guess what name we modified the whl name # to. @@ -218,7 +225,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s whl_library_args = dict( repo = pip_name, dep_template = "@{}//{{name}}:{{target}}".format(hub_name), - requirement = requirement_line, + requirement = requirement.requirement_line, ) maybe_args = dict( # The following values are safe to omit if they have false like values @@ -228,7 +235,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s environment = pip_attr.environment, envsubst = pip_attr.envsubst, experimental_target_platforms = pip_attr.experimental_target_platforms, - extra_pip_args = extra_pip_args, + extra_pip_args = requirement.extra_pip_args, group_deps = group_deps, group_name = group_name, pip_data_exclude = pip_attr.pip_data_exclude, @@ -249,11 +256,9 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s whl_library_args.update({k: v for k, (v, default) in maybe_args_with_default.items() if v == default}) if index_urls: - srcs = get_simpleapi_sources(requirement_line) - whls = [] sdist = None - for sha256 in srcs.shas: + for sha256 in requirement.srcs.shas: # For now if the artifact is marked as yanked we just ignore it. # # See https://packaging.python.org/en/latest/specifications/simple-repository-api/#adding-yank-support-to-the-simple-api @@ -279,12 +284,11 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # Older python versions have wheels for the `*m` ABI. "cp" + major_minor.replace(".", "") + "m", ], - want_os = module_ctx.os.name, - want_cpu = module_ctx.os.arch, + want_platform = repository_platform, ) or sdist if distribution: - whl_library_args["requirement"] = srcs.requirement + whl_library_args["requirement"] = requirement.srcs.requirement whl_library_args["urls"] = [distribution.url] whl_library_args["sha256"] = distribution.sha256 whl_library_args["filename"] = distribution.filename @@ -299,7 +303,7 @@ def _create_whl_repos(module_ctx, pip_attr, whl_map, whl_overrides, group_map, s # This is no-op because pip is not used to download the wheel. whl_library_args.pop("download_only", None) else: - print("WARNING: falling back to pip for installing the right file for {}".format(requirement_line)) # buildifier: disable=print + print("WARNING: falling back to pip for installing the right file for {}".format(requirement.requirement_line)) # buildifier: disable=print # We sort so that the lock-file remains the same no matter the order of how the # args are manipulated in the code going before. diff --git a/python/private/normalize_platform.bzl b/python/private/normalize_platform.bzl new file mode 100644 index 0000000000..633062f399 --- /dev/null +++ b/python/private/normalize_platform.bzl @@ -0,0 +1,13 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/python/private/parse_requirements.bzl b/python/private/parse_requirements.bzl new file mode 100644 index 0000000000..f9d7a05386 --- /dev/null +++ b/python/private/parse_requirements.bzl @@ -0,0 +1,374 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Requirements parsing for whl_library creation. + +Use cases that the code needs to cover: +* A single requirements_lock file that is used for the host platform. +* Per-OS requirements_lock files that are used for the host platform. +* A target platform specific requirements_lock that is used with extra + pip arguments with --platform, etc and download_only = True. + +In the last case only a single `requirements_lock` file is allowed, in all +other cases we assume that there may be a desire to resolve the requirements +file for the host platform to be backwards compatible with the legacy +behavior. +""" + +load("//python/pip_install:requirements_parser.bzl", "parse") +load(":normalize_name.bzl", "normalize_name") +load(":pypi_index_sources.bzl", "get_simpleapi_sources") +load(":whl_target_platforms.bzl", "whl_target_platforms") + +# This includes the vendored _translate_cpu and _translate_os from +# @platforms//host:extension.bzl at version 0.0.9 so that we don't +# force the users to depend on it. + +def _translate_cpu(arch): + if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]: + return "x86_32" + if arch in ["amd64", "x86_64", "x64"]: + return "x86_64" + if arch in ["ppc", "ppc64", "ppc64le"]: + return "ppc" + if arch in ["arm", "armv7l"]: + return "arm" + if arch in ["aarch64"]: + return "aarch64" + if arch in ["s390x", "s390"]: + return "s390x" + if arch in ["mips64el", "mips64"]: + return "mips64" + if arch in ["riscv64"]: + return "riscv64" + return arch + +def _translate_os(os): + if os.startswith("mac os"): + return "osx" + if os.startswith("freebsd"): + return "freebsd" + if os.startswith("openbsd"): + return "openbsd" + if os.startswith("linux"): + return "linux" + if os.startswith("windows"): + return "windows" + return os + +# TODO @aignas 2024-05-13: consider using the same platform tags as are used in +# the //python:versions.bzl +DEFAULT_PLATFORMS = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", +] + +def _default_platforms(*, filter): + if not filter: + fail("Must specific a filter string, got: {}".format(filter)) + + sanitized = filter.replace("*", "").replace("_", "") + if sanitized and not sanitized.isalnum(): + fail("The platform filter can only contain '*', '_' and alphanumerics") + + if "*" in filter: + prefix = filter.rstrip("*") + if "*" in prefix: + fail("The filter can only contain '*' at the end of it") + + if not prefix: + return DEFAULT_PLATFORMS + + return [p for p in DEFAULT_PLATFORMS if p.startswith(prefix)] + else: + return [p for p in DEFAULT_PLATFORMS if filter in p] + +def _platforms_from_args(extra_pip_args): + platform_values = [] + + for arg in extra_pip_args: + if platform_values and platform_values[-1] == "": + platform_values[-1] = arg + continue + + if arg == "--platform": + platform_values.append("") + continue + + if not arg.startswith("--platform"): + continue + + _, _, plat = arg.partition("=") + if not plat: + _, _, plat = arg.partition(" ") + if plat: + platform_values.append(plat) + else: + platform_values.append("") + + if not platform_values: + return [] + + platforms = { + p.target_platform: None + for arg in platform_values + for p in whl_target_platforms(arg) + } + return list(platforms.keys()) + +def parse_requirements( + ctx, + *, + requirements_by_platform = {}, + requirements_osx = None, + requirements_linux = None, + requirements_lock = None, + requirements_windows = None, + extra_pip_args = [], + fail_fn = fail): + """Get the requirements with platforms that the requirements apply to. + + Args: + ctx: A context that has .read function that would read contents from a label. + requirements_by_platform (label_keyed_string_dict): a way to have + different package versions (or different packages) for different + os, arch combinations. + requirements_osx (label): The requirements file for the osx OS. + requirements_linux (label): The requirements file for the linux OS. + requirements_lock (label): The requirements file for all OSes, or used as a fallback. + requirements_windows (label): The requirements file for windows OS. + extra_pip_args (string list): Extra pip arguments to perform extra validations and to + be joined with args fined in files. + fail_fn (Callable[[str], None]): A failure function used in testing failure cases. + + Returns: + A tuple where the first element a dict of dicts where the first key is + the normalized distribution name (with underscores) and the second key + is the requirement_line, then value and the keys are structs with the + following attributes: + * distribution: The non-normalized distribution name. + * srcs: The Simple API downloadable source list. + * requirement_line: The original requirement line. + * target_platforms: The list of target platforms that this package is for. + + The second element is extra_pip_args should be passed to `whl_library`. + """ + if not ( + requirements_lock or + requirements_linux or + requirements_osx or + requirements_windows or + requirements_by_platform + ): + fail_fn( + "A 'requirements_lock' attribute must be specified, a platform-specific lockfiles " + + "via 'requirements_by_platform' or an os-specific lockfiles must be specified " + + "via 'requirements_*' attributes", + ) + return None + + platforms = _platforms_from_args(extra_pip_args) + + if platforms: + lock_files = [ + f + for f in [ + requirements_lock, + requirements_linux, + requirements_osx, + requirements_windows, + ] + list(requirements_by_platform.keys()) + if f + ] + + if len(lock_files) > 1: + # If the --platform argument is used, check that we are using + # a single `requirements_lock` file instead of the OS specific ones as that is + # the only correct way to use the API. + fail_fn("only a single 'requirements_lock' file can be used when using '--platform' pip argument, consider specifying it via 'requirements_lock' attribute") + return None + + files_by_platform = [ + (lock_files[0], platforms), + ] + else: + files_by_platform = { + file: [ + platform + for filter_or_platform in specifier.split(",") + for platform in (_default_platforms(filter = filter_or_platform) if filter_or_platform.endswith("*") else [filter_or_platform]) + ] + for file, specifier in requirements_by_platform.items() + }.items() + + for f in [ + # If the users need a greater span of the platforms, they should consider + # using the 'requirements_by_platform' attribute. + (requirements_linux, _default_platforms(filter = "linux_*")), + (requirements_osx, _default_platforms(filter = "osx_*")), + (requirements_windows, _default_platforms(filter = "windows_*")), + (requirements_lock, None), + ]: + if f[0]: + files_by_platform.append(f) + + configured_platforms = {} + + options = {} + requirements = {} + for file, plats in files_by_platform: + if plats: + for p in plats: + if p in configured_platforms: + fail_fn( + "Expected the platform '{}' to be map only to a single requirements file, but got multiple: '{}', '{}'".format( + p, + configured_platforms[p], + file, + ), + ) + return None + configured_platforms[p] = file + else: + plats = [ + p + for p in DEFAULT_PLATFORMS + if p not in configured_platforms + ] + + contents = ctx.read(file) + + # Parse the requirements file directly in starlark to get the information + # needed for the whl_libary declarations later. + parse_result = parse(contents) + + # Replicate a surprising behavior that WORKSPACE builds allowed: + # Defining a repo with the same name multiple times, but only the last + # definition is respected. + # The requirement lines might have duplicate names because lines for extras + # are returned as just the base package name. e.g., `foo[bar]` results + # in an entry like `("foo", "foo[bar] == 1.0 ...")`. + requirements_dict = { + normalize_name(entry[0]): entry + for entry in sorted( + parse_result.requirements, + # Get the longest match and fallback to original WORKSPACE sorting, + # which should get us the entry with most extras. + # + # FIXME @aignas 2024-05-13: The correct behaviour might be to get an + # entry with all aggregated extras, but it is unclear if we + # should do this now. + key = lambda x: (len(x[1].partition("==")[0]), x), + ) + }.values() + + tokenized_options = [] + for opt in parse_result.options: + for p in opt.split(" "): + tokenized_options.append(p) + + pip_args = tokenized_options + extra_pip_args + for p in plats: + requirements[p] = requirements_dict + options[p] = pip_args + + requirements_by_platform = {} + for target_platform, reqs_ in requirements.items(): + extra_pip_args = options[target_platform] + + for distribution, requirement_line in reqs_: + for_whl = requirements_by_platform.setdefault( + normalize_name(distribution), + {}, + ) + + for_req = for_whl.setdefault( + (requirement_line, ",".join(extra_pip_args)), + struct( + distribution = distribution, + srcs = get_simpleapi_sources(requirement_line), + requirement_line = requirement_line, + target_platforms = [], + extra_pip_args = extra_pip_args, + download = len(platforms) > 0, + ), + ) + for_req.target_platforms.append(target_platform) + + return { + whl_name: [ + struct( + distribution = r.distribution, + srcs = r.srcs, + requirement_line = r.requirement_line, + target_platforms = sorted(r.target_platforms), + extra_pip_args = r.extra_pip_args, + download = r.download, + ) + for r in sorted(reqs.values(), key = lambda r: r.requirement_line) + ] + for whl_name, reqs in requirements_by_platform.items() + } + +def select_requirement(requirements, *, platform): + """A simple function to get a requirement for a particular platform. + + Args: + requirements (list[struct]): The list of requirements as returned by + the `parse_requirements` function above. + platform (str): The host platform. Usually an output of the + `host_platform` function. + + Returns: + None if not found or a struct returned as one of the values in the + parse_requirements function. The requirement that should be downloaded + by the host platform will be returned. + """ + maybe_requirement = [ + req + for req in requirements + if platform in req.target_platforms or req.download + ] + if not maybe_requirement: + # Sometimes the package is not present for host platform if there + # are whls specified only in particular requirements files, in that + # case just continue, however, if the download_only flag is set up, + # then the user can also specify the target platform of the wheel + # packages they want to download, in that case there will be always + # a requirement here, so we will not be in this code branch. + return None + + return maybe_requirement[0] + +def host_platform(repository_os): + """Return a string representation of the repository OS. + + Args: + repository_os (struct): The `module_ctx.os` or `repository_ctx.os` attribute. + See https://bazel.build/rules/lib/builtins/repository_os.html + + Returns: + The string representation of the platform that we can later used in the `pip` + machinery. + """ + return "{}_{}".format( + _translate_os(repository_os.name.lower()), + _translate_cpu(repository_os.arch.lower()), + ) diff --git a/python/private/pypi_index.bzl b/python/private/pypi_index.bzl index 28f1007b48..64d908e32b 100644 --- a/python/private/pypi_index.bzl +++ b/python/private/pypi_index.bzl @@ -17,8 +17,6 @@ A file that houses private functions used in the `bzlmod` extension with the sam """ load("@bazel_features//:features.bzl", "bazel_features") -load("@bazel_skylib//lib:sets.bzl", "sets") -load("//python/pip_install:requirements_parser.bzl", parse_requirements = "parse") load(":auth.bzl", "get_auth") load(":envsubst.bzl", "envsubst") load(":normalize_name.bzl", "normalize_name") @@ -68,7 +66,7 @@ def simpleapi_download(ctx, *, attr, cache, parallel_download = True): async_downloads = {} contents = {} index_urls = [attr.index_url] + attr.extra_index_urls - for pkg in get_packages_from_requirements(attr.sources): + for pkg in attr.sources: pkg_normalized = normalize_name(pkg) success = False @@ -204,62 +202,6 @@ def _read_index_result(ctx, result, output, url, cache, cache_key): else: return struct(success = False) -def get_packages_from_requirements(requirements_files): - """Get Simple API sources from a list of requirements files and merge them. - - Args: - requirements_files(list[str]): A list of requirements files contents. - - Returns: - A list. - """ - want_packages = sets.make() - for contents in requirements_files: - parse_result = parse_requirements(contents) - for distribution, _ in parse_result.requirements: - # NOTE: we'll be querying the PyPI servers multiple times if the - # requirements contains non-normalized names, but this is what user - # is specifying to us. - sets.insert(want_packages, distribution) - - return sets.to_list(want_packages) - -def get_simpleapi_sources(line): - """Get PyPI sources from a requirements.txt line. - - We interpret the spec described in - https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers - - Args: - line(str): The requirements.txt entry. - - Returns: - A struct with shas attribute containing a list of shas to download from pypi_index. - """ - head, _, maybe_hashes = line.partition(";") - _, _, version = head.partition("==") - version = version.partition(" ")[0].strip() - - if "@" in head: - shas = [] - else: - maybe_hashes = maybe_hashes or line - shas = [ - sha.strip() - for sha in maybe_hashes.split("--hash=sha256:")[1:] - ] - - if head == line: - head = line.partition("--hash=")[0].strip() - else: - head = head + ";" + maybe_hashes.partition("--hash=")[0].strip() - - return struct( - requirement = line if not shas else head, - version = version, - shas = sorted(shas), - ) - def parse_simple_api_html(*, url, content): """Get the package URLs for given shas by parsing the Simple API HTML. diff --git a/python/private/pypi_index_sources.bzl b/python/private/pypi_index_sources.bzl new file mode 100644 index 0000000000..470a8c9f5a --- /dev/null +++ b/python/private/pypi_index_sources.bzl @@ -0,0 +1,53 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +A file that houses private functions used in the `bzlmod` extension with the same name. +""" + +def get_simpleapi_sources(line): + """Get PyPI sources from a requirements.txt line. + + We interpret the spec described in + https://pip.pypa.io/en/stable/reference/requirement-specifiers/#requirement-specifiers + + Args: + line(str): The requirements.txt entry. + + Returns: + A struct with shas attribute containing a list of shas to download from pypi_index. + """ + head, _, maybe_hashes = line.partition(";") + _, _, version = head.partition("==") + version = version.partition(" ")[0].strip() + + if "@" in head: + shas = [] + else: + maybe_hashes = maybe_hashes or line + shas = [ + sha.strip() + for sha in maybe_hashes.split("--hash=sha256:")[1:] + ] + + if head == line: + head = line.partition("--hash=")[0].strip() + else: + head = head + ";" + maybe_hashes.partition("--hash=")[0].strip() + + return struct( + requirement = line if not shas else head, + version = version, + shas = sorted(shas), + ) diff --git a/python/private/whl_target_platforms.bzl b/python/private/whl_target_platforms.bzl index 4e17f2b4c7..14e178a66b 100644 --- a/python/private/whl_target_platforms.bzl +++ b/python/private/whl_target_platforms.bzl @@ -33,39 +33,6 @@ _LEGACY_ALIASES = { "manylinux2014_x86_64": "manylinux_2_17_x86_64", } -# _translate_cpu and _translate_os from @platforms//host:extension.bzl -def _translate_cpu(arch): - if arch in ["i386", "i486", "i586", "i686", "i786", "x86"]: - return "x86_32" - if arch in ["amd64", "x86_64", "x64"]: - return "x86_64" - if arch in ["ppc", "ppc64", "ppc64le"]: - return "ppc" - if arch in ["arm", "armv7l"]: - return "arm" - if arch in ["aarch64"]: - return "aarch64" - if arch in ["s390x", "s390"]: - return "s390x" - if arch in ["mips64el", "mips64"]: - return "mips64" - if arch in ["riscv64"]: - return "riscv64" - return None - -def _translate_os(os): - if os.startswith("mac os"): - return "osx" - if os.startswith("freebsd"): - return "freebsd" - if os.startswith("openbsd"): - return "openbsd" - if os.startswith("linux"): - return "linux" - if os.startswith("windows"): - return "windows" - return None - # The order of the dictionaries is to keep definitions with their aliases next to each # other _CPU_ALIASES = { @@ -151,14 +118,13 @@ def _whl_priority(value): # Windows does not have multiple wheels for the same target platform return (False, False, 0, 0) -def select_whl(*, whls, want_abis, want_os, want_cpu): +def select_whl(*, whls, want_abis, want_platform): """Select a suitable wheel from a list. Args: whls(list[struct]): A list of candidates. want_abis(list[str]): A list of ABIs that are supported. - want_os(str): The module_ctx.os.name. - want_cpu(str): The module_ctx.os.arch. + want_platform(str): The target platform. Returns: None or a struct with `url`, `sha256` and `filename` attributes for the @@ -209,10 +175,7 @@ def select_whl(*, whls, want_abis, want_os, want_cpu): target_plats[p] = sorted(platform_tags, key = _whl_priority) - want = target_plats.get("{}_{}".format( - _translate_os(want_os), - _translate_cpu(want_cpu), - )) + want = target_plats.get(want_platform) if not want: return want diff --git a/tests/private/parse_requirements/BUILD.bazel b/tests/private/parse_requirements/BUILD.bazel new file mode 100644 index 0000000000..3d7976e406 --- /dev/null +++ b/tests/private/parse_requirements/BUILD.bazel @@ -0,0 +1,3 @@ +load(":parse_requirements_tests.bzl", "parse_requirements_test_suite") + +parse_requirements_test_suite(name = "parse_requirements_tests") diff --git a/tests/private/parse_requirements/parse_requirements_tests.bzl b/tests/private/parse_requirements/parse_requirements_tests.bzl new file mode 100644 index 0000000000..0d6cd4e0e0 --- /dev/null +++ b/tests/private/parse_requirements/parse_requirements_tests.bzl @@ -0,0 +1,374 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:parse_requirements.bzl", "parse_requirements", "select_requirement") # buildifier: disable=bzl-visibility + +def _mock_ctx(): + testdata = { + "requirements_direct": """\ +foo[extra] @ https://some-url +""", + "requirements_linux": """\ +foo==0.0.3 --hash=sha256:deadbaaf +""", + "requirements_lock": """\ +foo[extra]==0.0.1 --hash=sha256:deadbeef +""", + "requirements_lock_dupe": """\ +foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef +foo==0.0.1 --hash=sha256:deadbeef +foo[extra]==0.0.1 --hash=sha256:deadbeef +""", + "requirements_osx": """\ +foo==0.0.3 --hash=sha256:deadbaaf +""", + "requirements_windows": """\ +foo[extra]==0.0.2 --hash=sha256:deadbeef +bar==0.0.1 --hash=sha256:deadb00f +""", + } + + return struct( + os = struct( + name = "linux", + arch = "x86_64", + ), + read = lambda x: testdata[x], + ) + +_tests = [] + +def _test_fail_no_requirements(env): + errors = [] + parse_requirements( + ctx = _mock_ctx(), + fail_fn = errors.append, + ) + env.expect.that_str(errors[0]).equals("""\ +A 'requirements_lock' attribute must be specified, a platform-specific lockfiles via 'requirements_by_platform' or an os-specific lockfiles must be specified via 'requirements_*' attributes""") + +_tests.append(_test_fail_no_requirements) + +def _test_simple(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_lock = "requirements_lock", + ) + got_alternative = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_lock": "*", + }, + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra]==0.0.1", + shas = ["deadbeef"], + version = "0.0.1", + ), + target_platforms = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + ), + ], + }) + env.expect.that_dict(got).contains_exactly(got_alternative) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "linux_ppc", + ).srcs.version, + ).equals("0.0.1") + +_tests.append(_test_simple) + +def _test_dupe_requirements(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_lock = "requirements_lock_dupe", + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra,extra_2]==0.0.1 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra,extra_2]==0.0.1", + shas = ["deadbeef"], + version = "0.0.1", + ), + target_platforms = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + ), + ], + }) + +_tests.append(_test_dupe_requirements) + +def _test_multi_os(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_linux = "requirements_linux", + requirements_osx = "requirements_osx", + requirements_windows = "requirements_windows", + ) + + # This is an alternative way to express the same intent + got_alternative = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_linux": "linux_*", + "requirements_osx": "osx_*", + "requirements_windows": "windows_*", + }, + ) + + env.expect.that_dict(got).contains_exactly({ + "bar": [ + struct( + distribution = "bar", + download = False, + extra_pip_args = [], + requirement_line = "bar==0.0.1 --hash=sha256:deadb00f", + srcs = struct( + requirement = "bar==0.0.1", + shas = ["deadb00f"], + version = "0.0.1", + ), + target_platforms = ["windows_x86_64"], + ), + ], + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + srcs = struct( + requirement = "foo==0.0.3", + shas = ["deadbaaf"], + version = "0.0.3", + ), + target_platforms = [ + "linux_aarch64", + "linux_arm", + "linux_ppc", + "linux_s390x", + "linux_x86_64", + "osx_aarch64", + "osx_x86_64", + ], + ), + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.2 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra]==0.0.2", + shas = ["deadbeef"], + version = "0.0.2", + ), + target_platforms = ["windows_x86_64"], + ), + ], + }) + env.expect.that_dict(got).contains_exactly(got_alternative) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "windows_x86_64", + ).srcs.version, + ).equals("0.0.2") + +_tests.append(_test_multi_os) + +def _test_fail_duplicate_platforms(env): + errors = [] + parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_linux": "linux_x86_64", + "requirements_lock": "*", + }, + fail_fn = errors.append, + ) + env.expect.that_collection(errors).has_size(1) + env.expect.that_str(",".join(errors)).equals("Expected the platform 'linux_x86_64' to be map only to a single requirements file, but got multiple: 'requirements_linux', 'requirements_lock'") + +_tests.append(_test_fail_duplicate_platforms) + +def _test_multi_os_download_only_platform(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_lock = "requirements_linux", + extra_pip_args = [ + "--platform", + "manylinux_2_27_x86_64", + "--platform=manylinux_2_12_x86_64", + "--platform manylinux_2_5_x86_64", + ], + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = True, + extra_pip_args = [ + "--platform", + "manylinux_2_27_x86_64", + "--platform=manylinux_2_12_x86_64", + "--platform manylinux_2_5_x86_64", + ], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + srcs = struct( + requirement = "foo==0.0.3", + shas = ["deadbaaf"], + version = "0.0.3", + ), + target_platforms = ["linux_x86_64"], + ), + ], + }) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "windows_x86_64", + ).srcs.version, + ).equals("0.0.3") + +_tests.append(_test_multi_os_download_only_platform) + +def _test_fail_download_only_bad_attr(env): + errors = [] + parse_requirements( + ctx = _mock_ctx(), + requirements_linux = "requirements_linux", + requirements_osx = "requirements_osx", + extra_pip_args = [ + "--platform", + "manylinux_2_27_x86_64", + "--platform=manylinux_2_12_x86_64", + "--platform manylinux_2_5_x86_64", + ], + fail_fn = errors.append, + ) + env.expect.that_str(errors[0]).equals("only a single 'requirements_lock' file can be used when using '--platform' pip argument, consider specifying it via 'requirements_lock' attribute") + +_tests.append(_test_fail_download_only_bad_attr) + +def _test_os_arch_requirements_with_default(env): + got = parse_requirements( + ctx = _mock_ctx(), + requirements_by_platform = { + "requirements_direct": "linux_super_exotic", + "requirements_linux": "linux_x86_64,linux_aarch64", + }, + requirements_lock = "requirements_lock", + ) + env.expect.that_dict(got).contains_exactly({ + "foo": [ + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo==0.0.3 --hash=sha256:deadbaaf", + srcs = struct( + requirement = "foo==0.0.3", + shas = ["deadbaaf"], + version = "0.0.3", + ), + target_platforms = ["linux_aarch64", "linux_x86_64"], + ), + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra] @ https://some-url", + srcs = struct( + requirement = "foo[extra] @ https://some-url", + shas = [], + version = "", + ), + target_platforms = ["linux_super_exotic"], + ), + struct( + distribution = "foo", + download = False, + extra_pip_args = [], + requirement_line = "foo[extra]==0.0.1 --hash=sha256:deadbeef", + srcs = struct( + requirement = "foo[extra]==0.0.1", + shas = ["deadbeef"], + version = "0.0.1", + ), + target_platforms = [ + "linux_arm", + "linux_ppc", + "linux_s390x", + "osx_aarch64", + "osx_x86_64", + "windows_x86_64", + ], + ), + ], + }) + env.expect.that_str( + select_requirement( + got["foo"], + platform = "windows_x86_64", + ).srcs.version, + ).equals("0.0.1") + env.expect.that_str( + select_requirement( + got["foo"], + platform = "linux_x86_64", + ).srcs.version, + ).equals("0.0.3") + +_tests.append(_test_os_arch_requirements_with_default) + +def parse_requirements_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/private/pypi_index/pypi_index_tests.bzl b/tests/private/pypi_index/pypi_index_tests.bzl index e2122b5eeb..fa381065b1 100644 --- a/tests/private/pypi_index/pypi_index_tests.bzl +++ b/tests/private/pypi_index/pypi_index_tests.bzl @@ -16,42 +16,10 @@ load("@rules_testing//lib:test_suite.bzl", "test_suite") load("@rules_testing//lib:truth.bzl", "subjects") -load("//python/private:pypi_index.bzl", "get_simpleapi_sources", "parse_simple_api_html") # buildifier: disable=bzl-visibility +load("//python/private:pypi_index.bzl", "parse_simple_api_html") # buildifier: disable=bzl-visibility _tests = [] -def _test_no_simple_api_sources(env): - inputs = [ - "foo==0.0.1", - "foo==0.0.1 @ https://someurl.org", - "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", - "foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef", - ] - for input in inputs: - got = get_simpleapi_sources(input) - env.expect.that_collection(got.shas).contains_exactly([]) - env.expect.that_str(got.version).equals("0.0.1") - -_tests.append(_test_no_simple_api_sources) - -def _test_simple_api_sources(env): - tests = { - "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef": [ - "deadbeef", - "deafbeef", - ], - "foo[extra]==0.0.2; (python_version < 2.7 or something_else == \"@\") --hash=sha256:deafbeef --hash=sha256:deadbeef": [ - "deadbeef", - "deafbeef", - ], - } - for input, want_shas in tests.items(): - got = get_simpleapi_sources(input) - env.expect.that_collection(got.shas).contains_exactly(want_shas) - env.expect.that_str(got.version).equals("0.0.2") - -_tests.append(_test_simple_api_sources) - def _generate_html(*items): return """\ diff --git a/tests/private/pypi_index_sources/BUILD.bazel b/tests/private/pypi_index_sources/BUILD.bazel new file mode 100644 index 0000000000..212615f480 --- /dev/null +++ b/tests/private/pypi_index_sources/BUILD.bazel @@ -0,0 +1,3 @@ +load(":pypi_index_sources_tests.bzl", "pypi_index_sources_test_suite") + +pypi_index_sources_test_suite(name = "pypi_index_sources_tests") diff --git a/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl b/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl new file mode 100644 index 0000000000..48d790fc68 --- /dev/null +++ b/tests/private/pypi_index_sources/pypi_index_sources_tests.bzl @@ -0,0 +1,60 @@ +# Copyright 2023 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"" + +load("@rules_testing//lib:test_suite.bzl", "test_suite") +load("//python/private:pypi_index_sources.bzl", "get_simpleapi_sources") # buildifier: disable=bzl-visibility + +_tests = [] + +def _test_no_simple_api_sources(env): + inputs = [ + "foo==0.0.1", + "foo==0.0.1 @ https://someurl.org", + "foo==0.0.1 @ https://someurl.org --hash=sha256:deadbeef", + "foo==0.0.1 @ https://someurl.org; python_version < 2.7 --hash=sha256:deadbeef", + ] + for input in inputs: + got = get_simpleapi_sources(input) + env.expect.that_collection(got.shas).contains_exactly([]) + env.expect.that_str(got.version).equals("0.0.1") + +_tests.append(_test_no_simple_api_sources) + +def _test_simple_api_sources(env): + tests = { + "foo==0.0.2 --hash=sha256:deafbeef --hash=sha256:deadbeef": [ + "deadbeef", + "deafbeef", + ], + "foo[extra]==0.0.2; (python_version < 2.7 or something_else == \"@\") --hash=sha256:deafbeef --hash=sha256:deadbeef": [ + "deadbeef", + "deafbeef", + ], + } + for input, want_shas in tests.items(): + got = get_simpleapi_sources(input) + env.expect.that_collection(got.shas).contains_exactly(want_shas) + env.expect.that_str(got.version).equals("0.0.2") + +_tests.append(_test_simple_api_sources) + +def pypi_index_sources_test_suite(name): + """Create the test suite. + + Args: + name: the name of the test suite + """ + test_suite(name = name, basic_tests = _tests) diff --git a/tests/private/whl_target_platforms/select_whl_tests.bzl b/tests/private/whl_target_platforms/select_whl_tests.bzl index 0d6f97d7a5..bed6d6633c 100644 --- a/tests/private/whl_target_platforms/select_whl_tests.bzl +++ b/tests/private/whl_target_platforms/select_whl_tests.bzl @@ -83,37 +83,37 @@ def _match(env, got, want_filename): _tests = [] def _test_selecting(env): - got = select_whl(whls = WHL_LIST, want_abis = ["none"], want_os = "ignored", want_cpu = "ignored") + got = select_whl(whls = WHL_LIST, want_abis = ["none"], want_platform = "ignored") _match(env, got, "pkg-0.0.1-py3-none-any.whl") - got = select_whl(whls = WHL_LIST, want_abis = ["abi3"], want_os = "ignored", want_cpu = "ignored") + got = select_whl(whls = WHL_LIST, want_abis = ["abi3"], want_platform = "ignored") _match(env, got, "pkg-0.0.1-py3-abi3-any.whl") # Check the selection failure - got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_os = "fancy", want_cpu = "exotic") + got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_platform = "fancy_exotic") _match(env, got, None) # Check we match the ABI and not the py version - got = select_whl(whls = WHL_LIST, want_abis = ["cp37m"], want_os = "linux", want_cpu = "amd64") + got = select_whl(whls = WHL_LIST, want_abis = ["cp37m"], want_platform = "linux_x86_64") _match(env, got, "pkg-0.0.1-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") # Check we can select a filename with many platform tags - got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_os = "linux", want_cpu = "i686") + got = select_whl(whls = WHL_LIST, want_abis = ["cp39"], want_platform = "linux_x86_32") _match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl") # Check that we prefer the specific wheel - got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_os = "mac os", want_cpu = "x86_64") + got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_platform = "osx_x86_64") _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_x86_64.whl") - got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_os = "mac os", want_cpu = "aarch64") + got = select_whl(whls = WHL_LIST, want_abis = ["cp311"], want_platform = "osx_aarch64") _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_11_0_arm64.whl") # Check that we can use the universal2 if the arm wheel is not available - got = select_whl(whls = [w for w in WHL_LIST if "arm64" not in w.filename], want_abis = ["cp311"], want_os = "mac os", want_cpu = "aarch64") + got = select_whl(whls = [w for w in WHL_LIST if "arm64" not in w.filename], want_abis = ["cp311"], want_platform = "osx_aarch64") _match(env, got, "pkg-0.0.1-cp311-cp311-macosx_10_9_universal2.whl") # Check we prefer platform specific wheels - got = select_whl(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_os = "linux", want_cpu = "x86_64") + got = select_whl(whls = WHL_LIST, want_abis = ["none", "abi3", "cp39"], want_platform = "linux_x86_64") _match(env, got, "pkg-0.0.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl") _tests.append(_test_selecting)