From 0b4ca4b8b8d56e713b4bf0721fbcb3ff9953c1fb Mon Sep 17 00:00:00 2001 From: Michael Krasnyk Date: Sat, 1 Jun 2024 20:47:07 +0200 Subject: [PATCH] fix: use PyInfo from rules_package, fixes #60 Related: https://github.com/aspect-build/rules_py/issues/338 --- .bazelrc | 4 +-- .bazelversion | 1 + MODULE.bazel | 5 ++-- README.md | 2 +- examples/{markers => }/.gitignore | 0 examples/aspect_rules_py/.bazelrc | 1 + examples/aspect_rules_py/BUILD.bazel | 9 +++++++ examples/aspect_rules_py/MODULE.bazel | 27 +++++++++++++++++++ .../.bazelrc => aspect_rules_py/WORKSPACE} | 0 examples/aspect_rules_py/poetry.lock | 17 ++++++++++++ examples/aspect_rules_py/pyproject.toml | 10 +++++++ examples/aspect_rules_py/test.py | 10 +++++++ examples/markers/MODULE.bazel | 2 +- examples/simple/.gitignore | 1 - examples/simple/MODULE.bazel | 2 +- examples/simple/poetry.lock | 16 +++++------ examples/torch/MODULE.bazel | 2 +- examples/transitions/.gitignore | 1 - examples/transitions/MODULE.bazel | 2 +- lib/private/py_zip.bzl | 3 ++- python/poetry.bzl | 3 ++- python/poetry_deps.bzl | 23 +++++++++++++--- python/py_venv.bzl | 14 +++++++--- python/repositories.bzl | 2 +- 24 files changed, 127 insertions(+), 30 deletions(-) create mode 100644 .bazelversion rename examples/{markers => }/.gitignore (100%) create mode 100644 examples/aspect_rules_py/.bazelrc create mode 100644 examples/aspect_rules_py/BUILD.bazel create mode 100644 examples/aspect_rules_py/MODULE.bazel rename examples/{transitions/.bazelrc => aspect_rules_py/WORKSPACE} (100%) create mode 100644 examples/aspect_rules_py/poetry.lock create mode 100644 examples/aspect_rules_py/pyproject.toml create mode 100644 examples/aspect_rules_py/test.py delete mode 100644 examples/simple/.gitignore delete mode 100644 examples/transitions/.gitignore diff --git a/.bazelrc b/.bazelrc index d90b796..ab11bc1 100644 --- a/.bazelrc +++ b/.bazelrc @@ -1,7 +1,7 @@ startup --windows_enable_symlinks common --enable_bzlmod -query --deleted_packages=examples/simple,examples/torch,examples/transitions,examples/markers -build --deleted_packages=examples/simple,examples/torch,examples/transitions,examples/markers +query --deleted_packages=examples/simple,examples/torch,examples/transitions,examples/markers,examples/aspect_rules_py +build --deleted_packages=examples/simple,examples/torch,examples/transitions,examples/markers,examples/aspect_rules_py build --enable_runfiles build --incompatible_default_to_explicit_init_py test --test_output=errors diff --git a/.bazelversion b/.bazelversion new file mode 100644 index 0000000..0e7b60d --- /dev/null +++ b/.bazelversion @@ -0,0 +1 @@ +7.1.2 \ No newline at end of file diff --git a/MODULE.bazel b/MODULE.bazel index 494347d..ae49d02 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -4,8 +4,9 @@ module( compatibility_level = 1, ) -bazel_dep(name = "bazel_skylib", version = "1.5.0") -bazel_dep(name = "platforms", version = "0.0.8") +bazel_dep(name = "bazel_skylib", version = "1.7.0") +bazel_dep(name = "platforms", version = "0.0.10") +bazel_dep(name = "rules_python", version = "0.32.2") internal_deps = use_extension("@rules_poetry//python:extensions.bzl", "internal_deps") use_repo(internal_deps, "rules_poetry_deps", "rules_poetry_pip") diff --git a/README.md b/README.md index f5140e6..8460ffa 100644 --- a/README.md +++ b/README.md @@ -14,7 +14,7 @@ This allows to use platform information of resolved Python toolchains and build To import `rules_poetry` in your project, you first need to add it to your `MODULE.bazel` file ```python -bazel_dep(name = "rules_python", version = "0.31.0") +bazel_dep(name = "rules_python", version = "0.32.2") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/markers/.gitignore b/examples/.gitignore similarity index 100% rename from examples/markers/.gitignore rename to examples/.gitignore diff --git a/examples/aspect_rules_py/.bazelrc b/examples/aspect_rules_py/.bazelrc new file mode 100644 index 0000000..b112879 --- /dev/null +++ b/examples/aspect_rules_py/.bazelrc @@ -0,0 +1 @@ +build --sandbox_default_allow_network=false \ No newline at end of file diff --git a/examples/aspect_rules_py/BUILD.bazel b/examples/aspect_rules_py/BUILD.bazel new file mode 100644 index 0000000..cfff156 --- /dev/null +++ b/examples/aspect_rules_py/BUILD.bazel @@ -0,0 +1,9 @@ +load("@aspect_rules_py//py:defs.bzl", "py_binary") + +py_test( + name = "test", + srcs = ["test.py"], + deps = [ + "@poetry//:absl-py", + ], +) diff --git a/examples/aspect_rules_py/MODULE.bazel b/examples/aspect_rules_py/MODULE.bazel new file mode 100644 index 0000000..9b71230 --- /dev/null +++ b/examples/aspect_rules_py/MODULE.bazel @@ -0,0 +1,27 @@ +module( + name = "poetry_demo", + version = "0.1.0", +) + +bazel_dep(name = "aspect_rules_py", version = "0.7.3") +bazel_dep(name = "rules_python", version = "0.32.2") + +# Register a hermetic Python toolchain rather than rely on a locally-installed +# interpreter. +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +python.toolchain( + configure_coverage_tool = True, + is_default = True, + python_version = "3.11", +) +use_repo(python, "python_3_11") + +# https://registry.bazel.build/modules/rules_poetry +bazel_dep(name = "rules_poetry", version = "0.3.4") + +poetry = use_extension("@rules_poetry//python:extensions.bzl", "poetry") +poetry.parse( + name = "poetry", + lock = "//:poetry.lock", +) +use_repo(poetry, "poetry") diff --git a/examples/transitions/.bazelrc b/examples/aspect_rules_py/WORKSPACE similarity index 100% rename from examples/transitions/.bazelrc rename to examples/aspect_rules_py/WORKSPACE diff --git a/examples/aspect_rules_py/poetry.lock b/examples/aspect_rules_py/poetry.lock new file mode 100644 index 0000000..50884ed --- /dev/null +++ b/examples/aspect_rules_py/poetry.lock @@ -0,0 +1,17 @@ +# This file is automatically @generated by Poetry 1.6.1 and should not be changed by hand. + +[[package]] +name = "absl-py" +version = "2.1.0" +description = "Abseil Python Common Libraries, see https://github.com/abseil/abseil-py." +optional = false +python-versions = ">=3.7" +files = [ + {file = "absl-py-2.1.0.tar.gz", hash = "sha256:7820790efbb316739cde8b4e19357243fc3608a152024288513dd968d7d959ff"}, + {file = "absl_py-2.1.0-py3-none-any.whl", hash = "sha256:526a04eadab8b4ee719ce68f204172ead1027549089702d99b9059f129ff1308"}, +] + +[metadata] +lock-version = "2.0" +python-versions = "~3.11" +content-hash = "584e8c8c5d2185988d8085eb7893baa9d3e47312bff55541068f2aba58797d5d" diff --git a/examples/aspect_rules_py/pyproject.toml b/examples/aspect_rules_py/pyproject.toml new file mode 100644 index 0000000..ccd8669 --- /dev/null +++ b/examples/aspect_rules_py/pyproject.toml @@ -0,0 +1,10 @@ +[tool.poetry] +name = "poetry-demo" +version = "0.1.0" +authors = [] +description = "Bazel Python example using Poetry" +package-mode = false + +[tool.poetry.dependencies] +python = "~3.11" +absl-py = "^2.1.0" \ No newline at end of file diff --git a/examples/aspect_rules_py/test.py b/examples/aspect_rules_py/test.py new file mode 100644 index 0000000..1544e98 --- /dev/null +++ b/examples/aspect_rules_py/test.py @@ -0,0 +1,10 @@ +from absl import app + + +def main(argv): + print("Hello, Python and Poetry!") + return 0 + + +if __name__ == "__main__": + app.run(main) diff --git a/examples/markers/MODULE.bazel b/examples/markers/MODULE.bazel index 082c110..864eb88 100644 --- a/examples/markers/MODULE.bazel +++ b/examples/markers/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.31.0") +bazel_dep(name = "rules_python", version = "0.32.2") python_version = "3_12" diff --git a/examples/simple/.gitignore b/examples/simple/.gitignore deleted file mode 100644 index ac51a05..0000000 --- a/examples/simple/.gitignore +++ /dev/null @@ -1 +0,0 @@ -bazel-* diff --git a/examples/simple/MODULE.bazel b/examples/simple/MODULE.bazel index b1d11a4..e619924 100644 --- a/examples/simple/MODULE.bazel +++ b/examples/simple/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.31.0") +bazel_dep(name = "rules_python", version = "0.32.2") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/simple/poetry.lock b/examples/simple/poetry.lock index 8fe67c8..41878f6 100644 --- a/examples/simple/poetry.lock +++ b/examples/simple/poetry.lock @@ -1,18 +1,18 @@ -# This file is automatically @generated by Poetry 1.8.2 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. [[package]] name = "boto3" -version = "1.34.93" +version = "1.34.117" description = "The AWS SDK for Python" optional = false python-versions = ">=3.8" files = [ - {file = "boto3-1.34.93-py3-none-any.whl", hash = "sha256:b59355bf4a1408563969526f314611dbeacc151cf90ecb22af295dcc4fe18def"}, - {file = "boto3-1.34.93.tar.gz", hash = "sha256:e39516e4ca21612932599819662759c04485d53ca457996a913163da11f052a4"}, + {file = "boto3-1.34.117-py3-none-any.whl", hash = "sha256:1506589e30566bbb2f4997b60968ff7d4ef8a998836c31eedd36437ac3b7408a"}, + {file = "boto3-1.34.117.tar.gz", hash = "sha256:c8a383b904d6faaf7eed0c06e31b423db128e4c09ce7bd2afc39d1cd07030a51"}, ] [package.dependencies] -botocore = ">=1.34.93,<1.35.0" +botocore = ">=1.34.117,<1.35.0" jmespath = ">=0.7.1,<2.0.0" s3transfer = ">=0.10.0,<0.11.0" @@ -21,13 +21,13 @@ crt = ["botocore[crt] (>=1.21.0,<2.0a0)"] [[package]] name = "botocore" -version = "1.34.93" +version = "1.34.117" description = "Low-level, data-driven core of boto 3." optional = false python-versions = ">=3.8" files = [ - {file = "botocore-1.34.93-py3-none-any.whl", hash = "sha256:6fbd5a53a2adc9b3d4ebd90ae0ede83a91a41d96231f8a5984051f75495f246d"}, - {file = "botocore-1.34.93.tar.gz", hash = "sha256:79d39b0b87e962991c6dd55e78ce15155099f6fb741be88b1b8a456a702cc150"}, + {file = "botocore-1.34.117-py3-none-any.whl", hash = "sha256:26a431997f882bcdd1e835f44c24b2a1752b1c4e5183c2ce62999ce95d518d6c"}, + {file = "botocore-1.34.117.tar.gz", hash = "sha256:4637ca42e6c51aebc4d9a2d92f97bf4bdb042e3f7985ff31a659a11e4c170e73"}, ] [package.dependencies] diff --git a/examples/torch/MODULE.bazel b/examples/torch/MODULE.bazel index b1d11a4..e619924 100644 --- a/examples/torch/MODULE.bazel +++ b/examples/torch/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.31.0") +bazel_dep(name = "rules_python", version = "0.32.2") python = use_extension("@rules_python//python/extensions:python.bzl", "python") python.toolchain(python_version = "3.12") diff --git a/examples/transitions/.gitignore b/examples/transitions/.gitignore deleted file mode 100644 index ac51a05..0000000 --- a/examples/transitions/.gitignore +++ /dev/null @@ -1 +0,0 @@ -bazel-* diff --git a/examples/transitions/MODULE.bazel b/examples/transitions/MODULE.bazel index a0c97c6..bfcb625 100644 --- a/examples/transitions/MODULE.bazel +++ b/examples/transitions/MODULE.bazel @@ -1,4 +1,4 @@ -bazel_dep(name = "rules_python", version = "0.31.0") +bazel_dep(name = "rules_python", version = "0.32.2") bazel_dep(name = "platforms", version = "0.0.8") python = use_extension("@rules_python//python/extensions:python.bzl", "python") diff --git a/lib/private/py_zip.bzl b/lib/private/py_zip.bzl index 7b2602a..57800db 100644 --- a/lib/private/py_zip.bzl +++ b/lib/private/py_zip.bzl @@ -3,6 +3,7 @@ Support for serverless deployments. """ load("@bazel_skylib//lib:paths.bzl", "paths") +load("//python:poetry_deps.bzl", _get_imports = "get_imports") load(":runfiles.bzl", _matches = "matches") DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3" @@ -66,7 +67,7 @@ def _py_zip_impl(ctx): ctx.actions.write(manifest_file, "\n".join(args)) ## Genrate a JSON files with enviroment variables for downstream consumers - python_paths = [workspace_dir] + [path for path in target[PyInfo].imports.to_list()] + python_paths = [workspace_dir] + _get_imports(target).to_list() json_file = ctx.actions.declare_file(basename + ".json") ctx.actions.write(json_file, json.encode({"environment": {"PYTHONPATH": ":".join(python_paths)}})) diff --git a/python/poetry.bzl b/python/poetry.bzl index ee3c47a..9aa0221 100644 --- a/python/poetry.bzl +++ b/python/poetry.bzl @@ -1,4 +1,5 @@ load("@rules_poetry_deps//:defs.bzl", _python = "python") +load("//python:poetry_deps.bzl", _get_imports = "get_imports") def _poetry_update_impl(ctx): script = """#!{python} @@ -23,7 +24,7 @@ if __name__ == "__main__": runpy.run_module("poetry", run_name="__main__", alter_sys=True) """.format( python = _python, - deps = repr(["../{}".format(path) for path in ctx.attr._poetry_deps[PyInfo].imports.to_list()]), + deps = repr(["../{}".format(path) for path in _get_imports(ctx.attr._poetry_deps).to_list()]), toml = ctx.attr.toml.files.to_list().pop().short_path, lock = ctx.attr.lock.files.to_list().pop().short_path, update = "" if ctx.attr.update else ', "--no-update"', diff --git a/python/poetry_deps.bzl b/python/poetry_deps.bzl index 7a10107..b3a1129 100644 --- a/python/poetry_deps.bzl +++ b/python/poetry_deps.bzl @@ -1,5 +1,6 @@ load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//lib:versions.bzl", "versions") +load("@rules_python//python:defs.bzl", StarPyInfo = "PyInfo") load("//python:markers.bzl", "evaluate", "parse") # Environment Markers https://peps.python.org/pep-0508/#environment-markers @@ -122,18 +123,20 @@ def _package_impl(ctx): ) deps = [dep for dep in ctx.attr.deps if _include_dep(dep, ctx.attr.markers, tags)] - transitive_imports = [dep[PyInfo].imports for dep in deps] - transitive_depsets = [dep[PyInfo].transitive_sources for dep in deps] + transitive_imports = [get_imports(dep) for dep in deps] + transitive_depsets = [get_transitive_sources(dep) for dep in deps] runfiles = [output] + [item for dep in transitive_depsets for item in dep.to_list()] files = depset([output], transitive = transitive_depsets) + imports = depset([output.short_path.replace("../", "")], transitive = transitive_imports) return [ DefaultInfo(files = files, runfiles = ctx.runfiles(files = runfiles)), - PyInfo(transitive_sources = files, imports = depset([output.short_path.replace("../", "")], transitive = transitive_imports)), + PyInfo(transitive_sources = files, imports = imports), + StarPyInfo(transitive_sources = files, imports = imports), ] package = rule( implementation = _package_impl, - provides = [PyInfo], + provides = [PyInfo, StarPyInfo], attrs = { "constraint": attr.string(mandatory = True, doc = "The package version constraint string"), "deps": attr.label_list(doc = "The package dependencies list"), @@ -154,3 +157,15 @@ package = rule( "@bazel_tools//tools/python:toolchain_type", ], ) + +def get_imports(target): + for info in [StarPyInfo, PyInfo]: + if info in target: + return target[info].imports + return depset() + +def get_transitive_sources(target): + for info in [StarPyInfo, PyInfo]: + if info in target: + return target[info].transitive_sources + return depset() diff --git a/python/py_venv.bzl b/python/py_venv.bzl index 4f8dec5..99934af 100644 --- a/python/py_venv.bzl +++ b/python/py_venv.bzl @@ -1,3 +1,6 @@ +load("@rules_python//python:defs.bzl", StarPyInfo = "PyInfo") +load("//python:poetry_deps.bzl", _get_imports = "get_imports", _get_transitive_sources = "get_transitive_sources") + def _py_venv_impl(ctx): """ Rule to link Python package into a virtual environment. @@ -17,7 +20,8 @@ def _py_venv_impl(ctx): deps = ctx.attr.deps output = ctx.actions.declare_directory("venv/{}".format(ctx.label.name)) - import_paths = ["{}/external/{}".format(ctx.bin_dir.path, path) for dep in deps for path in dep[PyInfo].imports.to_list()] + import_depsets = depset(transitive = [_get_imports(dep) for dep in deps]) + import_paths = ["{}/external/{}".format(ctx.bin_dir.path, path) for path in import_depsets.to_list()] ctx.actions.run( outputs = [output], @@ -29,18 +33,20 @@ def _py_venv_impl(ctx): executable = ctx.executable._py_venv, ) - transitive_depsets = [dep[PyInfo].transitive_sources for dep in deps] + transitive_depsets = [_get_transitive_sources(dep) for dep in deps] runfiles = [output] + [item for dep in transitive_depsets for item in dep.to_list()] files = depset([output], transitive = transitive_depsets) + imports = depset(["_main/" + output.short_path]) return [ DefaultInfo(files = files, runfiles = ctx.runfiles(files = runfiles)), - PyInfo(transitive_sources = files, imports = depset(["_main/" + output.short_path])), + PyInfo(transitive_sources = files, imports = imports), + StarPyInfo(transitive_sources = files, imports = imports), ] py_venv = rule( implementation = _py_venv_impl, - provides = [PyInfo], + provides = [PyInfo, StarPyInfo], attrs = { "deps": attr.label_list(doc = "The package dependencies list"), "_py_venv": attr.label(default = ":py_venv", cfg = "exec", executable = True), diff --git a/python/repositories.bzl b/python/repositories.bzl index 306c6bf..cde5d7f 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -14,7 +14,7 @@ _POETRY_INTERNAL_DEPS = [ ] def _poetry_deps_repo_impl(ctx): - poetry_version = "1.8.2" + poetry_version = "1.8.3" # Intentionally use a host default interpreter as the repository only used in host tooling targets # This may lead to inconsistency if the repository will be used with a different toolchain