diff --git a/.bazelrc b/.bazelrc index bf13baf132..0833d8d979 100644 --- a/.bazelrc +++ b/.bazelrc @@ -4,8 +4,8 @@ # (Note, we cannot use `common --deleted_packages` because the bazel version command doesn't support it) # To update these lines, execute # `bazel run @rules_bazel_integration_test//tools:update_deleted_packages` -build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered -query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +build --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered +query --deleted_packages=examples/build_file_generation,examples/build_file_generation/random_number_generator,examples/bzlmod,examples/bzlmod_build_file_generation,examples/bzlmod_build_file_generation/other_module/other_module/pkg,examples/bzlmod_build_file_generation/runfiles,examples/bzlmod/entry_points,examples/bzlmod/entry_points/tests,examples/bzlmod/libs/my_lib,examples/bzlmod/other_module,examples/bzlmod/other_module/other_module/pkg,examples/bzlmod/patches,examples/bzlmod/py_proto_library,examples/bzlmod/py_proto_library/example.com/another_proto,examples/bzlmod/py_proto_library/example.com/proto,examples/bzlmod/runfiles,examples/bzlmod/tests,examples/bzlmod/tests/other_module,examples/bzlmod/whl_mods,examples/multi_python_versions/libs/my_lib,examples/multi_python_versions/requirements,examples/multi_python_versions/tests,examples/pip_parse,examples/pip_parse_vendored,examples/pip_repository_annotations,examples/py_proto_library,examples/py_proto_library/example.com/another_proto,examples/py_proto_library/example.com/proto,tests/integration/compile_pip_requirements,tests/integration/compile_pip_requirements_test_from_external_repo,tests/integration/custom_commands,tests/integration/ignore_root_user_error,tests/integration/ignore_root_user_error/submodule,tests/integration/local_toolchains,tests/integration/pip_parse,tests/integration/pip_parse/empty,tests/integration/py_cc_toolchain_registered test --test_output=errors diff --git a/docs/sphinx/glossary.md b/docs/sphinx/glossary.md index f54034db2d..9afbcffb92 100644 --- a/docs/sphinx/glossary.md +++ b/docs/sphinx/glossary.md @@ -7,6 +7,30 @@ common attributes [Common attributes](https://bazel.build/reference/be/common-definitions#common-attributes) for a complete listing +in-build runtime +: An in-build runtime is one where the Python runtime, and all its files, are +known to the build system and a Python binary includes all the necessary parts +of the runtime in its runfiles. Such runtimes may be remotely downloaded, part +of your source control, or mapped in from local files by repositories. + +The main advantage of in-build runtimes is they ensure you know what Python +runtime will be used, since it's part of the build itself and included in +the resulting binary. The main disadvantage is the additional work it adds to +building. The whole Python runtime is included in a Python binary's runfiles, +which can be a significant number of files. + +platform runtime +: A platform runtime is a Python runtime that is assumed to be installed on the +system where a Python binary runs, whereever that may be. For example, using `/usr/bin/python3` +as the interpreter is a platform runtime -- it assumes that, wherever the binary +runs (your local machine, a remote worker, within a container, etc), that path +is available. Such runtimes are _not_ part of a binary's runfiles. + +The main advantage of platform runtimes is they are lightweight insofar as +building the binary is concerned. All Bazel has to do is pass along a string +path to the interpreter. The disadvantage is, if you don't control the systems +being run on, you may get different Python installations than expected. + rule callable : A function that behaves like a rule. This includes, but is not is not limited to: @@ -26,3 +50,4 @@ simple label nonconfigurable : A nonconfigurable value cannot use `select`. See Bazel's [configurable attributes](https://bazel.build/reference/be/common-definitions#configurable-attributes) documentation. + diff --git a/python/private/full_version.bzl b/python/private/full_version.bzl index 68c969416e..98eeee59a1 100644 --- a/python/private/full_version.bzl +++ b/python/private/full_version.bzl @@ -40,4 +40,4 @@ def full_version(version): ), ) else: - fail("Unknown version format: {}".format(version)) + fail("Unknown version format: '{}'".format(version)) diff --git a/python/private/get_local_runtime_info.py b/python/private/get_local_runtime_info.py new file mode 100644 index 0000000000..0207f56bef --- /dev/null +++ b/python/private/get_local_runtime_info.py @@ -0,0 +1,49 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import sys +import sysconfig + +data = { + "major": sys.version_info.major, + "minor": sys.version_info.minor, + "micro": sys.version_info.micro, + "include": sysconfig.get_path("include"), + "implementation_name": sys.implementation.name, +} + +config_vars = [ + # The libpythonX.Y.so file. Usually? + # It might be a static archive (.a) file instead. + "LDLIBRARY", + # The directory with library files. Supposedly. + # It's not entirely clear how to get the directory with libraries. + # There's several types of libraries with different names and a plethora + # of settings. + # https://stackoverflow.com/questions/47423246/get-pythons-lib-path + # For now, it seems LIBDIR has what is needed, so just use that. + "LIBDIR", + # The versioned libpythonX.Y.so.N file. Usually? + # It might be a static archive (.a) file instead. + "INSTSONAME", + # The libpythonX.so file. Usually? + # It might be a static archive (a.) file instead. + "PY3LIBRARY", + # The platform-specific filename suffix for library files. + # Includes the dot, e.g. `.so` + "SHLIB_SUFFIX", +] +data.update(zip(config_vars, sysconfig.get_config_vars(*config_vars))) +print(json.dumps(data)) diff --git a/python/private/local_runtime_repo.bzl b/python/private/local_runtime_repo.bzl new file mode 100644 index 0000000000..f6bca6cc2c --- /dev/null +++ b/python/private/local_runtime_repo.bzl @@ -0,0 +1,252 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create a repository for a locally installed Python runtime.""" + +load("//python/private:enum.bzl", "enum") +load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") + +# buildifier: disable=name-conventions +_OnFailure = enum( + SKIP = "skip", + WARN = "warn", + FAIL = "fail", +) + +_TOOLCHAIN_IMPL_TEMPLATE = """\ +# Generated by python/private/local_runtime_repo.bzl + +load("@rules_python//python/private:local_runtime_repo_setup.bzl", "define_local_runtime_toolchain_impl") + +define_local_runtime_toolchain_impl( + name = "local_runtime", + lib_ext = "{lib_ext}", + major = "{major}", + minor = "{minor}", + micro = "{micro}", + interpreter_path = "{interpreter_path}", + implementation_name = "{implementation_name}", + os = "{os}", +) +""" + +def _local_runtime_repo_impl(rctx): + logger = repo_utils.logger(rctx) + on_failure = rctx.attr.on_failure + + platforms_os_name = repo_utils.get_platforms_os_name(rctx) + if not platforms_os_name: + if on_failure == "fail": + fail("Unrecognized host platform '{}': cannot determine OS constraint".format( + rctx.os.name, + )) + + if on_failure == "warn": + logger.warn(lambda: "Unrecognized host platform '{}': cannot determine OS constraint".format( + rctx.os.name, + )) + + # else, on_failure must be skip + rctx.file("BUILD.bazel", _expand_incompatible_template()) + return + + result = _resolve_interpreter_path(rctx) + if not result.resolved_path: + if on_failure == "fail": + fail("interpreter not found: {}".format(result.describe_failure())) + + if on_failure == "warn": + logger.warn(lambda: "interpreter not found: {}".format(result.describe_failure())) + + # else, on_failure must be skip + rctx.file("BUILD.bazel", _expand_incompatible_template()) + return + else: + interpreter_path = result.resolved_path + + logger.info(lambda: "resolved interpreter {} to {}".format(rctx.attr.interpreter_path, interpreter_path)) + + exec_result = repo_utils.execute_unchecked( + rctx, + op = "local_runtime_repo.GetPythonInfo({})".format(rctx.name), + arguments = [ + interpreter_path, + rctx.path(rctx.attr._get_local_runtime_info), + ], + quiet = True, + ) + if exec_result.return_code != 0: + if on_failure == "fail": + fail("GetPythonInfo failed: {}".format(exec_result.describe_failure())) + if on_failure == "warn": + logger.warn(lambda: "GetPythonInfo failed: {}".format(exec_result.describe_failure())) + + # else, on_failure must be skip + rctx.file("BUILD.bazel", _expand_incompatible_template()) + return + + info = json.decode(exec_result.stdout) + logger.info(lambda: _format_get_info_result(info)) + + # NOTE: Keep in sync with recursive glob in define_local_runtime_toolchain_impl + repo_utils.watch_tree(rctx, rctx.path(info["include"])) + + # The cc_library.includes values have to be non-absolute paths, otherwise + # the toolchain will give an error. Work around this error by making them + # appear as part of this repo. + rctx.symlink(info["include"], "include") + + shared_lib_names = [ + info["PY3LIBRARY"], + info["LDLIBRARY"], + info["INSTSONAME"], + ] + + # In some cases, the value may be empty. Not clear why. + shared_lib_names = [v for v in shared_lib_names if v] + + # In some cases, the same value is returned for multiple keys. Not clear why. + shared_lib_names = {v: None for v in shared_lib_names}.keys() + shared_lib_dir = info["LIBDIR"] + + # The specific files are symlinked instead of the whole directory + # because it can point to a directory that has more than just + # the Python runtime shared libraries, e.g. /usr/lib, or a Python + # specific directory with pip-installed shared libraries. + rctx.report_progress("Symlinking external Python shared libraries") + for name in shared_lib_names: + origin = rctx.path("{}/{}".format(shared_lib_dir, name)) + + # The reported names don't always exist; it depends on the particulars + # of the runtime installation. + if origin.exists: + repo_utils.watch(rctx, origin) + rctx.symlink(origin, "lib/" + name) + + rctx.file("WORKSPACE", "") + rctx.file("MODULE.bazel", "") + rctx.file("REPO.bazel", "") + rctx.file("BUILD.bazel", _TOOLCHAIN_IMPL_TEMPLATE.format( + major = info["major"], + minor = info["minor"], + micro = info["micro"], + interpreter_path = interpreter_path, + lib_ext = info["SHLIB_SUFFIX"], + implementation_name = info["implementation_name"], + os = "@platforms//os:{}".format(repo_utils.get_platforms_os_name(rctx)), + )) + +local_runtime_repo = repository_rule( + implementation = _local_runtime_repo_impl, + doc = """ +Use a locally installed Python runtime as a toolchain implementation. + +Note this uses the runtime as a *platform runtime*. A platform runtime means +means targets don't include the runtime itself as part of their runfiles or +inputs. Instead, users must assure that where the targets run have the runtime +pre-installed or otherwise available. + +This results in lighter weight binaries (in particular, Bazel doesn't have to +create thousands of files for every `py_test`), at the risk of having to rely on +a system having the necessary Python installed. +""", + attrs = { + "interpreter_path": attr.string( + doc = """ +An absolute path or program name on the `PATH` env var. + +Values with slashes are assumed to be the path to a program. Otherwise, it is +treated as something to search for on `PATH` + +Note that, when a plain program name is used, the path to the interpreter is +resolved at repository evalution time, not runtime of any resulting binaries. +""", + default = "python3", + ), + "on_failure": attr.string( + default = _OnFailure.SKIP, + values = sorted(_OnFailure.__members__.values()), + doc = """ +How to handle errors when trying to automatically determine settings. + +* `skip` will silently skip creating a runtime. Instead, a non-functional + runtime will be generated and marked as incompatible so it cannot be used. + This is best if a local runtime is known not to work or be available + in certain cases and that's OK. e.g., one use windows paths when there + are people running on linux. +* `warn` will print a warning message. This is useful when you expect + a runtime to be available, but are OK with it missing and falling back + to some other runtime. +* `fail` will result in a failure. This is only recommended if you must + ensure the runtime is available. +""", + ), + "_get_local_runtime_info": attr.label( + allow_single_file = True, + default = "//python/private:get_local_runtime_info.py", + ), + "_rule_name": attr.string(default = "local_runtime_repo"), + }, + environ = ["PATH", REPO_DEBUG_ENV_VAR], +) + +def _expand_incompatible_template(): + return _TOOLCHAIN_IMPL_TEMPLATE.format( + interpreter_path = "/incompatible", + implementation_name = "incompatible", + lib_ext = "incompatible", + major = "0", + minor = "0", + micro = "0", + os = "@platforms//:incompatible", + ) + +def _resolve_interpreter_path(rctx): + """Find the absolute path for an interpreter. + + Args: + rctx: A repository_ctx object + + Returns: + `struct` with the following fields: + * `resolved_path`: `path` object of a path that exists + * `describe_failure`: `Callable | None`. If a path that doesn't exist, + returns a description of why it couldn't be resolved + A path object or None. The path may not exist. + """ + if "/" not in rctx.attr.interpreter_path and "\\" not in rctx.attr.interpreter_path: + # Provide a bit nicer integration with pyenv: recalculate the runtime if the + # user changes the python version using e.g. `pyenv shell` + repo_utils.getenv(rctx, "PYENV_VERSION") + result = repo_utils.which_unchecked(rctx, rctx.attr.interpreter_path) + resolved_path = result.binary + describe_failure = result.describe_failure + else: + repo_utils.watch(rctx, rctx.attr.interpreter_path) + resolved_path = rctx.path(rctx.attr.interpreter_path) + if not resolved_path.exists: + describe_failure = lambda: "Path not found: {}".format(repr(rctx.attr.interpreter_path)) + else: + describe_failure = None + + return struct( + resolved_path = resolved_path, + describe_failure = describe_failure, + ) + +def _format_get_info_result(info): + lines = ["GetPythonInfo result:"] + for key, value in sorted(info.items()): + lines.append(" {}: {}".format(key, value if value != "" else "")) + return "\n".join(lines) diff --git a/python/private/local_runtime_repo_setup.bzl b/python/private/local_runtime_repo_setup.bzl new file mode 100644 index 0000000000..23fa99dfa9 --- /dev/null +++ b/python/private/local_runtime_repo_setup.bzl @@ -0,0 +1,141 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Setup code called by the code generated by `local_runtime_repo`.""" + +load("@bazel_skylib//lib:selects.bzl", "selects") +load("@rules_cc//cc:defs.bzl", "cc_library") +load("@rules_python//python:py_runtime.bzl", "py_runtime") +load("@rules_python//python:py_runtime_pair.bzl", "py_runtime_pair") +load("@rules_python//python/cc:py_cc_toolchain.bzl", "py_cc_toolchain") +load("@rules_python//python/private:py_exec_tools_toolchain.bzl", "py_exec_tools_toolchain") + +_PYTHON_VERSION_FLAG = Label("@rules_python//python/config_settings:python_version") + +def define_local_runtime_toolchain_impl( + name, + lib_ext, + major, + minor, + micro, + interpreter_path, + implementation_name, + os): + """Defines a toolchain implementation for a local Python runtime. + + Generates public targets: + * `python_runtimes`: The target toolchain type implementation + * `py_exec_tools_toolchain`: The exec tools toolchain type implementation + * `py_cc_toolchain`: The py cc toolchain type implementation + * `os`: A constraint (or alias to one) for the `target_compatible_with` this + toolchain is compatible with. + * `is_matching_python_version`: A `config_setting` for `target_settings` + this toolchain is compatible with. + + Args: + name: `str` Only present to satisfy tooling + lib_ext: `str` The file extension for the `libpython` shared libraries + major: `str` The major Python version, e.g. `3` of `3.9.1`. + minor: `str` The minor Python version, e.g. `9` of `3.9.1`. + micro: `str` The micro Python version, e.g. "1" of `3.9.1`. + interpreter_path: `str` Absolute path to the interpreter. + implementation_name: `str` The implementation name, as returned by + `sys.implementation.name`. + os: `str` A label to the OS constraint (e.g. `@platforms//os:linux`) for + this runtime. + """ + major_minor = "{}.{}".format(major, minor) + major_minor_micro = "{}.{}".format(major_minor, micro) + + cc_library( + name = "_python_headers", + # NOTE: Keep in sync with watch_tree() called in local_runtime_repo + srcs = native.glob(["include/**/*.h"]), + includes = ["include"], + ) + + cc_library( + name = "_libpython", + # Don't use a recursive glob because the lib/ directory usually contains + # a subdirectory of the stdlib -- lots of unrelated files + srcs = native.glob([ + "lib/*{}".format(lib_ext), # Match libpython*.so + "lib/*{}*".format(lib_ext), # Also match libpython*.so.1.0 + ]), + hdrs = [":_python_headers"], + ) + + py_runtime( + name = "_py3_runtime", + interpreter_path = interpreter_path, + python_version = "PY3", + interpreter_version_info = { + "major": major, + "micro": micro, + "minor": minor, + }, + implementation_name = implementation_name, + ) + + py_runtime_pair( + name = "python_runtimes", + py2_runtime = None, + py3_runtime = ":_py3_runtime", + visibility = ["//visibility:public"], + ) + + py_exec_tools_toolchain( + name = "py_exec_tools_toolchain", + visibility = ["//visibility:public"], + precompiler = "@rules_python//tools/precompiler:precompiler", + ) + + py_cc_toolchain( + name = "py_cc_toolchain", + headers = ":_python_headers", + libs = ":_libpython", + python_version = major_minor_micro, + visibility = ["//visibility:public"], + ) + + native.alias( + name = "os", + # Call Label() to force the string to evaluate in the context of + # rules_python, not the calling BUILD-file code. This is because + # the value is an `@platforms//foo` string, which @rules_python has + # visibility to, but the calling repo may not. + actual = Label(os), + visibility = ["//visibility:public"], + ) + + native.config_setting( + name = "_is_major_minor", + flag_values = { + _PYTHON_VERSION_FLAG: major_minor, + }, + ) + native.config_setting( + name = "_is_major_minor_micro", + flag_values = { + _PYTHON_VERSION_FLAG: major_minor_micro, + }, + ) + selects.config_setting_group( + name = "is_matching_python_version", + match_any = [ + ":_is_major_minor", + ":_is_major_minor_micro", + ], + visibility = ["//visibility:public"], + ) diff --git a/python/private/local_runtime_toolchains_repo.bzl b/python/private/local_runtime_toolchains_repo.bzl new file mode 100644 index 0000000000..880fbfe224 --- /dev/null +++ b/python/private/local_runtime_toolchains_repo.bzl @@ -0,0 +1,93 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Create a repository to hold a local Python toolchain definitions.""" + +load("//python/private:text_util.bzl", "render") +load(":repo_utils.bzl", "REPO_DEBUG_ENV_VAR", "repo_utils") + +_TOOLCHAIN_TEMPLATE = """ +# Generated by local_runtime_toolchains_repo.bzl + +load("@rules_python//python/private:py_toolchain_suite.bzl", "define_local_toolchain_suites") + +define_local_toolchain_suites( + name = "toolchains", + version_aware_repo_names = {version_aware_names}, + version_unaware_repo_names = {version_unaware_names}, +) +""" + +def _local_runtime_toolchains_repo(rctx): + logger = repo_utils.logger(rctx) + rctx.file("WORKSPACE", "") + rctx.file("MODULE.bazel", "") + rctx.file("REPO.bazel", "") + + logger.info(lambda: _format_toolchains_for_logging(rctx)) + + rctx.file("BUILD.bazel", _TOOLCHAIN_TEMPLATE.format( + version_aware_names = render.list(rctx.attr.runtimes), + version_unaware_names = render.list(rctx.attr.default_runtimes or rctx.attr.runtimes), + )) + +local_runtime_toolchains_repo = repository_rule( + implementation = _local_runtime_toolchains_repo, + doc = """ +Create a repo of toolchains definitions for local runtimes. + +This is intended to be used on the toolchain implemenations generated by +`local_runtime_repo`. + +NOTE: This does not call `native.register_toolchains` -- the caller is +responsible for registering the toolchains this defines. +""", + attrs = { + "default_runtimes": attr.string_list( + doc = """ +The repo names of `local_runtime_repo` repos to define as toolchains. + +These will be defined as *version-unaware* toolchains. This means they will +match any Python version. As such, they are registered after the version-aware +toolchains defined by the `runtimes` attribute. + +Note that order matters: it determines the toolchain priority within the +package. +""", + ), + "runtimes": attr.string_list( + doc = """ +The repo names of `local_runtime_repo` repos to define as toolchains. + +These will be defined as *version-aware* toolchains. This means they require the +`--//python/config_settings:python_version` to be set in order to match. These +are registered before `default_runtimes`. + +Note that order matters: it determines the toolchain priority within the +package. +""", + ), + "_rule_name": attr.string(default = "local_toolchains_repo"), + }, + environ = [REPO_DEBUG_ENV_VAR], +) + +def _format_toolchains_for_logging(rctx): + lines = ["Local toolchain priority order:"] + i = 0 + for i, name in enumerate(rctx.attr.runtimes, start = i): + lines.append(" {}: {} (version aware)".format(i, name)) + for i, name in enumerate(rctx.attr.default_runtimes, start = i): + lines.append(" {}: {} (version unaware)".format(i, name)) + return "\n".join(lines) diff --git a/python/private/py_exec_tools_toolchain.bzl b/python/private/py_exec_tools_toolchain.bzl index b3d0fb2634..a4516d86eb 100644 --- a/python/private/py_exec_tools_toolchain.bzl +++ b/python/private/py_exec_tools_toolchain.bzl @@ -14,6 +14,7 @@ """Rule that defines a toolchain for build tools.""" +load("@bazel_skylib//lib:paths.bzl", "paths") load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//python/private:toolchain_types.bzl", "TARGET_TOOLCHAIN_TYPE") load(":py_exec_tools_info.bzl", "PyExecToolsInfo") @@ -53,11 +54,15 @@ py_exec_tools_toolchain = rule( def _current_interpreter_executable_impl(ctx): toolchain = ctx.toolchains[TARGET_TOOLCHAIN_TYPE] runtime = toolchain.py3_runtime + + # NOTE: We name the output filename after the underlying file name + # because of things like pyenv: they use $0 to determine what to + # re-exec. If it's not a recognized name, then they fail. if runtime.interpreter: - executable = ctx.actions.declare_file(ctx.label.name) + executable = ctx.actions.declare_file(runtime.interpreter.basename) ctx.actions.symlink(output = executable, target_file = runtime.interpreter, is_executable = True) else: - executable = ctx.actions.declare_symlink(ctx.label.name) + executable = ctx.actions.declare_symlink(paths.basename(runtime.interpreter_path)) ctx.actions.symlink(output = executable, target_path = runtime.interpreter_path) return [ toolchain, diff --git a/python/private/py_toolchain_suite.bzl b/python/private/py_toolchain_suite.bzl index 564e0ee45d..3fead95069 100644 --- a/python/private/py_toolchain_suite.bzl +++ b/python/private/py_toolchain_suite.bzl @@ -15,6 +15,7 @@ """Create the toolchain defs in a BUILD.bazel file.""" load("@bazel_skylib//lib:selects.bzl", "selects") +load("//python/private:text_util.bzl", "render") load( ":toolchain_types.bzl", "EXEC_TOOLS_TOOLCHAIN_TYPE", @@ -24,7 +25,15 @@ load( _IS_EXEC_TOOLCHAIN_ENABLED = Label("//python/config_settings:is_exec_tools_toolchain_enabled") -def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_python_version_constraint, flag_values, **kwargs): +# buildifier: disable=unnamed-macro +def py_toolchain_suite( + *, + prefix, + user_repository_name, + python_version, + set_python_version_constraint, + flag_values, + target_compatible_with = []): """For internal use only. Args: @@ -33,8 +42,7 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth python_version: The full (X.Y.Z) version of the interpreter. set_python_version_constraint: True or False as a string. flag_values: Extra flag values to match for this toolchain. - **kwargs: extra args passed to the `toolchain` calls. - + target_compatible_with: list constraints the toolchains are compatible with. """ # We have to use a String value here because bzlmod is passing in a @@ -82,30 +90,38 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth repr(set_python_version_constraint), )) + _internal_toolchain_suite( + prefix = prefix, + runtime_repo_name = user_repository_name, + target_settings = target_settings, + target_compatible_with = target_compatible_with, + ) + +def _internal_toolchain_suite(prefix, runtime_repo_name, target_compatible_with, target_settings): native.toolchain( name = "{prefix}_toolchain".format(prefix = prefix), - toolchain = "@{user_repository_name}//:python_runtimes".format( - user_repository_name = user_repository_name, + toolchain = "@{runtime_repo_name}//:python_runtimes".format( + runtime_repo_name = runtime_repo_name, ), toolchain_type = TARGET_TOOLCHAIN_TYPE, target_settings = target_settings, - **kwargs + target_compatible_with = target_compatible_with, ) native.toolchain( name = "{prefix}_py_cc_toolchain".format(prefix = prefix), - toolchain = "@{user_repository_name}//:py_cc_toolchain".format( - user_repository_name = user_repository_name, + toolchain = "@{runtime_repo_name}//:py_cc_toolchain".format( + runtime_repo_name = runtime_repo_name, ), toolchain_type = PY_CC_TOOLCHAIN_TYPE, target_settings = target_settings, - **kwargs + target_compatible_with = target_compatible_with, ) native.toolchain( name = "{prefix}_py_exec_tools_toolchain".format(prefix = prefix), - toolchain = "@{user_repository_name}//:py_exec_tools_toolchain".format( - user_repository_name = user_repository_name, + toolchain = "@{runtime_repo_name}//:py_exec_tools_toolchain".format( + runtime_repo_name = runtime_repo_name, ), toolchain_type = EXEC_TOOLS_TOOLCHAIN_TYPE, target_settings = select({ @@ -118,10 +134,46 @@ def py_toolchain_suite(*, prefix, user_repository_name, python_version, set_pyth # the RHS must be a `config_setting`. "//conditions:default": [_IS_EXEC_TOOLCHAIN_ENABLED], }), - exec_compatible_with = kwargs.get("target_compatible_with"), + exec_compatible_with = target_compatible_with, ) # NOTE: When adding a new toolchain, for WORKSPACE builds to see the # toolchain, the name must be added to the native.register_toolchains() # call in python/repositories.bzl. Bzlmod doesn't need anything; it will # register `:all`. + +def define_local_toolchain_suites(name, version_aware_repo_names, version_unaware_repo_names): + """Define toolchains for `local_runtime_repo` backed toolchains. + + This generates `toolchain` targets that can be registered using `:all`. The + specific names of the toolchain targets are not defined. The priority order + of the toolchains is the order that is passed in, with version-aware having + higher priority than version-unaware. + + Args: + name: `str` Unused; only present to satisfy tooling. + version_aware_repo_names: `list[str]` of the repo names that will have + version-aware toolchains defined. + version_unaware_repo_names: `list[str]` of the repo names that will have + version-unaware toolchains defined. + """ + i = 0 + for i, repo in enumerate(version_aware_repo_names, start = i): + prefix = render.left_pad_zero(i, 4) + _internal_toolchain_suite( + prefix = prefix, + runtime_repo_name = repo, + target_compatible_with = ["@{}//:os".format(repo)], + target_settings = ["@{}//:is_matching_python_version".format(repo)], + ) + + # The version unaware entries must go last because they will match any Python + # version. + for i, repo in enumerate(version_unaware_repo_names, start = i + 1): + prefix = render.left_pad_zero(i, 4) + _internal_toolchain_suite( + prefix = prefix, + runtime_repo_name = repo, + target_settings = [], + target_compatible_with = ["@{}//:os".format(repo)], + ) diff --git a/python/private/repo_utils.bzl b/python/private/repo_utils.bzl index 54ad45c2f1..9d76e19833 100644 --- a/python/private/repo_utils.bzl +++ b/python/private/repo_utils.bzl @@ -29,7 +29,7 @@ def _is_repo_debug_enabled(rctx): Returns: True if enabled, False if not. """ - return rctx.os.environ.get(REPO_DEBUG_ENV_VAR) == "1" + return _getenv(rctx, REPO_DEBUG_ENV_VAR) == "1" def _debug_print(rctx, message_cb): """Prints a message if repo debugging is enabled. @@ -46,7 +46,8 @@ def _logger(rctx): """Creates a logger instance for printing messages. Args: - rctx: repository_ctx object. + rctx: repository_ctx object. If the attribute `_rule_name` is + present, it will be included in log messages. Returns: A struct with attributes logging: trace, debug, info, warn, fail. @@ -65,11 +66,20 @@ def _logger(rctx): "TRACE": 3, }.get(verbosity_level, 0) - def _log(enabled_on_verbosity, level, message_cb): + def _log(enabled_on_verbosity, level, message_cb_or_str): if verbosity < enabled_on_verbosity: return + rule_name = getattr(rctx.attr, "_rule_name", "?") + if type(message_cb_or_str) == "string": + message = message_cb_or_str + else: + message = message_cb_or_str() - print("\nrules_python: {}: ".format(level.upper()), message_cb()) # buildifier: disable=print + print("\nrules_python:{}(@@{}) {}:".format( + rule_name, + rctx.name, + level.upper(), + ), message) # buildifier: disable=print return struct( trace = lambda message_cb: _log(3, "TRACE", message_cb), @@ -117,6 +127,7 @@ def _execute_internal( env_str = _env_to_str(environment), )) + rctx.report_progress("Running {}".format(op)) result = rctx.execute(arguments, environment = environment, **kwargs) if fail_on_error and result.return_code != 0: @@ -150,7 +161,18 @@ def _execute_internal( output = _outputs_to_str(result), )) - return result + result_kwargs = {k: getattr(result, k) for k in dir(result)} + return struct( + describe_failure = lambda: _execute_describe_failure( + op = op, + arguments = arguments, + result = result, + rctx = rctx, + kwargs = kwargs, + environment = environment, + ), + **result_kwargs + ) def _execute_unchecked(*args, **kwargs): """Execute a subprocess. @@ -185,6 +207,25 @@ def _execute_checked_stdout(*args, **kwargs): """Calls execute_checked, but only returns the stdout value.""" return _execute_checked(*args, **kwargs).stdout +def _execute_describe_failure(*, op, arguments, result, rctx, kwargs, environment): + return ( + "repo.execute: {op}: failure:\n" + + " command: {cmd}\n" + + " return code: {return_code}\n" + + " working dir: {cwd}\n" + + " timeout: {timeout}\n" + + " environment:{env_str}\n" + + "{output}" + ).format( + op = op, + cmd = _args_to_str(arguments), + return_code = result.return_code, + cwd = _cwd_to_str(rctx, kwargs), + timeout = _timeout_to_str(kwargs), + env_str = _env_to_str(environment), + output = _outputs_to_str(result), + ) + def _which_checked(rctx, binary_name): """Tests to see if a binary exists, and otherwise fails with a message. @@ -195,16 +236,54 @@ def _which_checked(rctx, binary_name): Returns: rctx.Path for the binary. """ + result = _which_unchecked(rctx, binary_name) + if result.binary == None: + fail(result.describe_failure()) + return result.binary + +def _which_unchecked(rctx, binary_name): + """Tests to see if a binary exists. + + This is also watch the `PATH` environment variable. + + Args: + binary_name: name of the binary to find. + rctx: repository context. + + Returns: + `struct` with attributes: + * `binary`: `repository_ctx.Path` + * `describe_failure`: `Callable | None`; takes no args. If the + binary couldn't be found, provides a detailed error description. + """ + path = _getenv(rctx, "PATH", "") binary = rctx.which(binary_name) - if binary == None: - fail(( - "Unable to find the binary '{binary_name}' on PATH.\n" + - " PATH = {path}" - ).format( - binary_name = binary_name, - path = rctx.os.environ.get("PATH"), - )) - return binary + if binary: + _watch(rctx, binary) + describe_failure = None + else: + describe_failure = lambda: _which_describe_failure(binary_name, path) + + return struct( + binary = binary, + describe_failure = describe_failure, + ) + +def _which_describe_failure(binary_name, path): + return ( + "Unable to find the binary '{binary_name}' on PATH.\n" + + " PATH = {path}" + ).format( + binary_name = binary_name, + path = path, + ) + +def _getenv(rctx, name, default = None): + # Bazel 7+ API + if hasattr(rctx, "getenv"): + return rctx.getenv(name, default) + else: + return rctx.os.environ.get("PATH", default) def _args_to_str(arguments): return " ".join([_arg_repr(a) for a in arguments]) @@ -262,12 +341,50 @@ def _outputs_to_str(result): lines.append("<{} empty>".format(name)) return "\n".join(lines) +def _get_platforms_os_name(rctx): + """Return the name in @platforms//os for the host os. + + Args: + rctx: repository_ctx + + Returns: + `str | None`. The target name if it maps to known platforms + value, otherwise None. + """ + os = rctx.os.name.lower() + if "linux" in os: + return os + if "windows" in os: + return "windows" + if "mac" in os: + return "osx" + + return None + +# TODO: Remove after Bazel 6 support dropped +def _watch(rctx, *args, **kwargs): + """Calls rctx.watch, if available.""" + if hasattr(rctx, "watch"): + rctx.watch(*args, **kwargs) + +# TODO: Remove after Bazel 6 support dropped +def _watch_tree(rctx, *args, **kwargs): + """Calls rctx.watch_tree, if available.""" + if hasattr(rctx, "watch_tree"): + rctx.watch_tree(*args, **kwargs) + repo_utils = struct( + # keep sorted + debug_print = _debug_print, execute_checked = _execute_checked, - execute_unchecked = _execute_unchecked, execute_checked_stdout = _execute_checked_stdout, + execute_unchecked = _execute_unchecked, + get_platforms_os_name = _get_platforms_os_name, + getenv = _getenv, is_repo_debug_enabled = _is_repo_debug_enabled, - debug_print = _debug_print, - which_checked = _which_checked, logger = _logger, + watch = _watch, + watch_tree = _watch_tree, + which_checked = _which_checked, + which_unchecked = _which_unchecked, ) diff --git a/python/private/text_util.bzl b/python/private/text_util.bzl index 8a018e7969..38f2b0e404 100644 --- a/python/private/text_util.bzl +++ b/python/private/text_util.bzl @@ -20,6 +20,15 @@ def _indent(text, indent = " " * 4): return "\n".join([indent + line for line in text.splitlines()]) +def _hanging_indent(text, indent = " " * 4): + if "\n" not in text: + return text + + lines = text.splitlines() + for i, line in enumerate(lines): + lines[i] = (indent if i != 0 else "") + line + return "\n".join(lines) + def _render_alias(name, actual, *, visibility = None): args = [ "name = \"{}\",".format(name), @@ -67,14 +76,24 @@ def _render_select(selects, *, no_match_error = None, key_repr = repr, value_rep return "{}({})".format(name, args) -def _render_list(items): +def _render_list(items, *, hanging_indent = ""): + """Convert a list to formatted text. + + Args: + items: list of items. + hanging_indent: str, indent to apply to second and following lines of + the formatted text. + + Returns: + The list pretty formatted as a string. + """ if not items: return "[]" if len(items) == 1: return "[{}]".format(repr(items[0])) - return "\n".join([ + text = "\n".join([ "[", _indent("\n".join([ "{},".format(repr(item)) @@ -82,6 +101,12 @@ def _render_list(items): ])), "]", ]) + if hanging_indent: + text = _hanging_indent(text, hanging_indent) + return text + +def _render_str(value): + return repr(value) def _render_tuple(items, *, value_repr = repr): if not items: @@ -116,10 +141,12 @@ def _left_pad_zero(index, length): render = struct( alias = _render_alias, dict = _render_dict, + hanging_indent = _hanging_indent, indent = _indent, left_pad_zero = _left_pad_zero, list = _render_list, select = _render_select, - tuple = _render_tuple, + str = _render_str, toolchain_prefix = _toolchain_prefix, + tuple = _render_tuple, ) diff --git a/tests/integration/BUILD.bazel b/tests/integration/BUILD.bazel index ac475da534..8724b25280 100644 --- a/tests/integration/BUILD.bazel +++ b/tests/integration/BUILD.bazel @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_binaries//:defs.bzl", "bazel_binaries") load("@rules_bazel_integration_test//bazel_integration_test:defs.bzl", "default_test_runner") load("//python:py_library.bzl", "py_library") load(":integration_test.bzl", "rules_python_integration_test") @@ -84,6 +85,15 @@ rules_python_integration_test( workspace_path = "ignore_root_user_error", ) +rules_python_integration_test( + name = "local_toolchains_test", + bazel_versions = [ + version + for version in bazel_binaries.versions.all + if not version.startswith("6.") + ], +) + rules_python_integration_test( name = "pip_parse_test", ) diff --git a/tests/integration/README.md b/tests/integration/README.md new file mode 100644 index 0000000000..e36e363224 --- /dev/null +++ b/tests/integration/README.md @@ -0,0 +1,21 @@ +# Bazel-in-Bazel integration tests + +The tests in this directory are Bazel-in-Bazel integration tests. These are +necessary because our CI has a limit of 80 jobs, and our test matrix uses most +of those for more important end-to-end tests of user-facing examples. + +The tests in here are more for testing internal aspects of the rules that aren't +easily tested as tests run by Bazel itself (basically anything that happens +prior to the analysis phase). + +## Adding a new directory + +When adding a new diretory, a couple files need to be updated to tell the outer +Bazel to ignore the nested workspace. + +* Add the directory to the `--deleted_packages` flag. Run `pre-commit` and it + will do this for you. This also allows the integration test to see the + nested workspace files correctly. +* Update `.bazelignore` and add `tests/integration//bazel-`. + This prevents Bazel from following infinite symlinks and freezing. +* Add a `rules_python_integration_test` target to the BUILD file. diff --git a/tests/integration/integration_test.bzl b/tests/integration/integration_test.bzl index 7a8070aa1c..8606f66bb3 100644 --- a/tests/integration/integration_test.bzl +++ b/tests/integration/integration_test.bzl @@ -28,6 +28,7 @@ def rules_python_integration_test( gazelle_plugin = False, tags = None, py_main = None, + bazel_versions = None, **kwargs): """Runs a bazel-in-bazel integration test. @@ -42,6 +43,8 @@ def rules_python_integration_test( py_main: Optional `.py` file to run tests using. When specified, a python based test runner is used, and this source file is the main entry point and responsible for executing tests. + bazel_versions: `list[str] | None`, the bazel versions to test. I + not specified, defaults to all configured bazel versions. **kwargs: Passed to the upstream `bazel_integration_tests` rule. """ workspace_path = workspace_path or name.removesuffix("_test") @@ -90,7 +93,7 @@ def rules_python_integration_test( name = name, workspace_path = workspace_path, test_runner = test_runner, - bazel_versions = bazel_binaries.versions.all, + bazel_versions = bazel_versions or bazel_binaries.versions.all, workspace_files = [name + "_workspace_files"], # Override the tags so that the `manual` tag isn't applied. tags = (tags or []) + [ diff --git a/tests/integration/local_toolchains/.bazelrc b/tests/integration/local_toolchains/.bazelrc new file mode 100644 index 0000000000..551df401b3 --- /dev/null +++ b/tests/integration/local_toolchains/.bazelrc @@ -0,0 +1,5 @@ +common --action_env=RULES_PYTHON_BZLMOD_DEBUG=1 +common --lockfile_mode=off +test --test_output=errors +# Windows requires these for multi-python support: +build --enable_runfiles diff --git a/tests/integration/local_toolchains/BUILD.bazel b/tests/integration/local_toolchains/BUILD.bazel new file mode 100644 index 0000000000..6fbf548901 --- /dev/null +++ b/tests/integration/local_toolchains/BUILD.bazel @@ -0,0 +1,20 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load("@rules_python//python:py_test.bzl", "py_test") + +py_test( + name = "test", + srcs = ["test.py"], +) diff --git a/tests/integration/local_toolchains/MODULE.bazel b/tests/integration/local_toolchains/MODULE.bazel new file mode 100644 index 0000000000..d4ef12e952 --- /dev/null +++ b/tests/integration/local_toolchains/MODULE.bazel @@ -0,0 +1,40 @@ +# Copyright 2024 The Bazel Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +module(name = "module_under_test") + +bazel_dep(name = "rules_python", version = "0.0.0") +local_path_override( + module_name = "rules_python", + path = "../../..", +) + +local_runtime_repo = use_repo_rule("@rules_python//python/private:local_runtime_repo.bzl", "local_runtime_repo") + +local_runtime_toolchains_repo = use_repo_rule("@rules_python//python/private:local_runtime_toolchains_repo.bzl", "local_runtime_toolchains_repo") + +local_runtime_repo( + name = "local_python3", + interpreter_path = "python3", + on_failure = "fail", +) + +local_runtime_toolchains_repo( + name = "local_toolchains", + runtimes = ["local_python3"], +) + +python = use_extension("@rules_python//python/extensions:python.bzl", "python") +use_repo(python, "rules_python_bzlmod_debug") + +register_toolchains("@local_toolchains//:all") diff --git a/tests/integration/local_toolchains/REPO.bazel b/tests/integration/local_toolchains/REPO.bazel new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/local_toolchains/WORKSPACE b/tests/integration/local_toolchains/WORKSPACE new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/local_toolchains/WORKSPACE.bzlmod b/tests/integration/local_toolchains/WORKSPACE.bzlmod new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/integration/local_toolchains/test.py b/tests/integration/local_toolchains/test.py new file mode 100644 index 0000000000..63771cf78d --- /dev/null +++ b/tests/integration/local_toolchains/test.py @@ -0,0 +1,26 @@ +import shutil +import subprocess +import sys +import unittest + + +class LocalToolchainTest(unittest.TestCase): + maxDiff = None + + def test_python_from_path_used(self): + shell_path = shutil.which("python3") + + # We call the interpreter and print its executable because of + # things like pyenv: they install a shim that re-execs python. + # The shim is e.g. /home/user/.pyenv/shims/python3, which then + # runs e.g. /usr/bin/python3 + expected = subprocess.check_output( + [shell_path, "-c", "import sys; print(sys.executable)"], + text=True, + ) + expected = expected.strip() + self.assertEqual(expected, sys.executable) + + +if __name__ == "__main__": + unittest.main()