From b6507095667d277c78b7251a2611714db304ca7e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Thu, 23 May 2024 20:28:37 -0700 Subject: [PATCH 1/2] fix: don't require system Python to perform bootstrapping This is a pretty major, but surprisingly non-invasive, overhaul of how binaries are started. It fixes several issues and lays ground work for future improvements. In brief: * A system Python is no longer needed to perform bootstrapping. * Errors due to `PYTHONPATH` exceeding environment variable size limits is no longer an issue. * Coverage integration is now cleaner and more direct. * The zipapp `__main__.py` entry point generation is separate from the Bazel binary bootstrap generation. * Self-executable zips now have actual bootstrap logic. The way all of this is accomplished is using a two stage bootstrap process. The first stage is responsible for locating the interpreter, and the second stage is responsible for configuring the runtime environment (e.g. import paths). This allows the first stage to be relatively simple (basically find a file in runfiles), so implementing it in cross-platform shell is feasible. The second stage, because it's running under the desired interpreter, can then do things like setting up import paths, and use the `runpy` module to call the program's real main. This also fixes the issue of long `PYTHONPATH` environment variables causing an error. Instead of passing the import paths using an environment variable, they are embedded into the second stage bootstrap, which can then add them to sys.path. This also switches from running coverage as a subprocess to using its APIs directly. This is possible because of the second stage bootstrap, which can rely on `import coverage` occurring in the correct environment. This new bootstrap method is disabled by default. It can be enabled by setting `--@rules_python//python/config_settings:bootstrap_impl=two_stage`. Once the new APIs are released, a subsequent release will make it the default. This is to allow easier upgrades for people defining their own toolchains. --- CHANGELOG.md | 15 +- CONTRIBUTING.md | 1 + .../api/python/config_settings/index.md | 31 ++ docs/sphinx/api/python/index.md | 20 + docs/sphinx/bazel_inventory.txt | 5 +- docs/sphinx/pip.md | 2 +- docs/sphinx/support.md | 3 +- docs/sphinx/toolchains.md | 23 +- examples/bzlmod/test.py | 44 +- python/BUILD.bazel | 8 +- python/config_settings/BUILD.bazel | 10 + python/private/BUILD.bazel | 45 ++ python/private/autodetecting_toolchain.bzl | 2 +- python/private/common/common.bzl | 4 +- python/private/common/providers.bzl | 93 +++- python/private/common/py_executable.bzl | 8 +- python/private/common/py_executable_bazel.bzl | 201 +++++-- python/private/common/py_runtime_rule.bzl | 39 +- python/private/flags.bzl | 10 + python/private/python_bootstrap_template.txt | 2 +- python/private/stage1_bootstrap_template.sh | 118 ++++ python/private/stage2_bootstrap_template.py | 510 ++++++++++++++++++ python/private/zip_main_template.py | 293 ++++++++++ python/repositories.bzl | 8 +- tests/base_rules/py_executable_base_tests.bzl | 4 +- tests/base_rules/py_test/py_test_tests.bzl | 21 +- tests/support/support.bzl | 3 + 27 files changed, 1442 insertions(+), 81 deletions(-) create mode 100644 python/private/stage1_bootstrap_template.sh create mode 100644 python/private/stage2_bootstrap_template.py create mode 100644 python/private/zip_main_template.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 3fdd039a5d..e331a8613d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,6 @@ +:::{default-domain} bzl +::: + # rules_python Changelog This is a human-friendly changelog in a keepachangelog.com style format. @@ -31,7 +34,7 @@ A brief description of the categories of changes: marked as `reproducible` and will not include any lock file entries from now on. -* (gazelle): Remove gazelle plugin's python deps and make it hermetic. +* (gazelle): Remove gazelle plugin's python deps and make it hermetic. Introduced a new Go-based helper leveraging tree-sitter for syntax analysis. Implemented the use of `pypi/stdlib-list` for standard library module verification. @@ -80,6 +83,16 @@ A brief description of the categories of changes: invalid usage previously but we were not failing the build. From now on this is explicitly disallowed. * (toolchains) Added riscv64 platform definition for python toolchains. +* (rules) A new bootstrap implementation that doesn't require a system Python + is available. It can be enabled by setting + {obj}`--@rules_python//python:config_settings:bootstrap_impl=two_phase`. It + will become the default in a subsequent release. + ([#691](https://github.com/bazelbuild/rules_python/issues/691)) +* (providers) `PyRuntimeInfo` has two new attributes: + {obj}`PyRuntimeInfo.stage2_bootstrap_template` and + {obj}`PyRuntimeInfo.zip_main_template`. +* (toolchains) A replacement for the Bazel-builtn autodetecting toolchain is + available. The `//python:autodetecting_toolchain` alias now uses it. [precompile-docs]: /precompiling diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10d1149cc7..cb123bfee0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -175,6 +175,7 @@ Issues should be triaged as follows: functionality, should also be filed in this repository but without the `core-rules` label. +(breaking-changes)= ## Breaking Changes Breaking changes are generally permitted, but we follow a 3-step process for diff --git a/docs/sphinx/api/python/config_settings/index.md b/docs/sphinx/api/python/config_settings/index.md index 82a5b2a520..29779fd813 100644 --- a/docs/sphinx/api/python/config_settings/index.md +++ b/docs/sphinx/api/python/config_settings/index.md @@ -1,3 +1,5 @@ +:::{default-domain} bzl +::: :::{bzl:currentfile} //python/config_settings:BUILD.bazel ::: @@ -66,3 +68,32 @@ Values: * `include_pyc`: Include `PyInfo.transitive_pyc_files` as part of the binary. * `disabled`: Don't include `PyInfo.transitive_pyc_files` as part of the binary. ::: + +::::{bzl:flag} bootstrap_impl +Determine how programs implement their startup process. + +Values: +* `system_python`: Use a bootstrap that requires a system Python available + in order to start programs. This requires + {obj}`PyRuntimeInfo.bootstrap_template` to be a Python program. +* `script`: Use a bootstrap that uses an arbitrary executable script (usually a + shell script) instead of requiring it be a Python program. + +:::{note} +The `script` bootstrap requires the toolchain to provide the `PyRuntimeInfo` +provider from `rules_python`. This loosely translates to using Bazel 7+ with a +toolchain created by rules_python. Most notably, WORKSPACE builds default to +using a legacy toolchain built into Bazel itself which doesn't support the +script bootstrap. If not available, the `system_python` bootstrap will be used +instead. +::: + +:::{seealso} +{obj}`PyRuntimeInfo.bootstrap_template` and +{obj}`PyRuntimeInfo.stage2_bootstrap_template` +::: + +:::{versionadded} 0.33.0 +::: + +:::: diff --git a/docs/sphinx/api/python/index.md b/docs/sphinx/api/python/index.md index 8026a7f145..494e7b4a02 100644 --- a/docs/sphinx/api/python/index.md +++ b/docs/sphinx/api/python/index.md @@ -1,3 +1,5 @@ +:::{default-domain} bzl +::: :::{bzl:currentfile} //python:BUILD.bazel ::: @@ -21,3 +23,21 @@ provides: * `PyRuntimeInfo`: The consuming target's target toolchain information ::: + +::::{target} autodetecting_toolchain + +A simple toolchain that simply uses `python3` from the runtime environment. + +Note that this toolchain provides no build-time information, which makes it of +limited utility. + +This is only provided to aid migration off the builtin Bazel toolchain +(`@bazel_tools//python:autodetecting_toolchain`), and is largely only applicable +to WORKSPACE builds. + +:::{deprecated} unspecified + +Switch to using a hermetic toolchain or manual toolchain configuration instead. +::: + +:::: diff --git a/docs/sphinx/bazel_inventory.txt b/docs/sphinx/bazel_inventory.txt index 62cbdf8926..c4aaabc074 100644 --- a/docs/sphinx/bazel_inventory.txt +++ b/docs/sphinx/bazel_inventory.txt @@ -10,7 +10,7 @@ bool bzl:type 1 rules/lib/bool - int bzl:type 1 rules/lib/int - depset bzl:type 1 rules/lib/depset - dict bzl:type 1 rules/lib/dict - -label bzl:doc 1 concepts/labels - +label bzl:type 1 concepts/labels - attr.bool bzl:type 1 rules/lib/toplevel/attr#bool - attr.int bzl:type 1 rules/lib/toplevel/attr#int - attr.label bzl:type 1 rules/lib/toplevel/attr#label - @@ -21,6 +21,7 @@ list bzl:type 1 rules/lib/list - python bzl:doc 1 reference/be/python - str bzl:type 1 rules/lib/string - struct bzl:type 1 rules/lib/builtins/struct - -target-name bzl:doc 1 concepts/labels#target-names - +Name bzl:type 1 concepts/labels#target-names - CcInfo bzl:provider 1 rules/lib/providers/CcInfo - CcInfo.linking_context bzl:provider-field 1 rules/lib/providers/CcInfo#linking_context - +ToolchainInfo bzl:type 1 rules/lib/providers/ToolchainInfo.html - diff --git a/docs/sphinx/pip.md b/docs/sphinx/pip.md index e1c8e343f0..fc29e41b5e 100644 --- a/docs/sphinx/pip.md +++ b/docs/sphinx/pip.md @@ -150,7 +150,7 @@ ARG=$1 # but we don't do anything with it as it's always "get" # formatting is optional echo '{' echo ' "headers": {' -echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="] +echo ' "Authorization": ["Basic dGVzdDoxMjPCow=="]' echo ' }' echo '}' ``` diff --git a/docs/sphinx/support.md b/docs/sphinx/support.md index a2b8e3ae20..ea099650bd 100644 --- a/docs/sphinx/support.md +++ b/docs/sphinx/support.md @@ -46,7 +46,8 @@ incremental fashion. Breaking changes are allowed, but follow a process to introduce them over a series of releases to so users can still incrementally upgrade. See the -[Breaking Changes](contributing#breaking-changes) doc for the process. +[Breaking Changes](#breaking-changes) doc for the process. + ## Experimental Features diff --git a/docs/sphinx/toolchains.md b/docs/sphinx/toolchains.md index bac89660bb..e3be22f97b 100644 --- a/docs/sphinx/toolchains.md +++ b/docs/sphinx/toolchains.md @@ -1,3 +1,6 @@ +:::{default-domain} bzl +::: + # Configuring Python toolchains and runtimes This documents how to configure the Python toolchain and runtimes for different @@ -193,7 +196,7 @@ load("@rules_python//python:repositories.bzl", "py_repositories") py_repositories() ``` -#### Workspace toolchain registration +### Workspace toolchain registration To register a hermetic Python toolchain rather than rely on a system-installed interpreter for runtime execution, you can add to the `WORKSPACE` file: @@ -221,3 +224,21 @@ pip_parse( After registration, your Python targets will use the toolchain's interpreter during execution, but a system-installed interpreter is still used to 'bootstrap' Python targets (see https://github.com/bazelbuild/rules_python/issues/691). You may also find some quirks while using this toolchain. Please refer to [python-build-standalone documentation's _Quirks_ section](https://gregoryszorc.com/docs/python-build-standalone/main/quirks.html). + +## Autodetecting toolchain + +The autodetecting toolchain is a deprecated toolchain that is built into Bazel. +It's name is a bit misleading: it doesn't autodetect anything. All it does is +use `python3` from the environment a binary runs within. This provides extremely +limited functionality to the rules (at build time, nothing is knowable about +the Python runtime). + +Bazel itself automatically registers `@bazel_tools//python:autodetecting_toolchain` +as the lowest priority toolchain. For WORKSPACE builds, if no other toolchain +is registered, that toolchain will be used. For bzlmod builds, rules_python +automatically registers a higher-priority toolchain; it won't be used unless +there is a toolchain misconfiguration somewhere. + +To aid migration off the Bazel-builtin toolchain, rules_python provides +{obj}`@rules_python//python:autodetecting_toolchain`. This is an equivalent +toolchain, but is implemented using rules_python's objects. diff --git a/examples/bzlmod/test.py b/examples/bzlmod/test.py index 533187557d..950c002919 100644 --- a/examples/bzlmod/test.py +++ b/examples/bzlmod/test.py @@ -14,6 +14,7 @@ import os import pathlib +import re import sys import unittest @@ -63,16 +64,47 @@ def test_coverage_sys_path(self): first_item.endswith("coverage"), f"Expected the first item in sys.path '{first_item}' to not be related to coverage", ) + + # We're trying to make sure that the coverage library added by the + # toolchain is _after_ any user-provided dependencies. This lets users + # override what coverage version they're using. + first_coverage_index = None + last_user_dep_index = None + for i, path in enumerate(sys.path): + if re.search("rules_python.*~pip~", path): + last_user_dep_index = i + if first_coverage_index is None and re.search( + ".*rules_python.*~python~.*coverage.*", path + ): + first_coverage_index = i + if os.environ.get("COVERAGE_MANIFEST"): + self.assertIsNotNone( + first_coverage_index, + "Expected to find toolchain coverage, but " + + f"it was not found.\nsys.path:\n{all_paths}", + ) + self.assertIsNotNone( + first_coverage_index, + "Expected to find at least one uiser dep, " + + "but none were found.\nsys.path:\n{all_paths}", + ) # we are running under the 'bazel coverage :test' - self.assertTrue( - "_coverage" in last_item, - f"Expected {last_item} to be related to coverage", + self.assertGreater( + first_coverage_index, + last_user_dep_index, + "Expected coverage provided by the toolchain to be after " + + "user provided dependencies.\n" + + f"Found coverage at index: {first_coverage_index}\n" + + f"Last user dep at index: {last_user_dep_index}\n" + + f"Full sys.path:\n{all_paths}", ) - self.assertEqual(pathlib.Path(last_item).name, "coverage") else: - self.assertFalse( - "coverage" in last_item, f"Expected coverage tooling to not be present" + self.assertIsNone( + first_coverage_index, + "Expected toolchain coverage to not be present\n" + + f"Found coverage at index: {first_coverage_index}\n" + + f"Full sys.path:\n{all_paths}", ) def test_main(self): diff --git a/python/BUILD.bazel b/python/BUILD.bazel index 5d31df5e9a..cbf29964fb 100644 --- a/python/BUILD.bazel +++ b/python/BUILD.bazel @@ -24,6 +24,7 @@ that @rules_python//python is only concerned with the core rules. """ load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("//python/private:autodetecting_toolchain.bzl", "define_autodetecting_toolchain") load(":current_py_toolchain.bzl", "current_py_toolchain") package(default_visibility = ["//visibility:public"]) @@ -318,14 +319,11 @@ toolchain_type( # safe if you know for a fact that your build is completely compatible with the # version of the `python` command installed on the target platform. -alias( - name = "autodetecting_toolchain", - actual = "@bazel_tools//tools/python:autodetecting_toolchain", -) +define_autodetecting_toolchain(name = "autodetecting_toolchain") alias( name = "autodetecting_toolchain_nonstrict", - actual = "@bazel_tools//tools/python:autodetecting_toolchain_nonstrict", + actual = ":autodetecting_toolchain", ) # ========= Packaging rules ========= diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index a0e59f70c0..a91ba12764 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -1,6 +1,7 @@ load("@bazel_skylib//rules:common_settings.bzl", "string_flag") load( "//python/private:flags.bzl", + "BootstrapImplFlag", "PrecompileAddToRunfilesFlag", "PrecompileFlag", "PrecompileSourceRetentionFlag", @@ -52,3 +53,12 @@ string_flag( # NOTE: Only public because its an implicit dependency visibility = ["//visibility:public"], ) + +string_flag( + name = "bootstrap_impl", + # todo: default to deprecated_one_stage initially + build_setting_default = BootstrapImplFlag.SCRIPT, + values = sorted(BootstrapImplFlag.__members__.values()), + # NOTE: Only public because its an implicit dependency + visibility = ["//visibility:public"], +) diff --git a/python/private/BUILD.bazel b/python/private/BUILD.bazel index 3e56208859..1dc6c88ae8 100644 --- a/python/private/BUILD.bazel +++ b/python/private/BUILD.bazel @@ -376,9 +376,54 @@ exports_files( visibility = ["//visibility:public"], ) +filegroup( + name = "stage1_bootstrap_template", + srcs = ["stage1_bootstrap_template.sh"], + # Not actually public. Only public because it's an implicit dependency of + # py_runtime. + visibility = ["//visibility:public"], +) + +filegroup( + name = "stage2_bootstrap_template", + srcs = ["stage2_bootstrap_template.py"], + # Not actually public. Only public because it's an implicit dependency of + # py_runtime. + visibility = ["//visibility:public"], +) + +filegroup( + name = "zip_main_template", + srcs = ["zip_main_template.py"], + # Not actually public. Only public because it's an implicit dependency of + # py_runtime. + visibility = ["//visibility:public"], +) + +# NOTE: Windows builds don't use this bootstrap. Instead, a native Windows +# program locates some Python exe and runs `python.exe foo.zip` which +# runs the __main__.py in the zip file. +alias( + name = "bootstrap_template", + actual = select({ + ":is_script_bootstrap_enabled": "stage1_bootstrap_template.sh", + "//conditions:default": "python_bootstrap_template.txt", + }), + # Not actually public. Only public because it's an implicit dependency of + # py_runtime. + visibility = ["//visibility:public"], +) + # Used to determine the use of `--stamp` in Starlark rules stamp_build_setting(name = "stamp") +config_setting( + name = "is_script_bootstrap_enabled", + flag_values = { + "//python/config_settings:bootstrap_impl": "script", + }, +) + print_toolchains_checksums(name = "print_toolchains_checksums") # Used for py_console_script_gen rule diff --git a/python/private/autodetecting_toolchain.bzl b/python/private/autodetecting_toolchain.bzl index 3caa5aa8ca..55c95699c9 100644 --- a/python/private/autodetecting_toolchain.bzl +++ b/python/private/autodetecting_toolchain.bzl @@ -32,7 +32,7 @@ def define_autodetecting_toolchain(name): # buildifier: disable=native-py py_runtime( name = "_autodetecting_py3_runtime", - interpreter = ":py3wrapper.sh", + interpreter = "//python/private:autodetecting_toolchain_interpreter.sh", python_version = "PY3", stub_shebang = "#!/usr/bin/env python3", visibility = ["//visibility:private"], diff --git a/python/private/common/common.bzl b/python/private/common/common.bzl index cfa7db7a2d..0ac9187b79 100644 --- a/python/private/common/common.bzl +++ b/python/private/common/common.bzl @@ -182,7 +182,7 @@ def create_cc_details_struct( cc_toolchain = cc_toolchain, ) -def create_executable_result_struct(*, extra_files_to_build, output_groups): +def create_executable_result_struct(*, extra_files_to_build, output_groups, extra_runfiles = None): """Creates a `CreateExecutableResult` struct. This is the return value type of the semantics create_executable function. @@ -192,6 +192,7 @@ def create_executable_result_struct(*, extra_files_to_build, output_groups): included as default outputs. output_groups: dict[str, depset[File]]; additional output groups that should be returned. + extra_runfiles: A runfiles object of additional runfiles to include. Returns: A `CreateExecutableResult` struct. @@ -199,6 +200,7 @@ def create_executable_result_struct(*, extra_files_to_build, output_groups): return struct( extra_files_to_build = extra_files_to_build, output_groups = output_groups, + extra_runfiles = extra_runfiles, ) def union_attrs(*attr_dicts, allow_none = False): diff --git a/python/private/common/providers.bzl b/python/private/common/providers.bzl index 5b84549185..e1876ff9d3 100644 --- a/python/private/common/providers.bzl +++ b/python/private/common/providers.bzl @@ -18,7 +18,7 @@ load("//python/private:util.bzl", "IS_BAZEL_6_OR_HIGHER") DEFAULT_STUB_SHEBANG = "#!/usr/bin/env python3" -DEFAULT_BOOTSTRAP_TEMPLATE = Label("//python/private:python_bootstrap_template.txt") +DEFAULT_BOOTSTRAP_TEMPLATE = Label("//python/private:bootstrap_template") _PYTHON_VERSION_VALUES = ["PY2", "PY3"] @@ -78,7 +78,9 @@ def _PyRuntimeInfo_init( python_version, stub_shebang = None, bootstrap_template = None, - interpreter_version_info = None): + interpreter_version_info = None, + stage2_bootstrap_template = None, + zip_main_template = None): if (interpreter_path and interpreter) or (not interpreter_path and not interpreter): fail("exactly one of interpreter or interpreter_path must be specified") @@ -126,7 +128,9 @@ def _PyRuntimeInfo_init( "interpreter_version_info": interpreter_version_info_struct_from_dict(interpreter_version_info), "pyc_tag": pyc_tag, "python_version": python_version, + "stage2_bootstrap_template": stage2_bootstrap_template, "stub_shebang": stub_shebang, + "zip_main_template": zip_main_template, } # TODO(#15897): Rename this to PyRuntimeInfo when we're ready to replace the Java @@ -147,7 +151,45 @@ the same conventions as the standard CPython interpreter. "bootstrap_template": """ :type: File -See py_runtime_rule.bzl%py_runtime.bootstrap_template for docs. +A template of code responsible for the initial startup of a program. + +This code is responsible for: + +* Locating the target interpreter. Typically it is in runfiles, but not always. +* Setting necessary environment variables, command line flags, or other + configuration that can't be modified after the interpreter starts. +* Invoking the appropriate entry point. This is usually a second-stage bootstrap + that performs additional setup prior to running a program's actual entry point. + +The {obj}`--bootstrap_impl` flag affects how this stage 1 bootstrap +is expected to behave and the substutitions performed. + +* `--bootstrap_impl=system_python` substitutions: `%is_zipfile%`, `%python_binary%`, + `%target%`, `%workspace_name`, `%coverage_tool%`, `%import_all%`, `%imports%`, + `%main%`, `%shebang%` +* `--bootstrap_impl=script` substititions: `%is_zipfile%`, `%python_binary%`, + `%target%`, `%workspace_name`, `%shebang%, `%stage2_bootstrap%` + +Substitution definitions: + +* `%shebang%`: The shebang to use with the bootstrap; the bootstrap template + may choose to ignore this. +* `%stage2_bootstrap%`: A runfiles-relative path to the stage 2 bootstrap. +* `%python_binary%`: The path to the target Python interpreter. There are three + types of paths: + * An absolute path to a system interpreter (e.g. begins with `/`). + * A runfiles-relative path to an interpreter (e.g. `somerepo/bin/python3`) + * A program to search for on PATH, i.e. a word without spaces, e.g. `python3`. +* `%workspace_name%`: The name of the workspace the target belongs to. +* `%is_zipfile%`: The string `1` if this template is prepended to a zipfile to + create a self-executable zip file. The string `0` otherwise. + +For the other substitution definitions, see the {obj}`stage2_bootstrap_template` +docs. + +:::{versionchanged} 0.33.0 +The set of substitutions depends on {obj}`--bootstrap_impl` +::: """, "coverage_files": """ :type: depset[File] | None @@ -216,6 +258,30 @@ correctly. Indicates whether this runtime uses Python major version 2 or 3. Valid values are (only) `"PY2"` and `"PY3"`. +""", + "stage2_bootstrap_template": """ +:type: File + +A template of Python code that runs under the desired interpreter and is +responsible for orchestrating calling the program's actual main code. This +bootstrap is responsible for affecting the current runtime's state, such as +import paths or enabling coverage, so that, when it runs the program's actual +main code, it works properly under Bazel. + +The following substitutions are made during template expansion: +* `%main%`: A runfiles-relative path to the program's actual main file. This + can be a `.py` or `.pyc` file, depending on precompile settings. +* `%coverage_tool%`: Runfiles-relative path to the coverage library's entry point. + If coverage is not enabled or available, an empty string. +* `%import_all%`: The string `True` if all repositories in the runfiles should + be added to sys.path. The string `False` otherwise. +* `%imports%`: A colon-delimited string of runfiles-relative paths to add to + sys.path. +* `%target%`: The name of the target this is for. +* `%workspace_name%`: The name of the workspace the target belongs to. + +:::{versionadded} 0.33.0 +::: """, "stub_shebang": """ :type: str @@ -223,6 +289,27 @@ are (only) `"PY2"` and `"PY3"`. "Shebang" expression prepended to the bootstrapping Python stub script used when executing {obj}`py_binary` targets. Does not apply to Windows. +""", + "zip_main_template": """ +:type: File + +A template of Python code that becomes a zip file's top-level `__main__.py` +file. The top-level `__main__.py` file is used when the zip file is explicitly +passed to a Python interpreter. See PEP 441 for more information about zipapp +support. Note that py_binary-generated zip files are self-executing and +skip calling `__main__.py`. + +The following substitutions are made during template expansion: +* `%stage2_bootstrap%`: A runfiles-relative string to the stage 2 bootstrap file. +* `%python_binary%`: The path to the target Python interpreter. There are three + types of paths: + * An absolute path to a system interpreter (e.g. begins with `/`). + * A runfiles-relative path to an interpreter (e.g. `somerepo/bin/python3`) + * A program to search for on PATH, i.e. a word without spaces, e.g. `python3`. +* `%workspace_name%`: The name of the workspace for the built target. + +:::{versionadded} 0.33.0 +::: """, }, ) diff --git a/python/private/common/py_executable.bzl b/python/private/common/py_executable.bzl index cf7d6fad50..ff1f74de99 100644 --- a/python/private/common/py_executable.bzl +++ b/python/private/common/py_executable.bzl @@ -118,6 +118,10 @@ Valid values are: values = ["PY2", "PY3"], doc = "Defunct, unused, does nothing.", ), + "_bootstrap_impl_flag": attr.label( + default = "//python/config_settings:bootstrap_impl", + providers = [BuildSettingInfo], + ), "_pyc_collection_flag": attr.label( default = "//python/config_settings:pyc_collection", providers = [BuildSettingInfo], @@ -212,7 +216,9 @@ def py_executable_base_impl(ctx, *, semantics, is_test, inherited_environment = runfiles_details = runfiles_details, ) - extra_exec_runfiles = ctx.runfiles(transitive_files = exec_result.extra_files_to_build) + extra_exec_runfiles = exec_result.extra_runfiles.merge( + ctx.runfiles(transitive_files = exec_result.extra_files_to_build), + ) runfiles_details = struct( default_runfiles = runfiles_details.default_runfiles.merge(extra_exec_runfiles), data_runfiles = runfiles_details.data_runfiles.merge(extra_exec_runfiles), diff --git a/python/private/common/py_executable_bazel.bzl b/python/private/common/py_executable_bazel.bzl index 1c41fc15e5..53d70f00b9 100644 --- a/python/private/common/py_executable_bazel.bzl +++ b/python/private/common/py_executable_bazel.bzl @@ -15,6 +15,7 @@ load("@bazel_skylib//lib:dicts.bzl", "dicts") load("@bazel_skylib//lib:paths.bzl", "paths") +load("//python/private:flags.bzl", "BootstrapImplFlag") load(":attributes_bazel.bzl", "IMPORTS_ATTRS") load( ":common.bzl", @@ -166,12 +167,6 @@ def _create_executable( runfiles_details): _ = is_test, cc_details, native_deps_details # @unused - common_bootstrap_template_kwargs = dict( - main_py = main_py, - imports = imports, - runtime_details = runtime_details, - ) - is_windows = target_platform_has_any_constraint(ctx, ctx.attr._windows_constraints) if is_windows: @@ -181,21 +176,47 @@ def _create_executable( else: base_executable_name = executable.basename - zip_bootstrap = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) - zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) + # The check for stage2_bootstrap_template is to support legacy + # BuiltinPyRuntimeInfo providers, which is likely to come from + # @bazel_tools//tools/python:autodetecting_toolchain, the toolchain used + # for workspace builds when no rules_python toolchain is configured. + if (BootstrapImplFlag.get_value(ctx) == BootstrapImplFlag.SCRIPT and + runtime_details.effective_runtime and + hasattr(runtime_details.effective_runtime, "stage2_bootstrap_template")): + stage2_bootstrap = _create_stage2_bootstrap( + ctx, + output_prefix = base_executable_name, + output_sibling = executable, + main_py = main_py, + imports = imports, + runtime_details = runtime_details, + ) + extra_runfiles = ctx.runfiles([stage2_bootstrap]) + zip_main = _create_zip_main( + ctx, + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + ) + else: + stage2_bootstrap = None + extra_runfiles = ctx.runfiles() + zip_main = ctx.actions.declare_file(base_executable_name + ".temp", sibling = executable) + _create_stage1_bootstrap( + ctx, + output = zip_main, + main_py = main_py, + imports = imports, + is_for_zip = True, + runtime_details = runtime_details, + ) - _expand_bootstrap_template( - ctx, - output = zip_bootstrap, - is_for_zip = True, - **common_bootstrap_template_kwargs - ) + zip_file = ctx.actions.declare_file(base_executable_name + ".zip", sibling = executable) _create_zip_file( ctx, output = zip_file, original_nonzip_executable = executable, - executable_for_zip_file = zip_bootstrap, - runfiles = runfiles_details.default_runfiles, + zip_main = zip_main, + runfiles = runfiles_details.default_runfiles.merge(extra_runfiles), ) extra_files_to_build = [] @@ -244,13 +265,23 @@ def _create_executable( if bootstrap_output != None: fail("Should not occur: bootstrap_output should not be used " + "when creating an executable zip") - _create_executable_zip_file(ctx, output = executable, zip_file = zip_file) + _create_executable_zip_file( + ctx, + output = executable, + zip_file = zip_file, + python_binary_path = runtime_details.executable_interpreter_path, + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + ) elif bootstrap_output: - _expand_bootstrap_template( + _create_stage1_bootstrap( ctx, output = bootstrap_output, - is_for_zip = build_zip_enabled, - **common_bootstrap_template_kwargs + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + is_for_zip = False, + imports = imports, + main_py = main_py, ) else: # Otherwise, this should be the Windows case of launcher + zip. @@ -268,16 +299,40 @@ def _create_executable( return create_executable_result_struct( extra_files_to_build = depset(extra_files_to_build), output_groups = {"python_zip_file": depset([zip_file])}, + extra_runfiles = extra_runfiles, + ) + +def _create_zip_main(ctx, *, stage2_bootstrap, runtime_details): + # The location of this file doesn't really matter. It's added to + # the zip file as the top-level __main__.py file and not included + # elsewhere. + output = ctx.actions.declare_file(ctx.label.name + "_zip__main__.py") + ctx.actions.expand_template( + template = runtime_details.effective_runtime.zip_main_template, + output = output, + substitutions = { + "%python_binary%": runtime_details.executable_interpreter_path, + "%stage2_bootstrap%": "{}/{}".format( + ctx.workspace_name, + stage2_bootstrap.short_path, + ), + "%workspace_name%": ctx.workspace_name, + }, ) + return output -def _expand_bootstrap_template( +def _create_stage2_bootstrap( ctx, *, - output, + output_prefix, + output_sibling, main_py, imports, - is_for_zip, runtime_details): + output = ctx.actions.declare_file( + "{}_stage2_bootstrap.py".format(output_prefix), + sibling = output_sibling, + ) runtime = runtime_details.effective_runtime if (ctx.configuration.coverage_enabled and runtime and @@ -289,12 +344,7 @@ def _expand_bootstrap_template( else: coverage_tool_runfiles_path = "" - if runtime: - shebang = runtime.stub_shebang - template = runtime.bootstrap_template - else: - shebang = DEFAULT_STUB_SHEBANG - template = ctx.file._bootstrap_template + template = runtime.stage2_bootstrap_template ctx.actions.expand_template( template = template, @@ -303,18 +353,66 @@ def _expand_bootstrap_template( "%coverage_tool%": coverage_tool_runfiles_path, "%import_all%": "True" if ctx.fragments.bazel_py.python_import_all_repositories else "False", "%imports%": ":".join(imports.to_list()), - "%is_zipfile%": "True" if is_for_zip else "False", - "%main%": "{}/{}".format( - ctx.workspace_name, - main_py.short_path, - ), - "%python_binary%": runtime_details.executable_interpreter_path, - "%shebang%": shebang, + "%main%": "{}/{}".format(ctx.workspace_name, main_py.short_path), "%target%": str(ctx.label), "%workspace_name%": ctx.workspace_name, }, is_executable = True, ) + return output + +def _create_stage1_bootstrap( + ctx, + *, + output, + main_py = None, + stage2_bootstrap = None, + imports = None, + is_for_zip, + runtime_details): + runtime = runtime_details.effective_runtime + + subs = { + "%is_zipfile%": "1" if is_for_zip else "0", + "%python_binary%": runtime_details.executable_interpreter_path, + "%target%": str(ctx.label), + "%workspace_name%": ctx.workspace_name, + } + + if stage2_bootstrap: + subs["%stage2_bootstrap%"] = "{}/{}".format( + ctx.workspace_name, + stage2_bootstrap.short_path, + ) + template = runtime.bootstrap_template + subs["%shebang%"] = runtime.stub_shebang + else: + if (ctx.configuration.coverage_enabled and + runtime and + runtime.coverage_tool): + coverage_tool_runfiles_path = "{}/{}".format( + ctx.workspace_name, + runtime.coverage_tool.short_path, + ) + else: + coverage_tool_runfiles_path = "" + if runtime: + subs["%shebang%"] = runtime.stub_shebang + template = runtime.bootstrap_template + else: + subs["%shebang%"] = DEFAULT_STUB_SHEBANG + template = ctx.file._bootstrap_template + + subs["%coverage_tool%"] = coverage_tool_runfiles_path + subs["%import_all%"] = ("True" if ctx.fragments.bazel_py.python_import_all_repositories else "False") + subs["%imports%"] = ":".join(imports.to_list()) + subs["%main%"] = "{}/{}".format(ctx.workspace_name, main_py.short_path) + + ctx.actions.expand_template( + template = template, + output = output, + substitutions = subs, + ) def _create_windows_exe_launcher( ctx, @@ -346,7 +444,7 @@ def _create_windows_exe_launcher( use_default_shell_env = True, ) -def _create_zip_file(ctx, *, output, original_nonzip_executable, executable_for_zip_file, runfiles): +def _create_zip_file(ctx, *, output, original_nonzip_executable, zip_main, runfiles): workspace_name = ctx.workspace_name legacy_external_runfiles = _py_builtins.get_legacy_external_runfiles(ctx) @@ -354,7 +452,7 @@ def _create_zip_file(ctx, *, output, original_nonzip_executable, executable_for_ manifest.use_param_file("@%s", use_always = True) manifest.set_param_file_format("multiline") - manifest.add("__main__.py={}".format(executable_for_zip_file.path)) + manifest.add("__main__.py={}".format(zip_main.path)) manifest.add("__init__.py=") manifest.add( "{}=".format( @@ -375,7 +473,7 @@ def _create_zip_file(ctx, *, output, original_nonzip_executable, executable_for_ manifest.add_all(runfiles.files, map_each = map_zip_runfiles, allow_closure = True) - inputs = [executable_for_zip_file] + inputs = [zip_main] if _py_builtins.is_bzlmod_enabled(ctx): zip_repo_mapping_manifest = ctx.actions.declare_file( output.basename + ".repo_mapping", @@ -424,17 +522,32 @@ def _get_zip_runfiles_path(path, workspace_name, legacy_external_runfiles): zip_runfiles_path = paths.normalize("{}/{}".format(workspace_name, path)) return "{}/{}".format(_ZIP_RUNFILES_DIRECTORY_NAME, zip_runfiles_path) -def _create_executable_zip_file(ctx, *, output, zip_file): +def _create_executable_zip_file(ctx, *, output, zip_file, stage2_bootstrap, runtime_details): + prelude = ctx.actions.declare_file( + "{}_zip_prelude.sh".format(output.basename), + sibling = output, + ) + if stage2_bootstrap: + _create_stage1_bootstrap( + ctx, + output = prelude, + stage2_bootstrap = stage2_bootstrap, + runtime_details = runtime_details, + is_for_zip = True, + ) + else: + ctx.actions.write(prelude, "#!/usr/bin/env python3\n") + ctx.actions.run_shell( - command = "echo '{shebang}' | cat - {zip} > {output}".format( - shebang = "#!/usr/bin/env python3", + command = "cat {prelude} {zip} > {output}".format( + prelude = prelude.path, zip = zip_file.path, output = output.path, ), - inputs = [zip_file], + inputs = [prelude, zip_file], outputs = [output], use_default_shell_env = True, - mnemonic = "BuildBinary", + mnemonic = "PyBuildExecutableZip", progress_message = "Build Python zip executable: %{label}", ) diff --git a/python/private/common/py_runtime_rule.bzl b/python/private/common/py_runtime_rule.bzl index 53d925cdba..a7eeb7e3ec 100644 --- a/python/private/common/py_runtime_rule.bzl +++ b/python/private/common/py_runtime_rule.bzl @@ -102,19 +102,20 @@ def _py_runtime_impl(ctx): files = runtime_files if hermetic else None, coverage_tool = coverage_tool, coverage_files = coverage_files, - pyc_tag = pyc_tag, python_version = python_version, stub_shebang = ctx.attr.stub_shebang, bootstrap_template = ctx.file.bootstrap_template, - interpreter_version_info = interpreter_version_info, - implementation_name = ctx.attr.implementation_name, ) builtin_py_runtime_info_kwargs = dict(py_runtime_info_kwargs) - # Pop these because they don't exist on BuiltinPyRuntimeInfo - builtin_py_runtime_info_kwargs.pop("interpreter_version_info") - builtin_py_runtime_info_kwargs.pop("pyc_tag") - builtin_py_runtime_info_kwargs.pop("implementation_name") + # There are all args that BuiltinPyRuntimeInfo doesn't support + py_runtime_info_kwargs.update(dict( + implementation_name = ctx.attr.implementation_name, + interpreter_version_info = interpreter_version_info, + pyc_tag = pyc_tag, + stage2_bootstrap_template = ctx.file.stage2_bootstrap_template, + zip_main_template = ctx.file.zip_main_template, + )) if not IS_BAZEL_7_OR_HIGHER: builtin_py_runtime_info_kwargs.pop("bootstrap_template") @@ -290,6 +291,17 @@ However, in the future this attribute will be mandatory and have no default value. """, ), + "stage2_bootstrap_template": attr.label( + default = "//python/private:stage2_bootstrap_template", + allow_single_file = True, + doc = """ +The template to use when two stage bootstrapping is enabled + +:::{seealso} +{obj}`PyRuntimeInfo.stage2_bootstrap_template` and {obj}`--bootstrap_impl` +::: +""", + ), "stub_shebang": attr.string( default = DEFAULT_STUB_SHEBANG, doc = """ @@ -300,6 +312,19 @@ See https://github.com/bazelbuild/bazel/issues/8685 for motivation. Does not apply to Windows. +""", + ), + "zip_main_template": attr.label( + default = "//python/private:zip_main_template", + allow_single_file = True, + doc = """ +The template to use for a zip's top-level `__main__.py` file. + +This becomes the entry point executed when `python foo.zip` is run. + +:::{seealso} +The {obj}`PyRuntimeInfo.zip_main_template` field. +::: """, ), }), diff --git a/python/private/flags.bzl b/python/private/flags.bzl index 36d305da8a..d141f72eee 100644 --- a/python/private/flags.bzl +++ b/python/private/flags.bzl @@ -21,6 +21,16 @@ unnecessary files when all that are needed are flag definitions. load("@bazel_skylib//rules:common_settings.bzl", "BuildSettingInfo") load("//python/private:enum.bzl", "enum") +def _bootstrap_impl_flag_get_value(ctx): + return ctx.attr._bootstrap_impl_flag[BuildSettingInfo].value + +# buildifier: disable=name-conventions +BootstrapImplFlag = enum( + SYSTEM_PYTHON = "system_python", + SCRIPT = "script", + get_value = _bootstrap_impl_flag_get_value, +) + def _precompile_flag_get_effective_value(ctx): value = ctx.attr._precompile_flag[BuildSettingInfo].value if value == PrecompileFlag.AUTO: diff --git a/python/private/python_bootstrap_template.txt b/python/private/python_bootstrap_template.txt index 8eaedbc4dc..0f9c90b3b3 100644 --- a/python/private/python_bootstrap_template.txt +++ b/python/private/python_bootstrap_template.txt @@ -91,7 +91,7 @@ def FindPythonBinary(module_space): def PrintVerbose(*args): if os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE"): - print("bootstrap:", *args, file=sys.stderr) + print("bootstrap:", *args, file=sys.stderr, flush=True) def PrintVerboseCoverage(*args): """Print output if VERBOSE_COVERAGE is non-empty in the environment.""" diff --git a/python/private/stage1_bootstrap_template.sh b/python/private/stage1_bootstrap_template.sh new file mode 100644 index 0000000000..fb46cc696c --- /dev/null +++ b/python/private/stage1_bootstrap_template.sh @@ -0,0 +1,118 @@ +#!/bin/bash + +set -e + +if [[ -n "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then + set -x +fi + +# runfiles-relative path +STAGE2_BOOTSTRAP="%stage2_bootstrap%" + +# runfiles-relative path, absolute path, or single word +PYTHON_BINARY='%python_binary%' + +# 0 or 1 +IS_ZIPFILE="%is_zipfile%" + +if [[ "$IS_ZIPFILE" == "1" ]]; then + zip_dir=$(mktemp -d --suffix Bazel.runfiles_) + + if [[ -n "$zip_dir" && -z "${RULES_PYTHON_BOOTSTRAP_VERBOSE:-}" ]]; then + trap 'rm -fr "$zip_dir"' EXIT + fi + # unzip emits a warning and exits with code 1 when there is extraneous data, + # like this bootstrap prelude code, but otherwise successfully extracts, so + # we have to ignore its exit code and suppress stderr. + # The alternative requires having to copy ourselves elsewhere with the prelude + # stripped (because zip can't extract from a stream). We avoid that because + # it's wasteful. + ( unzip -q -d "$zip_dir" "$0" 2>/dev/null || /bin/true ) + + RUNFILES_DIR="$zip_dir/runfiles" + if [[ ! -d "$RUNFILES_DIR" ]]; then + echo "Runfiles dir not found: zip extraction likely failed" + echo "Run with RULES_PYTHON_BOOTSTRAP_VERBOSE=1 to aid debugging" + exit 1 + fi + +else + function find_runfiles_root() { + if [[ -n "${RUNFILES_DIR:-}" ]]; then + echo "$RUNFILES_DIR" + return 0 + elif [[ "${RUNFILES_MANIFEST_FILE:-}" = *".runfiles_manifest" ]]; then + echo "${RUNFILES_MANIFEST_FILE%%.runfiles_manifest}" + return 0 + elif [[ "${RUNFILES_MANIFEST_FILE:-}" = *".runfiles/MANIFEST" ]]; then + echo "${RUNFILES_MANIFEST_FILE%%.runfiles/MANIFEST}" + return 0 + fi + + stub_filename="$1" + # A relative path to our executable, as happens with + # a build action or bazel-bin/ invocation + if [[ "$stub_filename" != /* ]]; then + stub_filename="$PWD/$stub_filename" + fi + + while true; do + module_space="${stub_filename}.runfiles" + if [[ -d "$module_space" ]]; then + echo "$module_space" + return 0 + fi + if [[ "$stub_filename" == *.runfiles/* ]]; then + echo "${stub_filename%.runfiles*}.runfiles" + return 0 + fi + if [[ ! -L "$stub_filename" ]]; then + break + fi + target=$(realpath $maybe_runfiles_root) + stub_filename="$target" + done + echo >&2 "Unable to find runfiles directory for $1" + exit 1 + } + RUNFILES_DIR=$(find_runfiles_root $0) +fi + + +function find_python_interpreter() { + runfiles_root="$1" + interpreter_path="$2" + if [[ "$interpreter_path" == /* ]]; then + # An absolute path, i.e. platform runtime + echo "$interpreter_path" + elif [[ "$interpreter_path" == */* ]]; then + # A runfiles-relative path + echo "$runfiles_root/$interpreter_path" + else + # A plain word, e.g. "python3". Rely on searching PATH + echo "$interpreter_path" + fi +} + +python_exe=$(find_python_interpreter $RUNFILES_DIR $PYTHON_BINARY) +stage2_bootstrap="$RUNFILES_DIR/$STAGE2_BOOTSTRAP" + +declare -a interpreter_env +declare -a interpreter_args + +# Don't prepend a potentially unsafe path to sys.path +# See: https://docs.python.org/3.11/using/cmdline.html#envvar-PYTHONSAFEPATH +# NOTE: Only works for 3.11+ +interpreter_env+=("PYTHONSAFEPATH=1") + +export RUNFILES_DIR +# NOTE: We use <(...) to pass the Python program as a file so that stdin can +# still be passed along as normal. +env \ + "${interpreter_env[@]}" \ + "$python_exe" \ + "${interpreter_args[@]}" \ + "$stage2_bootstrap" \ + "$@" + +exit $? diff --git a/python/private/stage2_bootstrap_template.py b/python/private/stage2_bootstrap_template.py new file mode 100644 index 0000000000..69c0dec0e5 --- /dev/null +++ b/python/private/stage2_bootstrap_template.py @@ -0,0 +1,510 @@ +# This is a "stage 2" bootstrap. We can assume we've running under the desired +# interpreter, with some of the basic interpreter options/envvars set. +# However, more setup is required to make the app's real main file runnable. + +import sys + +# The Python interpreter unconditionally prepends the directory containing this +# script (following symlinks) to the import path. This is the cause of #9239, +# and is a special case of #7091. We therefore explicitly delete that entry. +# TODO(#7091): Remove this hack when no longer necessary. +# TODO: Use sys.flags.safe_path to determine whether this removal should be +# performed +del sys.path[0] + +import contextlib +import os +import re +import runpy +import subprocess +import uuid + +# ===== Template substitutions start ===== +# We just put them in one place so its easy to tell which are used. + +# Runfiles-relative path to the main Python source file. +MAIN = "%main%" +# Colon-delimited string of runfiles-relative import paths to add +IMPORTS_STR = "%imports%" +WORKSPACE_NAME = "%workspace_name%" +# Though the import all value is the correct literal, we quote it +# so this file is parsable by tools. +IMPORT_ALL = True if "%import_all%" == "True" else False +# Runfiles-relative path to the coverage tool entry point, if any. +COVERAGE_TOOL = "%coverage_tool%" + +# ===== Template substitutions end ===== + + +# Return True if running on Windows +def is_windows(): + return os.name == "nt" + + +def get_windows_path_with_unc_prefix(path): + path = path.strip() + + # No need to add prefix for non-Windows platforms. + if not is_windows() or sys.version_info[0] < 3: + return path + + # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been + # removed from common Win32 file and directory functions. + # Related doc: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later + import platform + + if platform.win32_ver()[1] >= "10.0.14393": + return path + + # import sysconfig only now to maintain python 2.6 compatibility + import sysconfig + + if sysconfig.get_platform() == "mingw": + return path + + # Lets start the unicode fun + if path.startswith(unicode_prefix): + return path + + # os.path.abspath returns a normalized absolute path + return unicode_prefix + os.path.abspath(path) + + +def search_path(name): + """Finds a file in a given search path.""" + search_path = os.getenv("PATH", os.defpath).split(os.pathsep) + for directory in search_path: + if directory: + path = os.path.join(directory, name) + if os.path.isfile(path) and os.access(path, os.X_OK): + return path + return None + + +def is_verbose(): + return bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")) + + +def print_verbose(*args, mapping=None, values=None): + if is_verbose(): + if mapping is not None: + for key, value in sorted((mapping or {}).items()): + print( + "bootstrap: stage 2:", + *args, + f"{key}={value!r}", + file=sys.stderr, + flush=True, + ) + elif values is not None: + for i, v in enumerate(values): + print( + "bootstrap: stage 2:", + *args, + f"[{i}] {v!r}", + file=sys.stderr, + flush=True, + ) + else: + print("bootstrap: stage 2:", *args, file=sys.stderr, flush=True) + + +def print_verbose_coverage(*args): + """Print output if VERBOSE_COVERAGE is non-empty in the environment.""" + if os.environ.get("VERBOSE_COVERAGE"): + print(*args, file=sys.stderr, flush=True) + + +def is_verbose_coverage(): + """Returns True if VERBOSE_COVERAGE is non-empty in the environment.""" + return os.environ.get("VERBOSE_COVERAGE") or is_verbose() + + +def find_coverage_entry_point(module_space): + cov_tool = COVERAGE_TOOL + if cov_tool: + print_verbose_coverage("Using toolchain coverage_tool %r" % cov_tool) + else: + cov_tool = os.environ.get("PYTHON_COVERAGE") + if cov_tool: + print_verbose_coverage("PYTHON_COVERAGE: %r" % cov_tool) + if cov_tool: + return find_binary(module_space, cov_tool) + return None + + +def find_binary(module_space, bin_name): + """Finds the real binary if it's not a normal absolute path.""" + if not bin_name: + return None + if bin_name.startswith("//"): + # Case 1: Path is a label. Not supported yet. + raise AssertionError( + "Bazel does not support execution of Python interpreters via labels yet" + ) + elif os.path.isabs(bin_name): + # Case 2: Absolute path. + return bin_name + # Use normpath() to convert slashes to os.sep on Windows. + elif os.sep in os.path.normpath(bin_name): + # Case 3: Path is relative to the repo root. + return os.path.join(module_space, bin_name) + else: + # Case 4: Path has to be looked up in the search path. + return search_path(bin_name) + + +def create_python_path_entries(python_imports, module_space): + parts = python_imports.split(":") + return [module_space] + ["%s/%s" % (module_space, path) for path in parts] + + +def find_runfiles_root(main_rel_path): + """Finds the runfiles tree.""" + # When the calling process used the runfiles manifest to resolve the + # location of this stub script, the path may be expanded. This means + # argv[0] may no longer point to a location inside the runfiles + # directory. We should therefore respect RUNFILES_DIR and + # RUNFILES_MANIFEST_FILE set by the caller. + runfiles_dir = os.environ.get("RUNFILES_DIR", None) + if not runfiles_dir: + runfiles_manifest_file = os.environ.get("RUNFILES_MANIFEST_FILE", "") + if runfiles_manifest_file.endswith( + ".runfiles_manifest" + ) or runfiles_manifest_file.endswith(".runfiles/MANIFEST"): + runfiles_dir = runfiles_manifest_file[:-9] + # Be defensive: the runfiles dir should contain our main entry point. If + # it doesn't, then it must not be our runfiles directory. + if runfiles_dir and os.path.exists(os.path.join(runfiles_dir, main_rel_path)): + return runfiles_dir + + stub_filename = sys.argv[0] + if not os.path.isabs(stub_filename): + stub_filename = os.path.join(os.getcwd(), stub_filename) + + while True: + module_space = stub_filename + (".exe" if is_windows() else "") + ".runfiles" + if os.path.isdir(module_space): + return module_space + + runfiles_pattern = r"(.*\.runfiles)" + (r"\\" if is_windows() else "/") + ".*" + matchobj = re.match(runfiles_pattern, stub_filename) + if matchobj: + return matchobj.group(1) + + if not os.path.islink(stub_filename): + break + target = os.readlink(stub_filename) + if os.path.isabs(target): + stub_filename = target + else: + stub_filename = os.path.join(os.path.dirname(stub_filename), target) + + raise AssertionError("Cannot find .runfiles directory for %s" % sys.argv[0]) + + +# Returns repository roots to add to the import path. +def get_repositories_imports(module_space, import_all): + if import_all: + repo_dirs = [os.path.join(module_space, d) for d in os.listdir(module_space)] + repo_dirs.sort() + return [d for d in repo_dirs if os.path.isdir(d)] + return [os.path.join(module_space, WORKSPACE_NAME)] + + +def runfiles_envvar(module_space): + """Finds the runfiles manifest or the runfiles directory. + + Returns: + A tuple of (var_name, var_value) where var_name is either 'RUNFILES_DIR' or + 'RUNFILES_MANIFEST_FILE' and var_value is the path to that directory or + file, or (None, None) if runfiles couldn't be found. + """ + # If this binary is the data-dependency of another one, the other sets + # RUNFILES_MANIFEST_FILE or RUNFILES_DIR for our sake. + runfiles = os.environ.get("RUNFILES_MANIFEST_FILE", None) + if runfiles: + return ("RUNFILES_MANIFEST_FILE", runfiles) + + runfiles = os.environ.get("RUNFILES_DIR", None) + if runfiles: + return ("RUNFILES_DIR", runfiles) + + # Look for the runfiles "output" manifest, argv[0] + ".runfiles_manifest" + runfiles = module_space + "_manifest" + if os.path.exists(runfiles): + return ("RUNFILES_MANIFEST_FILE", runfiles) + + # Look for the runfiles "input" manifest, argv[0] + ".runfiles/MANIFEST" + # Normally .runfiles_manifest and MANIFEST are both present, but the + # former will be missing for zip-based builds or if someone copies the + # runfiles tree elsewhere. + runfiles = os.path.join(module_space, "MANIFEST") + if os.path.exists(runfiles): + return ("RUNFILES_MANIFEST_FILE", runfiles) + + # If running in a sandbox and no environment variables are set, then + # Look for the runfiles next to the binary. + if module_space.endswith(".runfiles") and os.path.isdir(module_space): + return ("RUNFILES_DIR", module_space) + + return (None, None) + + +def deduplicate(items): + """Efficiently filter out duplicates, keeping the first element only.""" + seen = set() + for it in items: + if it not in seen: + seen.add(it) + yield it + + +def instrumented_file_paths(): + """Yields tuples of realpath of each instrumented file with the relative path.""" + manifest_filename = os.environ.get("COVERAGE_MANIFEST") + if not manifest_filename: + return + with open(manifest_filename, "r") as manifest: + for line in manifest: + filename = line.strip() + if not filename: + continue + try: + realpath = os.path.realpath(filename) + except OSError: + print( + "Could not find instrumented file {}".format(filename), + file=sys.stderr, + flush=True, + ) + continue + if realpath != filename: + print_verbose_coverage("Fixing up {} -> {}".format(realpath, filename)) + yield (realpath, filename) + + +def unresolve_symlinks(output_filename): + # type: (str) -> None + """Replace realpath of instrumented files with the relative path in the lcov output. + + Though we are asking coveragepy to use relative file names, currently + ignore that for purposes of generating the lcov report (and other reports + which are not the XML report), so we need to go and fix up the report. + + This function is a workaround for that issue. Once that issue is fixed + upstream and the updated version is widely in use, this should be removed. + + See https://github.com/nedbat/coveragepy/issues/963. + """ + substitutions = list(instrumented_file_paths()) + if substitutions: + unfixed_file = output_filename + ".tmp" + os.rename(output_filename, unfixed_file) + with open(unfixed_file, "r") as unfixed: + with open(output_filename, "w") as output_file: + for line in unfixed: + if line.startswith("SF:"): + for realpath, filename in substitutions: + line = line.replace(realpath, filename) + output_file.write(line) + os.unlink(unfixed_file) + + +def _run_py(main_filename, *, args, cwd=None): + # type: (str, str, list[str], dict[str, str]) -> ... + """Executes the given Python file using the various environment settings.""" + + orig_argv = sys.argv + orig_cwd = os.getcwd() + try: + sys.argv = [main_filename] + args + if cwd: + os.chdir(cwd) + print_verbose("run_py: cwd:", os.getcwd()) + print_verbose("run_py: sys.argv: ", values=sys.argv) + print_verbose("run_py: os.environ:", mapping=os.environ) + print_verbose("run_py: sys.path:", values=sys.path) + runpy.run_path(main_filename, run_name="__main__") + finally: + os.chdir(orig_cwd) + sys.argv = orig_argv + + +@contextlib.contextmanager +def _maybe_collect_coverage(enable): + if not enable: + yield + return + + import uuid + + import coverage + + coverage_dir = os.environ["COVERAGE_DIR"] + unique_id = uuid.uuid4() + + # We need for coveragepy to use relative paths. This can only be configured + rcfile_name = os.path.join(coverage_dir, ".coveragerc_{}".format(unique_id)) + with open(rcfile_name, "w") as rcfile: + rcfile.write( + """[run] +relative_files = True +""" + ) + try: + cov = coverage.Coverage( + config_file=rcfile_name, + branch=True, + # NOTE: The messages arg controls what coverage prints to stdout/stderr, + # which can interfere with the Bazel coverage command. Enabling message + # output is only useful for debugging coverage support. + messages=is_verbose_coverage(), + omit=[ + # Pipes can't be read back later, which can cause coverage to + # throw an error when trying to get its source code. + "/dev/fd/*", + ], + ) + cov.start() + try: + yield + finally: + cov.stop() + lcov_path = os.path.join(coverage_dir, "pylcov.dat") + cov.lcov_report( + outfile=lcov_path, + # Ignore errors because sometimes instrumented files aren't + # readable afterwards. e.g. if they come from /dev/fd or if + # they were transient code-under-test in /tmp + ignore_errors=True, + ) + if os.path.isfile(lcov_path): + unresolve_symlinks(lcov_path) + finally: + try: + os.unlink(rcfile_name) + except OSError as err: + # It's possible that the profiled program might execute another Python + # binary through a wrapper that would then delete the rcfile. Not much + # we can do about that, besides ignore the failure here. + print_verbose_coverage("Error removing temporary coverage rc file:", err) + + +def main(): + print_verbose("initial argv:", values=sys.argv) + print_verbose("initial cwd:", os.getcwd()) + print_verbose("initial environ:", mapping=os.environ) + print_verbose("initial sys.path:", values=sys.path) + + main_rel_path = MAIN + if is_windows(): + main_rel_path = main_rel_path.replace("/", os.sep) + + module_space = find_runfiles_root(main_rel_path) + print_verbose("runfiles root:", module_space) + + # Recreate the "add main's dir to sys.path[0]" behavior to match the + # system-python bootstrap / typical Python behavior. + # + # Without safe path enabled, when `python foo/bar.py` is run, python will + # resolve the foo/bar.py symlink to its real path, then add the directory + # of that path to sys.path. But, the resolved directory for the symlink + # depends on if the file is generated or not. + # + # When foo/bar.py is a source file, then it's a symlink pointing + # back to the client source directory. This means anything from that source + # directory becomes importable, i.e. most code is importable. + # + # When foo/bar.py is a generated file, then it's a symlink pointing to + # somewhere under bazel-out/.../bin, i.e. where generated files are. This + # means only other generated files are importable (not source files). + # + # To replicate this behavior, we add main's directory within the runfiles + # when safe path isn't enabled. + if not getattr(sys.flags, "safe_path", False): + prepend_path_entries = [ + os.path.join(module_space, os.path.dirname(main_rel_path)) + ] + else: + prepend_path_entries = [] + python_path_entries = create_python_path_entries(IMPORTS_STR, module_space) + python_path_entries += get_repositories_imports(module_space, IMPORT_ALL) + python_path_entries = [ + get_windows_path_with_unc_prefix(d) for d in python_path_entries + ] + + # Remove duplicates to avoid overly long PYTHONPATH (#10977). Preserve order, + # keep first occurrence only. + python_path_entries = deduplicate(python_path_entries) + + if is_windows(): + python_path_entries = [p.replace("/", os.sep) for p in python_path_entries] + else: + # deduplicate returns a generator, but we need a list after this. + python_path_entries = list(python_path_entries) + + # We're emulating PYTHONPATH being set, so we insert at the start + # This isn't a great idea (it can shadow the stdlib), but is the historical + # behavior. + runfiles_envkey, runfiles_envvalue = runfiles_envvar(module_space) + if runfiles_envkey: + os.environ[runfiles_envkey] = runfiles_envvalue + + main_filename = os.path.join(module_space, main_rel_path) + main_filename = get_windows_path_with_unc_prefix(main_filename) + assert os.path.exists(main_filename), ( + "Cannot exec() %r: file not found." % main_filename + ) + assert os.access(main_filename, os.R_OK), ( + "Cannot exec() %r: file not readable." % main_filename + ) + + # COVERAGE_DIR is set if coverage is enabled and instrumentation is configured + # for something, though it could be another program executing this one or + # one executed by this one (e.g. an extension module). + if os.environ.get("COVERAGE_DIR"): + cov_tool = find_coverage_entry_point(module_space) + if cov_tool is None: + print_verbose_coverage( + "Coverage was enabled, but python coverage tool was not configured." + + "To enable coverage, consult the docs at " + + "https://rules-python.readthedocs.io/en/latest/coverage.html" + ) + else: + # Inhibit infinite recursion: + if "PYTHON_COVERAGE" in os.environ: + del os.environ["PYTHON_COVERAGE"] + + if not os.path.exists(cov_tool): + raise EnvironmentError( + "Python coverage tool %r not found. " + "Try running with VERBOSE_COVERAGE=1 to collect more information." + % cov_tool + ) + + # coverage library expects sys.path[0] to contain the library, and replaces + # it with the directory of the program it starts. Our actual sys.path[0] is + # the runfiles directory, which must not be replaced. + # CoverageScript.do_execute() undoes this sys.path[0] setting. + # + # Update sys.path such that python finds the coverage package. The coverage + # entry point is coverage.coverage_main, so we need to do twice the dirname. + coverage_dir = os.path.dirname(os.path.dirname(cov_tool)) + print_verbose("coverage: adding to sys.path:", coverage_dir) + python_path_entries.append(coverage_dir) + python_path_entries = deduplicate(python_path_entries) + else: + cov_tool = None + + sys.stdout.flush() + # NOTE: The sys.path must be modified before coverage is imported/activated + sys.path[0:0] = prepend_path_entries + sys.path.extend(python_path_entries) + with _maybe_collect_coverage(enable=cov_tool is not None): + # The first arg is this bootstrap, so drop that for the re-invocation. + _run_py(main_filename, args=sys.argv[1:]) + sys.exit(0) + + +main() diff --git a/python/private/zip_main_template.py b/python/private/zip_main_template.py new file mode 100644 index 0000000000..962ec4b110 --- /dev/null +++ b/python/private/zip_main_template.py @@ -0,0 +1,293 @@ +# Template for the __main__.py file inserted into zip files +# +# NOTE: This file is a "stage 1" bootstrap, so it's responsible for locating the +# desired runtime and having it run the stage 2 bootstrap. This means it can't +# assume much about the current runtime and environment. e.g, the current +# runtime may not be the correct one, the zip may not have been extract, the +# runfiles env vars may not be set, etc. +# +# NOTE: This program must retain compatibility with a wide variety of Python +# versions since it is run by an unknown Python interpreter. + +import sys + +# The Python interpreter unconditionally prepends the directory containing this +# script (following symlinks) to the import path. This is the cause of #9239, +# and is a special case of #7091. We therefore explicitly delete that entry. +# TODO(#7091): Remove this hack when no longer necessary. +del sys.path[0] + +import os +import shutil +import subprocess +import tempfile +import zipfile + +_STAGE2_BOOTSTRAP = "%stage2_bootstrap%" +_PYTHON_BINARY = "%python_binary%" +_WORKSPACE_NAME = "%workspace_name%" + + +# Return True if running on Windows +def is_windows(): + return os.name == "nt" + + +def get_windows_path_with_unc_prefix(path): + """Adds UNC prefix after getting a normalized absolute Windows path. + + No-op for non-Windows platforms or if running under python2. + """ + path = path.strip() + + # No need to add prefix for non-Windows platforms. + # And \\?\ doesn't work in python 2 or on mingw + if not is_windows() or sys.version_info[0] < 3: + return path + + # Starting in Windows 10, version 1607(OS build 14393), MAX_PATH limitations have been + # removed from common Win32 file and directory functions. + # Related doc: https://docs.microsoft.com/en-us/windows/win32/fileio/maximum-file-path-limitation?tabs=cmd#enable-long-paths-in-windows-10-version-1607-and-later + import platform + + if platform.win32_ver()[1] >= "10.0.14393": + return path + + # import sysconfig only now to maintain python 2.6 compatibility + import sysconfig + + if sysconfig.get_platform() == "mingw": + return path + + # Lets start the unicode fun + unicode_prefix = "\\\\?\\" + if path.startswith(unicode_prefix): + return path + + # os.path.abspath returns a normalized absolute path + return unicode_prefix + os.path.abspath(path) + + +def has_windows_executable_extension(path): + return path.endswith(".exe") or path.endswith(".com") or path.endswith(".bat") + + +if is_windows() and not has_windows_executable_extension(_PYTHON_BINARY): + _PYTHON_BINARY = _PYTHON_BINARY + ".exe" + + +def search_path(name): + """Finds a file in a given search path.""" + search_path = os.getenv("PATH", os.defpath).split(os.pathsep) + for directory in search_path: + if directory: + path = os.path.join(directory, name) + if os.path.isfile(path) and os.access(path, os.X_OK): + return path + return None + + +def find_python_binary(module_space): + """Finds the real Python binary if it's not a normal absolute path.""" + return find_binary(module_space, _PYTHON_BINARY) + + +def print_verbose(*args, mapping=None, values=None): + if bool(os.environ.get("RULES_PYTHON_BOOTSTRAP_VERBOSE")): + if mapping is not None: + for key, value in sorted((mapping or {}).items()): + print( + "bootstrap: stage 1:", + *args, + f"{key}={value!r}", + file=sys.stderr, + flush=True, + ) + elif values is not None: + for i, v in enumerate(values): + print( + "bootstrap: stage 1:", + *args, + f"[{i}] {v!r}", + file=sys.stderr, + flush=True, + ) + else: + print("bootstrap: stage 1:", *args, file=sys.stderr, flush=True) + + +def find_binary(module_space, bin_name): + """Finds the real binary if it's not a normal absolute path.""" + if not bin_name: + return None + if bin_name.startswith("//"): + # Case 1: Path is a label. Not supported yet. + raise AssertionError( + "Bazel does not support execution of Python interpreters via labels yet" + ) + elif os.path.isabs(bin_name): + # Case 2: Absolute path. + return bin_name + # Use normpath() to convert slashes to os.sep on Windows. + elif os.sep in os.path.normpath(bin_name): + # Case 3: Path is relative to the repo root. + return os.path.join(module_space, bin_name) + else: + # Case 4: Path has to be looked up in the search path. + return search_path(bin_name) + + +def extract_zip(zip_path, dest_dir): + """Extracts the contents of a zip file, preserving the unix file mode bits. + + These include the permission bits, and in particular, the executable bit. + + Ideally the zipfile module should set these bits, but it doesn't. See: + https://bugs.python.org/issue15795. + + Args: + zip_path: The path to the zip file to extract + dest_dir: The path to the destination directory + """ + zip_path = get_windows_path_with_unc_prefix(zip_path) + dest_dir = get_windows_path_with_unc_prefix(dest_dir) + with zipfile.ZipFile(zip_path) as zf: + for info in zf.infolist(): + zf.extract(info, dest_dir) + # UNC-prefixed paths must be absolute/normalized. See + # https://docs.microsoft.com/en-us/windows/desktop/fileio/naming-a-file#maximum-path-length-limitation + file_path = os.path.abspath(os.path.join(dest_dir, info.filename)) + # The Unix st_mode bits (see "man 7 inode") are stored in the upper 16 + # bits of external_attr. Of those, we set the lower 12 bits, which are the + # file mode bits (since the file type bits can't be set by chmod anyway). + attrs = info.external_attr >> 16 + if attrs != 0: # Rumor has it these can be 0 for zips created on Windows. + os.chmod(file_path, attrs & 0o7777) + + +# Create the runfiles tree by extracting the zip file +def create_module_space(): + temp_dir = tempfile.mkdtemp("", "Bazel.runfiles_") + extract_zip(os.path.dirname(__file__), temp_dir) + # IMPORTANT: Later code does `rm -fr` on dirname(module_space) -- it's + # important that deletion code be in sync with this directory structure + return os.path.join(temp_dir, "runfiles") + + +def execute_file( + python_program, + main_filename, + args, + env, + module_space, + workspace, +): + # type: (str, str, list[str], dict[str, str], str, str|None, str|None) -> ... + """Executes the given Python file using the various environment settings. + + This will not return, and acts much like os.execv, except is much + more restricted, and handles Bazel-related edge cases. + + Args: + python_program: (str) Path to the Python binary to use for execution + main_filename: (str) The Python file to execute + args: (list[str]) Additional args to pass to the Python file + env: (dict[str, str]) A dict of environment variables to set for the execution + module_space: (str) Path to the module space/runfiles tree directory + workspace: (str|None) Name of the workspace to execute in. This is expected to be a + directory under the runfiles tree. + """ + # We want to use os.execv instead of subprocess.call, which causes + # problems with signal passing (making it difficult to kill + # Bazel). However, these conditions force us to run via + # subprocess.call instead: + # + # - On Windows, os.execv doesn't handle arguments with spaces + # correctly, and it actually starts a subprocess just like + # subprocess.call. + # - When running in a workspace or zip file, we need to clean up the + # workspace after the process finishes so control must return here. + try: + subprocess_argv = [python_program, main_filename] + args + print_verbose("subprocess argv:", values=subprocess_argv) + print_verbose("subprocess env:", mapping=env) + print_verbose("subprocess cwd:", workspace) + ret_code = subprocess.call(subprocess_argv, env=env, cwd=workspace) + sys.exit(ret_code) + finally: + # NOTE: dirname() is called because create_module_space() creates a + # sub-directory within a temporary directory, and we want to remove the + # whole temporary directory. + shutil.rmtree(os.path.dirname(module_space), True) + + +def main(): + print_verbose("running zip main bootstrap") + print_verbose("initial argv:", values=sys.argv) + print_verbose("initial environ:", mapping=os.environ) + print_verbose("initial sys.executable", sys.executable) + print_verbose("initial sys.version", sys.version) + + args = sys.argv[1:] + + new_env = {} + + # The main Python source file. + # The magic string percent-main-percent is replaced with the runfiles-relative + # filename of the main file of the Python binary in BazelPythonSemantics.java. + main_rel_path = _STAGE2_BOOTSTRAP + if is_windows(): + main_rel_path = main_rel_path.replace("/", os.sep) + + # todo: need to cleanup the created directory + module_space = create_module_space() + print_verbose("extracted runfiles to:", module_space) + + new_env["RUNFILES_DIR"] = module_space + + # Don't prepend a potentially unsafe path to sys.path + # See: https://docs.python.org/3.11/using/cmdline.html#envvar-PYTHONSAFEPATH + new_env["PYTHONSAFEPATH"] = "1" + + main_filename = os.path.join(module_space, main_rel_path) + main_filename = get_windows_path_with_unc_prefix(main_filename) + assert os.path.exists(main_filename), ( + "Cannot exec() %r: file not found." % main_filename + ) + assert os.access(main_filename, os.R_OK), ( + "Cannot exec() %r: file not readable." % main_filename + ) + + program = python_program = find_python_binary(module_space) + if python_program is None: + raise AssertionError("Could not find python binary: " + _PYTHON_BINARY) + + # Some older Python versions on macOS (namely Python 3.7) may unintentionally + # leave this environment variable set after starting the interpreter, which + # causes problems with Python subprocesses correctly locating sys.executable, + # which subsequently causes failure to launch on Python 3.11 and later. + if "__PYVENV_LAUNCHER__" in os.environ: + del os.environ["__PYVENV_LAUNCHER__"] + + new_env.update((key, val) for key, val in os.environ.items() if key not in new_env) + + workspace = None + # If RUN_UNDER_RUNFILES equals 1, it means we need to + # change directory to the right runfiles directory. + # (So that the data files are accessible) + if os.environ.get("RUN_UNDER_RUNFILES") == "1": + workspace = os.path.join(module_space, _WORKSPACE_NAME) + + sys.stdout.flush() + execute_file( + python_program, + main_filename, + args, + new_env, + module_space, + workspace, + ) + + +if __name__ == "__main__": + main() diff --git a/python/repositories.bzl b/python/repositories.bzl index 26081a6b48..4ffadd050a 100644 --- a/python/repositories.bzl +++ b/python/repositories.bzl @@ -185,7 +185,10 @@ def _python_repository_impl(rctx): elif rctx.attr.distutils_content: rctx.file(distutils_path, rctx.attr.distutils_content) - # Make the Python installation read-only. + # Make the Python installation read-only. This is to prevent issues due to + # pycs being generated at runtime: + # * The pycs are not deterministic (they contain timestamps) + # * Multiple processes trying to write the same pycs can result in errors. if not rctx.attr.ignore_root_user_error: if "windows" not in platform: lib_dir = "lib" if "windows" not in platform else "Lib" @@ -200,6 +203,9 @@ def _python_repository_impl(rctx): op = "python_repository.TestReadOnly", arguments = [repo_utils.which_checked(rctx, "touch"), "{}/.test".format(lib_dir)], ) + + # The issue with running as root is the installation is no longer + # read-only, so the problems due to pyc can resurface. if exec_result.return_code == 0: stdout = repo_utils.execute_checked_stdout( rctx, diff --git a/tests/base_rules/py_executable_base_tests.bzl b/tests/base_rules/py_executable_base_tests.bzl index b6f28026db..43e800a99f 100644 --- a/tests/base_rules/py_executable_base_tests.bzl +++ b/tests/base_rules/py_executable_base_tests.bzl @@ -20,7 +20,7 @@ load("@rules_testing//lib:truth.bzl", "matching") load("@rules_testing//lib:util.bzl", rt_util = "util") load("//tests/base_rules:base_tests.bzl", "create_base_tests") load("//tests/base_rules:util.bzl", "WINDOWS_ATTR", pt_util = "util") -load("//tests/support:support.bzl", "WINDOWS") +load("//tests/support:support.bzl", "WINDOWS_X86_64") _BuiltinPyRuntimeInfo = PyRuntimeInfo @@ -50,7 +50,7 @@ def _test_basic_windows(name, config): "//command_line_option:cpu": "windows_x86_64", "//command_line_option:crosstool_top": Label("//tests/cc:cc_toolchain_suite"), "//command_line_option:extra_toolchains": [str(Label("//tests/cc:all"))], - "//command_line_option:platforms": [WINDOWS], + "//command_line_option:platforms": [WINDOWS_X86_64], }, attr_values = {"target_compatible_with": target_compatible_with}, ) diff --git a/tests/base_rules/py_test/py_test_tests.bzl b/tests/base_rules/py_test/py_test_tests.bzl index 50c1db27cf..c77bd7eb04 100644 --- a/tests/base_rules/py_test/py_test_tests.bzl +++ b/tests/base_rules/py_test/py_test_tests.bzl @@ -21,13 +21,26 @@ load( "create_executable_tests", ) load("//tests/base_rules:util.bzl", pt_util = "util") -load("//tests/support:support.bzl", "LINUX", "MAC") +load("//tests/support:support.bzl", "LINUX_X86_64", "MAC_X86_64") # Explicit Label() calls are required so that it resolves in @rules_python # context instead of @rules_testing context. _FAKE_CC_TOOLCHAIN = Label("//tests/cc:cc_toolchain_suite") _FAKE_CC_TOOLCHAINS = [str(Label("//tests/cc:all"))] +# The Windows CI currently runs as root, which breaks when +# the analysis tests try to install (but not use, because +# these are analysis tests) a runtime for another platform. +# This is because the toolchain install has an assert to +# verify the runtime install is read-only, which it can't +# be when running as root. +_SKIP_WINDOWS = { + "target_compatible_with": select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), +} + _tests = [] def _test_mac_requires_darwin_for_execution(name, config): @@ -52,8 +65,9 @@ def _test_mac_requires_darwin_for_execution(name, config): "//command_line_option:cpu": "darwin_x86_64", "//command_line_option:crosstool_top": _FAKE_CC_TOOLCHAIN, "//command_line_option:extra_toolchains": _FAKE_CC_TOOLCHAINS, - "//command_line_option:platforms": [MAC], + "//command_line_option:platforms": [MAC_X86_64], }, + attr_values = _SKIP_WINDOWS, ) def _test_mac_requires_darwin_for_execution_impl(env, target): @@ -84,8 +98,9 @@ def _test_non_mac_doesnt_require_darwin_for_execution(name, config): "//command_line_option:cpu": "k8", "//command_line_option:crosstool_top": _FAKE_CC_TOOLCHAIN, "//command_line_option:extra_toolchains": _FAKE_CC_TOOLCHAINS, - "//command_line_option:platforms": [LINUX], + "//command_line_option:platforms": [LINUX_X86_64], }, + attr_values = _SKIP_WINDOWS, ) def _test_non_mac_doesnt_require_darwin_for_execution_impl(env, target): diff --git a/tests/support/support.bzl b/tests/support/support.bzl index 14a743b8a2..4bcc554854 100644 --- a/tests/support/support.bzl +++ b/tests/support/support.bzl @@ -20,8 +20,11 @@ # places. MAC = Label("//tests/support:mac") +MAC_X86_64 = Label("//tests/support:mac_x86_64") LINUX = Label("//tests/support:linux") +LINUX_X86_64 = Label("//tests/support:linux_x86_64") WINDOWS = Label("//tests/support:windows") +WINDOWS_X86_64 = Label("//tests/support:windows_x86_64") PLATFORM_TOOLCHAIN = str(Label("//tests/support:platform_toolchain")) CC_TOOLCHAIN = str(Label("//tests/cc:all")) From b7b7b3fc66d270dc22197b3dec0621f9ebd08d4e Mon Sep 17 00:00:00 2001 From: Richard Levasseur Date: Sun, 2 Jun 2024 07:21:56 -0700 Subject: [PATCH 2/2] default to sys py; remove todo --- python/config_settings/BUILD.bazel | 3 +-- python/private/zip_main_template.py | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/python/config_settings/BUILD.bazel b/python/config_settings/BUILD.bazel index a91ba12764..9dab53c039 100644 --- a/python/config_settings/BUILD.bazel +++ b/python/config_settings/BUILD.bazel @@ -56,8 +56,7 @@ string_flag( string_flag( name = "bootstrap_impl", - # todo: default to deprecated_one_stage initially - build_setting_default = BootstrapImplFlag.SCRIPT, + build_setting_default = BootstrapImplFlag.SYSTEM_PYTHON, values = sorted(BootstrapImplFlag.__members__.values()), # NOTE: Only public because its an implicit dependency visibility = ["//visibility:public"], diff --git a/python/private/zip_main_template.py b/python/private/zip_main_template.py index 962ec4b110..18eaed9630 100644 --- a/python/private/zip_main_template.py +++ b/python/private/zip_main_template.py @@ -239,7 +239,6 @@ def main(): if is_windows(): main_rel_path = main_rel_path.replace("/", os.sep) - # todo: need to cleanup the created directory module_space = create_module_space() print_verbose("extracted runfiles to:", module_space)