fix: add flag to use runtime venv creation when using bootstrap=script

CHANGELOG.md

@@ -77,6 +77,10 @@ Unreleased changes template.
   The related issue is [#908](https://github.com/bazelbuild/rules_python/issue/908).
 * (sphinxdocs) Do not crash when `tag_class` does not have a populated `doc` value.
   Fixes ([#2579](https://github.com/bazelbuild/rules_python/issues/2579)).
+* (binaries/tests) Fix packaging when using `--bootstrap_impl=script`: set
+  {obj}`--venvs_use_declare_symlink=no` to have it not create symlinks at
+  build time (they will be created at runtime instead).
+  (Fixes [#2489](https://github.com/bazelbuild/rules_python/issues/2489))
 
 {#v0-0-0-added}
 ### Added

MODULE.bazel

@@ -84,6 +84,7 @@ bazel_dep(name = "rules_testing", version = "0.6.0", dev_dependency = True)
 bazel_dep(name = "rules_shell", version = "0.3.0", dev_dependency = True)
 bazel_dep(name = "rules_multirun", version = "0.9.0", dev_dependency = True)
 bazel_dep(name = "bazel_ci_rules", version = "1.0.0", dev_dependency = True)
+bazel_dep(name = "rules_pkg", version = "1.0.1", dev_dependency = True)
 
 # Extra gazelle plugin deps so that WORKSPACE.bzlmod can continue including it for e2e tests.
 # We use `WORKSPACE.bzlmod` because it is impossible to have dev-only local overrides.

docs/api/rules_python/python/config_settings/index.md

@@ -212,6 +212,7 @@ Values:
 :::
 ::::
 
+
 ::::{bzl:flag} bootstrap_impl
 Determine how programs implement their startup process.
 
@@ -258,3 +259,26 @@ Values:
 :::
 
 ::::
+
+::::{bzl:flag} venvs_use_declare_symlink
+
+Determines if relative symlinks are created using `declare_symlink()` at build
+time.
+
+This is only intended to work around
+[#2489](https://github.com/bazelbuild/rules_python/issues/2489), where some
+packaging rules don't support `declare_symlink()` artifacts.
+
+Values:
+* `yes`: Use `declare_symlink()` and create relative symlinks at build time.
+* `no`: Do not use `declare_symlink()`. Instead, the venv will be created at
+  runtime.
+
+:::{seealso}
+{envvar}`RULES_PYTHON_EXTRACT_ROOT` for customizing where the runtime venv
+is created.
+:::
+
+:::{versionadded} VERSION_NEXT_PATCH
+:::
+::::

docs/environment-variables.md

@@ -1,28 +1,56 @@
 # Environment Variables
 
-:::{envvar} RULES_PYTHON_REPO_DEBUG
+:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE
 
-When `1`, repository rules will print debug information about what they're
+When `1`, debug information about bootstrapping of a program is printed to
+stderr.
+:::
+
+:::{envvar} RULES_PYTHON_BZLMOD_DEBUG
+
+When `1`, bzlmod extensions will print debug information about what they're
 doing. This is mostly useful for development to debug errors.
 :::
 
-:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY
+:::{envvar} RULES_PYTHON_DEPRECATION_WARNINGS
 
-Determines the verbosity of logging output for repo rules. Valid values:
+When `1`, the rules_python will warn users about deprecated functionality that will
+be removed in a subsequent major `rules_python` version. Defaults to `0` if unset.
+:::
 
-* `DEBUG`
-* `INFO`
-* `TRACE`
+:::{envvar} RULES_PYTHON_ENABLE_PYSTAR
+
+When `1`, the rules_python Starlark implementation of the core rules is used
+instead of the Bazel-builtin rules. Note this requires Bazel 7+.
 :::
 
-:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH
+::::{envvar} RULES_PYTHON_EXTRACT_ROOT
 
-Determines the python interpreter platform to be used for a particular
-interpreter `(version, os, arch)` triple to be used in repository rules.
-Replace the `VERSION_OS_ARCH` part with actual values when using, e.g.
-`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the
-os, arch values are the same as the ones mentioned in the
-`//python:versions.bzl` file.
+Directory to use as the root for creating files necessary for bootstrapping so
+that a binary can run.
+
+Only applicable when {bzl:flag}`--venvs_use_declare_symlink=no` is used.
+
+When set, a binary will attempt to find a unique, reusable, location within this
+directory for the files it needs to create to aid startup. The files may not be
+deleted upon program exit; it is the responsibility of the caller to ensure
+cleanup.
+
+Manually specifying the directory is useful to lower the overhead of
+extracting/creating files on every program execution. By using a location
+outside /tmp, longer lived programs don't have to worry about files in /tmp
+being cleaned up by the OS.
+
+If not set, then a temporary directory will be created and deleted upon program
+exit.
+
+:::{versionadded} VERSION_NEXT_PATCH
+:::
+::::
+
+:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE
+
+When `1`, debug information from gazelle is printed to stderr.
 :::
 
 :::{envvar} RULES_PYTHON_PIP_ISOLATED
@@ -34,37 +62,32 @@ Valid values:
 * Other non-empty values mean to use isolated mode.
 :::
 
-:::{envvar} RULES_PYTHON_BZLMOD_DEBUG
+:::{envvar} RULES_PYTHON_REPO_DEBUG
 
-When `1`, bzlmod extensions will print debug information about what they're
+When `1`, repository rules will print debug information about what they're
 doing. This is mostly useful for development to debug errors.
 :::
 
-:::{envvar} RULES_PYTHON_DEPRECATION_WARNINGS
-
-When `1`, the rules_python will warn users about deprecated functionality that will
-be removed in a subsequent major `rules_python` version. Defaults to `0` if unset.
-:::
+:::{envvar} RULES_PYTHON_REPO_DEBUG_VERBOSITY
 
-:::{envvar} RULES_PYTHON_ENABLE_PYSTAR
+Determines the verbosity of logging output for repo rules. Valid values:
 
-When `1`, the rules_python Starlark implementation of the core rules is used
-instead of the Bazel-builtin rules. Note this requires Bazel 7+.
+* `DEBUG`
+* `INFO`
+* `TRACE`
 :::
 
-:::{envvar} RULES_PYTHON_BOOTSTRAP_VERBOSE
+:::{envvar} RULES_PYTHON_REPO_TOOLCHAIN_VERSION_OS_ARCH
 
-When `1`, debug information about bootstrapping of a program is printed to
-stderr.
+Determines the python interpreter platform to be used for a particular
+interpreter `(version, os, arch)` triple to be used in repository rules.
+Replace the `VERSION_OS_ARCH` part with actual values when using, e.g.
+`3_13_0_linux_x86_64`. The version values must have `_` instead of `.` and the
+os, arch values are the same as the ones mentioned in the
+`//python:versions.bzl` file.
 :::
 
 :::{envvar} VERBOSE_COVERAGE
 
 When `1`, debug information about coverage behavior is printed to stderr.
 :::
-
-
-:::{envvar} RULES_PYTHON_GAZELLE_VERBOSE
-
-When `1`, debug information from gazelle is printed to stderr.
-:::

python/config_settings/BUILD.bazel

@@ -9,6 +9,7 @@ load(
     "LibcFlag",
     "PrecompileFlag",
     "PrecompileSourceRetentionFlag",
+    "VenvsUseDeclareSymlinkFlag",
 )
 load(
     "//python/private/pypi:flags.bzl",
@@ -121,6 +122,13 @@ config_setting(
     visibility = ["//visibility:public"],
 )
 
+string_flag(
+    name = "venvs_use_declare_symlink",
+    build_setting_default = VenvsUseDeclareSymlinkFlag.YES,
+    values = VenvsUseDeclareSymlinkFlag.flag_values(),
+    visibility = ["//visibility:public"],
+)
+
 # pip.parse related flags
 
 string_flag(

python/private/flags.bzl

@@ -123,6 +123,21 @@ PrecompileSourceRetentionFlag = enum(
     get_effective_value = _precompile_source_retention_flag_get_effective_value,
 )
 
+def _venvs_use_declare_symlink_flag_get_value(ctx):
+    return ctx.attr._venvs_use_declare_symlink_flag[BuildSettingInfo].value
+
+# Decides if the venv created by bootstrap=script uses declare_file() to
+# create relative symlinks. Workaround for #2489 (packaging rules not supporting
+# declare_link() files).
+# buildifier: disable=name-conventions
+VenvsUseDeclareSymlinkFlag = FlagEnum(
+    # Use declare_file() and relative symlinks in the venv
+    YES = "yes",
+    # Do not use declare_file() and relative symlinks in the venv
+    NO = "no",
+    get_value = _venvs_use_declare_symlink_flag_get_value,
+)
+
 # Used for matching freethreaded toolchains and would have to be used in wheels
 # as well.
 # buildifier: disable=name-conventions

python/private/py_executable.bzl

@@ -51,7 +51,7 @@ load(
     "target_platform_has_any_constraint",
     "union_attrs",
 )
-load(":flags.bzl", "BootstrapImplFlag")
+load(":flags.bzl", "BootstrapImplFlag", "VenvsUseDeclareSymlinkFlag")
 load(":precompile.bzl", "maybe_precompile")
 load(":py_cc_link_params_info.bzl", "PyCcLinkParamsInfo")
 load(":py_executable_info.bzl", "PyExecutableInfo")
@@ -195,6 +195,10 @@ accepting arbitrary Python versions.
         "_python_version_flag": attr.label(
             default = "//python/config_settings:python_version",
         ),
+        "_venvs_use_declare_symlink_flag": attr.label(
+            default = "//python/config_settings:venvs_use_declare_symlink",
+            providers = [BuildSettingInfo],
+        ),
         "_windows_constraints": attr.label_list(
             default = [
                 "@platforms//os:windows",
@@ -512,7 +516,25 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
     ctx.actions.write(pyvenv_cfg, "")
 
     runtime = runtime_details.effective_runtime
-    if runtime.interpreter:
+    venvs_use_declare_symlink_enabled = (
+        VenvsUseDeclareSymlinkFlag.get_value(ctx) == VenvsUseDeclareSymlinkFlag.YES
+    )
+
+    if not venvs_use_declare_symlink_enabled:
+        if runtime.interpreter:
+            interpreter_actual_path = _runfiles_root_path(ctx, runtime.interpreter.short_path)
+        else:
+            interpreter_actual_path = runtime.interpreter_path
+
+        py_exe_basename = paths.basename(interpreter_actual_path)
+
+        # When the venv symlinks are disabled, the $venv/bin/python3 file isn't
+        # needed or used at runtime. However, the zip code uses the interpreter
+        # File object to figure out some paths.
+        interpreter = ctx.actions.declare_file("{}/bin/{}".format(venv, py_exe_basename))
+        ctx.actions.write(interpreter, "actual:{}".format(interpreter_actual_path))
+
+    elif runtime.interpreter:
         py_exe_basename = paths.basename(runtime.interpreter.short_path)
 
         # Even though ctx.actions.symlink() is used, using
@@ -571,6 +593,7 @@ def _create_venv(ctx, output_prefix, imports, runtime_details):
 
     return struct(
         interpreter = interpreter,
+        recreate_venv_at_runtime = not venvs_use_declare_symlink_enabled,
         # Runfiles root relative path or absolute path
         interpreter_actual_path = interpreter_actual_path,
         files_without_interpreter = [pyvenv_cfg, pth, site_init],
@@ -657,15 +680,13 @@ def _create_stage1_bootstrap(
     else:
         python_binary_path = runtime_details.executable_interpreter_path
 
-    if is_for_zip and venv:
-        python_binary_actual = venv.interpreter_actual_path
-    else:
-        python_binary_actual = ""
+    python_binary_actual = venv.interpreter_actual_path if venv else ""
 
     subs = {
         "%is_zipfile%": "1" if is_for_zip else "0",
         "%python_binary%": python_binary_path,
         "%python_binary_actual%": python_binary_actual,
+        "%recreate_venv_at_runtime%": str(int(venv.recreate_venv_at_runtime)) if venv else "0",
         "%target%": str(ctx.label),
         "%workspace_name%": ctx.workspace_name,
     }