Merge branch 'main' into faas-add-pex-args

docs/docs/python/overview/lockfiles.mdx

@@ -184,10 +184,7 @@ It is strongly recommended that these tools be installed from a hermetic lockfil
 
 The only time you need to think about this is if you want to customize the tool requirements that Pants uses. This might be the case if you want to modify the version of a tool or add extra requirements (for example, tool plugins).
 
-:::caution Exporting tools requires a custom lockfile
-:::
-
-If you want a tool to be installed from some resolve, instead of from the built-in lockfile, you set `install_from_resolve` and `requirements` on the tool's config section:
+Tools can also be installed from a specific resolve instead of from the built-in lockfile. This is useful for specifying a version of the tool and including extra packages. To do this, set `install_from_resolve` and `requirements` on the tool's config section:
 
 ```toml title="pants.toml"
 [python.resolves]

docs/docs/shell/self-extractable-archives.mdx

@@ -0,0 +1,265 @@
+---
+    title: Self-extractable archives
+    sidebar_position: 2
+---
+
+Self-extractable archives with [`makeself`](https://github.com/megastep/makeself)
+
+---
+
+Pants integrates with [`makeself`](https://github.com/megastep/makeself) tool
+to allow you to easily build self-extractable archives. To enable the
+integration activate the `makeself` backend in `pants.toml`:
+
+```toml title="pants.toml"
+[GLOBAL]
+backend_packages = [
+    ...
+    "pants.backend.experimental.makeself",
+]
+```
+
+## Minimal example
+
+The [`makeself_archive`](../../reference/targets/makeself_archive.mdx) target
+allows you to bundle files and packages into a single executable archive.
+
+Here is a minimal example:
+
+```python title="BUILD"
+makeself_archive(
+    name="arc",
+    startup_script=["echo", "hello pants"],
+)
+```
+
+To build the archive use the `package` goal:
+
+```bash
+pants package :arc
+```
+```
+[INFO] Wrote dist/arc.run
+Built Makeself binary: arc.run
+```
+
+Now run the archive just like a regular executable:
+
+```bash
+dist/arc.run
+```
+```
+Verifying archive integrity...  100%   MD5 checksums are OK. All good.
+Uncompressing arc.run  100%  
+hello pants
+```
+
+The built archive supports a bunch of parameters, you can inspect them manually:
+
+```bash
+dist/arc.run --help
+```
+
+Or refer to the [`makeself`](https://github.com/megastep/makeself) documentation.
+
+## Bundling multiple files
+
+You can bundle multiple shell scripts using
+[`files`](../../reference/targets/makeself_archive.mdx#files) field:
+
+```python tab={"label":"BUILD"}
+shell_sources(name="src")
+
+makeself_archive(
+    name="arc",
+    files=["lib.sh:src", "entrypoint.sh:src"],
+    startup_script=["./entrypoint.sh"],
+)
+```
+
+```bash tab={"label":"entrypoint.sh"} 
+#!/bin/bash
+
+. lib.sh
+echo $@ "one two three" | first_column
+```
+
+```bash tab={"label":"lib.sh"} 
+#!/bin/bash
+
+function first_column {
+    awk '{print $1}'
+}
+```
+
+Notice that we need to use a relative path to the `./entrypoint.sh`.
+
+```bash
+pants package :arc && dist/arc.run
+```
+```
+[INFO] Wrote dist/arc.run
+Built Makeself binary: arc.run
+Verifying archive integrity...  100%   MD5 checksums are OK. All good.
+Uncompressing arc.run  100%  
+one
+```
+
+To pass the arguments to the `startup_script` use `--`:
+
+```bash
+pants package :arc && dist/arc.run -- zero
+```
+```
+[INFO] Wrote dist/arc.run
+Built Makeself binary: arc.run
+Verifying archive integrity...  100%   MD5 checksums are OK. All good.
+Uncompressing arc.run  100%  
+zero
+```
+
+## `pants run`
+
+Instead of packaging and running `makeself_archive` manually, you can use the `run` goal instead:
+
+```bash
+pants run :arc
+```
+```
+Verifying archive integrity...  100%   MD5 checksums are OK. All good.
+Uncompressing arc.run  100%  
+one
+```
+
+To pass the arguments through the `pants run` goal you need `--`, then you need
+another `--` to pass arguments to the archive's `startup_script`, so you end up with
+two `--`:
+
+```bash
+pants run :arc -- -- zero
+```
+```
+Verifying archive integrity...  100%   MD5 checksums are OK. All good.
+Uncompressing arc.run  100%  
+zero
+```
+
+Similarly you can pass flags to the archive, for example, `quiet` flag to suppress progress messages:
+```bash
+pants run :arc -- -q -- zero
+```
+```
+zero
+```
+
+## Bundling packages like `pex_binary`
+
+You can put other packages like `pex_binary` into a makeself archive. 
+
+To configure `pex_binary`, first, update your `pants.toml`:
+
+```toml title="pants.toml"
+backend_packages = [
+    ...
+    "pants.backend.shell",
+    "pants.backend.experimental.makeself",
+    "pants.backend.python",
+]
+
+[python]
+interpreter_constraints = ["CPython==3.12.*"]
+```
+
+Now define the `pex_binary` and add it to the makeself archive via the
+`packages` field:
+
+```python tab={"label":"BUILD"}
+python_sources(name="py")
+pex_binary(
+    name="upper",
+    entry_point="upper.py",
+)
+shell_sources(name="sh")
+makeself_archive(
+    name="arc",
+    files=["lib.sh:sh", "entrypoint.sh:sh"],
+    packages=[":upper"],
+    startup_script=["./entrypoint.sh"],
+)
+```
+
+```python tab={"label":"upper.py"}
+print(input().upper())
+```
+
+```bash tab={"label":"entrypoint.sh"} 
+#!/bin/bash
+
+. lib.sh
+echo $@ "one two three" | first_column | ./upper.pex
+```
+
+```bash tab={"label":"lib.sh"} 
+#!/bin/bash
+
+function first_column {
+    awk '{print $1}'
+}
+```
+
+```bash
+pants run :arc -- -q -- zero
+```
+```
+/usr/bin/env: ‘python3.12’: No such file or directory
+```
+
+Oops! This happened because `pants run` is running in isolated environment, so
+we have to explicitly tell pants that we want to access the python interpreter
+and a couple of binaries used by pex:
+
+```python title="BUILD"
+...
+makeself_archive(
+    name="arc",
+    files=["lib.sh:sh", "entrypoint.sh:sh"],
+    packages=[":upper"],
+    startup_script=["./entrypoint.sh"],
+    tools=["python3.12", "grep", "sort"],
+)
+```
+
+Now it should work:
+
+```bash
+pants run :arc -- -q -- zero
+```
+```
+ZERO
+```
+
+Yay!
+
+## Using `makeself` build args
+
+To control the `makeself` archive creation you can provide `args` field:
+
+```python title="BUILD"
+makeself_archive(
+    name="arc",
+    startup_script=["echo", "Done"],
+    args=["--xz", "--nomd5"],
+    tools=["xz"],
+) 
+```
+```bash
+pants run :arc
+```
+```
+Verifying archive integrity...  100%   CRC checksums are OK. All good.
+Uncompressing arc.run  100%  
+Done
+```
+
+See full list of available options in the
+[docs](https://github.com/megastep/makeself#usage).

docs/docs/using-pants/setting-up-an-ide.mdx

@@ -49,7 +49,7 @@ The `--py-resolve-format=symlinked_immutable_virtualenv` option symlinks to an i
 
 ### Tool virtualenvs
 
-`pants export` can also create a virtualenv for each of the Python tools you use via Pants, such as `black`, `isort`, `pytest`, `mypy`, `flake8` and so on. This allows you to configure your editor to use the same version of the tool as Pants does for workflows like formatting on save. Follow [the instructions for creating a tool lockfile](../python/overview/lockfiles#lockfiles-for-tools).
+`pants export` can also create a virtualenv for each of the Python tools you use via Pants, such as `black`, `isort`, `pytest`, `mypy`, `flake8` and so on. This allows you to configure your editor to use the same version of the tool as Pants does for workflows like formatting on save. To use a custom version of these tools, follow [the instructions for creating a tool lockfile](../python/overview/lockfiles#lockfiles-for-tools).
 
 ## Generated code
 

docs/notes/2.22.x.md

@@ -30,6 +30,15 @@ A new experimental `SQL` backend was added along with the [sqlfluff
 linter](https://www.pantsbuild.org/2.22/reference/subsystems/sqlfluff). See
 docs [here](https://www.pantsbuild.org/2.22/docs/sql).
 
+#### NEW: Makeself
+
+A new experimental `pants.backend.experimental.makeself` backend was added to support
+[`makeself`](https://github.com/megastep/makeself) tool. A new
+[`makeself_archive`](https://www.pantsbuild.org/2.22/reference/targets/makeself_archive)
+target allows you to create self-extractable archives that bundle files, bash
+scripts or even pex binaries into a single executable archive! See examples in the
+[docs](https://www.pantsbuild.org/2.22/docs/shell/self-extractable-archives).
+
 #### Helm
 
 The default version of the [hikaru](https://github.com/haxsaw/hikaru) packages used to parse Kubernetes resources has been upgraded.  If you were previously installing `hikaru` from a custom resolve you should be aware that the upstream packaging strategy for supporting multiple Kubernetes versions [has changed](https://github.com/haxsaw/hikaru/blob/main/release_notes.rst#v110).
@@ -47,6 +56,8 @@ deploy jars, in addition to those specified on a per-jar basis in the `deploy_ja
 This option's default value excludes signature files from constituent jars, which are known to cause the deploy jar
 to fail to execute (since naturally it doesn't match those signatures).
 
+The internal code for exporting JVM tools was refactored.
+
 ##### Scala
 
 Setting the `orphan_files_behaviour = "ignore"` option for [`pants.backend.experimental.scala.lint.scalafix`](https://www.pantsbuild.org/2.22/reference/subsystems/scalafix#orphan_files_behavior) or [`pants.backend.experimental.scala.lint.scalafmt`](https://www.pantsbuild.org/2.22/reference/subsystems/scalafmt#orphan_files_behavior) backend is now properly silent. It previously showed spurious warnings.
@@ -73,6 +84,8 @@ The deprecation for the `platforms` field for the `pex_binary` and `pex_binaries
 
 Python tool subsystem docs and help text now include the default version of the tool, along with instructions on how to override this version using a custom lockfile. Additionally, the help text for the `install_from_resolve` option for Python tools now includes this same information.
 
+Python tools can be [exported from their default bundled lockfiles](https://www.pantsbuild.org/2.22/docs/using-pants/setting-up-an-ide#tool-virtualenvs). For instance, when using the default `black` subsystem, `pants export --resolve=black` will export a venv containing the version of black that Pants runs.
+
 New field `pex_build_extra_args` is available on FaaS targets [python_aws_lambda_function](https://www.pantsbuild.org/2.22/reference/targets/python_aws_lambda_function#pex_build_extra_args), 
 [python_aws_lambda_layer](https://www.pantsbuild.org/2.22/reference/targets/python_aws_lambda_layer#pex_build_extra_args), and [python_google_cloud_function]((https://www.pantsbuild.org/2.22/reference/targets/python_aws_lambda_layer#pex_build_extra_args). This allows passing arguments to the `pex` invocation for collecting sources and packaging.
 

src/python/pants/backend/cc/lint/clangformat/subsystem.py

@@ -8,6 +8,7 @@
 
 from pants.backend.python.subsystems.python_tool_base import PythonToolBase
 from pants.backend.python.target_types import ConsoleScript
+from pants.core.goals.resolves import ExportableTool
 from pants.core.util_rules.config_files import ConfigFilesRequest
 from pants.engine.rules import Rule, collect_rules
 from pants.engine.unions import UnionRule
@@ -51,4 +52,7 @@ def config_request(self, dirs: Iterable[str]) -> ConfigFilesRequest:
 
 
 def rules() -> Iterable[Rule | UnionRule]:
-    return collect_rules()
+    return [
+        *collect_rules(),
+        UnionRule(ExportableTool, ClangFormat),
+    ]

src/python/pants/backend/codegen/avro/java/rules.py

@@ -17,7 +17,7 @@
 from pants.backend.java.target_types import JavaSourceField
 from pants.base.glob_match_error_behavior import GlobMatchErrorBehavior
 from pants.build_graph.address import Address
-from pants.core.goals.generate_lockfiles import GenerateToolLockfileSentinel
+from pants.core.goals.resolves import ExportableTool
 from pants.engine.fs import (
     AddPrefix,
     CreateDigest,
@@ -53,7 +53,7 @@
 from pants.jvm.jdk_rules import InternalJdk, JvmProcess
 from pants.jvm.resolve import jvm_tool
 from pants.jvm.resolve.coursier_fetch import ToolClasspath, ToolClasspathRequest
-from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool, GenerateJvmToolLockfileSentinel
+from pants.jvm.resolve.jvm_tool import GenerateJvmLockfileFromTool
 from pants.jvm.subsystems import JvmSubsystem
 from pants.jvm.target_types import JvmResolveField, PrefixedJvmJdkField, PrefixedJvmResolveField
 from pants.source.source_root import SourceRoot, SourceRootRequest
@@ -66,10 +66,6 @@ class GenerateJavaFromAvroRequest(GenerateSourcesRequest):
     output = JavaSourceField
 
 
-class AvroToolLockfileSentinel(GenerateJvmToolLockfileSentinel):
-    resolve_name = AvroSubsystem.options_scope
-
-
 @dataclass(frozen=True)
 class CompileAvroSourceRequest:
     digest: Digest
@@ -116,7 +112,7 @@ async def compile_avro_source(
     output_dir = "_generated_files"
     toolcp_relpath = "__toolcp"
 
-    lockfile_request = await Get(GenerateJvmLockfileFromTool, AvroToolLockfileSentinel())
+    lockfile_request = GenerateJvmLockfileFromTool.create(avro_tools)
     tool_classpath, subsetted_input_digest, empty_output_dir = await MultiGet(
         Get(ToolClasspath, ToolClasspathRequest(lockfile=lockfile_request)),
         Get(
@@ -222,13 +218,6 @@ def make_avro_process(
     return CompiledAvroSource(normalized_digest)
 
 
-@rule
-def generate_avro_tools_lockfile_request(
-    _: AvroToolLockfileSentinel, tool: AvroSubsystem
-) -> GenerateJvmLockfileFromTool:
-    return GenerateJvmLockfileFromTool.create(tool)
-
-
 @dataclass(frozen=True)
 class AvroRuntimeDependencyInferenceFieldSet(FieldSet):
     required_fields = (
@@ -326,7 +315,7 @@ def rules():
         *jvm_tool.rules(),
         *jdk_rules.rules(),
         UnionRule(GenerateSourcesRequest, GenerateJavaFromAvroRequest),
-        UnionRule(GenerateToolLockfileSentinel, AvroToolLockfileSentinel),
+        UnionRule(ExportableTool, AvroSubsystem),
         UnionRule(InferDependenciesRequest, InferAvroRuntimeDependencyRequest),
         AvroSourceTarget.register_plugin_field(PrefixedJvmJdkField),
         AvroSourcesGeneratorTarget.register_plugin_field(PrefixedJvmJdkField),