feat: add trampoline for pixi global (#2381)

Co-authored-by: nichmor <nmorkotilo@gmail.com>
Co-authored-by: Wolf Vollprecht <w.vollprecht@gmail.com>
Co-authored-by: Ruben Arts <ruben.arts@hotmail.com>

.github/workflows/trampoline.yaml

@@ -0,0 +1,116 @@
+name: Update Trampoline Binary
+
+on:
+  push:
+    paths:
+      - 'crates/pixi_trampoline/**'
+  workflow_dispatch:
+  pull_request:
+    paths:
+      - 'crates/pixi_trampoline/**'
+
+permissions:
+  contents: write  # Allow write permissions for contents (like pushing to the repo)
+  pull-requests: write
+
+jobs:
+  build:
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        working-directory: crates/pixi_trampoline
+    strategy:
+      fail-fast: true
+      matrix:
+        include:
+          - { name: "Linux-x86_64", target: x86_64-unknown-linux-musl, os: ubuntu-latest  }
+          - { name: "Linux-aarch64", target: aarch64-unknown-linux-musl, os: ubuntu-latest }
+          - { name: "macOS-x86", target: x86_64-apple-darwin, os: macos-13 }
+          - { name: "macOS-arm", target: aarch64-apple-darwin, os: macos-14 }
+          - { name: "Windows", target: x86_64-pc-windows-msvc, os: windows-latest }
+          - { name: "Windows-arm", target: aarch64-pc-windows-msvc, os: windows-latest }
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0 # Fetch full history so we have branch information
+
+      - name: Set up Rust
+        uses: taiki-e/setup-cross-toolchain-action@v1
+        with:
+          target: ${{ matrix.target }}
+
+      - name: Build trampoline binary
+        run: cargo build --release --target ${{ matrix.target }}
+
+      - name: Move trampoline binary on windows
+        if: startsWith(matrix.name, 'Windows')
+        run: |
+          mkdir -p trampolines-binaries
+          mv target/${{ matrix.target }}/release/pixi_trampoline.exe trampolines-binaries/pixi-trampoline-${{ matrix.target }}.exe
+
+      - name: Move trampoline binary on unix
+        if: startsWith(matrix.name, 'Windows') == false
+        run: |
+          mkdir -p trampolines-binaries
+          mv target/${{ matrix.target }}/release/pixi_trampoline trampolines-binaries/pixi-trampoline-${{ matrix.target }}
+
+      - name: Upload binary artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: trampoline-${{ matrix.target }}
+          path: crates/pixi_trampoline/trampolines-binaries/
+
+  aggregate:
+    runs-on: ubuntu-latest
+    defaults:
+      run:
+        working-directory: crates/pixi_trampoline
+    needs: build  # This ensures the aggregation job runs after the build jobs
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Download all binaries
+        uses: actions/download-artifact@v3
+        with:
+          path: crates/pixi_trampoline/trampolines-binaries/
+
+      - name: List downloaded files
+        run: ls -R trampolines-binaries/
+
+      - name: Move trampolines
+        run: |
+          mkdir -p trampolines
+          # Iterate through all files in trampolines directory and its subdirectories
+          find trampolines-binaries -type f  -name 'pixi-trampoline-*' -exec mv -f {} trampolines/ \;
+          # now iterate through all files in trampolines directory and compress them using zstd
+          # and remove the original file
+          # by using -f we allow overwriting the file
+          for file in trampolines/*; do
+            zstd "$file" -f
+            rm "$file"
+          done
+          ls -R trampolines/
+
+      - name: Upload binary artifact
+        uses: actions/upload-artifact@v3
+        with:
+          name: trampolines
+          path: crates/pixi_trampoline/trampolines/
+
+
+
+      - name: Commit and push updated binaries
+        # Don't run on forks
+        if: github.repository == 'prefix-dev/pixi' && startsWith(github.ref, 'reaf/heads')
+        run: |
+          # Set the repository to push to the repository the workflow is running on
+          git config user.name "GitHub Actions"
+          git config user.email "actions@github.com"
+          git add trampolines/
+          git commit -m "[CI]: Update trampoline binaries for all targets"
+          # Push changes to the branch that triggered the workflow
+          BRANCH=$(echo "${GITHUB_REF#refs/heads/}")
+          git push origin HEAD:$BRANCH

Cargo.toml

@@ -1,5 +1,9 @@
 [workspace]
 members = ["crates/*"]
+# exclude pixi_trampoline as it's not possible
+# to override some of the profile settings in workspace
+# https://github.com/rust-lang/cargo/issues/8264
+exclude = ["crates/pixi_trampoline"]
 
 [workspace.package]
 authors = ["pixi contributors <hi@prefix.dev>"]
@@ -123,6 +127,7 @@ uv-types = { git = "https://github.com/astral-sh/uv", tag = "0.4.30" }
 winapi = { version = "0.3.9", default-features = false }
 xxhash-rust = "0.8.10"
 zip = { version = "2.2.0", default-features = false }
+zstd = { version = "0.13.2", default-features = false }
 
 fancy_display = { path = "crates/fancy_display" }
 pixi_config = { path = "crates/pixi_config" }
@@ -284,6 +289,7 @@ uv-resolver = { workspace = true }
 uv-types = { workspace = true }
 xxhash-rust = { workspace = true }
 zip = { workspace = true, features = ["deflate", "time"] }
+zstd = { workspace = true }
 
 
 [target.'cfg(unix)'.dependencies]

crates/pixi_trampoline/Cargo.toml

@@ -0,0 +1,29 @@
+[package]
+authors = ["pixi contributors <hi@prefix.dev>"]
+description = "Trampoline binary that is used to run binaries instaled by pixi global"
+edition = "2021"
+homepage = "https://github.com/prefix-dev/pixi"
+license = "BSD-3-Clause"
+name = "pixi_trampoline"
+readme = "README.md"
+repository = "https://github.com/prefix-dev/pixi"
+version = "0.1.0"
+
+[profile.release]
+# Enable Link Time Optimization.
+lto = true
+# Reduce number of codegen units to increase optimizations.
+codegen-units = 1
+# Optimize for size.
+opt-level = "z"
+# Abort on panic.
+panic = "abort"
+# Automatically strip symbols from the binary.
+debug = false
+strip = true
+
+
+[dependencies]
+ctrlc = "3.4"
+serde = { version = "1.0.210", features = ["derive"] }
+serde_json = "1.0.128"

crates/pixi_trampoline/README.md

@@ -0,0 +1,22 @@
+# A small trampoline binary that allow to run executables installed by pixi global install.
+
+
+This is the configuration used by trampoline to set the env variables, and run the executable.
+
+```js
+{
+    // Full path to the executable
+    "exe": "/Users/wolfv/.pixi/envs/conda-smithy/bin/conda-smithy",
+    // One or more path segments to prepend to the PATH environment variable
+    "path": "/Users/wolfv/.pixi/envs/conda-smithy/bin",
+    // One or more environment variables to set
+    "env": {
+        "CONDA_PREFIX": "/Users/wolfv/.pixi/envs/conda-smithy"
+    }
+}
+```
+
+# How to build it?
+You can use `trampoline.yaml` workflow to build the binary for all the platforms and architectures supported by pixi.
+In case of building it manually, you can use the following command, after executing the `cargo build --release`, you need to compress it using `zstd`.
+If running it manually or triggered by changes in `crates/pixi_trampoline` from the main repo, they will be automatically committed to the branch.

crates/pixi_trampoline/src/main.rs

@@ -0,0 +1,85 @@
+use serde::Deserialize;
+use std::collections::HashMap;
+use std::env;
+use std::fs::File;
+#[cfg(target_family = "unix")]
+use std::os::unix::process::CommandExt;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+
+
+// trampoline configuration folder name
+pub const TRAMPOLINE_CONFIGURATION: &str = "trampoline_configuration";
+
+#[derive(Deserialize, Debug)]
+struct Metadata {
+    exe: String,
+    path: String,
+    env: HashMap<String, String>,
+}
+
+fn read_metadata(current_exe: &Path) -> Metadata {
+    // the metadata file is next to the current executable parent folder,
+    // under trampoline_configuration/current_exe_name.json
+    let exe_parent = current_exe.parent().expect("should have a parent");
+    let exe_name = current_exe.file_stem().expect("should have a file name");
+    let metadata_path = exe_parent.join(TRAMPOLINE_CONFIGURATION).join(format!("{}{}", exe_name.to_string_lossy(), ".json"));
+    let metadata_file = File::open(metadata_path).unwrap();
+    let metadata: Metadata = serde_json::from_reader(metadata_file).unwrap();
+    metadata
+}
+
+fn prepend_path(extra_path: &str) -> String {
+    let path = env::var("PATH").unwrap();
+    let mut split_path = env::split_paths(&path).collect::<Vec<_>>();
+    split_path.insert(0, PathBuf::from(extra_path));
+    let new_path = env::join_paths(split_path).unwrap();
+    new_path.to_string_lossy().into_owned()
+}
+
+fn main() -> () {
+    // Get command-line arguments (excluding the program name)
+    let args: Vec<String> = env::args().collect();
+    let current_exe = env::current_exe().expect("Failed to get current executable path");
+
+    // ignore any ctrl-c signals
+    ctrlc::set_handler(move || {}).expect("Error setting Ctrl-C handler");
+
+    let metadata = read_metadata(&current_exe);
+
+    // Create a new Command for the specified executable
+    let mut cmd = Command::new(metadata.exe);
+
+    let new_path = prepend_path(&metadata.path);
+
+    // Set the PATH environment variable
+    cmd.env("PATH", new_path);
+
+    // Set any additional environment variables
+    for (key, value) in metadata.env.iter() {
+        cmd.env(key, value);
+    }
+
+    // Add any additional arguments
+    cmd.args(&args[1..]);
+
+    // Configure stdin, stdout, and stderr to use the current process's streams
+    cmd.stdin(Stdio::inherit())
+        .stdout(Stdio::inherit())
+        .stderr(Stdio::inherit());
+
+    // Spawn the child process
+    #[cfg(target_family = "unix")]
+    cmd.exec();
+
+    #[cfg(target_os = "windows")]
+    {
+        let mut child = cmd.spawn().expect("process spawn should succeed");
+
+        // Wait for the child process to complete
+        let status = child.wait().expect("failed to wait on child");
+
+        // Exit with the same status code as the child process
+        std::process::exit(status.code().unwrap_or(1));
+    }
+}

docs/features/global_tools.md

@@ -150,6 +150,17 @@ You can `remove` dependencies by running:
 pixi global remove --environment my-env package-a package-b
 ```
 
+### Trampolines
+
+To increase efficiency, `pixi` uses *trampolines*—small, specialized binary files that manage configuration and environment setup before executing the main binary. The trampoline approach allows for skipping the execution of activation scripts that have a significant performance impact.
+
+When you execute a global install binary, a trampoline performs the following sequence of steps:
+
+* Each trampoline first reads a configuration file named after the binary being executed. This configuration file, in JSON format (e.g., `python.json`), contains key information about how the environment should be set up. The configuration file is stored in `.pixi/bin/trampoline_configuration`.
+* Once the configuration is loaded and the environment is set, the trampoline executes the original binary with the correct environment settings.
+* When installing a new binary, a new trampoline is placed in the `.pixi/bin` directory and is hardlinked to the `.pixi/bin/trampoline_configuration/trampoline_bin`. This optimizes storage space and avoids duplication of the same trampoline.
+
+
 ### Example: Adding a series of tools at once
 Without specifying an environment, you can add multiple tools at once:
 ```shell