Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add a trampoline variant that just executes python #8637

Merged
merged 8 commits into from
Oct 29, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 10 additions & 3 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -395,16 +395,23 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
workspaces: ${{ github.workspace }}/crates/uv-trampoline
- name: "Test committed binaries"
working-directory: ${{ github.workspace }}
run: |
rustup target add ${{ matrix.target-arch }}-pc-windows-msvc
cargo test -p uv-trampoline-builder --target ${{ matrix.target-arch }}-pc-windows-msvc
# Build and copy the new binaries
- name: "Build"
working-directory: ${{ github.workspace }}/crates/uv-trampoline
run: |
cargo build --target ${{ matrix.target-arch }}-pc-windows-msvc
cp target/${{ matrix.target-arch }}-pc-windows-msvc/debug/uv-trampoline-console.exe trampolines/uv-trampoline-${{ matrix.target-arch }}-console.exe
cp target/${{ matrix.target-arch }}-pc-windows-msvc/debug/uv-trampoline-gui.exe trampolines/uv-trampoline-${{ matrix.target-arch }}-gui.exe
- name: "Test"
working-directory: ${{ github.workspace }}/crates/uv-trampoline
run: cargo test --target ${{ matrix.target-arch }}-pc-windows-msvc --test *
- name: "Test new binaries"
working-directory: ${{ github.workspace }}
run: |
# We turn off the default "production" test feature since these are debug binaries
cargo test -p uv-trampoline-builder --target ${{ matrix.target-arch }}-pc-windows-msvc --no-default-features

typos:
runs-on: ubuntu-latest
Expand Down
15 changes: 15 additions & 0 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -61,6 +61,7 @@ uv-settings = { path = "crates/uv-settings" }
uv-shell = { path = "crates/uv-shell" }
uv-state = { path = "crates/uv-state" }
uv-static = { path = "crates/uv-static" }
uv-trampoline-builder = { path = "crates/uv-trampoline-builder" }
uv-tool = { path = "crates/uv-tool" }
uv-types = { path = "crates/uv-types" }
uv-version = { path = "crates/uv-version" }
Expand Down
1 change: 1 addition & 0 deletions crates/uv-install-wheel/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ uv-normalize = { workspace = true }
uv-pep440 = { workspace = true }
uv-platform-tags = { workspace = true }
uv-pypi-types = { workspace = true }
uv-trampoline-builder = { workspace = true }
uv-warnings = { workspace = true }

clap = { workspace = true, optional = true, features = ["derive"] }
Expand Down
2 changes: 2 additions & 0 deletions crates/uv-install-wheel/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -97,4 +97,6 @@ pub enum Error {
MismatchedVersion(Version, Version),
#[error("Invalid egg-link")]
InvalidEggLink(PathBuf),
#[error(transparent)]
LauncherError(#[from] uv_trampoline_builder::Error),
}
170 changes: 8 additions & 162 deletions crates/uv-install-wheel/src/wheel.rs
Original file line number Diff line number Diff line change
@@ -1,51 +1,26 @@
use std::collections::HashMap;
use std::io::{BufReader, Cursor, Read, Seek, Write};
use std::io;
use std::io::{BufReader, Read, Seek, Write};
use std::path::{Path, PathBuf};
use std::{env, io};

use crate::record::RecordEntry;
use crate::script::Script;
use crate::{Error, Layout};
use data_encoding::BASE64URL_NOPAD;
use fs_err as fs;
use fs_err::{DirEntry, File};
use mailparse::parse_headers;
use rustc_hash::FxHashMap;
use sha2::{Digest, Sha256};
use tracing::{instrument, warn};
use walkdir::WalkDir;

use uv_cache_info::CacheInfo;
use uv_fs::{relative_to, Simplified};
use uv_normalize::PackageName;
use uv_pypi_types::DirectUrl;
use walkdir::WalkDir;
use zip::write::FileOptions;
use zip::ZipWriter;

const LAUNCHER_MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V'];
use uv_trampoline_builder::windows_script_launcher;

#[cfg(all(windows, target_arch = "x86"))]
const LAUNCHER_I686_GUI: &[u8] =
include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-gui.exe");

#[cfg(all(windows, target_arch = "x86"))]
const LAUNCHER_I686_CONSOLE: &[u8] =
include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-console.exe");

#[cfg(all(windows, target_arch = "x86_64"))]
const LAUNCHER_X86_64_GUI: &[u8] =
include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe");

#[cfg(all(windows, target_arch = "x86_64"))]
const LAUNCHER_X86_64_CONSOLE: &[u8] =
include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe");

#[cfg(all(windows, target_arch = "aarch64"))]
const LAUNCHER_AARCH64_GUI: &[u8] =
include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe");

#[cfg(all(windows, target_arch = "aarch64"))]
const LAUNCHER_AARCH64_CONSOLE: &[u8] =
include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe");
use crate::record::RecordEntry;
use crate::script::Script;
use crate::{Error, Layout};

/// Wrapper script template function
///
Expand Down Expand Up @@ -158,87 +133,6 @@ fn format_shebang(executable: impl AsRef<Path>, os_name: &str, relocatable: bool
format!("#!{executable}")
}

/// A Windows script is a minimal .exe launcher binary with the python entrypoint script appended as
/// stored zip file.
///
/// <https://github.com/pypa/pip/blob/fd0ea6bc5e8cb95e518c23d901c26ca14db17f89/src/pip/_vendor/distlib/scripts.py#L248-L262>
#[allow(unused_variables)]
pub(crate) fn windows_script_launcher(
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This moved ~unchanged into uv-trampoline-builder

launcher_python_script: &str,
is_gui: bool,
python_executable: impl AsRef<Path>,
) -> Result<Vec<u8>, Error> {
// This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain
// compilation on all platforms.
if cfg!(not(windows)) {
return Err(Error::NotWindows);
}

let launcher_bin: &[u8] = match env::consts::ARCH {
#[cfg(all(windows, target_arch = "x86"))]
"x86" => {
if is_gui {
LAUNCHER_I686_GUI
} else {
LAUNCHER_I686_CONSOLE
}
}
#[cfg(all(windows, target_arch = "x86_64"))]
"x86_64" => {
if is_gui {
LAUNCHER_X86_64_GUI
} else {
LAUNCHER_X86_64_CONSOLE
}
}
#[cfg(all(windows, target_arch = "aarch64"))]
"aarch64" => {
if is_gui {
LAUNCHER_AARCH64_GUI
} else {
LAUNCHER_AARCH64_CONSOLE
}
}
#[cfg(windows)]
arch => {
return Err(Error::UnsupportedWindowsArch(arch));
}
#[cfg(not(windows))]
arch => &[],
};

let mut payload: Vec<u8> = Vec::new();
{
// We're using the zip writer, but with stored compression
// https://github.com/njsmith/posy/blob/04927e657ca97a5e35bb2252d168125de9a3a025/src/trampolines/mod.rs#L75-L82
// https://github.com/pypa/distlib/blob/8ed03aab48add854f377ce392efffb79bb4d6091/PC/launcher.c#L259-L271
let stored = FileOptions::default().compression_method(zip::CompressionMethod::Stored);
let mut archive = ZipWriter::new(Cursor::new(&mut payload));
let error_msg = "Writing to Vec<u8> should never fail";
archive.start_file("__main__.py", stored).expect(error_msg);
archive
.write_all(launcher_python_script.as_bytes())
.expect(error_msg);
archive.finish().expect(error_msg);
}

let python = python_executable.as_ref();
let python_path = python.simplified_display().to_string();

let mut launcher: Vec<u8> = Vec::with_capacity(launcher_bin.len() + payload.len());
launcher.extend_from_slice(launcher_bin);
launcher.extend_from_slice(&payload);
launcher.extend_from_slice(python_path.as_bytes());
launcher.extend_from_slice(
&u32::try_from(python_path.as_bytes().len())
.expect("File Path to be smaller than 4GB")
.to_le_bytes(),
);
launcher.extend_from_slice(&LAUNCHER_MAGIC_NUMBER);

Ok(launcher)
}

/// Returns a [`PathBuf`] to `python[w].exe` for script execution.
///
/// <https://github.com/pypa/pip/blob/76e82a43f8fb04695e834810df64f2d9a2ff6020/src/pip/_vendor/distlib/scripts.py#L121-L126>
Expand Down Expand Up @@ -1075,54 +969,6 @@ mod test {
Ok(())
}

#[test]
#[cfg(all(windows, target_arch = "x86"))]
fn test_launchers_are_small() {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This moved into uv-trampoline-builder, size limit bumped

// At time of writing, they are 45kb~ bytes.
assert!(
super::LAUNCHER_I686_GUI.len() < 45 * 1024,
"GUI launcher: {}",
super::LAUNCHER_I686_GUI.len()
);
assert!(
super::LAUNCHER_I686_CONSOLE.len() < 45 * 1024,
"CLI launcher: {}",
super::LAUNCHER_I686_CONSOLE.len()
);
}

#[test]
#[cfg(all(windows, target_arch = "x86_64"))]
fn test_launchers_are_small() {
// At time of writing, they are 45kb~ bytes.
assert!(
super::LAUNCHER_X86_64_GUI.len() < 45 * 1024,
"GUI launcher: {}",
super::LAUNCHER_X86_64_GUI.len()
);
assert!(
super::LAUNCHER_X86_64_CONSOLE.len() < 45 * 1024,
"CLI launcher: {}",
super::LAUNCHER_X86_64_CONSOLE.len()
);
}

#[test]
#[cfg(all(windows, target_arch = "aarch64"))]
fn test_launchers_are_small() {
// At time of writing, they are 45kb~ bytes.
assert!(
super::LAUNCHER_AARCH64_GUI.len() < 45 * 1024,
"GUI launcher: {}",
super::LAUNCHER_AARCH64_GUI.len()
);
assert!(
super::LAUNCHER_AARCH64_CONSOLE.len() < 45 * 1024,
"CLI launcher: {}",
super::LAUNCHER_AARCH64_CONSOLE.len()
);
}

#[test]
fn test_script_executable() -> Result<()> {
// Test with adjacent pythonw.exe
Expand Down
34 changes: 34 additions & 0 deletions crates/uv-trampoline-builder/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,34 @@
[package]
name = "uv-trampoline-builder"
version = "0.0.1"
publish = false
description = "Builds launchers for `uv-trampoline`"

edition = { workspace = true }
rust-version = { workspace = true }
homepage = { workspace = true }
documentation = { workspace = true }
repository = { workspace = true }
authors = { workspace = true }
license = { workspace = true }

[features]
default = ["production"]

# Expect tests to run against production builds of `uv-trampoline` binaries, rather than debug builds
production = []

[lints]
workspace = true

[dependencies]
uv-fs = { workspace = true }
thiserror = { workspace = true }
zip = { workspace = true }

[dev-dependencies]
assert_cmd = { version = "2.0.16" }
assert_fs = { version = "1.1.2" }
anyhow = { version = "1.0.89" }
fs-err = { workspace = true }
which = { workspace = true }
Loading
Loading