diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 7f8777503aeb..bcd663f165fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -395,6 +395,11 @@ jobs: - uses: Swatinem/rust-cache@v2 with: workspaces: ${{ github.workspace }}/crates/uv-trampoline + - name: "Test committed binaries" + working-directory: ${{ github.workspace }} + run: | + rustup target add ${{ matrix.target-arch }}-pc-windows-msvc + cargo test -p uv-trampoline-builder --target ${{ matrix.target-arch }}-pc-windows-msvc # Build and copy the new binaries - name: "Build" working-directory: ${{ github.workspace }}/crates/uv-trampoline @@ -402,9 +407,11 @@ jobs: cargo build --target ${{ matrix.target-arch }}-pc-windows-msvc cp target/${{ matrix.target-arch }}-pc-windows-msvc/debug/uv-trampoline-console.exe trampolines/uv-trampoline-${{ matrix.target-arch }}-console.exe cp target/${{ matrix.target-arch }}-pc-windows-msvc/debug/uv-trampoline-gui.exe trampolines/uv-trampoline-${{ matrix.target-arch }}-gui.exe - - name: "Test" - working-directory: ${{ github.workspace }}/crates/uv-trampoline - run: cargo test --target ${{ matrix.target-arch }}-pc-windows-msvc --test * + - name: "Test new binaries" + working-directory: ${{ github.workspace }} + run: | + # We turn off the default "production" test feature since these are debug binaries + cargo test -p uv-trampoline-builder --target ${{ matrix.target-arch }}-pc-windows-msvc --no-default-features typos: runs-on: ubuntu-latest diff --git a/Cargo.lock b/Cargo.lock index 4ebbcfc9aab0..66411d51456c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4770,6 +4770,7 @@ dependencies = [ "uv-pep440", "uv-platform-tags", "uv-pypi-types", + "uv-trampoline-builder", "uv-warnings", "walkdir", "zip", @@ -5268,6 +5269,20 @@ dependencies = [ "uv-virtualenv", ] +[[package]] +name = "uv-trampoline-builder" +version = "0.0.1" +dependencies = [ + "anyhow", + "assert_cmd", + "assert_fs", + "fs-err", + "thiserror", + "uv-fs", + "which", + "zip", +] + [[package]] name = "uv-types" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index a2358c19f266..3f4a71dca1c6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -61,6 +61,7 @@ uv-settings = { path = "crates/uv-settings" } uv-shell = { path = "crates/uv-shell" } uv-state = { path = "crates/uv-state" } uv-static = { path = "crates/uv-static" } +uv-trampoline-builder = { path = "crates/uv-trampoline-builder" } uv-tool = { path = "crates/uv-tool" } uv-types = { path = "crates/uv-types" } uv-version = { path = "crates/uv-version" } diff --git a/crates/uv-install-wheel/Cargo.toml b/crates/uv-install-wheel/Cargo.toml index 97f5cf57a909..d7dcd8fd1db7 100644 --- a/crates/uv-install-wheel/Cargo.toml +++ b/crates/uv-install-wheel/Cargo.toml @@ -28,6 +28,7 @@ uv-normalize = { workspace = true } uv-pep440 = { workspace = true } uv-platform-tags = { workspace = true } uv-pypi-types = { workspace = true } +uv-trampoline-builder = { workspace = true } uv-warnings = { workspace = true } clap = { workspace = true, optional = true, features = ["derive"] } diff --git a/crates/uv-install-wheel/src/lib.rs b/crates/uv-install-wheel/src/lib.rs index 83300d42da76..69249229aab6 100644 --- a/crates/uv-install-wheel/src/lib.rs +++ b/crates/uv-install-wheel/src/lib.rs @@ -97,4 +97,6 @@ pub enum Error { MismatchedVersion(Version, Version), #[error("Invalid egg-link")] InvalidEggLink(PathBuf), + #[error(transparent)] + LauncherError(#[from] uv_trampoline_builder::Error), } diff --git a/crates/uv-install-wheel/src/wheel.rs b/crates/uv-install-wheel/src/wheel.rs index 559237e16769..cf640348e7fc 100644 --- a/crates/uv-install-wheel/src/wheel.rs +++ b/crates/uv-install-wheel/src/wheel.rs @@ -1,11 +1,8 @@ use std::collections::HashMap; -use std::io::{BufReader, Cursor, Read, Seek, Write}; +use std::io; +use std::io::{BufReader, Read, Seek, Write}; use std::path::{Path, PathBuf}; -use std::{env, io}; -use crate::record::RecordEntry; -use crate::script::Script; -use crate::{Error, Layout}; use data_encoding::BASE64URL_NOPAD; use fs_err as fs; use fs_err::{DirEntry, File}; @@ -13,39 +10,17 @@ use mailparse::parse_headers; use rustc_hash::FxHashMap; use sha2::{Digest, Sha256}; use tracing::{instrument, warn}; +use walkdir::WalkDir; + use uv_cache_info::CacheInfo; use uv_fs::{relative_to, Simplified}; use uv_normalize::PackageName; use uv_pypi_types::DirectUrl; -use walkdir::WalkDir; -use zip::write::FileOptions; -use zip::ZipWriter; - -const LAUNCHER_MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V']; +use uv_trampoline_builder::windows_script_launcher; -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_GUI: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-gui.exe"); - -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_CONSOLE: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-console.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_GUI: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_CONSOLE: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_GUI: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_CONSOLE: &[u8] = - include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe"); +use crate::record::RecordEntry; +use crate::script::Script; +use crate::{Error, Layout}; /// Wrapper script template function /// @@ -158,87 +133,6 @@ fn format_shebang(executable: impl AsRef, os_name: &str, relocatable: bool format!("#!{executable}") } -/// A Windows script is a minimal .exe launcher binary with the python entrypoint script appended as -/// stored zip file. -/// -/// -#[allow(unused_variables)] -pub(crate) fn windows_script_launcher( - launcher_python_script: &str, - is_gui: bool, - python_executable: impl AsRef, -) -> Result, Error> { - // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain - // compilation on all platforms. - if cfg!(not(windows)) { - return Err(Error::NotWindows); - } - - let launcher_bin: &[u8] = match env::consts::ARCH { - #[cfg(all(windows, target_arch = "x86"))] - "x86" => { - if is_gui { - LAUNCHER_I686_GUI - } else { - LAUNCHER_I686_CONSOLE - } - } - #[cfg(all(windows, target_arch = "x86_64"))] - "x86_64" => { - if is_gui { - LAUNCHER_X86_64_GUI - } else { - LAUNCHER_X86_64_CONSOLE - } - } - #[cfg(all(windows, target_arch = "aarch64"))] - "aarch64" => { - if is_gui { - LAUNCHER_AARCH64_GUI - } else { - LAUNCHER_AARCH64_CONSOLE - } - } - #[cfg(windows)] - arch => { - return Err(Error::UnsupportedWindowsArch(arch)); - } - #[cfg(not(windows))] - arch => &[], - }; - - let mut payload: Vec = Vec::new(); - { - // We're using the zip writer, but with stored compression - // https://github.com/njsmith/posy/blob/04927e657ca97a5e35bb2252d168125de9a3a025/src/trampolines/mod.rs#L75-L82 - // https://github.com/pypa/distlib/blob/8ed03aab48add854f377ce392efffb79bb4d6091/PC/launcher.c#L259-L271 - let stored = FileOptions::default().compression_method(zip::CompressionMethod::Stored); - let mut archive = ZipWriter::new(Cursor::new(&mut payload)); - let error_msg = "Writing to Vec should never fail"; - archive.start_file("__main__.py", stored).expect(error_msg); - archive - .write_all(launcher_python_script.as_bytes()) - .expect(error_msg); - archive.finish().expect(error_msg); - } - - let python = python_executable.as_ref(); - let python_path = python.simplified_display().to_string(); - - let mut launcher: Vec = Vec::with_capacity(launcher_bin.len() + payload.len()); - launcher.extend_from_slice(launcher_bin); - launcher.extend_from_slice(&payload); - launcher.extend_from_slice(python_path.as_bytes()); - launcher.extend_from_slice( - &u32::try_from(python_path.as_bytes().len()) - .expect("File Path to be smaller than 4GB") - .to_le_bytes(), - ); - launcher.extend_from_slice(&LAUNCHER_MAGIC_NUMBER); - - Ok(launcher) -} - /// Returns a [`PathBuf`] to `python[w].exe` for script execution. /// /// @@ -1075,54 +969,6 @@ mod test { Ok(()) } - #[test] - #[cfg(all(windows, target_arch = "x86"))] - fn test_launchers_are_small() { - // At time of writing, they are 45kb~ bytes. - assert!( - super::LAUNCHER_I686_GUI.len() < 45 * 1024, - "GUI launcher: {}", - super::LAUNCHER_I686_GUI.len() - ); - assert!( - super::LAUNCHER_I686_CONSOLE.len() < 45 * 1024, - "CLI launcher: {}", - super::LAUNCHER_I686_CONSOLE.len() - ); - } - - #[test] - #[cfg(all(windows, target_arch = "x86_64"))] - fn test_launchers_are_small() { - // At time of writing, they are 45kb~ bytes. - assert!( - super::LAUNCHER_X86_64_GUI.len() < 45 * 1024, - "GUI launcher: {}", - super::LAUNCHER_X86_64_GUI.len() - ); - assert!( - super::LAUNCHER_X86_64_CONSOLE.len() < 45 * 1024, - "CLI launcher: {}", - super::LAUNCHER_X86_64_CONSOLE.len() - ); - } - - #[test] - #[cfg(all(windows, target_arch = "aarch64"))] - fn test_launchers_are_small() { - // At time of writing, they are 45kb~ bytes. - assert!( - super::LAUNCHER_AARCH64_GUI.len() < 45 * 1024, - "GUI launcher: {}", - super::LAUNCHER_AARCH64_GUI.len() - ); - assert!( - super::LAUNCHER_AARCH64_CONSOLE.len() < 45 * 1024, - "CLI launcher: {}", - super::LAUNCHER_AARCH64_CONSOLE.len() - ); - } - #[test] fn test_script_executable() -> Result<()> { // Test with adjacent pythonw.exe diff --git a/crates/uv-trampoline-builder/Cargo.toml b/crates/uv-trampoline-builder/Cargo.toml new file mode 100644 index 000000000000..f4c54e15a7af --- /dev/null +++ b/crates/uv-trampoline-builder/Cargo.toml @@ -0,0 +1,34 @@ +[package] +name = "uv-trampoline-builder" +version = "0.0.1" +publish = false +description = "Builds launchers for `uv-trampoline`" + +edition = { workspace = true } +rust-version = { workspace = true } +homepage = { workspace = true } +documentation = { workspace = true } +repository = { workspace = true } +authors = { workspace = true } +license = { workspace = true } + +[features] +default = ["production"] + +# Expect tests to run against production builds of `uv-trampoline` binaries, rather than debug builds +production = [] + +[lints] +workspace = true + +[dependencies] +uv-fs = { workspace = true } +thiserror = { workspace = true } +zip = { workspace = true } + +[dev-dependencies] +assert_cmd = { version = "2.0.16" } +assert_fs = { version = "1.1.2" } +anyhow = { version = "1.0.89" } +fs-err = { workspace = true } +which = { workspace = true } diff --git a/crates/uv-trampoline-builder/src/lib.rs b/crates/uv-trampoline-builder/src/lib.rs new file mode 100644 index 000000000000..3b1503f7c3c3 --- /dev/null +++ b/crates/uv-trampoline-builder/src/lib.rs @@ -0,0 +1,406 @@ +use std::io::{Cursor, Write}; +use std::path::Path; + +use thiserror::Error; +use uv_fs::Simplified; +use zip::write::FileOptions; +use zip::ZipWriter; + +#[cfg(all(windows, target_arch = "x86"))] +const LAUNCHER_I686_GUI: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-gui.exe"); + +#[cfg(all(windows, target_arch = "x86"))] +const LAUNCHER_I686_CONSOLE: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-i686-console.exe"); + +#[cfg(all(windows, target_arch = "x86_64"))] +const LAUNCHER_X86_64_GUI: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe"); + +#[cfg(all(windows, target_arch = "x86_64"))] +const LAUNCHER_X86_64_CONSOLE: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe"); + +#[cfg(all(windows, target_arch = "aarch64"))] +const LAUNCHER_AARCH64_GUI: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe"); + +#[cfg(all(windows, target_arch = "aarch64"))] +const LAUNCHER_AARCH64_CONSOLE: &[u8] = + include_bytes!("../../uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe"); + +/// The kind of trampoline launcher to create. +/// +/// See [`uv-trampoline::bounce::TrampolineKind`]. +enum LauncherKind { + /// The trampoline should execute itself, it's a zipped Python script. + Script, + /// The trampoline should just execute Python, it's a proxy Python executable. + Python, +} + +impl LauncherKind { + const fn magic_number(&self) -> &'static [u8; 4] { + match self { + Self::Script => b"UVSC", + Self::Python => b"UVPY", + } + } +} + +/// Note: The caller is responsible for adding the path of the wheel we're installing. +#[derive(Error, Debug)] +pub enum Error { + #[error( + "Unable to create Windows launcher for: {0} (only x86_64, x86, and arm64 are supported)" + )] + UnsupportedWindowsArch(&'static str), + #[error("Unable to create Windows launcher on non-Windows platform")] + NotWindows, +} + +#[allow(clippy::unnecessary_wraps, unused_variables)] +fn get_launcher_bin(gui: bool) -> Result<&'static [u8], Error> { + Ok(match std::env::consts::ARCH { + #[cfg(all(windows, target_arch = "x86"))] + "x86" => { + if gui { + LAUNCHER_I686_GUI + } else { + LAUNCHER_I686_CONSOLE + } + } + #[cfg(all(windows, target_arch = "x86_64"))] + "x86_64" => { + if gui { + LAUNCHER_X86_64_GUI + } else { + LAUNCHER_X86_64_CONSOLE + } + } + #[cfg(all(windows, target_arch = "aarch64"))] + "aarch64" => { + if gui { + LAUNCHER_AARCH64_GUI + } else { + LAUNCHER_AARCH64_CONSOLE + } + } + #[cfg(windows)] + arch => { + return Err(Error::UnsupportedWindowsArch(arch)); + } + #[cfg(not(windows))] + _ => &[], + }) +} + +/// A Windows script is a minimal .exe launcher binary with the python entrypoint script appended as +/// stored zip file. +/// +/// +#[allow(unused_variables)] +pub fn windows_script_launcher( + launcher_python_script: &str, + is_gui: bool, + python_executable: impl AsRef, +) -> Result, Error> { + // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain + // compilation on all platforms. + if cfg!(not(windows)) { + return Err(Error::NotWindows); + } + + let launcher_bin: &[u8] = get_launcher_bin(is_gui)?; + + let mut payload: Vec = Vec::new(); + { + // We're using the zip writer, but with stored compression + // https://github.com/njsmith/posy/blob/04927e657ca97a5e35bb2252d168125de9a3a025/src/trampolines/mod.rs#L75-L82 + // https://github.com/pypa/distlib/blob/8ed03aab48add854f377ce392efffb79bb4d6091/PC/launcher.c#L259-L271 + let stored = FileOptions::default().compression_method(zip::CompressionMethod::Stored); + let mut archive = ZipWriter::new(Cursor::new(&mut payload)); + let error_msg = "Writing to Vec should never fail"; + archive.start_file("__main__.py", stored).expect(error_msg); + archive + .write_all(launcher_python_script.as_bytes()) + .expect(error_msg); + archive.finish().expect(error_msg); + } + + let python = python_executable.as_ref(); + let python_path = python.simplified_display().to_string(); + + let mut launcher: Vec = Vec::with_capacity(launcher_bin.len() + payload.len()); + launcher.extend_from_slice(launcher_bin); + launcher.extend_from_slice(&payload); + launcher.extend_from_slice(python_path.as_bytes()); + launcher.extend_from_slice( + &u32::try_from(python_path.as_bytes().len()) + .expect("file path should be smaller than 4GB") + .to_le_bytes(), + ); + launcher.extend_from_slice(LauncherKind::Script.magic_number()); + + Ok(launcher) +} + +/// A minimal .exe launcher binary for Python. +/// +/// Sort of equivalent to a `python` symlink on Unix. +#[allow(unused_variables)] +pub fn windows_python_launcher( + python_executable: impl AsRef, + is_gui: bool, +) -> Result, Error> { + // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain + // compilation on all platforms. + if cfg!(not(windows)) { + return Err(Error::NotWindows); + } + + let launcher_bin: &[u8] = get_launcher_bin(is_gui)?; + + let python = python_executable.as_ref(); + let python_path = python.simplified_display().to_string(); + + let mut launcher: Vec = Vec::with_capacity(launcher_bin.len()); + launcher.extend_from_slice(launcher_bin); + launcher.extend_from_slice(python_path.as_bytes()); + launcher.extend_from_slice( + &u32::try_from(python_path.as_bytes().len()) + .expect("file path should be smaller than 4GB") + .to_le_bytes(), + ); + launcher.extend_from_slice(LauncherKind::Python.magic_number()); + + Ok(launcher) +} + +#[cfg(all(test, windows))] +#[allow(clippy::print_stdout)] +mod test { + use std::io::Write; + use std::path::Path; + use std::process::Command; + + use anyhow::Result; + use assert_cmd::prelude::OutputAssertExt; + use assert_fs::prelude::PathChild; + use fs_err::File; + + use which::which; + + use super::{windows_python_launcher, windows_script_launcher}; + + #[test] + #[cfg(all(windows, target_arch = "x86", feature = "production"))] + fn test_launchers_are_small() { + // At time of writing, they are ~45kb. + assert!( + super::LAUNCHER_I686_GUI.len() < 45 * 1024, + "GUI launcher: {}", + super::LAUNCHER_I686_GUI.len() + ); + assert!( + super::LAUNCHER_I686_CONSOLE.len() < 45 * 1024, + "CLI launcher: {}", + super::LAUNCHER_I686_CONSOLE.len() + ); + } + + #[test] + #[cfg(all(windows, target_arch = "x86_64", feature = "production"))] + fn test_launchers_are_small() { + // At time of writing, they are ~45kb. + assert!( + super::LAUNCHER_X86_64_GUI.len() < 45 * 1024, + "GUI launcher: {}", + super::LAUNCHER_X86_64_GUI.len() + ); + assert!( + super::LAUNCHER_X86_64_CONSOLE.len() < 45 * 1024, + "CLI launcher: {}", + super::LAUNCHER_X86_64_CONSOLE.len() + ); + } + + #[test] + #[cfg(all(windows, target_arch = "aarch64", feature = "production"))] + fn test_launchers_are_small() { + // At time of writing, they are ~45kb. + assert!( + super::LAUNCHER_AARCH64_GUI.len() < 45 * 1024, + "GUI launcher: {}", + super::LAUNCHER_AARCH64_GUI.len() + ); + assert!( + super::LAUNCHER_AARCH64_CONSOLE.len() < 45 * 1024, + "CLI launcher: {}", + super::LAUNCHER_AARCH64_CONSOLE.len() + ); + } + + /// Utility script for the test. + fn get_script_launcher(shebang: &str, is_gui: bool) -> String { + if is_gui { + format!( + r##"{shebang} +# -*- coding: utf-8 -*- +import re +import sys + +def make_gui() -> None: + from tkinter import Tk, ttk + root = Tk() + root.title("uv Test App") + frm = ttk.Frame(root, padding=10) + frm.grid() + ttk.Label(frm, text="Hello from uv-trampoline-gui.exe").grid(column=0, row=0) + root.mainloop() + +if __name__ == "__main__": + sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) + sys.exit(make_gui()) +"## + ) + } else { + format!( + r##"{shebang} +# -*- coding: utf-8 -*- +import re +import sys + +def main_console() -> None: + print("Hello from uv-trampoline-console.exe", file=sys.stdout) + print("Hello from uv-trampoline-console.exe", file=sys.stderr) + for arg in sys.argv[1:]: + print(arg, file=sys.stderr) + +if __name__ == "__main__": + sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) + sys.exit(main_console()) +"## + ) + } + } + + /// See [`uv-install-wheel::wheel::format_shebang`]. + fn format_shebang(executable: impl AsRef) -> String { + // Convert the executable to a simplified path. + let executable = executable.as_ref().display().to_string(); + format!("#!{executable}") + } + + #[test] + fn console_script_launcher() -> Result<()> { + // Create Temp Dirs + let temp_dir = assert_fs::TempDir::new()?; + let console_bin_path = temp_dir.child("launcher.console.exe"); + + // Locate an arbitrary python installation from PATH + let python_executable_path = which("python")?; + + // Generate Launcher Script + let launcher_console_script = + get_script_launcher(&format_shebang(&python_executable_path), false); + + // Generate Launcher Payload + let console_launcher = + windows_script_launcher(&launcher_console_script, false, &python_executable_path)?; + + // Create Launcher + File::create(console_bin_path.path())?.write_all(console_launcher.as_ref())?; + + println!( + "Wrote Console Launcher in {}", + console_bin_path.path().display() + ); + + let stdout_predicate = "Hello from uv-trampoline-console.exe\r\n"; + let stderr_predicate = "Hello from uv-trampoline-console.exe\r\n"; + + // Test Console Launcher + #[cfg(windows)] + Command::new(console_bin_path.path()) + .assert() + .success() + .stdout(stdout_predicate) + .stderr(stderr_predicate); + + let args_to_test = vec!["foo", "bar", "foo bar", "foo \"bar\"", "foo 'bar'"]; + let stderr_predicate = format!("{}{}\r\n", stderr_predicate, args_to_test.join("\r\n")); + + // Test Console Launcher (with args) + Command::new(console_bin_path.path()) + .args(args_to_test) + .assert() + .success() + .stdout(stdout_predicate) + .stderr(stderr_predicate); + + Ok(()) + } + + #[test] + fn console_python_launcher() -> Result<()> { + // Create Temp Dirs + let temp_dir = assert_fs::TempDir::new()?; + let console_bin_path = temp_dir.child("launcher.console.exe"); + + // Locate an arbitrary python installation from PATH + let python_executable_path = which("python")?; + + // Generate Launcher Payload + let console_launcher = windows_python_launcher(&python_executable_path, false)?; + + // Create Launcher + File::create(console_bin_path.path())?.write_all(console_launcher.as_ref())?; + + println!( + "Wrote Python Launcher in {}", + console_bin_path.path().display() + ); + + // Test Console Launcher + Command::new(console_bin_path.path()) + .arg("-c") + .arg("print('Hello from Python Launcher')") + .assert() + .success() + .stdout("Hello from Python Launcher\r\n"); + + Ok(()) + } + + #[test] + #[ignore] + fn gui_launcher() -> Result<()> { + // Create Temp Dirs + let temp_dir = assert_fs::TempDir::new()?; + let gui_bin_path = temp_dir.child("launcher.gui.exe"); + + // Locate an arbitrary pythonw installation from PATH + let pythonw_executable_path = which("pythonw")?; + + // Generate Launcher Script + let launcher_gui_script = + get_script_launcher(&format_shebang(&pythonw_executable_path), true); + + // Generate Launcher Payload + let gui_launcher = + windows_script_launcher(&launcher_gui_script, true, &pythonw_executable_path)?; + + // Create Launcher + File::create(gui_bin_path.path())?.write_all(gui_launcher.as_ref())?; + + println!("Wrote GUI Launcher in {}", gui_bin_path.path().display()); + + // Test GUI Launcher + // NOTICE: This will spawn a GUI and will wait until you close the window. + Command::new(gui_bin_path.path()).assert().success(); + + Ok(()) + } +} diff --git a/crates/uv-trampoline/Cargo.lock b/crates/uv-trampoline/Cargo.lock index 0611026d7986..5ae4861c14e5 100644 --- a/crates/uv-trampoline/Cargo.lock +++ b/crates/uv-trampoline/Cargo.lock @@ -1,6 +1,6 @@ # This file is automatically @generated by Cargo. # It is not intended for manual editing. -version = 3 +version = 4 [[package]] name = "adler" @@ -245,9 +245,9 @@ dependencies = [ [[package]] name = "libc" -version = "0.2.155" +version = "0.2.161" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97b3888a4aecf77e811145cadf6eef5901f4782c53886191b2f693f24761847c" +checksum = "8e9489c2807c139ffd9c1794f4af0ebe86a828db53ecdc7fea2111d0fed085d1" [[package]] name = "linux-raw-sys" @@ -346,9 +346,9 @@ checksum = "7a66a03ae7c801facd77a29370b4faec201768915ac14a721ba36f20bc9c209b" [[package]] name = "rustix" -version = "0.38.34" +version = "0.38.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70dc5ec042f7a43c4a73241207cecc9873a06d45debb38b329f8541d85c2730f" +checksum = "aa260229e6538e52293eeb577aabd09945a09d6d9cc0fc550ed7529056c2e32a" dependencies = [ "bitflags", "errno", diff --git a/crates/uv-trampoline/README.md b/crates/uv-trampoline/README.md index 8ff40aa82a5d..3a4de1933aef 100644 --- a/crates/uv-trampoline/README.md +++ b/crates/uv-trampoline/README.md @@ -12,17 +12,20 @@ LLD and add the `rustup` targets: ```shell sudo apt install llvm clang lld -rustup target add i686-pc-windows-msvc -rustup target add x86_64-pc-windows-msvc -rustup target add aarch64-pc-windows-msvc +cargo install cargo-xwin +rustup toolchain install nightly-2024-10-27 +rustup component add rust-src --toolchain nightly-2024-10-27-x86_64-unknown-linux-gnu +rustup target add --toolchain nightly-2024-10-27 i686-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 x86_64-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 aarch64-pc-windows-msvc ``` -Then, build the trampolines for both supported architectures: +Then, build the trampolines for all supported architectures: ```shell -cargo +nightly-2024-06-08 xwin build --xwin-arch x86 --release --target i686-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target x86_64-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target aarch64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --xwin-arch x86 --release --target i686-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target x86_64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target aarch64-pc-windows-msvc ``` ### Cross-compiling from macOS @@ -32,22 +35,25 @@ LLVM and add the `rustup` targets: ```shell brew install llvm -rustup target add i686-pc-windows-msvc -rustup target add x86_64-pc-windows-msvc -rustup target add aarch64-pc-windows-msvc +cargo install cargo-xwin +rustup toolchain install nightly-2024-10-27 +rustup component add rust-src --toolchain nightly-2024-10-27-aarch64-apple-darwin +rustup target add --toolchain nightly-2024-10-27 i686-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 x86_64-pc-windows-msvc +rustup target add --toolchain nightly-2024-10-27 aarch64-pc-windows-msvc ``` -Then, build the trampolines for both supported architectures: +Then, build the trampolines for all supported architectures: ```shell -cargo +nightly-2024-06-08 xwin build --release --target i686-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target x86_64-pc-windows-msvc -cargo +nightly-2024-06-08 xwin build --release --target aarch64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target i686-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target x86_64-pc-windows-msvc +cargo +nightly-2024-10-27 xwin build --release --target aarch64-pc-windows-msvc ``` ### Updating the prebuilt executables -After building the trampolines for both supported architectures: +After building the trampolines for all supported architectures: ```shell cp target/aarch64-pc-windows-msvc/release/uv-trampoline-console.exe trampolines/uv-trampoline-aarch64-console.exe diff --git a/crates/uv-trampoline/rust-toolchain.toml b/crates/uv-trampoline/rust-toolchain.toml index a0c8420807eb..fce0096acafe 100644 --- a/crates/uv-trampoline/rust-toolchain.toml +++ b/crates/uv-trampoline/rust-toolchain.toml @@ -1,2 +1,2 @@ [toolchain] -channel = "nightly-2024-06-08" +channel = "nightly-2024-10-27" diff --git a/crates/uv-trampoline/src/bounce.rs b/crates/uv-trampoline/src/bounce.rs index 22c59340a0bf..a1cdd176d077 100644 --- a/crates/uv-trampoline/src/bounce.rs +++ b/crates/uv-trampoline/src/bounce.rs @@ -33,27 +33,60 @@ use windows::Win32::{ use crate::{eprintln, format}; -const MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V']; const PATH_LEN_SIZE: usize = size_of::(); const MAX_PATH_LEN: u32 = 32 * 1024; -/// Transform ` ` to `python `. +/// The kind of trampoline. +enum TrampolineKind { + /// The trampoline should execute itself, it's a zipped Python script. + Script, + /// The trampoline should just execute Python, it's a proxy Python executable. + Python, +} + +impl TrampolineKind { + const fn magic_number(&self) -> &'static [u8; 4] { + match self { + Self::Script => b"UVSC", + Self::Python => b"UVPY", + } + } + + fn from_buffer(buffer: &[u8]) -> Option { + if buffer.ends_with(Self::Script.magic_number()) { + Some(Self::Script) + } else if buffer.ends_with(Self::Python.magic_number()) { + Some(Self::Python) + } else { + None + } + } +} + +/// Transform ` ` to `python ` or `python ` +/// depending on the [`TrampolineKind`]. fn make_child_cmdline() -> CString { let executable_name = std::env::current_exe().unwrap_or_else(|_| { eprintln!("Failed to get executable name"); exit_with_status(1); }); - let python_exe = find_python_exe(executable_name.as_ref()); + let (kind, python_exe) = read_trampoline_metadata(executable_name.as_ref()); let mut child_cmdline = Vec::::new(); push_quoted_path(python_exe.as_ref(), &mut child_cmdline); child_cmdline.push(b' '); - // Use the full executable name because CMD only passes the name of the executable (but not the path) - // when e.g. invoking `black` instead of `/Scripts/black` and Python then fails - // to find the file. Unfortunately, this complicates things because we now need to split the executable - // from the arguments string... - push_quoted_path(executable_name.as_ref(), &mut child_cmdline); + // Only execute the trampoline again if it's a script, otherwise, just invoke Python. + match kind { + TrampolineKind::Python => {} + TrampolineKind::Script => { + // Use the full executable name because CMD only passes the name of the executable (but not the path) + // when e.g. invoking `black` instead of `/Scripts/black` and Python then fails + // to find the file. Unfortunately, this complicates things because we now need to split the executable + // from the arguments string... + push_quoted_path(executable_name.as_ref(), &mut child_cmdline); + } + } push_arguments(&mut child_cmdline); @@ -86,17 +119,21 @@ fn push_quoted_path(path: &Path, command: &mut Vec) { command.extend(br#"""#); } -/// Reads the executable binary from the back to find the path to the Python executable that is written -/// after the ZIP file content. +/// Reads the executable binary from the back to find: +/// +/// * The path to the Python executable +/// * The kind of trampoline we are executing /// /// The executable is expected to have the following format: -/// * The file must end with the magic number 'UVUV'. +/// +/// * The file must end with the magic number 'UVPY' or 'UVSC' (identifying the trampoline kind) /// * The last 4 bytes (little endian) are the length of the path to the Python executable. /// * The path encoded as UTF-8 comes right before the length /// /// # Panics +/// /// If there's any IO error, or the file does not conform to the specified format. -fn find_python_exe(executable_name: &Path) -> PathBuf { +fn read_trampoline_metadata(executable_name: &Path) -> (TrampolineKind, PathBuf) { let mut file_handle = File::open(executable_name).unwrap_or_else(|_| { print_last_error_and_exit(&format!( "Failed to open executable '{}'", @@ -117,6 +154,7 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { let mut buffer: Vec = Vec::new(); let mut bytes_to_read = 1024.min(u32::try_from(file_size).unwrap_or(u32::MAX)); + let mut kind; let path: String = loop { // SAFETY: Casting to usize is safe because we only support 64bit systems where usize is guaranteed to be larger than u32. buffer.resize(bytes_to_read as usize, 0); @@ -135,13 +173,14 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { // Truncate the buffer to the actual number of bytes read. buffer.truncate(read_bytes); - if !buffer.ends_with(&MAGIC_NUMBER) { - eprintln!("Magic number 'UVUV' not found at the end of the file. Did you append the magic number, the length and the path to the python executable at the end of the file?"); + let Some(inner_kind) = TrampolineKind::from_buffer(&buffer) else { + eprintln!("Magic number 'UVSC' or 'UVPY' not found at the end of the file. Did you append the magic number, the length and the path to the python executable at the end of the file?"); exit_with_status(1); - } + }; + kind = inner_kind; // Remove the magic number - buffer.truncate(buffer.len() - MAGIC_NUMBER.len()); + buffer.truncate(buffer.len() - kind.magic_number().len()); let path_len = match buffer.get(buffer.len() - PATH_LEN_SIZE..) { Some(path_len) => { @@ -177,7 +216,7 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { } else { // SAFETY: Casting to u32 is safe because `path_len` is guaranteed to be less than 32KBs, // MAGIC_NUMBER is 4 bytes and PATH_LEN_SIZE is 4 bytes. - bytes_to_read = (path_len + MAGIC_NUMBER.len() + PATH_LEN_SIZE) as u32; + bytes_to_read = (path_len + kind.magic_number().len() + PATH_LEN_SIZE) as u32; if u64::from(bytes_to_read) > file_size { eprintln!("The length of the python executable path exceeds the file size. Verify that the path length is appended to the end of the launcher script as a u32 in little endian"); @@ -201,10 +240,12 @@ fn find_python_exe(executable_name: &Path) -> PathBuf { }; // NOTICE: dunce adds 5kb~ - dunce::canonicalize(path.as_path()).unwrap_or_else(|_| { + let path = dunce::canonicalize(path.as_path()).unwrap_or_else(|_| { eprintln!("Failed to canonicalize script path"); exit_with_status(1); - }) + }); + + (kind, path) } fn push_arguments(output: &mut Vec) { diff --git a/crates/uv-trampoline/tests/harness.rs b/crates/uv-trampoline/tests/harness.rs deleted file mode 100644 index 0af6bdc04515..000000000000 --- a/crates/uv-trampoline/tests/harness.rs +++ /dev/null @@ -1,269 +0,0 @@ -use std::io::{Cursor, Write}; -use std::path::Path; -use std::process::Command; -use std::{env, io}; - -use anyhow::Result; -use assert_cmd::prelude::OutputAssertExt; -use assert_fs::prelude::PathChild; -use fs_err::File; -use thiserror::Error; -use which::which; -use zip::write::FileOptions; -use zip::ZipWriter; - -const LAUNCHER_MAGIC_NUMBER: [u8; 4] = [b'U', b'V', b'U', b'V']; - -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_GUI: &[u8] = include_bytes!("../trampolines/uv-trampoline-i686-gui.exe"); - -#[cfg(all(windows, target_arch = "x86"))] -const LAUNCHER_I686_CONSOLE: &[u8] = - include_bytes!("../trampolines/uv-trampoline-i686-console.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_GUI: &[u8] = include_bytes!("../trampolines/uv-trampoline-x86_64-gui.exe"); - -#[cfg(all(windows, target_arch = "x86_64"))] -const LAUNCHER_X86_64_CONSOLE: &[u8] = - include_bytes!("../trampolines/uv-trampoline-x86_64-console.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_GUI: &[u8] = include_bytes!("../trampolines/uv-trampoline-aarch64-gui.exe"); - -#[cfg(all(windows, target_arch = "aarch64"))] -const LAUNCHER_AARCH64_CONSOLE: &[u8] = - include_bytes!("../trampolines/uv-trampoline-aarch64-console.exe"); - -/// Note: The caller is responsible for adding the path of the wheel we're installing. -#[derive(Error, Debug)] -pub enum Error { - #[error(transparent)] - Io(#[from] io::Error), - #[error( - "Unable to create Windows launcher for: {0} (only x86_64, x86, and arm64 are supported)" - )] - UnsupportedWindowsArch(&'static str), - #[error("Unable to create Windows launcher on non-Windows platform")] - NotWindows, -} - -/// Wrapper script template function -/// -/// -fn get_script_launcher(shebang: &str, is_gui: bool) -> String { - if is_gui { - format!( - r##"{shebang} -# -*- coding: utf-8 -*- -import re -import sys - -def make_gui() -> None: - from tkinter import Tk, ttk - root = Tk() - root.title("uv Test App") - frm = ttk.Frame(root, padding=10) - frm.grid() - ttk.Label(frm, text="Hello from uv-trampoline-gui.exe").grid(column=0, row=0) - root.mainloop() - -if __name__ == "__main__": - sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) - sys.exit(make_gui()) -"## - ) - } else { - format!( - r##"{shebang} -# -*- coding: utf-8 -*- -import re -import sys - -def main_console() -> None: - print("Hello from uv-trampoline-console.exe", file=sys.stdout) - print("Hello from uv-trampoline-console.exe", file=sys.stderr) - for arg in sys.argv[1:]: - print(arg, file=sys.stderr) - -if __name__ == "__main__": - sys.argv[0] = re.sub(r"(-script\.pyw|\.exe)?$", "", sys.argv[0]) - sys.exit(main_console()) -"## - ) - } -} - -/// Format the shebang for a given Python executable. -/// -/// Like pip, if a shebang is non-simple (too long or contains spaces), we use `/bin/sh` as the -/// executable. -/// -/// See: -fn format_shebang(executable: impl AsRef) -> String { - // Convert the executable to a simplified path. - let executable = executable.as_ref().display().to_string(); - format!("#!{executable}") -} - -/// A Windows script is a minimal .exe launcher binary with the python entrypoint script appended as -/// stored zip file. -/// -/// -#[allow(unused_variables)] -fn windows_script_launcher( - launcher_python_script: &str, - is_gui: bool, - python_executable: impl AsRef, -) -> Result, Error> { - // This method should only be called on Windows, but we avoid `#[cfg(windows)]` to retain - // compilation on all platforms. - if cfg!(not(windows)) { - return Err(Error::NotWindows); - } - - let launcher_bin: &[u8] = match env::consts::ARCH { - #[cfg(all(windows, target_arch = "x86"))] - "x86" => { - if is_gui { - LAUNCHER_I686_GUI - } else { - LAUNCHER_I686_CONSOLE - } - } - #[cfg(all(windows, target_arch = "x86_64"))] - "x86_64" => { - if is_gui { - LAUNCHER_X86_64_GUI - } else { - LAUNCHER_X86_64_CONSOLE - } - } - #[cfg(all(windows, target_arch = "aarch64"))] - "aarch64" => { - if is_gui { - LAUNCHER_AARCH64_GUI - } else { - LAUNCHER_AARCH64_CONSOLE - } - } - #[cfg(windows)] - arch => { - return Err(Error::UnsupportedWindowsArch(arch)); - } - #[cfg(not(windows))] - arch => &[], - }; - - let mut payload: Vec = Vec::new(); - { - // We're using the zip writer, but with stored compression - // https://github.com/njsmith/posy/blob/04927e657ca97a5e35bb2252d168125de9a3a025/src/trampolines/mod.rs#L75-L82 - // https://github.com/pypa/distlib/blob/8ed03aab48add854f377ce392efffb79bb4d6091/PC/launcher.c#L259-L271 - let stored = FileOptions::default().compression_method(zip::CompressionMethod::Stored); - let mut archive = ZipWriter::new(Cursor::new(&mut payload)); - let error_msg = "Writing to Vec should never fail"; - archive.start_file("__main__.py", stored).expect(error_msg); - archive - .write_all(launcher_python_script.as_bytes()) - .expect(error_msg); - archive.finish().expect(error_msg); - } - - let python = python_executable.as_ref(); - let python_path = python.display().to_string(); - - let mut launcher: Vec = Vec::with_capacity(launcher_bin.len() + payload.len()); - launcher.extend_from_slice(launcher_bin); - launcher.extend_from_slice(&payload); - launcher.extend_from_slice(python_path.as_bytes()); - launcher.extend_from_slice( - &u32::try_from(python_path.as_bytes().len()) - .expect("File Path to be smaller than 4GB") - .to_le_bytes(), - ); - launcher.extend_from_slice(&LAUNCHER_MAGIC_NUMBER); - - Ok(launcher) -} - -#[test] -fn generate_console_launcher() -> Result<()> { - // Create Temp Dirs - let temp_dir = assert_fs::TempDir::new()?; - let console_bin_path = temp_dir.child("launcher.console.exe"); - - // Locate an arbitrary python installation from PATH - let python_executable_path = which("python")?; - - // Generate Launcher Script - let launcher_console_script = - get_script_launcher(&format_shebang(&python_executable_path), false); - - // Generate Launcher Payload - let console_launcher = - windows_script_launcher(&launcher_console_script, false, &python_executable_path)?; - - // Create Launcher - File::create(console_bin_path.path())?.write_all(console_launcher.as_ref())?; - - println!( - "Wrote Console Launcher in {}", - console_bin_path.path().display() - ); - - let stdout_predicate = "Hello from uv-trampoline-console.exe\r\n"; - let stderr_predicate = "Hello from uv-trampoline-console.exe\r\n"; - - // Test Console Launcher - #[cfg(windows)] - Command::new(console_bin_path.path()) - .assert() - .success() - .stdout(stdout_predicate) - .stderr(stderr_predicate); - - let args_to_test = vec!["foo", "bar", "foo bar", "foo \"bar\"", "foo 'bar'"]; - let stderr_predicate = format!("{}{}\r\n", stderr_predicate, args_to_test.join("\r\n")); - - // Test Console Launcher (with args) - #[cfg(windows)] - Command::new(console_bin_path.path()) - .args(args_to_test) - .assert() - .success() - .stdout(stdout_predicate) - .stderr(stderr_predicate); - - Ok(()) -} - -#[test] -#[ignore] -fn generate_gui_launcher() -> Result<()> { - // Create Temp Dirs - let temp_dir = assert_fs::TempDir::new()?; - let gui_bin_path = temp_dir.child("launcher.gui.exe"); - - // Locate an arbitrary pythonw installation from PATH - let pythonw_executable_path = which("pythonw")?; - - // Generate Launcher Script - let launcher_gui_script = get_script_launcher(&format_shebang(&pythonw_executable_path), true); - - // Generate Launcher Payload - let gui_launcher = - windows_script_launcher(&launcher_gui_script, true, &pythonw_executable_path)?; - - // Create Launcher - File::create(gui_bin_path.path())?.write_all(gui_launcher.as_ref())?; - - println!("Wrote GUI Launcher in {}", gui_bin_path.path().display()); - - // Test GUI Launcher - // NOTICE: This will spawn a GUI and will wait until you close the window. - #[cfg(windows)] - Command::new(gui_bin_path.path()).assert().success(); - - Ok(()) -} diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe index 5b6ea9cab831..773c2f9ba5ac 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-console.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe index 1f6524111b1e..f0fe2a3bb16e 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-aarch64-gui.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe b/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe index 3f0bed6a18a4..ee1e70e71313 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-i686-console.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe b/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe index 936aedb5e003..6edd2f511478 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-i686-gui.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe index 8ba2f2b5acd0..f437f03f0c57 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-console.exe differ diff --git a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe index 9f503fe512c9..39069c7431ec 100755 Binary files a/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe and b/crates/uv-trampoline/trampolines/uv-trampoline-x86_64-gui.exe differ