From 5dd61c2a538410461180c41526be4ed69ac663f0 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 21 Dec 2021 22:15:23 +0800 Subject: [PATCH 1/4] Update lddtree to 0.2.0 --- Cargo.lock | 32 +++++++++++--------------------- Cargo.toml | 2 +- src/auditwheel/repair.rs | 5 +++-- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 86fac6c74..774693896 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1022,9 +1022,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646" [[package]] name = "lddtree" -version = "0.1.4" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f5d1501bcbb2572cd5aaec64307faf409b52f25c0d8adbaeaa8afe01387f467" +checksum = "c530d2918615bf5a01051451a5cd07be67d076fc8242c99e877cf9abc90fdf07" dependencies = [ "fs-err", "glob", @@ -1282,9 +1282,9 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.13.0" +version = "1.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05499f3756671c15885fee9034446956fff3f243d6077b91e5767df161f766b3" +checksum = "19e64526ebdee182341572e50e9ad03965aa510cd94427a4549448f285e957a1" dependencies = [ "hermit-abi", "libc", @@ -1579,9 +1579,9 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.11.7" +version = "0.11.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07bea77bc708afa10e59905c3d4af7c8fd43c9214251673095ff8b14345fcbc5" +checksum = "7c4e0a76dc12a116108933f6301b95e83634e0c47b0afbed6abbaa0601e99258" dependencies = [ "base64", "bytes", @@ -1666,7 +1666,7 @@ dependencies = [ "log", "ring", "sct", - "webpki 0.22.0", + "webpki", ] [[package]] @@ -2093,7 +2093,7 @@ checksum = "a27d5f2b839802bd8267fa19b0530f5a08b9c08cd417976be2a65d130fe1c11b" dependencies = [ "rustls", "tokio", - "webpki 0.22.0", + "webpki", ] [[package]] @@ -2368,16 +2368,6 @@ dependencies = [ "wasm-bindgen", ] -[[package]] -name = "webpki" -version = "0.21.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b8e38c0608262c46d4a56202ebabdeb094cef7e560ca7a226c6bf055188aa4ea" -dependencies = [ - "ring", - "untrusted", -] - [[package]] name = "webpki" version = "0.22.0" @@ -2390,11 +2380,11 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "0.21.1" +version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aabe153544e473b775453675851ecc86863d2a81d786d741f6b76778f2a48940" +checksum = "c475786c6f47219345717a043a37ec04cb4bc185e28853adcc4fa0a947eba630" dependencies = [ - "webpki 0.21.4", + "webpki", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index 80c3e7d5b..40f41b6b0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -57,7 +57,7 @@ ignore = "0.4.18" dialoguer = "0.9.0" console = "0.15.0" minijinja = "0.8.2" -lddtree = "0.1.4" +lddtree = "0.2.0" [dev-dependencies] indoc = "1.0.3" diff --git a/src/auditwheel/repair.rs b/src/auditwheel/repair.rs index db8c9082f..6600826de 100644 --- a/src/auditwheel/repair.rs +++ b/src/auditwheel/repair.rs @@ -5,13 +5,14 @@ use fs_err as fs; use lddtree::DependencyAnalyzer; use sha2::{Digest, Sha256}; use std::io; -use std::path::Path; +use std::path::{Path, PathBuf}; pub fn get_external_libs( artifact: impl AsRef, policy: &Policy, ) -> Result, AuditWheelError> { - let dep_analyzer = DependencyAnalyzer::new(); + let root = PathBuf::from("/"); + let dep_analyzer = DependencyAnalyzer::new(root); let deps = dep_analyzer.analyze(artifact).unwrap(); let mut ext_libs = Vec::new(); for (name, lib) in deps.libraries { From 9295d8f23ca5f46c80378a3e525b43361db22a62 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 21 Dec 2021 22:35:44 +0800 Subject: [PATCH 2/4] Revert "No repair support for cross compiled wheel yet" This reverts commit 436ee3b6fff70ed98740ca51f90c587c618bb31f. --- src/auditwheel/audit.rs | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) diff --git a/src/auditwheel/audit.rs b/src/auditwheel/audit.rs index def57de55..f29b5d7d2 100644 --- a/src/auditwheel/audit.rs +++ b/src/auditwheel/audit.rs @@ -256,7 +256,6 @@ pub fn auditwheel_rs( if !target.is_linux() || platform_tag == Some(PlatformTag::Linux) { return Ok((Policy::default(), false)); } - let cross_compiling = target.cross_compiling(); let arch = target.target_arch().to_string(); let mut file = File::open(path).map_err(AuditWheelError::IoError)?; let mut buffer = Vec::new(); @@ -300,15 +299,10 @@ pub fn auditwheel_rs( should_repair = false; break; } - Err(err @ AuditWheelError::LinksForbiddenLibrariesError(..)) => { - // TODO: support repair for cross compiled wheels - if !cross_compiling { - highest_policy = Some(policy.clone()); - should_repair = true; - break; - } else { - return Err(err); - } + Err(AuditWheelError::LinksForbiddenLibrariesError(..)) => { + highest_policy = Some(policy.clone()); + should_repair = true; + break; } Err(AuditWheelError::VersionedSymbolTooNewError(..)) | Err(AuditWheelError::BlackListedSymbolsError(..)) @@ -340,14 +334,9 @@ pub fn auditwheel_rs( should_repair = false; Ok(policy) } - Err(err @ AuditWheelError::LinksForbiddenLibrariesError(..)) => { - // TODO: support repair for cross compiled wheels - if !cross_compiling { - should_repair = true; - Ok(policy) - } else { - Err(err) - } + Err(AuditWheelError::LinksForbiddenLibrariesError(..)) => { + should_repair = true; + Ok(policy) } Err(err) => Err(err), } From e3238be0514b9762be6e127bd0029a7abdfd8b15 Mon Sep 17 00:00:00 2001 From: messense Date: Tue, 21 Dec 2021 23:12:59 +0800 Subject: [PATCH 3/4] Add support for repairing cross compiled linux wheels --- Cargo.lock | 1 + Cargo.toml | 1 + Changelog.md | 2 ++ src/auditwheel/repair.rs | 4 ++-- src/build_context.rs | 48 +++++++++++++++++++++++++++++++++++++++- 5 files changed, 53 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 774693896..55de3da76 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1072,6 +1072,7 @@ dependencies = [ "bytesize", "cargo_metadata", "cbindgen", + "cc", "configparser", "console", "dialoguer", diff --git a/Cargo.toml b/Cargo.toml index 40f41b6b0..d8c055c6b 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,7 @@ dialoguer = "0.9.0" console = "0.15.0" minijinja = "0.8.2" lddtree = "0.2.0" +cc = "1.0.72" [dev-dependencies] indoc = "1.0.3" diff --git a/Changelog.md b/Changelog.md index 3834e1b27..acf94b533 100644 --- a/Changelog.md +++ b/Changelog.md @@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +* Add support for repairing cross compiled linux wheels in [#754](https://github.com/PyO3/maturin/pull/754) + ## [0.12.5] - 2021-12-20 * Fix docs for `new` and `init` commands in `maturin --help` in [#734](https://github.com/PyO3/maturin/pull/734) diff --git a/src/auditwheel/repair.rs b/src/auditwheel/repair.rs index 6600826de..f36dc581c 100644 --- a/src/auditwheel/repair.rs +++ b/src/auditwheel/repair.rs @@ -10,9 +10,9 @@ use std::path::{Path, PathBuf}; pub fn get_external_libs( artifact: impl AsRef, policy: &Policy, + sysroot: PathBuf, ) -> Result, AuditWheelError> { - let root = PathBuf::from("/"); - let dep_analyzer = DependencyAnalyzer::new(root); + let dep_analyzer = DependencyAnalyzer::new(sysroot); let deps = dep_analyzer.analyze(artifact).unwrap(); let mut ext_libs = Vec::new(); for (name, lib) in deps.libraries { diff --git a/src/build_context.rs b/src/build_context.rs index f85a3b613..f3a7d4e58 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -15,6 +15,7 @@ use lddtree::Library; use std::borrow::Cow; use std::collections::HashMap; use std::path::{Path, PathBuf}; +use std::process::{Command, Stdio}; /// The way the rust code is used in the wheel #[derive(Clone, Debug, PartialEq, Eq)] @@ -272,7 +273,8 @@ impl BuildContext { } })?; let external_libs = if should_repair && !self.editable { - get_external_libs(&artifact, &policy).with_context(|| { + let sysroot = get_sysroot_path(&self.target)?; + get_external_libs(&artifact, &policy, sysroot).with_context(|| { if let Some(platform_tag) = platform_tag { format!("Error repairing wheel for {} compliance", platform_tag) } else { @@ -685,6 +687,50 @@ fn relpath(to: &Path, from: &Path) -> PathBuf { result } +/// Get sysroot path from target C compiler +/// +/// Currently only gcc is supported, clang doesn't have a `--print-sysroot` option +/// TODO: allow specify sysroot from environment variable? +fn get_sysroot_path(target: &Target) -> Result { + use crate::target::get_host_target; + + let host_triple = get_host_target()?; + let target_triple = target.target_triple(); + if host_triple != target_triple { + let mut build = cc::Build::new(); + build + // Suppress cargo metadata for example env vars printing + .cargo_metadata(false) + // opt_level, host and target are required + .opt_level(0) + .host(&host_triple) + .target(target_triple); + let compiler = build + .try_get_compiler() + .with_context(|| format!("Failed to get compiler for {}", target_triple))?; + let path = compiler.path(); + let out = Command::new(path) + .arg("--print-sysroot") + .stdout(Stdio::piped()) + .stderr(Stdio::null()) + .output() + .with_context(|| format!("Failed to run `{} --print-sysroot`", path.display()))?; + if out.status.success() { + let sysroot = String::from_utf8(out.stdout) + .context("Failed to read the sysroot path")? + .trim() + .to_owned(); + return Ok(PathBuf::from(sysroot)); + } else { + bail!( + "Failed to get the sysroot path: {}", + String::from_utf8(out.stderr)? + ); + } + } + Ok(PathBuf::from("/")) +} + #[cfg(test)] mod test { use super::relpath; From 2c875dbd49da5c1f9d9e25a2b6bdc5e9ac3b0e52 Mon Sep 17 00:00:00 2001 From: messense Date: Wed, 22 Dec 2021 10:03:48 +0800 Subject: [PATCH 4/4] Rename `get_external_libs` to `find_external_libs` --- src/auditwheel/mod.rs | 2 +- src/auditwheel/repair.rs | 2 +- src/build_context.rs | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/auditwheel/mod.rs b/src/auditwheel/mod.rs index e79bdbd6a..22e298ea6 100644 --- a/src/auditwheel/mod.rs +++ b/src/auditwheel/mod.rs @@ -8,4 +8,4 @@ mod repair; pub use audit::*; pub use platform_tag::PlatformTag; pub use policy::{Policy, MANYLINUX_POLICIES, MUSLLINUX_POLICIES}; -pub use repair::{get_external_libs, hash_file}; +pub use repair::{find_external_libs, hash_file}; diff --git a/src/auditwheel/repair.rs b/src/auditwheel/repair.rs index f36dc581c..1fce01da3 100644 --- a/src/auditwheel/repair.rs +++ b/src/auditwheel/repair.rs @@ -7,7 +7,7 @@ use sha2::{Digest, Sha256}; use std::io; use std::path::{Path, PathBuf}; -pub fn get_external_libs( +pub fn find_external_libs( artifact: impl AsRef, policy: &Policy, sysroot: PathBuf, diff --git a/src/build_context.rs b/src/build_context.rs index f3a7d4e58..53b07b4fb 100644 --- a/src/build_context.rs +++ b/src/build_context.rs @@ -1,5 +1,5 @@ use crate::auditwheel::{ - auditwheel_rs, get_external_libs, hash_file, patchelf, PlatformTag, Policy, + auditwheel_rs, find_external_libs, hash_file, patchelf, PlatformTag, Policy, }; use crate::compile::warn_missing_py_init; use crate::module_writer::{ @@ -274,7 +274,7 @@ impl BuildContext { })?; let external_libs = if should_repair && !self.editable { let sysroot = get_sysroot_path(&self.target)?; - get_external_libs(&artifact, &policy, sysroot).with_context(|| { + find_external_libs(&artifact, &policy, sysroot).with_context(|| { if let Some(platform_tag) = platform_tag { format!("Error repairing wheel for {} compliance", platform_tag) } else {