From 9bdbd1e5bb2e0b12898304a2653ce6d7fdf7f500 Mon Sep 17 00:00:00 2001 From: Jake Shadle Date: Wed, 31 Jan 2024 13:39:27 +0100 Subject: [PATCH] Add zlib-ng-no-cmake feature --- Cargo-zng.toml | 4 +- Cargo.toml | 10 +- README.md | 5 + build.rs | 24 +- build_zng.rs | 716 -------------------------- ci/test.bash | 7 +- src/compile_check/arm/aarch64_caps.c | 4 - src/compile_check/arm/arm_caps.c | 4 - src/compile_check/arm/arm_hwcaps.c | 5 - src/compile_check/arm/arm_hwneon.c | 4 - src/compile_check/arm/arm_neon.c | 4 - src/compile_check/arm/ld4.c | 8 - src/compile_check/avx512/basic.c | 6 - src/compile_check/avx512/mask.c | 3 - src/compile_check/avx512/vnni.c | 6 - src/compile_check/avx512/vpclmulqdq.c | 7 - src/compile_check/flag.c | 1 - systest/Cargo.toml | 1 + zng/cc.rs | 413 +++++++++++++++ zng/cmake.rs | 60 +++ 20 files changed, 508 insertions(+), 784 deletions(-) delete mode 100644 build_zng.rs delete mode 100644 src/compile_check/arm/aarch64_caps.c delete mode 100644 src/compile_check/arm/arm_caps.c delete mode 100644 src/compile_check/arm/arm_hwcaps.c delete mode 100644 src/compile_check/arm/arm_hwneon.c delete mode 100644 src/compile_check/arm/arm_neon.c delete mode 100644 src/compile_check/arm/ld4.c delete mode 100644 src/compile_check/avx512/basic.c delete mode 100644 src/compile_check/avx512/mask.c delete mode 100644 src/compile_check/avx512/vnni.c delete mode 100644 src/compile_check/avx512/vpclmulqdq.c delete mode 100644 src/compile_check/flag.c create mode 100644 zng/cc.rs create mode 100644 zng/cmake.rs diff --git a/Cargo-zng.toml b/Cargo-zng.toml index 1ba8628f..0c8c092c 100644 --- a/Cargo-zng.toml +++ b/Cargo-zng.toml @@ -26,7 +26,7 @@ exclude = [ "/systest", ] -build = "build_zng.rs" +build = "zng/cmake.rs" readme = "README-zng.md" [workspace] @@ -36,4 +36,4 @@ members = ["systest"] libc = "0.2.43" [build-dependencies] -cc = "1.0" +cmake = "0.1" diff --git a/Cargo.toml b/Cargo.toml index 84b38ed2..a1ca334a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -37,6 +37,7 @@ libc = { version = "0.2.43", optional = true } [build-dependencies] pkg-config = "0.3.9" cc = "1.0.18" +cmake = { version = "0.1.50", optional = true } vcpkg = "0.2" [features] @@ -58,12 +59,9 @@ default = ["libc", "stock-zlib"] # # This allows higher-level crates depending on your library to opt into zlib-ng # if desired. -zlib-ng = ["libc"] -# AVX-512 is nightly-only currently , -# enabling this feature means that AVX-512 support is probed for in the C compiler. -# -# No effect on non-x86 arches -zlib-ng-avx512 = ["zlib-ng"] +zlib-ng = ["libc", "cmake"] +# Builds zlib-ng from source using cc instead of cmake +zlib-ng-no-cmake = ["libc"] stock-zlib = [] # Deprecated: the assembly routines are outdated, and either reduce performance # or cause segfaults. diff --git a/README.md b/README.md index fcb48f09..a5493b12 100644 --- a/README.md +++ b/README.md @@ -43,6 +43,11 @@ libz-sys = { version = "1.1.0", default-features = false, features = ["libc"] } This allows higher-level crates depending on your library to opt into zlib-ng if desired. +Building zlib-ng requires `cmake` unless the `zlib-ng-no-cmake` feature is enabled, +in which case `cc` is used instead. Note that this option enables _all_ compiler +features that are supported for the given target, which may not compile on older +compilers or targets without certain headers. + Crates that don't require compatibility with the zlib C API, and use zlib exclusively from Rust or support the zlib-ng native C API (prefixed with `zng_`) can use [`libz-ng-sys`](https://crates.io/crates/libz-ng-sys) instead, diff --git a/build.rs b/build.rs index 1368a124..4b7e6501 100644 --- a/build.rs +++ b/build.rs @@ -5,12 +5,16 @@ use std::path::PathBuf; fn main() { println!("cargo:rerun-if-env-changed=LIBZ_SYS_STATIC"); println!("cargo:rerun-if-changed=build.rs"); + println!("cargo:rerun-if-changed=zng/cmake.rs"); + println!("cargo:rerun-if-changed=zng/cc.rs"); + let host = env::var("HOST").unwrap(); let target = env::var("TARGET").unwrap(); let host_and_target_contain = |s| host.contains(s) && target.contains(s); - let want_ng = cfg!(feature = "zlib-ng") && !cfg!(feature = "stock-zlib"); + let want_ng = cfg!(any(feature = "zlib-ng", feature = "zlib-ng-no-cmake")) + && !cfg!(feature = "stock-zlib"); if want_ng && target != "wasm32-unknown-unknown" { return build_zlib_ng(&target, true); @@ -161,13 +165,19 @@ fn build_zlib(cfg: &mut cc::Build, target: &str) { println!("cargo:include={}/include", dst.to_str().unwrap()); } -#[cfg(not(feature = "zlib-ng"))] -fn build_zlib_ng(_target: &str, _compat: bool) {} +#[cfg(any(feature = "zlib-ng", feature = "zlib-ng-no-cmake"))] +mod zng { + #[cfg_attr(feature = "zlib-ng", path = "cmake.rs")] + #[cfg_attr(feature = "zlib-ng-no-cmake", path = "cc.rs")] + mod build_zng; + + pub(super) use build_zng::build_zlib_ng; +} -#[cfg(feature = "zlib-ng")] -mod build_zng; -#[cfg(feature = "zlib-ng")] -use build_zng::build_zlib_ng; +fn build_zlib_ng(_target: &str, _compat: bool) { + #[cfg(any(feature = "zlib-ng", feature = "zlib-ng-no-cmake"))] + zng::build_zlib_ng(_target, _compat); +} fn try_vcpkg() -> bool { // see if there is a vcpkg tree with zlib installed diff --git a/build_zng.rs b/build_zng.rs deleted file mode 100644 index 32b988bf..00000000 --- a/build_zng.rs +++ /dev/null @@ -1,716 +0,0 @@ -use std::{ - collections::{BTreeMap, BTreeSet}, - env, fs, - hash::{Hash as _, Hasher as _}, - io::Write as _, - path::{Path, PathBuf}, -}; - -fn append(cfg: &mut cc::Build, root: Option<&str>, files: impl IntoIterator) { - let root = root.unwrap_or(""); - cfg.files( - files - .into_iter() - .map(|fname| format!("src/zlib-ng/{root}{fname}.c")), - ); -} - -/// Replicate the behavior of cmake/make/configure of stripping out the -/// @ZLIB_SYMBOL_PREFIX@ since we don't want or need it -fn strip_symbol_prefix(input: &Path, output: &Path, get_version: bool) -> String { - let contents = fs::read_to_string(input) - .map_err(|err| format!("failed to read {input:?}: {err}")) - .unwrap(); - let mut h = - std::io::BufWriter::new(fs::File::create(output).expect("failed to create zlib include")); - - use std::io::IoSlice; - let mut write = |bufs: &[IoSlice]| { - // write_all_vectored is unstable - for buf in bufs { - h.write_all(&buf).unwrap(); - } - }; - - let mut version = None; - for line in contents.lines() { - if let Some((begin, end)) = line.split_once("@ZLIB_SYMBOL_PREFIX@") { - write(&[ - IoSlice::new(begin.as_bytes()), - IoSlice::new(end.as_bytes()), - IoSlice::new(b"\n"), - ]); - } else { - write(&[IoSlice::new(line.as_bytes()), IoSlice::new(b"\n")]); - } - - if get_version { - if line.contains("ZLIBNG_VERSION") && line.contains("#define") { - version = Some(line.split('"').nth(1).unwrap().to_owned()); - } - } - } - - if get_version { - version.expect("failed to detect ZLIBNG_VERSION") - } else { - String::new() - } -} - -#[derive(Default)] -struct AppendState { - flags: BTreeSet<&'static str>, - defines: BTreeSet<&'static str>, - files: BTreeSet<&'static str>, -} - -#[derive(Debug, Hash, Copy, Clone)] -struct TargetFeature { - check: &'static str, - msvc_flags: &'static [&'static str], - flags: &'static [&'static str], - defines: &'static [&'static str], - files: &'static [&'static str], -} - -struct Ctx<'ctx> { - cfg: &'ctx mut cc::Build, - root: Option<&'ctx str>, - enabled: Option<&'ctx BTreeSet<&'ctx str>>, - msvc: bool, - append: AppendState, - cache: Option>, -} - -impl<'ctx> Ctx<'ctx> { - fn enable(&mut self, tf: TargetFeature) -> bool { - let enabled = self - .enabled - .as_ref() - .expect("target features not set for ctx") - .contains(tf.check); - - if enabled { - self.push(tf); - } - enabled - } - - fn compile_check(&mut self, subdir: &str, tf: TargetFeature) -> bool { - let dst = Self::compile_check_root(); - fs::create_dir_all(&dst).unwrap(); - - if let Some(res) = self.check_cache(&tf) { - println!("cargo:warning=using cached compile check"); - if res { - self.push(tf); - } - - return res; - } - - let mut cmd = self.cfg.get_compiler().to_command(); - - // compile only - cmd.arg(if self.msvc { "/c" } else { "-c" }); - - // set output file so we don't pollute the cwd - { - let path = { - let mut pb = dst.join(tf.check); - pb.set_extension(if self.msvc { "obj" } else { "o" }); - pb - }; - if self.msvc { - cmd.arg(format!("/Fo\"{}\"", path.display())); - } else { - cmd.arg("-o"); - cmd.arg(path); - } - } - - let flags = if self.msvc { tf.msvc_flags } else { tf.flags }; - cmd.args(flags); - - let path = { - let mut p = "src/compile_check/".to_owned(); - if !subdir.is_empty() { - p.push_str(subdir); - p.push('/'); - } - - p.push_str(tf.check); - p.push_str(".c"); - p - }; - - cmd.arg(path); - - let output = cmd.output().expect("failed to run compiler"); - - let result = if !output.status.success() { - println!("cargo:warning=failed to compile check '{tf:?}'"); - - for line in String::from_utf8_lossy(&output.stderr).lines() { - println!("cargo:warning={line}"); - } - - false - } else { - self.push(tf); - true - }; - - self.insert_cache(tf, result); - result - } - - fn push(&mut self, tf: TargetFeature) { - println!("cargo:warning=enabling {tf:?}"); - - let flags = if self.msvc { tf.msvc_flags } else { tf.flags }; - - self.append.flags.extend(flags); - self.append.defines.extend(tf.defines); - self.append.files.extend(tf.files); - } - - fn check_cache(&mut self, tf: &TargetFeature) -> Option { - let cache = self.cache.get_or_insert_with(|| { - let mut cache = Self::compile_check_root(); - cache.push("cache.txt"); - - let Ok(cache_contents) = fs::read_to_string(cache) else { - return BTreeMap::new(); - }; - - let mut cache = BTreeMap::new(); - for line in cache_contents.lines() { - let Some((hash, res)) = line.split_once(' ') else { - continue; - }; - let Ok(hash) = u64::from_str_radix(hash, 16) else { - continue; - }; - let res = match res { - "1" => true, - "0" => false, - _ => continue, - }; - - cache.insert(hash, res); - } - - cache - }); - - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - tf.hash(&mut hasher); - - cache.get(&hasher.finish()).cloned() - } - - fn insert_cache(&mut self, tf: TargetFeature, res: bool) { - let cache = self.cache.as_mut().unwrap(); - - let mut hasher = std::collections::hash_map::DefaultHasher::new(); - tf.hash(&mut hasher); - cache.insert(hasher.finish(), res); - } - - fn compile_check_root() -> PathBuf { - let mut pb = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - pb.push("compile_check"); - pb - } -} - -impl<'ctx> Drop for Ctx<'ctx> { - fn drop(&mut self) { - let app = std::mem::take(&mut self.append); - for flag in app.flags { - self.cfg.flag(flag); - } - - for def in app.defines { - self.cfg.define(def, None); - } - - append(self.cfg, self.root, app.files); - - if let Some(cache) = self.cache.take() { - let mut cstr = String::new(); - for (hash, res) in cache { - use std::fmt::Write as _; - writeln!(&mut cstr, "{hash:08x} {}", if res { "1" } else { "0" }).unwrap(); - } - - fs::write( - Self::compile_check_root().join("cache.txt"), - cstr.as_bytes(), - ) - .expect("failed to write cache.txt"); - } - } -} - -pub fn build_zlib_ng(target: &str, compat: bool) { - let mut cfg = cc::Build::new(); - - let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap()); - let lib = dst.join("lib"); - cfg.warnings(false).out_dir(&lib); - - append( - &mut cfg, - None, - [ - "adler32", - "adler32_fold", - "chunkset", - "compare256", - "compress", - "cpu_features", - "crc32_braid", - "crc32_braid_comb", - "crc32_fold", - "deflate", - "deflate_fast", - "deflate_huff", - "deflate_medium", - "deflate_quick", - "deflate_rle", - "deflate_slow", - "deflate_stored", - "functable", - // GZFILEOP - "gzlib", - "gzwrite", - "infback", - "inflate", - "inftrees", - "insert_string", - "insert_string_roll", - "slide_hash", - "trees", - "uncompr", - "zutil", - ], - ); - - if compat { - cfg.define("ZLIB_COMPAT", None); - } - - cfg.define("WITH_GZFILEOP", None); - - { - let mut build = dst.join("build"); - fs::create_dir_all(&build).unwrap(); - build.push("gzread.c"); - - strip_symbol_prefix(Path::new("src/zlib-ng/gzread.c.in"), &build, false); - cfg.file(build); - } - - let msvc = target.ends_with("pc-windows-msvc"); - - cfg.std("c11"); - - // This can be made configurable if it is an issue but most of these would - // only fail if the user was on a decade old+ libc impl - if !msvc { - cfg.define("HAVE_ALIGNED_ALLOC", None) - .define("HAVE_ATTRIBUTE_ALIGNED", None) - .define("HAVE_BUILTIN_CTZ", None) - .define("HAVE_BUILTIN_CTZLL", None) - .define("HAVE_THREAD_LOCAL", None) - .define("HAVE_VISIBILITY_HIDDEN", None) - .define("HAVE_VISIBILITY_INTERNAL", None) - .define("_LARGEFILE64_SOURCE", "1") - .define("__USE_LARGEFILE64", None); - } - - if !target.contains("windows") { - cfg.define("STDC", None) - .define("_POSIX_SOURCE", None) - .define("HAVE_POSIX_MEMALIGN", None) - .flag("-fvisibility=hidden"); - } - - if target.contains("apple") { - cfg.define("_C99_SOURCE", None); - } else if target.contains("solaris") { - cfg.define("_XOPEN_SOURCE", "700"); - } else if target.contains("s390x") { - // Enable hardware compression on s390x. - cfg.file("src/zlib-ng/arch/s390/dfltcc_deflate.c") - .flag("-DDFLTCC_LEVEL_MASK=0x7e"); - } - - let tf = env::var("CARGO_CFG_TARGET_FEATURE").unwrap_or_default(); - let target_features: BTreeSet<_> = tf.split(',').collect(); - - let arch = env::var("CARGO_CFG_TARGET_ARCH").expect("failed to retrieve target arch"); - match arch.as_str() { - "x86_64" | "i686" => { - cfg.define("X86_FEATURES", None); - cfg.file("src/zlib-ng/arch/x86/x86_features.c"); - - let is_64 = arch.as_str() == "x86_64"; - - let pclmulqdq = { - let mut ctx = Ctx { - cfg: &mut cfg, - root: Some("arch/x86/"), - enabled: Some(&target_features), - msvc, - append: Default::default(), - cache: None, - }; - - ctx.enable(TargetFeature { - check: "avx2", - defines: &["X86_AVX2"], - flags: &["-mavx2"], - msvc_flags: &["/arch:AVX2"], - files: &["chunkset_avx2", "compare256_avx2", "adler32_avx2"], - }); - if ctx.enable(TargetFeature { - check: "sse2", - defines: &["X86_SSE2"], - flags: &["-msse2"], - msvc_flags: if is_64 { &[] } else { &["/arch:SSE2"] }, - files: &["chunkset_sse2", "compare256_sse2", "slide_hash_sse2"], - }) { - if arch != "x86_64" { - ctx.cfg.define("X86_NOCHECK_SSE2", None); - } - } - - let sse3 = ctx.enable(TargetFeature { - check: "sse3", - defines: &["X86_SSSE3"], - flags: &["-msse3"], - msvc_flags: &["/arch:SSE3"], - files: &["adler32_ssse3", "chunkset_ssse3"], - }); - - let sse4 = ctx.enable(TargetFeature { - check: "sse4.2", - defines: &["X86_SSE42"], - flags: &["-msse4.2"], - msvc_flags: &["/arch:SSE4.2"], - files: &["adler32_sse42", "insert_string_sse42"], - }); - - let pclmulqdq = sse3 - && sse4 - && ctx.enable(TargetFeature { - check: "pclmulqdq", - defines: &["X86_PCLMULQDQ_CRC"], - flags: &["-mpclmul"], - msvc_flags: &[], - files: &["crc32_pclmulqdq"], - }); - - ctx.enable(TargetFeature { - check: "xsave", - defines: &[], - flags: &["-mxsave"], - msvc_flags: &[], - files: &[], - }); - - pclmulqdq - }; - - if env::var_os("CARGO_FEATURE_ZLIB_NG_AVX512").is_some() { - enable_avx512(&mut cfg, msvc, pclmulqdq); - } - } - "aarch64" | "arm" => { - let is_aarch64 = arch == "aarch64"; - - cfg.define("ARM_FEATURES", None); - cfg.file("src/zlib-ng/arch/arm/arm_features.c"); - - let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); - - let mut ctx = Ctx { - cfg: &mut cfg, - root: Some("arch/arm/"), - enabled: Some(&target_features), - msvc, - append: Default::default(), - cache: None, - }; - - // Support runtime detection on linux/android - 'caps: { - if matches!(target_os.as_str(), "linux" | "android") { - ctx.append.defines.insert("HAVE_SYS_AUXV_H"); - - if is_aarch64 { - ctx.compile_check( - "arm", - TargetFeature { - check: "aarch64_caps", - defines: &["ARM_AUXV_HAS_CRC32"], - files: &[], - flags: &[], - msvc_flags: &[], - }, - ); - break 'caps; - } - - if !ctx.compile_check( - "arm", - TargetFeature { - check: "arm_caps", - defines: &["ARM_AUXV_HAS_CRC32"], - files: &[], - flags: &[], - msvc_flags: &[], - }, - ) { - ctx.compile_check( - "arm", - TargetFeature { - check: "arm_hwcaps", - defines: &["ARM_AUXV_HAS_CRC32", "ARM_ASM_HWCAP"], - files: &[], - flags: &[], - msvc_flags: &[], - }, - ); - } - - if !ctx.compile_check( - "arm", - TargetFeature { - check: "arm_neon", - defines: &["ARM_AUXV_HAS_NEON"], - files: &[], - flags: &[], - msvc_flags: &[], - }, - ) { - ctx.compile_check( - "arm", - TargetFeature { - check: "arm_hwneon", - defines: &["ARM_AUXV_HAS_NEON"], - files: &[], - flags: &[], - msvc_flags: &[], - }, - ); - } - } - } - - // According to the cmake macro, MSVC is missing the crc32 intrinsic - // for arm, don't know if that is still true though - if !msvc || is_aarch64 { - ctx.enable(TargetFeature { - check: "crc", - defines: &["ARM_ACLE"], - files: &["crc32_acle", "insert_string_acle"], - flags: &["-march=armv8-a+crc"], - msvc_flags: &[], - }); - } - - const NEON_64: &[&str] = &["-march=armv8-a+simd"]; - const NEON: &[&str] = &["-mfpu=neon"]; - - let flags = if is_aarch64 { NEON_64 } else { NEON }; - - if ctx.enable(TargetFeature { - check: "neon", - defines: &["ARM_NEON"], - files: &[ - "adler32_neon", - "chunkset_neon", - "compare256_neon", - "slide_hash_neon", - ], - flags, - msvc_flags: &[], - }) { - if msvc { - ctx.append.defines.insert("__ARM_NEON__"); - } - - ctx.compile_check( - "arm", - TargetFeature { - check: "ld4", - defines: &["ARM_NEON_HASLD4"], - files: &[], - flags, - msvc_flags: &[], - }, - ); - } - } - _ => { - // TODO: powerpc, riscv, s390 - } - } - - let include = dst.join("include"); - - fs::create_dir_all(&include).unwrap(); - - let (zconf_h, zlib_h, mangle) = if compat { - ("zconf.h", "zlib.h", "zlib_name_mangling.h") - } else { - fs::copy("src/zlib-ng/zconf-ng.h.in", include.join("zconf-ng.h")).unwrap(); - ("zconf-ng.h", "zlib-ng.h", "zlib_name_mangling-ng.h") - }; - - if msvc { - fs::copy(format!("src/zlib-ng/{zconf_h}.in"), include.join(zconf_h)).unwrap(); - } else { - // If we don't do this then _some_ 32-bit targets will have an incorrect - // size for off_t if they don't _also_ define `HAVE_UNISTD_H`, so we - // copy configure/cmake here - let new_zconf = fs::read_to_string(format!("src/zlib-ng/{zconf_h}.in")) - .expect("failed to read zconf.h.in") - .replace( - "#ifdef HAVE_UNISTD_H /* may be set to #if 1 by configure/cmake/etc */", - &format!( - "#if 1 /* was set to #if 1 by {}:{}:{} */", - file!(), - line!(), - column!() - ), - ); - - fs::write(include.join(zconf_h), new_zconf).unwrap(); - } - - fs::copy( - "src/zlib-ng/zlib_name_mangling.h.empty", - include.join(mangle), - ) - .unwrap(); - - let version = strip_symbol_prefix( - Path::new(&format!("src/zlib-ng/{zlib_h}.in")), - &include.join(zlib_h), - true, - ); - - cfg.include(&include).include("src/zlib-ng"); - cfg.compile("z"); - - fs::create_dir_all(lib.join("pkgconfig")).unwrap(); - fs::write( - lib.join("pkgconfig/zlib.pc"), - fs::read_to_string("src/zlib-ng/zlib.pc.in") - .unwrap() - .replace("@prefix@", dst.to_str().unwrap()) - .replace("@includedir@", "${prefix}/include") - .replace("@libdir@", "${prefix}/lib") - .replace("@VERSION@", &version), - ) - .unwrap(); - - println!("cargo:root={}", dst.display()); - println!("cargo:rustc-link-search=native={}", lib.display()); - println!("cargo:include={}", include.display()); - - if !compat { - println!("cargo:rustc-cfg=zng"); - } -} - -/// Remove this once rustc stabilizes avx512 target features -/// -/// -fn enable_avx512(cfg: &mut cc::Build, msvc: bool, with_pclmulqdq: bool) { - const FEATURES: [TargetFeature; 4] = [ - TargetFeature { - check: "basic", - msvc_flags: &["/arch:AVX512"], - flags: &["-mavx512f", "-mavx512dq", "-mavx512bw", "-mavx512vl"], - defines: &["X86_AVX512"], - files: &["adler32_avx512"], - }, - TargetFeature { - check: "mask", - msvc_flags: &["/arch:AVX512"], - flags: &["-mavx512f", "-mavx512dq", "-mavx512bw", "-mavx512vl"], - defines: &["X86_MASK_INTRIN"], - files: &[], - }, - TargetFeature { - check: "vnni", - msvc_flags: &["/arch:AVX512"], - flags: &[ - "-mavx512f", - "-mavx512dq", - "-mavx512bw", - "-mavx512vl", - "-mavx512vnni", - ], - defines: &["X86_AVX512VNNI"], - files: &["adler32_avx512_vnni"], - }, - TargetFeature { - check: "vpclmulqdq", - msvc_flags: &["/arch:AVX512"], - flags: &["-mavx512f", "-mvpclmulqdq"], - defines: &["X86_VPCLMULQDQ_CRC"], - files: &["crc32_vpclmulqdq"], - }, - ]; - - let mut ctx = Ctx { - cfg, - root: Some("arch/x86/"), - enabled: None, - msvc, - append: Default::default(), - cache: None, - }; - - // The zlib-ng cmake scripts to check target features claim that GCC doesn't - // generate good code unless mtune is set, not sure if this is still the - // case, but we faithfully replicate it just in case - if !msvc { - for target in [&["-mtune=cascadelake"], &["-mtune=skylake-avx512"]] { - if ctx.compile_check( - "", - TargetFeature { - check: "flag", - defines: &[], - flags: target, - msvc_flags: &[], - files: &[], - }, - ) { - break; - } - } - } - - for tf in FEATURES { - if tf.check == "vpclmulqdq" && !with_pclmulqdq { - break; - } - - ctx.compile_check("avx512", tf); - } -} - -#[allow(dead_code)] -fn main() { - let target = env::var("TARGET").unwrap(); - build_zlib_ng(&target, false); -} diff --git a/ci/test.bash b/ci/test.bash index f6b60006..d32e0f73 100755 --- a/ci/test.bash +++ b/ci/test.bash @@ -34,6 +34,11 @@ $CROSS run --target $TARGET_TRIPLE --manifest-path systest/Cargo.toml echo === zlib-ng build === $CROSS test --target $TARGET_TRIPLE --no-default-features --features zlib-ng $CROSS run --target $TARGET_TRIPLE --manifest-path systest/Cargo.toml --no-default-features --features zlib-ng + +echo '=== zlib-ng-no-cmake build ===' +$CROSS test --target "$TARGET_TRIPLE" --no-default-features --features zlib-ng-no-cmake +$CROSS run --target "$TARGET_TRIPLE" --manifest-path systest/Cargo.toml --no-default-features --features zlib-ng-no-cmake + echo === libz-ng-sys build === mv Cargo-zng.toml Cargo.toml mv systest/Cargo-zng.toml systest/Cargo.toml @@ -67,4 +72,4 @@ set -x $CROSS test --features zlib --target $TARGET_TRIPLE $CROSS test --features zlib-default --no-default-features --target $TARGET_TRIPLE $CROSS test --features zlib-ng --no-default-features --target $TARGET_TRIPLE -$CROSS test --features zlib-ng-compat --no-default-features --target $TARGET_TRIPLE \ No newline at end of file +$CROSS test --features zlib-ng-compat --no-default-features --target $TARGET_TRIPLE diff --git a/src/compile_check/arm/aarch64_caps.c b/src/compile_check/arm/aarch64_caps.c deleted file mode 100644 index b16f3912..00000000 --- a/src/compile_check/arm/aarch64_caps.c +++ /dev/null @@ -1,4 +0,0 @@ -#include -int main() { - return (getauxval(AT_HWCAP2) & HWCAP2_CRC32); -} diff --git a/src/compile_check/arm/arm_caps.c b/src/compile_check/arm/arm_caps.c deleted file mode 100644 index 9a4850c1..00000000 --- a/src/compile_check/arm/arm_caps.c +++ /dev/null @@ -1,4 +0,0 @@ -#include -int main() { - return (getauxval(AT_HWCAP2) & HWCAP2_CRC32); -} \ No newline at end of file diff --git a/src/compile_check/arm/arm_hwcaps.c b/src/compile_check/arm/arm_hwcaps.c deleted file mode 100644 index 86765166..00000000 --- a/src/compile_check/arm/arm_hwcaps.c +++ /dev/null @@ -1,5 +0,0 @@ -#include -#include -int main() { - return (getauxval(AT_HWCAP2) & HWCAP2_CRC32); -} \ No newline at end of file diff --git a/src/compile_check/arm/arm_hwneon.c b/src/compile_check/arm/arm_hwneon.c deleted file mode 100644 index 032c134b..00000000 --- a/src/compile_check/arm/arm_hwneon.c +++ /dev/null @@ -1,4 +0,0 @@ -#include -int main() { - return (getauxval(AT_HWCAP) & HWCAP_NEON); -} diff --git a/src/compile_check/arm/arm_neon.c b/src/compile_check/arm/arm_neon.c deleted file mode 100644 index f24c84e5..00000000 --- a/src/compile_check/arm/arm_neon.c +++ /dev/null @@ -1,4 +0,0 @@ -#include -int main() { - return (getauxval(AT_HWCAP) & HWCAP_ARM_NEON); -} diff --git a/src/compile_check/arm/ld4.c b/src/compile_check/arm/ld4.c deleted file mode 100644 index 9ae0f5d5..00000000 --- a/src/compile_check/arm/ld4.c +++ /dev/null @@ -1,8 +0,0 @@ -// Check whether compiler supports loading 4 neon vecs into a register range -#if defined(_MSC_VER) && (defined(_M_ARM64) || defined(_M_ARM64EC)) - #include -#else - #include -#endif -int32x4x4_t f(int var[16]) { return vld1q_s32_x4(var); } -int main(void) { return 0; } diff --git a/src/compile_check/avx512/basic.c b/src/compile_check/avx512/basic.c deleted file mode 100644 index 9fc0aa35..00000000 --- a/src/compile_check/avx512/basic.c +++ /dev/null @@ -1,6 +0,0 @@ -#include -__m512i f(__m512i y) { -__m512i x = _mm512_set1_epi8(2); - return _mm512_sub_epi8(x, y); -} -int main(void) { return 0; } diff --git a/src/compile_check/avx512/mask.c b/src/compile_check/avx512/mask.c deleted file mode 100644 index 3734f130..00000000 --- a/src/compile_check/avx512/mask.c +++ /dev/null @@ -1,3 +0,0 @@ -#include -__mmask16 f(__mmask16 x) { return _knot_mask16(x); } -int main(void) { return 0; } \ No newline at end of file diff --git a/src/compile_check/avx512/vnni.c b/src/compile_check/avx512/vnni.c deleted file mode 100644 index 49aeeb9b..00000000 --- a/src/compile_check/avx512/vnni.c +++ /dev/null @@ -1,6 +0,0 @@ -#include -__m512i f(__m512i x, __m512i y) { - __m512i z = _mm512_setzero_epi32(); - return _mm512_dpbusd_epi32(z, x, y); -} -int main(void) { return 0; } diff --git a/src/compile_check/avx512/vpclmulqdq.c b/src/compile_check/avx512/vpclmulqdq.c deleted file mode 100644 index c13fc070..00000000 --- a/src/compile_check/avx512/vpclmulqdq.c +++ /dev/null @@ -1,7 +0,0 @@ -#include -#include -__m512i f(__m512i a) { - __m512i b = _mm512_setzero_si512(); - return _mm512_clmulepi64_epi128(a, b, 0x10); -} -int main(void) { return 0; } diff --git a/src/compile_check/flag.c b/src/compile_check/flag.c deleted file mode 100644 index 76e81970..00000000 --- a/src/compile_check/flag.c +++ /dev/null @@ -1 +0,0 @@ -int main() { return 0; } diff --git a/systest/Cargo.toml b/systest/Cargo.toml index 8cc1a8c4..7123f97b 100644 --- a/systest/Cargo.toml +++ b/systest/Cargo.toml @@ -14,3 +14,4 @@ ctest2 = "0.4.4" [features] libz-static = ["libz-sys/static"] zlib-ng = ["libz-sys/zlib-ng"] +zlib-ng-no-cmake = ["libz-sys/zlib-ng-no-cmake"] diff --git a/zng/cc.rs b/zng/cc.rs new file mode 100644 index 00000000..42edc123 --- /dev/null +++ b/zng/cc.rs @@ -0,0 +1,413 @@ +use std::{ + env, fs, + io::Write as _, + path::{Path, PathBuf}, +}; + +struct Build { + cfg: cc::Build, + is_msvc: bool, +} + +impl Build { + fn new(cfg: cc::Build) -> Self { + let is_msvc = cfg.get_compiler().is_like_msvc(); + Self { cfg, is_msvc } + } + + fn append(&mut self, root: Option<&str>, files: &[&str]) { + let root = root.map_or(String::new(), |s| { + assert!(!s.ends_with('/'), "remove trailing slash"); + format!("{s}/") + }); + self.cfg.files( + files + .into_iter() + .map(|fname| format!("src/zlib-ng/{root}{fname}.c")), + ); + } + + fn mflag( + &mut self, + non_msvc: impl Into>, + msvc: impl Into>, + ) { + let Some(flag) = (if self.is_msvc { + msvc.into() + } else { + non_msvc.into() + }) else { + return; + }; + self.cfg.flag(flag); + } +} + +impl std::ops::Deref for Build { + type Target = cc::Build; + + fn deref(&self) -> &Self::Target { + &self.cfg + } +} + +impl std::ops::DerefMut for Build { + fn deref_mut(&mut self) -> &mut Self::Target { + &mut self.cfg + } +} + +/// Replicate the behavior of cmake/make/configure of stripping out the +/// @ZLIB_SYMBOL_PREFIX@ since we don't want or need it +fn strip_symbol_prefix(input: &Path, output: &Path, get_version: bool) -> String { + let contents = fs::read_to_string(input) + .map_err(|err| format!("failed to read {input:?}: {err}")) + .unwrap(); + let mut h = + std::io::BufWriter::new(fs::File::create(output).expect("failed to create zlib include")); + + use std::io::IoSlice; + let mut write = |bufs: &[IoSlice]| { + // write_all_vectored is unstable + for buf in bufs { + h.write_all(&buf).unwrap(); + } + }; + + let mut version = None; + for line in contents.lines() { + if let Some((begin, end)) = line.split_once("@ZLIB_SYMBOL_PREFIX@") { + write(&[ + IoSlice::new(begin.as_bytes()), + IoSlice::new(end.as_bytes()), + IoSlice::new(b"\n"), + ]); + } else { + write(&[IoSlice::new(line.as_bytes()), IoSlice::new(b"\n")]); + } + + if get_version { + if line.contains("ZLIBNG_VERSION") && line.contains("#define") { + version = Some(line.split('"').nth(1).unwrap().to_owned()); + } + } + } + + if get_version { + version.expect("failed to detect ZLIBNG_VERSION") + } else { + String::new() + } +} +pub fn build_zlib_ng(target: &str, compat: bool) { + let mut cfg = cc::Build::new(); + + let dst = PathBuf::from(env::var_os("OUT_DIR").unwrap()); + let lib = dst.join("lib"); + cfg.warnings(false).out_dir(&lib); + + let mut cfg = Build::new(cfg); + + cfg.append( + None, + &[ + "adler32", + "adler32_fold", + "chunkset", + "compare256", + "compress", + "cpu_features", + "crc32_braid", + "crc32_braid_comb", + "crc32_fold", + "deflate", + "deflate_fast", + "deflate_huff", + "deflate_medium", + "deflate_quick", + "deflate_rle", + "deflate_slow", + "deflate_stored", + "functable", + // GZFILEOP + "gzlib", + "gzwrite", + "infback", + "inflate", + "inftrees", + "insert_string", + "insert_string_roll", + "slide_hash", + "trees", + "uncompr", + "zutil", + ], + ); + + if compat { + cfg.define("ZLIB_COMPAT", None); + } + + cfg.define("WITH_GZFILEOP", None); + + { + let mut build = dst.join("build"); + fs::create_dir_all(&build).unwrap(); + build.push("gzread.c"); + + strip_symbol_prefix(Path::new("src/zlib-ng/gzread.c.in"), &build, false); + cfg.file(build); + } + + let msvc = target.ends_with("pc-windows-msvc"); + + cfg.std("c11"); + + // This can be made configurable if it is an issue but most of these would + // only fail if the user was on a decade old+ libc impl + if !msvc { + cfg.define("HAVE_ALIGNED_ALLOC", None) + .define("HAVE_ATTRIBUTE_ALIGNED", None) + .define("HAVE_BUILTIN_CTZ", None) + .define("HAVE_BUILTIN_CTZLL", None) + .define("HAVE_THREAD_LOCAL", None) + .define("HAVE_VISIBILITY_HIDDEN", None) + .define("HAVE_VISIBILITY_INTERNAL", None) + .define("_LARGEFILE64_SOURCE", "1") + .define("__USE_LARGEFILE64", None); + + // Turn implicit functions into errors, this would indicate eg. a + // define is not set + cfg.flag("-Werror-implicit-function-declaration"); + } + + if !target.contains("windows") { + cfg.define("STDC", None) + .define("_POSIX_SOURCE", None) + .define("HAVE_POSIX_MEMALIGN", None) + .flag("-fvisibility=hidden"); + } + + if target.contains("apple") { + cfg.define("_C99_SOURCE", None); + } else if target.contains("solaris") { + cfg.define("_XOPEN_SOURCE", "700"); + } else if target.contains("s390x") { + // Enable hardware compression on s390x. + cfg.file("src/zlib-ng/arch/s390/dfltcc_deflate.c") + .flag("-DDFLTCC_LEVEL_MASK=0x7e"); + } + + let arch = env::var("CARGO_CFG_TARGET_ARCH").expect("failed to retrieve target arch"); + match arch.as_str() { + "x86_64" | "i686" => { + cfg.define("X86_FEATURES", None); + cfg.file("src/zlib-ng/arch/x86/x86_features.c"); + + let is_64 = arch.as_str() == "x86_64"; + + // AVX2 + cfg.define("X86_AVX2", None); + cfg.append( + Some("arch/x86"), + &[ + "chunkset_avx2", + "compare256_avx2", + "adler32_avx2", + "slide_hash_avx2", + ], + ); + cfg.mflag("-mavx2", "/arch:AVX2"); + + // SSE2 + cfg.define("X86_SSE2", None); + cfg.append( + Some("arch/x86"), + &["chunkset_sse2", "compare256_sse2", "slide_hash_sse2"], + ); + cfg.mflag("-msse2", (!is_64).then_some("/arch:SSE2")); + + // SSE3 + cfg.define("X86_SSSE3", None); + cfg.append(Some("arch/x86"), &["adler32_ssse3", "chunkset_ssse3"]); + cfg.mflag("-msse3", "/arch:SSE3"); + + // SSE4.2 + cfg.define("X86_SSE42", None); + cfg.append(Some("arch/x86"), &["adler32_sse42", "insert_string_sse42"]); + cfg.mflag("-msse4.2", "/arch:SSE4.2"); + + // AVX-512 + { + for def in &[ + "X86_AVX512", + "X86_MASK_INTRIN", + "X86_AVX512VNNI", + "X86_VPCLMULQDQ_CRC", + ] { + cfg.define(def, None); + } + + cfg.append( + Some("arch/x86"), + &["adler32_avx512", "adler32_avx512_vnni", "crc32_vpclmulqdq"], + ); + + if cfg.is_msvc { + cfg.flag("/arch:AVX512"); + } else { + // The zlib-ng cmake scripts to check target features claim that GCC doesn't + // generate good code unless mtune is set, not sure if this is still the + // case, but we faithfully replicate it just in case + for flag in &[ + "-mavx512f", + "-mavx512dq", + "-mavx512bw", + "-mavx512vl", + "-mavx512vnni", + "-mvpclmulqdq", + "-mtune=cascadelake", + ] { + cfg.flag(flag); + } + } + } + + // Misc + cfg.define("X86_PCLMULQDQ_CRC", None); + cfg.append(Some("arch/x86"), &["crc32_pclmulqdq"]); + cfg.mflag("-mpclmul", None); + cfg.mflag("-mxsave", None); + } + "aarch64" | "arm" => { + let is_aarch64 = arch == "aarch64"; + + cfg.define("ARM_FEATURES", None); + cfg.file("src/zlib-ng/arch/arm/arm_features.c"); + + let target_os = env::var("CARGO_CFG_TARGET_OS").unwrap(); + + // Support runtime detection on linux/android + if matches!(target_os.as_str(), "linux" | "android") { + for def in &[ + "HAVE_SYS_AUXV_H", + "ARM_AUXV_HAS_CRC32", + "ARM_AUXV_HAS_CRC32", + ] { + cfg.define(def, None); + } + + if !is_aarch64 { + cfg.define("ARM_AUXV_HAS_NEON", None); + } + } + + // According to the cmake macro, MSVC is missing the crc32 intrinsic + // for arm, don't know if that is still true though + if !cfg.is_msvc || is_aarch64 { + cfg.define("ARM_ACLE", None); + cfg.append(Some("arch/arm"), &["crc32_acle", "insert_string_acle"]); + cfg.mflag("-march=armv8-a+crc", None); + } + + // neon + cfg.define("ARM_NEON", None).define("ARM_NEON_HASLD4", None); + if cfg.is_msvc { + cfg.define("__ARM_NEON__", None); + } + cfg.append( + Some("arch/arm"), + &[ + "adler32_neon", + "chunkset_neon", + "compare256_neon", + "slide_hash_neon", + ], + ); + cfg.mflag( + if is_aarch64 { + "-march=armv8-a+simd" + } else { + "-mfpu=neon" + }, + None, + ); + } + _ => { + // TODO: powerpc, riscv, s390 + } + } + + let include = dst.join("include"); + + fs::create_dir_all(&include).unwrap(); + + let (zconf_h, zlib_h, mangle) = if compat { + ("zconf.h", "zlib.h", "zlib_name_mangling.h") + } else { + fs::copy("src/zlib-ng/zconf-ng.h.in", include.join("zconf-ng.h")).unwrap(); + ("zconf-ng.h", "zlib-ng.h", "zlib_name_mangling-ng.h") + }; + + if msvc { + fs::copy(format!("src/zlib-ng/{zconf_h}.in"), include.join(zconf_h)).unwrap(); + } else { + // If we don't do this then _some_ 32-bit targets will have an incorrect + // size for off_t if they don't _also_ define `HAVE_UNISTD_H`, so we + // copy configure/cmake here + let new_zconf = fs::read_to_string(format!("src/zlib-ng/{zconf_h}.in")) + .expect("failed to read zconf.h.in") + .replace( + "#ifdef HAVE_UNISTD_H /* may be set to #if 1 by configure/cmake/etc */", + &format!( + "#if 1 /* was set to #if 1 by {}:{}:{} */", + file!(), + line!(), + column!() + ), + ); + + fs::write(include.join(zconf_h), new_zconf).unwrap(); + } + + fs::copy( + "src/zlib-ng/zlib_name_mangling.h.empty", + include.join(mangle), + ) + .unwrap(); + + let version = strip_symbol_prefix( + Path::new(&format!("src/zlib-ng/{zlib_h}.in")), + &include.join(zlib_h), + true, + ); + + cfg.include(&include).include("src/zlib-ng"); + cfg.compile("z"); + + fs::create_dir_all(lib.join("pkgconfig")).unwrap(); + fs::write( + lib.join("pkgconfig/zlib.pc"), + fs::read_to_string("src/zlib-ng/zlib.pc.in") + .unwrap() + .replace("@prefix@", dst.to_str().unwrap()) + .replace("@includedir@", "${prefix}/include") + .replace("@libdir@", "${prefix}/lib") + .replace("@VERSION@", &version), + ) + .unwrap(); + + println!("cargo:root={}", dst.display()); + println!("cargo:rustc-link-search=native={}", lib.display()); + println!("cargo:include={}", include.display()); + + if !compat { + println!("cargo:rustc-cfg=zng"); + } +} + +#[allow(dead_code)] +fn main() { + let target = env::var("TARGET").unwrap(); + build_zlib_ng(&target, false); +} diff --git a/zng/cmake.rs b/zng/cmake.rs new file mode 100644 index 00000000..25576253 --- /dev/null +++ b/zng/cmake.rs @@ -0,0 +1,60 @@ +use std::env; + +pub fn build_zlib_ng(target: &str, compat: bool) { + let mut cmake = cmake::Config::new("src/zlib-ng"); + cmake + .define("BUILD_SHARED_LIBS", "OFF") + .define("ZLIB_COMPAT", if compat { "ON" } else { "OFF" }) + .define("ZLIB_ENABLE_TESTS", "OFF") + .define("WITH_GZFILEOP", "ON"); + if target.contains("s390x") { + // Enable hardware compression on s390x. + cmake + .define("WITH_DFLTCC_DEFLATE", "1") + .define("WITH_DFLTCC_INFLATE", "1") + .cflag("-DDFLTCC_LEVEL_MASK=0x7e"); + } + if target == "i686-pc-windows-msvc" { + cmake.define("CMAKE_GENERATOR_PLATFORM", "Win32"); + } + + let install_dir = cmake.build(); + + let includedir = install_dir.join("include"); + let libdir = install_dir.join("lib"); + let libdir64 = install_dir.join("lib64"); + println!( + "cargo:rustc-link-search=native={}", + libdir.to_str().unwrap() + ); + println!( + "cargo:rustc-link-search=native={}", + libdir64.to_str().unwrap() + ); + let mut debug_suffix = ""; + let libname = if target.contains("windows") && target.contains("msvc") { + if env::var("OPT_LEVEL").unwrap() == "0" { + debug_suffix = "d"; + } + "zlibstatic" + } else { + "z" + }; + println!( + "cargo:rustc-link-lib=static={}{}{}", + libname, + if compat { "" } else { "-ng" }, + debug_suffix, + ); + println!("cargo:root={}", install_dir.to_str().unwrap()); + println!("cargo:include={}", includedir.to_str().unwrap()); + if !compat { + println!("cargo:rustc-cfg=zng"); + } +} + +#[allow(dead_code)] +fn main() { + let target = env::var("TARGET").unwrap(); + build_zlib_ng(&target, false); +}