From d865c17d7cc8c69a23439ac11edeb7925f707e49 Mon Sep 17 00:00:00 2001 From: Alex Crichton Date: Fri, 4 Jan 2019 11:43:08 -0800 Subject: [PATCH] Add ADX-related intrinsics Closes #322 --- coresimd/x86/adx.rs | 46 ++++++++++++++++++++++++ coresimd/x86/mod.rs | 3 ++ coresimd/x86_64/adx.rs | 46 ++++++++++++++++++++++++ coresimd/x86_64/mod.rs | 3 ++ crates/coresimd/src/lib.rs | 3 +- crates/stdsimd-verify/tests/x86-intel.rs | 4 +-- stdsimd/arch/detect/arch/x86.rs | 6 ++++ stdsimd/arch/detect/os/x86.rs | 6 ++++ 8 files changed, 114 insertions(+), 3 deletions(-) create mode 100644 coresimd/x86/adx.rs create mode 100644 coresimd/x86_64/adx.rs diff --git a/coresimd/x86/adx.rs b/coresimd/x86/adx.rs new file mode 100644 index 0000000000..c59743980f --- /dev/null +++ b/coresimd/x86/adx.rs @@ -0,0 +1,46 @@ +#[cfg(test)] +use stdsimd_test::assert_instr; + +#[allow(improper_ctypes)] +extern "unadjusted" { + #[link_name = "llvm.x86.addcarry.u32"] + fn llvm_addcarry_u32(a: u8, b: u32, c: u32) -> (u8, u32); + #[link_name = "llvm.x86.subborrow.u32"] + fn llvm_subborrow_u32(a: u8, b: u32, c: u32) -> (u8, u32); +} + +/// Add unsigned 32-bit integers a and b with unsigned 8-bit carry-in c_in +/// (carry flag), and store the unsigned 32-bit result in out, and the carry-out +/// is returned (carry or overflow flag). +#[inline] +#[cfg_attr(test, assert_instr(adc))] +#[stable(feature = "simd_x86_adx", since = "1.33.0")] +pub unsafe fn _addcarry_u32(c_in: u8, a: u32, b: u32, out: &mut u32) -> u8 { + let (a, b) = llvm_addcarry_u32(c_in, a, b); + *out = b; + a +} + +/// Add unsigned 32-bit integers a and b with unsigned 8-bit carry-in c_in +/// (carry or overflow flag), and store the unsigned 32-bit result in out, and +/// the carry-out is returned (carry or overflow flag). +#[inline] +#[target_feature(enable = "adx")] +#[cfg_attr(test, assert_instr(adc))] +#[stable(feature = "simd_x86_adx", since = "1.33.0")] +#[cfg(not(stage0))] +pub unsafe fn _addcarryx_u32(c_in: u8, a: u32, b: u32, out: &mut u32) -> u8 { + _addcarry_u32(c_in, a, b, out) +} + +/// Add unsigned 32-bit integers a and b with unsigned 8-bit carry-in c_in +/// (carry or overflow flag), and store the unsigned 32-bit result in out, and +/// the carry-out is returned (carry or overflow flag). +#[inline] +#[cfg_attr(test, assert_instr(sbb))] +#[stable(feature = "simd_x86_adx", since = "1.33.0")] +pub unsafe fn _subborrow_u32(c_in: u8, a: u32, b: u32, out: &mut u32) -> u8 { + let (a, b) = llvm_subborrow_u32(c_in, a, b); + *out = b; + a +} diff --git a/coresimd/x86/mod.rs b/coresimd/x86/mod.rs index 63cdecf4c0..e490c9442d 100644 --- a/coresimd/x86/mod.rs +++ b/coresimd/x86/mod.rs @@ -593,6 +593,9 @@ pub use self::rdrand::*; mod sha; pub use self::sha::*; +mod adx; +pub use self::adx::*; + #[cfg(test)] use stdsimd_test::assert_instr; diff --git a/coresimd/x86_64/adx.rs b/coresimd/x86_64/adx.rs new file mode 100644 index 0000000000..0343351b91 --- /dev/null +++ b/coresimd/x86_64/adx.rs @@ -0,0 +1,46 @@ +#[cfg(test)] +use stdsimd_test::assert_instr; + +#[allow(improper_ctypes)] +extern "unadjusted" { + #[link_name = "llvm.x86.addcarry.u64"] + fn llvm_addcarry_u64(a: u8, b: u64, c: u64) -> (u8, u64); + #[link_name = "llvm.x86.subborrow.u64"] + fn llvm_subborrow_u64(a: u8, b: u64, c: u64) -> (u8, u64); +} + +/// Add unsigned 64-bit integers a and b with unsigned 8-bit carry-in c_in +/// (carry flag), and store the unsigned 64-bit result in out, and the carry-out +/// is returned (carry or overflow flag). +#[inline] +#[cfg_attr(test, assert_instr(adc))] +#[stable(feature = "simd_x86_adx", since = "1.33.0")] +pub unsafe fn _addcarry_u64(c_in: u8, a: u64, b: u64, out: &mut u64) -> u8 { + let (a, b) = llvm_addcarry_u64(c_in, a, b); + *out = b; + a +} + +/// Add unsigned 64-bit integers a and b with unsigned 8-bit carry-in c_in +/// (carry or overflow flag), and store the unsigned 64-bit result in out, and +/// the carry-out is returned (carry or overflow flag). +#[inline] +#[target_feature(enable = "adx")] +#[cfg_attr(test, assert_instr(adc))] +#[stable(feature = "simd_x86_adx", since = "1.33.0")] +#[cfg(not(stage0))] +pub unsafe fn _addcarryx_u64(c_in: u8, a: u64, b: u64, out: &mut u64) -> u8 { + _addcarry_u64(c_in, a, b, out) +} + +/// Add unsigned 64-bit integers a and b with unsigned 8-bit carry-in c_in +/// (carry or overflow flag), and store the unsigned 64-bit result in out, and +/// the carry-out is returned (carry or overflow flag). +#[inline] +#[cfg_attr(test, assert_instr(sbb))] +#[stable(feature = "simd_x86_adx", since = "1.33.0")] +pub unsafe fn _subborrow_u64(c_in: u8, a: u64, b: u64, out: &mut u64) -> u8 { + let (a, b) = llvm_subborrow_u64(c_in, a, b); + *out = b; + a +} diff --git a/coresimd/x86_64/mod.rs b/coresimd/x86_64/mod.rs index 46859e92c9..ca16e4f17b 100644 --- a/coresimd/x86_64/mod.rs +++ b/coresimd/x86_64/mod.rs @@ -41,3 +41,6 @@ pub use self::rdrand::*; mod cmpxchg16b; pub use self::cmpxchg16b::*; + +mod adx; +pub use self::adx::*; diff --git a/crates/coresimd/src/lib.rs b/crates/coresimd/src/lib.rs index 4cc91236be..fdb4fb277d 100644 --- a/crates/coresimd/src/lib.rs +++ b/crates/coresimd/src/lib.rs @@ -37,7 +37,8 @@ mips_target_feature, powerpc_target_feature, wasm_target_feature, - abi_unadjusted + abi_unadjusted, + adx_target_feature )] // NB: When running nvptx/nvptx64 cross tests, enabling "integer_atomics" yields // a compile-time error: 'unknown feature `integer_atomics`'. This ought to be diff --git a/crates/stdsimd-verify/tests/x86-intel.rs b/crates/stdsimd-verify/tests/x86-intel.rs index 546dc1a33c..561ef95a5c 100644 --- a/crates/stdsimd-verify/tests/x86-intel.rs +++ b/crates/stdsimd-verify/tests/x86-intel.rs @@ -240,8 +240,8 @@ fn matches(rust: &Function, intel: &Intrinsic) -> Result<(), String> { // ensuring that we've actually enabled the right instruction // set for this intrinsic. match rust.name { - "_bswap" => {} - "_bswap64" => {} + "_bswap" | "_bswap64" | "_addcarry_u32" | "_addcarry_u64" | "_subborrow_u32" + | "_subborrow_u64" => {} _ => { if intel.cpuid.is_empty() { bail!("missing cpuid for {}", rust.name); diff --git a/stdsimd/arch/detect/arch/x86.rs b/stdsimd/arch/detect/arch/x86.rs index 7a202e7135..fdf37dac72 100644 --- a/stdsimd/arch/detect/arch/x86.rs +++ b/stdsimd/arch/detect/arch/x86.rs @@ -230,6 +230,10 @@ macro_rules! is_x86_feature_detected { cfg!(target_feature = "cmpxchg16b") || $crate::arch::detect::check_for( $crate::arch::detect::Feature::cmpxchg16b) }; + ("adx") => { + cfg!(target_feature = "adx") || $crate::arch::detect::check_for( + $crate::arch::detect::Feature::adx) + }; ($t:tt) => { compile_error!(concat!("unknown target feature: ", $t)) }; @@ -322,4 +326,6 @@ pub enum Feature { xsavec, /// CMPXCH16B, a 16-byte compare-and-swap instruction cmpxchg16b, + /// ADX, Intel ADX (Multi-Precision Add-Carry Instruction Extensions) + adx, } diff --git a/stdsimd/arch/detect/os/x86.rs b/stdsimd/arch/detect/os/x86.rs index c0fbf8b73b..ab49a6b4e7 100644 --- a/stdsimd/arch/detect/os/x86.rs +++ b/stdsimd/arch/detect/os/x86.rs @@ -124,6 +124,7 @@ fn detect_features() -> cache::Initializer { enable(proc_info_ecx, 1, Feature::pclmulqdq); enable(proc_info_ecx, 30, Feature::rdrand); enable(extended_features_ebx, 18, Feature::rdseed); + enable(extended_features_ebx, 19, Feature::adx); enable(proc_info_edx, 4, Feature::tsc); enable(proc_info_edx, 23, Feature::mmx); enable(proc_info_edx, 24, Feature::fxsr); @@ -290,6 +291,7 @@ mod tests { println!("xsaves: {:?}", is_x86_feature_detected!("xsaves")); println!("xsavec: {:?}", is_x86_feature_detected!("xsavec")); println!("cmpxchg16b: {:?}", is_x86_feature_detected!("cmpxchg16b")); + println!("adx: {:?}", is_x86_feature_detected!("adx")); } #[test] @@ -350,5 +352,9 @@ mod tests { is_x86_feature_detected!("cmpxchg16b"), information.cmpxchg16b(), ); + assert_eq!( + is_x86_feature_detected!("adx"), + information.adx(), + ); } }