diff --git a/crates/core_simd/src/intrinsics.rs b/crates/core_simd/src/intrinsics.rs index fafeed6a62a..13cda880a6c 100644 --- a/crates/core_simd/src/intrinsics.rs +++ b/crates/core_simd/src/intrinsics.rs @@ -67,4 +67,15 @@ extern "platform-intrinsic" { // {s,u}sub.sat pub(crate) fn simd_saturating_sub(x: T, y: T) -> T; + + // reductions + pub(crate) fn simd_reduce_add_ordered(x: T, y: U) -> U; + pub(crate) fn simd_reduce_mul_ordered(x: T, y: U) -> U; + pub(crate) fn simd_reduce_all(x: T) -> bool; + pub(crate) fn simd_reduce_any(x: T) -> bool; + pub(crate) fn simd_reduce_max(x: T) -> U; + pub(crate) fn simd_reduce_min(x: T) -> U; + pub(crate) fn simd_reduce_and(x: T) -> U; + pub(crate) fn simd_reduce_or(x: T) -> U; + pub(crate) fn simd_reduce_xor(x: T) -> U; } diff --git a/crates/core_simd/src/lib.rs b/crates/core_simd/src/lib.rs index 906ee3f06ae..0fc2641516d 100644 --- a/crates/core_simd/src/lib.rs +++ b/crates/core_simd/src/lib.rs @@ -11,6 +11,8 @@ mod first; mod permute; #[macro_use] mod transmute; +#[macro_use] +mod reduction; mod comparisons; mod fmt; diff --git a/crates/core_simd/src/masks/bitmask.rs b/crates/core_simd/src/masks/bitmask.rs index d7400699fde..b4d1b6d9557 100644 --- a/crates/core_simd/src/masks/bitmask.rs +++ b/crates/core_simd/src/masks/bitmask.rs @@ -1,7 +1,7 @@ use crate::LanesAtMost32; /// A mask where each lane is represented by a single bit. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, PartialOrd, PartialEq, Ord, Eq, Hash)] #[repr(transparent)] pub struct BitMask(u64) where @@ -14,7 +14,7 @@ where /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { if value { - Self(u64::MAX) + Self(u64::MAX >> (64 - LANES)) } else { Self(u64::MIN) } diff --git a/crates/core_simd/src/masks/full_masks.rs b/crates/core_simd/src/masks/full_masks.rs index cca077b14d0..a6689ce48c6 100644 --- a/crates/core_simd/src/masks/full_masks.rs +++ b/crates/core_simd/src/masks/full_masks.rs @@ -14,22 +14,27 @@ impl core::fmt::Display for TryFromMaskError { } macro_rules! define_mask { - { $(#[$attr:meta])* struct $name:ident($type:ty); } => { + { + $(#[$attr:meta])* + struct $name:ident( + crate::$type:ident<$lanes2:ident> + ); + } => { $(#[$attr])* #[derive(Default, PartialEq, PartialOrd, Eq, Ord, Hash)] #[repr(transparent)] - pub struct $name($type) + pub struct $name(crate::$type<$lanes2>) where - $type: crate::LanesAtMost32; + crate::$type: crate::LanesAtMost32; impl Copy for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, {} impl Clone for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { #[inline] fn clone(&self) -> Self { @@ -37,13 +42,13 @@ macro_rules! define_mask { } } - impl $name<$lanes> + impl $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { - Self(<$type>::splat( + Self(>::splat( if value { -1 } else { @@ -76,42 +81,51 @@ macro_rules! define_mask { } } - /// Creates a mask from an integer vector. + /// Converts the mask to the equivalent integer representation, where -1 represents + /// "set" and 0 represents "unset". + #[inline] + pub fn to_int(self) -> crate::$type { + self.0 + } + + /// Creates a mask from the equivalent integer representation, where -1 represents + /// "set" and 0 represents "unset". /// - /// # Safety - /// All lanes must be either 0 or -1. + /// Each provided lane must be either 0 or -1. #[inline] - pub unsafe fn from_int_unchecked(value: $type) -> Self { + pub unsafe fn from_int_unchecked(value: crate::$type) -> Self { Self(value) } - /// Creates a mask from an integer vector. + /// Creates a mask from the equivalent integer representation, where -1 represents + /// "set" and 0 represents "unset". /// /// # Panics /// Panics if any lane is not 0 or -1. #[inline] - pub fn from_int(value: $type) -> Self { + pub fn from_int(value: crate::$type) -> Self { use core::convert::TryInto; value.try_into().unwrap() } } - impl core::convert::From for $name<$lanes> + impl core::convert::From for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { fn from(value: bool) -> Self { Self::splat(value) } } - impl core::convert::TryFrom<$type> for $name<$lanes> + impl core::convert::TryFrom> for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Error = TryFromMaskError; - fn try_from(value: $type) -> Result { - if value.as_slice().iter().all(|x| *x == 0 || *x == -1) { + fn try_from(value: crate::$type) -> Result { + let valid = (value.lanes_eq(crate::$type::::splat(0)) | value.lanes_eq(crate::$type::::splat(-1))).all(); + if valid { Ok(Self(value)) } else { Err(TryFromMaskError(())) @@ -119,21 +133,21 @@ macro_rules! define_mask { } } - impl core::convert::From<$name<$lanes>> for $type + impl core::convert::From<$name> for crate::$type where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { - fn from(value: $name<$lanes>) -> Self { + fn from(value: $name) -> Self { value.0 } } - impl core::convert::From> for $name<$lanes> + impl core::convert::From> for $name where - $type: crate::LanesAtMost32, - crate::BitMask<$lanes>: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, + crate::BitMask: crate::LanesAtMost32, { - fn from(value: crate::BitMask<$lanes>) -> Self { + fn from(value: crate::BitMask) -> Self { // TODO use an intrinsic to do this efficiently (with LLVM's sext instruction) let mut mask = Self::splat(false); for lane in 0..LANES { @@ -143,10 +157,10 @@ macro_rules! define_mask { } } - impl core::convert::From<$name<$lanes>> for crate::BitMask<$lanes> + impl core::convert::From<$name> for crate::BitMask where - $type: crate::LanesAtMost32, - crate::BitMask<$lanes>: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, + crate::BitMask: crate::LanesAtMost32, { fn from(value: $name<$lanes>) -> Self { // TODO use an intrinsic to do this efficiently (with LLVM's trunc instruction) @@ -158,9 +172,9 @@ macro_rules! define_mask { } } - impl core::fmt::Debug for $name<$lanes> + impl core::fmt::Debug for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { f.debug_list() @@ -169,36 +183,36 @@ macro_rules! define_mask { } } - impl core::fmt::Binary for $name<$lanes> + impl core::fmt::Binary for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { core::fmt::Binary::fmt(&self.0, f) } } - impl core::fmt::Octal for $name<$lanes> + impl core::fmt::Octal for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { core::fmt::Octal::fmt(&self.0, f) } } - impl core::fmt::LowerHex for $name<$lanes> + impl core::fmt::LowerHex for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { core::fmt::LowerHex::fmt(&self.0, f) } } - impl core::fmt::UpperHex for $name<$lanes> + impl core::fmt::UpperHex for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { core::fmt::UpperHex::fmt(&self.0, f) @@ -207,7 +221,7 @@ macro_rules! define_mask { impl core::ops::BitAnd for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] @@ -218,7 +232,7 @@ macro_rules! define_mask { impl core::ops::BitAnd for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] @@ -229,7 +243,7 @@ macro_rules! define_mask { impl core::ops::BitAnd<$name> for bool where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = $name; #[inline] @@ -240,7 +254,7 @@ macro_rules! define_mask { impl core::ops::BitOr for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] @@ -251,7 +265,7 @@ macro_rules! define_mask { impl core::ops::BitOr for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] @@ -262,7 +276,7 @@ macro_rules! define_mask { impl core::ops::BitOr<$name> for bool where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = $name; #[inline] @@ -273,7 +287,7 @@ macro_rules! define_mask { impl core::ops::BitXor for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] @@ -284,7 +298,7 @@ macro_rules! define_mask { impl core::ops::BitXor for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = Self; #[inline] @@ -295,7 +309,7 @@ macro_rules! define_mask { impl core::ops::BitXor<$name> for bool where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = $name; #[inline] @@ -306,7 +320,7 @@ macro_rules! define_mask { impl core::ops::Not for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { type Output = $name; #[inline] @@ -317,7 +331,7 @@ macro_rules! define_mask { impl core::ops::BitAndAssign for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { #[inline] fn bitand_assign(&mut self, rhs: Self) { @@ -327,7 +341,7 @@ macro_rules! define_mask { impl core::ops::BitAndAssign for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { #[inline] fn bitand_assign(&mut self, rhs: bool) { @@ -337,7 +351,7 @@ macro_rules! define_mask { impl core::ops::BitOrAssign for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { #[inline] fn bitor_assign(&mut self, rhs: Self) { @@ -347,7 +361,7 @@ macro_rules! define_mask { impl core::ops::BitOrAssign for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { #[inline] fn bitor_assign(&mut self, rhs: bool) { @@ -357,7 +371,7 @@ macro_rules! define_mask { impl core::ops::BitXorAssign for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { #[inline] fn bitxor_assign(&mut self, rhs: Self) { @@ -367,13 +381,15 @@ macro_rules! define_mask { impl core::ops::BitXorAssign for $name where - $type: crate::LanesAtMost32, + crate::$type: crate::LanesAtMost32, { #[inline] fn bitxor_assign(&mut self, rhs: bool) { *self ^= Self::splat(rhs); } } + + impl_full_mask_reductions! { $name, $type } } } diff --git a/crates/core_simd/src/masks/mod.rs b/crates/core_simd/src/masks/mod.rs index 0b986aaf7e1..4503187e4b8 100644 --- a/crates/core_simd/src/masks/mod.rs +++ b/crates/core_simd/src/masks/mod.rs @@ -7,25 +7,27 @@ pub use full_masks::*; mod bitmask; pub use bitmask::*; -use crate::LanesAtMost32; +use crate::{LanesAtMost32, SimdI128, SimdI16, SimdI32, SimdI64, SimdI8, SimdIsize}; macro_rules! define_opaque_mask { { $(#[$attr:meta])* - struct $name:ident($inner_ty:ty); - @bits $bits_ty:ty + struct $name:ident($inner_ty:ident<$lanes2:ident>); + @bits $bits_ty:ident } => { $(#[$attr])* #[allow(non_camel_case_types)] - pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; + pub struct $name($inner_ty) where $bits_ty: LanesAtMost32; - impl $name<$lanes> + impl_opaque_mask_reductions! { $name, $inner_ty, $bits_ty } + + impl $name where - $bits_ty: LanesAtMost32 + $bits_ty: LanesAtMost32 { /// Construct a mask by setting all lanes to the given value. pub fn splat(value: bool) -> Self { - Self(<$inner_ty>::splat(value)) + Self(<$inner_ty>::splat(value)) } /// Converts an array to a SIMD vector. @@ -69,66 +71,72 @@ macro_rules! define_opaque_mask { } } - impl From> for $name<$lanes> + impl From> for $name where - $bits_ty: LanesAtMost32, - BitMask<$lanes>: LanesAtMost32, + $bits_ty: LanesAtMost32, + BitMask: LanesAtMost32, { - fn from(value: BitMask<$lanes>) -> Self { + fn from(value: BitMask) -> Self { Self(value.into()) } } - impl From<$name<$lanes>> for crate::BitMask<$lanes> + impl From<$name> for crate::BitMask where - $bits_ty: LanesAtMost32, - BitMask<$lanes>: LanesAtMost32, + $bits_ty: LanesAtMost32, + BitMask: LanesAtMost32, { - fn from(value: $name<$lanes>) -> Self { + fn from(value: $name) -> Self { value.0.into() } } - impl From<$inner_ty> for $name<$lanes> + impl From<$inner_ty> for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { - fn from(value: $inner_ty) -> Self { + fn from(value: $inner_ty) -> Self { Self(value) } } - impl From<$name<$lanes>> for $inner_ty + impl From<$name> for $inner_ty where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { - fn from(value: $name<$lanes>) -> Self { + fn from(value: $name) -> Self { value.0 } } // vector/array conversion - impl From<[bool; $lanes]> for $name<$lanes> where $bits_ty: crate::LanesAtMost32 { - fn from(array: [bool; $lanes]) -> Self { + impl From<[bool; LANES]> for $name + where + $bits_ty: crate::LanesAtMost32 + { + fn from(array: [bool; LANES]) -> Self { Self::from_array(array) } } - impl From<$name<$lanes>> for [bool; $lanes] where $bits_ty: crate::LanesAtMost32 { - fn from(vector: $name<$lanes>) -> Self { + impl From<$name> for [bool; LANES] + where + $bits_ty: crate::LanesAtMost32 + { + fn from(vector: $name) -> Self { vector.to_array() } } - impl Copy for $name<$lanes> + impl Copy for $name where - $inner_ty: Copy, - $bits_ty: LanesAtMost32, + $inner_ty: Copy, + $bits_ty: LanesAtMost32, {} - impl Clone for $name<$lanes> + impl Clone for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn clone(&self) -> Self { @@ -136,9 +144,9 @@ macro_rules! define_opaque_mask { } } - impl Default for $name<$lanes> + impl Default for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn default() -> Self { @@ -146,9 +154,9 @@ macro_rules! define_opaque_mask { } } - impl PartialEq for $name<$lanes> + impl PartialEq for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn eq(&self, other: &Self) -> bool { @@ -156,9 +164,9 @@ macro_rules! define_opaque_mask { } } - impl PartialOrd for $name<$lanes> + impl PartialOrd for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn partial_cmp(&self, other: &Self) -> Option { @@ -166,9 +174,9 @@ macro_rules! define_opaque_mask { } } - impl core::fmt::Debug for $name<$lanes> + impl core::fmt::Debug for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { fn fmt(&self, f: &mut core::fmt::Formatter) -> core::fmt::Result { core::fmt::Debug::fmt(&self.0, f) @@ -177,7 +185,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = Self; #[inline] @@ -188,7 +196,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = Self; #[inline] @@ -199,7 +207,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAnd<$name> for bool where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = $name; #[inline] @@ -210,7 +218,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = Self; #[inline] @@ -221,7 +229,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = Self; #[inline] @@ -232,7 +240,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOr<$name> for bool where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = $name; #[inline] @@ -243,7 +251,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = Self; #[inline] @@ -254,7 +262,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = Self; #[inline] @@ -265,7 +273,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXor<$name> for bool where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = $name; #[inline] @@ -276,7 +284,7 @@ macro_rules! define_opaque_mask { impl core::ops::Not for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { type Output = $name; #[inline] @@ -287,7 +295,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAndAssign for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn bitand_assign(&mut self, rhs: Self) { @@ -297,7 +305,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitAndAssign for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn bitand_assign(&mut self, rhs: bool) { @@ -307,7 +315,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOrAssign for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn bitor_assign(&mut self, rhs: Self) { @@ -317,7 +325,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitOrAssign for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn bitor_assign(&mut self, rhs: bool) { @@ -327,7 +335,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXorAssign for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn bitxor_assign(&mut self, rhs: Self) { @@ -337,7 +345,7 @@ macro_rules! define_opaque_mask { impl core::ops::BitXorAssign for $name where - $bits_ty: LanesAtMost32, + $bits_ty: LanesAtMost32, { #[inline] fn bitxor_assign(&mut self, rhs: bool) { @@ -352,7 +360,7 @@ define_opaque_mask! { /// /// The layout of this type is unspecified. struct Mask8(SimdMask8); - @bits crate::SimdI8 + @bits SimdI8 } define_opaque_mask! { @@ -360,7 +368,7 @@ define_opaque_mask! { /// /// The layout of this type is unspecified. struct Mask16(SimdMask16); - @bits crate::SimdI16 + @bits SimdI16 } define_opaque_mask! { @@ -368,7 +376,7 @@ define_opaque_mask! { /// /// The layout of this type is unspecified. struct Mask32(SimdMask32); - @bits crate::SimdI32 + @bits SimdI32 } define_opaque_mask! { @@ -376,7 +384,7 @@ define_opaque_mask! { /// /// The layout of this type is unspecified. struct Mask64(SimdMask64); - @bits crate::SimdI64 + @bits SimdI64 } define_opaque_mask! { @@ -384,7 +392,7 @@ define_opaque_mask! { /// /// The layout of this type is unspecified. struct Mask128(SimdMask128); - @bits crate::SimdI128 + @bits SimdI128 } define_opaque_mask! { @@ -392,7 +400,7 @@ define_opaque_mask! { /// /// The layout of this type is unspecified. struct MaskSize(SimdMaskSize); - @bits crate::SimdIsize + @bits SimdIsize } /// Vector of eight 8-bit masks diff --git a/crates/core_simd/src/reduction.rs b/crates/core_simd/src/reduction.rs new file mode 100644 index 00000000000..382d366dd3d --- /dev/null +++ b/crates/core_simd/src/reduction.rs @@ -0,0 +1,162 @@ +macro_rules! impl_integer_reductions { + { $name:ident, $scalar:ty } => { + impl crate::$name + where + Self: crate::LanesAtMost32 + { + /// Horizontal wrapping add. Returns the sum of the lanes of the vector, with wrapping addition. + #[inline] + pub fn horizontal_sum(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_add_ordered(self, 0) } + } + + /// Horizontal wrapping multiply. Returns the product of the lanes of the vector, with wrapping multiplication. + #[inline] + pub fn horizontal_product(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_mul_ordered(self, 1) } + } + + /// Horizontal bitwise "and". Returns the cumulative bitwise "and" across the lanes of + /// the vector. + #[inline] + pub fn horizontal_and(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_and(self) } + } + + /// Horizontal bitwise "or". Returns the cumulative bitwise "or" across the lanes of + /// the vector. + #[inline] + pub fn horizontal_or(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_or(self) } + } + + /// Horizontal bitwise "xor". Returns the cumulative bitwise "xor" across the lanes of + /// the vector. + #[inline] + pub fn horizontal_xor(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_xor(self) } + } + + /// Horizontal maximum. Returns the maximum lane in the vector. + #[inline] + pub fn horizontal_max(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_max(self) } + } + + /// Horizontal minimum. Returns the minimum lane in the vector. + #[inline] + pub fn horizontal_min(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_min(self) } + } + } + } +} + +macro_rules! impl_float_reductions { + { $name:ident, $scalar:ty } => { + impl crate::$name + where + Self: crate::LanesAtMost32 + { + + /// Horizontal add. Returns the sum of the lanes of the vector. + #[inline] + pub fn horizontal_sum(self) -> $scalar { + // LLVM sum is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_slice().iter().sum() + } else { + unsafe { crate::intrinsics::simd_reduce_add_ordered(self, 0.) } + } + } + + /// Horizontal multiply. Returns the product of the lanes of the vector. + #[inline] + pub fn horizontal_product(self) -> $scalar { + // LLVM product is inaccurate on i586 + if cfg!(all(target_arch = "x86", not(target_feature = "sse2"))) { + self.as_slice().iter().product() + } else { + unsafe { crate::intrinsics::simd_reduce_mul_ordered(self, 1.) } + } + } + + /// Horizontal maximum. Returns the maximum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. This function will not return `NaN` unless all lanes are `NaN`. + #[inline] + pub fn horizontal_max(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_max(self) } + } + + /// Horizontal minimum. Returns the minimum lane in the vector. + /// + /// Returns values based on equality, so a vector containing both `0.` and `-0.` may + /// return either. This function will not return `NaN` unless all lanes are `NaN`. + #[inline] + pub fn horizontal_min(self) -> $scalar { + unsafe { crate::intrinsics::simd_reduce_min(self) } + } + } + } +} + +macro_rules! impl_full_mask_reductions { + { $name:ident, $inner:ident } => { + impl crate::$name + where + crate::$inner: crate::LanesAtMost32 + { + /// Returns true if any lane is set, or false otherwise. + #[inline] + pub fn any(self) -> bool { + unsafe { crate::intrinsics::simd_reduce_any(self.to_int()) } + } + + /// Returns true if all lanes are set, or false otherwise. + #[inline] + pub fn all(self) -> bool { + unsafe { crate::intrinsics::simd_reduce_all(self.to_int()) } + } + } + } +} + +macro_rules! impl_opaque_mask_reductions { + { $name:ident, $inner:ident, $bits_ty:ident } => { + impl $name + where + $bits_ty: crate::LanesAtMost32 + { + /// Returns true if any lane is set, or false otherwise. + #[inline] + pub fn any(self) -> bool { + self.0.any() + } + + /// Returns true if all lanes are set, or false otherwise. + #[inline] + pub fn all(self) -> bool { + self.0.all() + } + } + } +} + +impl crate::BitMask +where + crate::BitMask: crate::LanesAtMost32, +{ + /// Returns true if any lane is set, or false otherwise. + #[inline] + pub fn any(self) -> bool { + self != Self::splat(false) + } + + /// Returns true if all lanes are set, or false otherwise. + #[inline] + pub fn all(self) -> bool { + self == Self::splat(true) + } +} diff --git a/crates/core_simd/src/vector/float.rs b/crates/core_simd/src/vector/float.rs index 393e39023d9..91585b46946 100644 --- a/crates/core_simd/src/vector/float.rs +++ b/crates/core_simd/src/vector/float.rs @@ -6,6 +6,7 @@ macro_rules! impl_float_vector { { $name:ident, $type:ty, $bits_ty:ident, $mask_ty:ident, $mask_impl_ty:ident } => { impl_vector! { $name, $type } + impl_float_reductions! { $name, $type } impl $name where diff --git a/crates/core_simd/src/vector/int.rs b/crates/core_simd/src/vector/int.rs index 5304d11cd6e..24f77cb3e10 100644 --- a/crates/core_simd/src/vector/int.rs +++ b/crates/core_simd/src/vector/int.rs @@ -4,6 +4,7 @@ macro_rules! impl_integer_vector { { $name:ident, $type:ty, $mask_ty:ident, $mask_impl_ty:ident } => { impl_vector! { $name, $type } + impl_integer_reductions! { $name, $type } impl Eq for $name where Self: crate::LanesAtMost32 {} diff --git a/crates/core_simd/src/vector/uint.rs b/crates/core_simd/src/vector/uint.rs index 71b5b295112..3866b9ca5c6 100644 --- a/crates/core_simd/src/vector/uint.rs +++ b/crates/core_simd/src/vector/uint.rs @@ -5,6 +5,7 @@ macro_rules! impl_unsigned_vector { { $name:ident, $type:ty } => { impl_vector! { $name, $type } + impl_integer_reductions! { $name, $type } impl Eq for $name where Self: crate::LanesAtMost32 {} diff --git a/crates/core_simd/tests/masks.rs b/crates/core_simd/tests/masks.rs index 03a835b9c66..59da77de622 100644 --- a/crates/core_simd/tests/masks.rs +++ b/crates/core_simd/tests/masks.rs @@ -59,6 +59,24 @@ macro_rules! test_mask_api { let mask = core_simd::$name::<8>::splat(false); let _ = mask.test(8); } + + #[test] + fn any() { + assert!(!core_simd::$name::<8>::splat(false).any()); + assert!(core_simd::$name::<8>::splat(true).any()); + let mut v = core_simd::$name::<8>::splat(false); + v.set(2, true); + assert!(v.any()); + } + + #[test] + fn all() { + assert!(!core_simd::$name::<8>::splat(false).all()); + assert!(core_simd::$name::<8>::splat(true).all()); + let mut v = core_simd::$name::<8>::splat(false); + v.set(2, true); + assert!(!v.all()); + } } } } diff --git a/crates/core_simd/tests/ops_macros.rs b/crates/core_simd/tests/ops_macros.rs index a70a8a9c48b..37f3b49a330 100644 --- a/crates/core_simd/tests/ops_macros.rs +++ b/crates/core_simd/tests/ops_macros.rs @@ -136,6 +136,83 @@ macro_rules! impl_binary_checked_op_test { }; } +#[macro_export] +macro_rules! impl_common_integer_tests { + { $vector:ident, $scalar:ident } => { + test_helpers::test_lanes! { + fn horizontal_sum() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).horizontal_sum(), + x.iter().copied().fold(0 as $scalar, $scalar::wrapping_add), + ); + Ok(()) + }); + } + + fn horizontal_product() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).horizontal_product(), + x.iter().copied().fold(1 as $scalar, $scalar::wrapping_mul), + ); + Ok(()) + }); + } + + fn horizontal_and() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).horizontal_and(), + x.iter().copied().fold(-1i8 as $scalar, <$scalar as core::ops::BitAnd>::bitand), + ); + Ok(()) + }); + } + + fn horizontal_or() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).horizontal_or(), + x.iter().copied().fold(0 as $scalar, <$scalar as core::ops::BitOr>::bitor), + ); + Ok(()) + }); + } + + fn horizontal_xor() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).horizontal_xor(), + x.iter().copied().fold(0 as $scalar, <$scalar as core::ops::BitXor>::bitxor), + ); + Ok(()) + }); + } + + fn horizontal_max() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).horizontal_max(), + x.iter().copied().max().unwrap(), + ); + Ok(()) + }); + } + + fn horizontal_min() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + $vector::::from_array(x).horizontal_min(), + x.iter().copied().min().unwrap(), + ); + Ok(()) + }); + } + } + } +} + /// Implement tests for signed integers. #[macro_export] macro_rules! impl_signed_tests { @@ -144,6 +221,8 @@ macro_rules! impl_signed_tests { type Vector = core_simd::$vector; type Scalar = $scalar; + impl_common_integer_tests! { Vector, Scalar } + test_helpers::test_lanes! { fn neg() { test_helpers::test_unary_elementwise( @@ -241,6 +320,8 @@ macro_rules! impl_unsigned_tests { type Vector = core_simd::$vector; type Scalar = $scalar; + impl_common_integer_tests! { Vector, Scalar } + test_helpers::test_lanes_panic! { fn rem_zero_panic() { let a = Vector::::splat(42); @@ -397,6 +478,50 @@ macro_rules! impl_float_tests { }, ).unwrap(); } + + fn horizontal_sum() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + Vector::::from_array(x).horizontal_sum(), + x.iter().sum(), + ); + Ok(()) + }); + } + + fn horizontal_product() { + test_helpers::test_1(&|x| { + test_helpers::prop_assert_biteq! ( + Vector::::from_array(x).horizontal_product(), + x.iter().product(), + ); + Ok(()) + }); + } + + fn horizontal_max() { + test_helpers::test_1(&|x| { + let vmax = Vector::::from_array(x).horizontal_max(); + let smax = x.iter().copied().fold(Scalar::NAN, Scalar::max); + // 0 and -0 are treated the same + if !(x.contains(&0.) && x.contains(&-0.) && vmax.abs() == 0. && smax.abs() == 0.) { + test_helpers::prop_assert_biteq!(vmax, smax); + } + Ok(()) + }); + } + + fn horizontal_min() { + test_helpers::test_1(&|x| { + let vmax = Vector::::from_array(x).horizontal_min(); + let smax = x.iter().copied().fold(Scalar::NAN, Scalar::min); + // 0 and -0 are treated the same + if !(x.contains(&0.) && x.contains(&-0.) && vmax.abs() == 0. && smax.abs() == 0.) { + test_helpers::prop_assert_biteq!(vmax, smax); + } + Ok(()) + }); + } } } } diff --git a/crates/test_helpers/src/biteq.rs b/crates/test_helpers/src/biteq.rs index 4a41fe3a16e..00350e22418 100644 --- a/crates/test_helpers/src/biteq.rs +++ b/crates/test_helpers/src/biteq.rs @@ -95,7 +95,7 @@ impl core::fmt::Debug for BitEqWrapper<'_, T> { #[macro_export] macro_rules! prop_assert_biteq { - { $a:expr, $b:expr } => { + { $a:expr, $b:expr $(,)? } => { { use $crate::biteq::BitEqWrapper; let a = $a;