XSAVE, CPUID, Runtime AVX-512 and ARM support

.travis.yml

@@ -8,7 +8,7 @@ matrix:
     - env: TARGET=i586-unknown-linux-gnu
     - env: TARGET=i686-unknown-linux-gnu
     - env: TARGET=x86_64-unknown-linux-gnu NO_ADD=1
-    - env: TARGET=x86_64-unknown-linux-gnu-emulated NO_ADD=1 STDSIMD_TEST_EVERYTHING=1
+    - env: TARGET=x86_64-unknown-linux-gnu-emulated NO_ADD=1 STDSIMD_TEST_EVERYTHING=1 FEATURES="intel_sde"
     - env: TARGET=arm-unknown-linux-gnueabihf
     - env: TARGET=armv7-unknown-linux-gnueabihf
     - env: TARGET=aarch64-unknown-linux-gnu
@@ -33,7 +33,7 @@ install:
 
 script:
   - cargo generate-lockfile
-  - ci/run-docker.sh $TARGET
+  - ci/run-docker.sh $TARGET $FEATURES
 
 notifications:
   email:

Cargo.toml

@@ -28,8 +28,9 @@ opt-level = 3
 
 [dev-dependencies]
 stdsimd-test = { version = "0.*", path = "stdsimd-test" }
-cupid = "0.3"
+cupid = "0.4.0"
 
 [features]
 strict = []
-std = []
+std = []
+intel_sde = []

ci/docker/aarch64-unknown-linux-gnu/Dockerfile

@@ -8,6 +8,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   qemu-user \
   make \
   file
+
 ENV CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER=aarch64-linux-gnu-gcc \
     CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_RUNNER="qemu-aarch64 -L /usr/aarch64-linux-gnu" \
     OBJDUMP=aarch64-linux-gnu-objdump

ci/run-docker.sh

@@ -4,10 +4,11 @@
 set -ex
 
 run() {
-    echo $1
+    echo "Building docker container for TARGET=${1}"
     docker build -t stdsimd ci/docker/$1
     mkdir -p target
     target=$(echo $1 | sed 's/-emulated//')
+    echo "Running docker"
     docker run \
       --user `id -u`:`id -g` \
       --rm \
@@ -16,6 +17,7 @@ run() {
       --env CARGO_HOME=/cargo \
       --volume `rustc --print sysroot`:/rust:ro \
       --env TARGET=$target \
+      --env FEATURES=$2 \
       --env STDSIMD_TEST_EVERYTHING \
       --volume `pwd`:/checkout:ro \
       --volume `pwd`/target:/checkout/target \
@@ -31,5 +33,5 @@ if [ -z "$1" ]; then
     run $d
   done
 else
-  run $1
+  run $1 $2
 fi

ci/run.sh

@@ -15,10 +15,15 @@ case ${TARGET} in
         ;;
 esac
 
+FEATURES="strict,$FEATURES"
+FEATURES_STD="${FEATURES},std"
+
 echo "RUSTFLAGS=${RUSTFLAGS}"
+echo "FEATURES=${FEATURES}"
+echo "OBJDUMP=${OBJDUMP}"
 
-cargo test --target $TARGET --features "strict"
-cargo test --release --target $TARGET --features "strict"
+cargo test --target $TARGET --features $FEATURES --verbose -- --nocapture
+cargo test --release --target $TARGET --features $FEATURES --verbose -- --nocapture
 
-cargo test --target $TARGET --features "strict,std"
-cargo test --release --target $TARGET --features "strict,std"
+cargo test --target $TARGET --features $FEATURES_STD --verbose -- --nocapture
+cargo test --release --target $TARGET --features $FEATURES_STD --verbose -- --nocapture

src/lib.rs

@@ -121,7 +121,7 @@
 #![feature(const_fn, link_llvm_intrinsics, platform_intrinsics, repr_simd,
            simd_ffi, target_feature, cfg_target_feature, i128_type, asm,
            const_atomic_usize_new, stmt_expr_attributes)]
-#![cfg_attr(test, feature(proc_macro, test))]
+#![cfg_attr(test, feature(proc_macro, test, repr_align, attr_literals))]
 #![cfg_attr(feature = "cargo-clippy",
             allow(inline_always, too_many_arguments, cast_sign_loss,
                   cast_lossless, cast_possible_wrap,
@@ -159,8 +159,29 @@ pub mod vendor {
     pub use aarch64::*;
 
     pub use nvptx::*;
+
+    #[cfg(any(
+        // x86/x86_64:
+        any(target_arch = "x86", target_arch = "x86_64"),
+        // linux + std + (arm|aarch64):
+        all(target_os = "linux",
+            feature = "std",
+            any(target_arch = "arm", target_arch = "aarch64"))
+    ))]
+    pub use runtime::{__unstable_detect_feature, __Feature};
 }
 
+#[cfg(any(
+    // x86/x86_64:
+    any(target_arch = "x86", target_arch = "x86_64"),
+    // linux + std + (arm|aarch64):
+    all(target_os = "linux",
+        feature = "std",
+        any(target_arch = "arm", target_arch = "aarch64"))
+))]
+#[macro_use]
+mod runtime;
+
 #[macro_use]
 mod macros;
 mod simd_llvm;
@@ -204,7 +225,6 @@ mod v16 {
 }
 
 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
-#[macro_use]
 mod x86;
 
 #[cfg(any(target_arch = "arm", target_arch = "aarch64"))]

src/macros.rs

@@ -373,56 +373,6 @@ macro_rules! define_casts {
     }
 }
 
-/// Is a feature supported by the host CPU?
-///
-/// This macro performs run-time feature detection. It returns true if the host
-/// CPU in which the binary is running on supports a particular feature.
-#[macro_export]
-macro_rules! cfg_feature_enabled {
-    ($name:tt) => (
-        {
-            #[cfg(target_feature = $name)]
-            {
-                true
-            }
-            #[cfg(not(target_feature = $name))]
-            {
-                __unstable_detect_feature!($name)
-            }
-        }
-    )
-}
-
-/// On ARM features are only detected at compile-time using
-/// cfg(target_feature), so if this macro is executed the
-/// feature is not supported.
-#[cfg(any(target_arch = "arm", target_arch = "aarch64"))]
-#[macro_export]
-#[doc(hidden)]
-macro_rules! __unstable_detect_feature {
-    ("neon") => { false };
-    ($t:tt) => { compile_error!(concat!("unknown target feature: ", $t)) };
-}
-
-/// In all unsupported architectures using the macro is an error
-#[cfg(not(any(target_arch = "x86", target_arch = "x86_64",
-              target_arch = "arm", target_arch = "aarch64")))]
-#[macro_export]
-#[doc(hidden)]
-macro_rules! __unstable_detect_feature {
-    ($t:tt) => { compile_error!(concat!("unknown target feature: ", $t)) };
-}
-
-#[cfg(test)]
-mod tests {
-    #[cfg(target_arch = "x86_64")]
-    #[test]
-    fn test_macros() {
-        assert!(cfg_feature_enabled!("sse"));
-    }
-}
-
-
 #[cfg(test)]
 #[macro_export]
 macro_rules! test_arithmetic_ {

src/runtime/aarch64.rs

@@ -0,0 +1,56 @@
+//! Run-time feature detection on ARM Aarch64.
+use super::{bit, linux};
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! __unstable_detect_feature {
+    ("neon") => {
+        // FIXME: this should be removed once we rename Aarch64 neon to asimd
+        $crate::vendor::__unstable_detect_feature($crate::vendor::__Feature::asimd{})
+    };
+    ("asimd") => {
+        $crate::vendor::__unstable_detect_feature($crate::vendor::__Feature::asimd{})
+    };
+    ("pmull") => {
+        $crate::vendor::__unstable_detect_feature($crate::vendor::__Feature::pmull{})
+    };
+    ($t:tt) => { compile_error!(concat!("unknown arm target feature: ", $t)) };
+}
+
+/// ARM Aarch64 CPU Feature enum. Each variant denotes a position in a bitset
+/// for a particular feature.
+///
+/// PLEASE: do not use this, it is an implementation detail subject to change.
+#[doc(hidden)]
+#[allow(non_camel_case_types)]
+#[repr(u8)]
+pub enum __Feature {
+    /// ARM Advanced SIMD (ASIMD) - Aarch64
+    asimd,
+    /// Polynomial Multiply
+    pmull,
+}
+
+pub fn detect_features<T: linux::FeatureQuery>(mut x: T) -> usize {
+    let value: usize = 0;
+    {
+        let mut enable_feature = |f| {
+            if x.has_feature(&f) {
+                bit::set(value, f as u32);
+            }
+        };
+        enable_feature(__Feature::asimd);
+        enable_feature(__Feature::pmull);
+    }
+    value
+}
+
+impl linux::FeatureQuery for linux::CpuInfo {
+    fn has_feature(&mut self, x: &__Feature) -> bool {
+        use self::__Feature::*;
+        match *x {
+            asimd => self.field("Features").has("asimd"),
+            pmull => self.field("Features").has("pmull"),
+        }
+    }
+}

src/runtime/arm.rs

@@ -0,0 +1,66 @@
+//! Run-time feature detection on ARM Aarch32.
+
+use super::{bit, linux};
+
+#[macro_export]
+#[doc(hidden)]
+macro_rules! __unstable_detect_feature {
+    ("neon") => {
+        $crate::vendor::__unstable_detect_feature($crate::vendor::__Feature::neon{})
+    };
+    ("pmull") => {
+        $crate::vendor::__unstable_detect_feature($crate::vendor::__Feature::pmull{})
+    };
+    ($t:tt) => { compile_error!(concat!("unknown arm target feature: ", $t)) };
+}
+
+/// ARM CPU Feature enum. Each variant denotes a position in a bitset for a
+/// particular feature.
+///
+/// PLEASE: do not use this, it is an implementation detail subject to change.
+#[doc(hidden)]
+#[allow(non_camel_case_types)]
+#[repr(u8)]
+pub enum __Feature {
+    /// ARM Advanced SIMD (NEON) - Aarch32
+    neon,
+    /// Polynomial Multiply
+    pmull,
+}
+
+pub fn detect_features<T: linux::FeatureQuery>(mut x: T) -> usize {
+    let value: usize = 0;
+    {
+        let mut enable_feature = |f| {
+            if x.has_feature(&f) {
+                bit::set(value, f as u32);
+            }
+        };
+        enable_feature(__Feature::neon);
+        enable_feature(__Feature::pmull);
+    }
+    value
+}
+
+/// Is the CPU known to have a broken NEON unit?
+///
+/// See https://crbug.com/341598.
+fn has_broken_neon(cpuinfo: &linux::CpuInfo) -> bool {
+    cpuinfo.field("CPU implementer") == "0x51"
+        && cpuinfo.field("CPU architecture") == "7"
+        && cpuinfo.field("CPU variant") == "0x1"
+        && cpuinfo.field("CPU part") == "0x04d"
+        && cpuinfo.field("CPU revision") == "0"
+}
+
+impl linux::FeatureQuery for linux::CpuInfo {
+    fn has_feature(&mut self, x: &__Feature) -> bool {
+        use self::__Feature::*;
+        match *x {
+            neon => {
+                self.field("Features").has("neon") && !has_broken_neon(self)
+            }
+            pmull => self.field("Features").has("pmull"),
+        }
+    }
+}

src/runtime/bit.rs

@@ -0,0 +1,11 @@
+//! Bit manipulation utilities
+
+/// Sets the `bit` of `x`.
+pub const fn set(x: usize, bit: u32) -> usize {
+    x | 1 << bit
+}
+
+/// Tests the `bit` of `x`.
+pub const fn test(x: usize, bit: u32) -> bool {
+    x & (1 << bit) != 0
+}

src/runtime/cache.rs

@@ -0,0 +1,29 @@
+//! Cache of run-time feature detection
+
+use super::bit;
+use std::sync::atomic::{AtomicUsize, Ordering};
+
+/// This global variable is a bitset used to cache the features supported by
+/// the
+/// CPU.
+static CACHE: AtomicUsize = AtomicUsize::new(::std::usize::MAX);
+
+/// Test the `bit` of the storage. If the storage has not been initialized,
+/// initializes it with the result of `f()`.
+///
+/// On its first invocation, it detects the CPU features and caches them in the
+/// `FEATURES` global variable as an `AtomicUsize`.
+///
+/// It uses the `__Feature` variant to index into this variable as a bitset. If
+/// the bit is set, the feature is enabled, and otherwise it is disabled.
+///
+/// PLEASE: do not use this, it is an implementation detail subject to change.
+pub fn test<F>(bit: u32, f: F) -> bool
+where
+    F: FnOnce() -> usize,
+{
+    if CACHE.load(Ordering::Relaxed) == ::std::usize::MAX {
+        CACHE.store(f(), Ordering::Relaxed);
+    }
+    bit::test(CACHE.load(Ordering::Relaxed), bit)
+}