x86_64: Update to stabilized cmpxchg16b_target_feature

README.md

@@ -125,7 +125,7 @@ See also [the `atomic128` module's readme](https://github.com/taiki-e/portable-a
   If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64).
 
   Note:
-  - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in 1.59+ (AVX) or nightly (CMPXCHG16B) for x86_64, and in nightly for other platforms, otherwise it works the same as when this cfg is set.
+  - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, otherwise it works the same as when this cfg is set.
   - If the required target features are enabled at compile-time, the atomic operations are inlined.
   - This is compatible with no-std (as with all features except `std`).
   - Some aarch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well.

bench/benches/bench.rs

@@ -1,7 +1,7 @@
 #![warn(rust_2018_idioms, single_use_lifetimes, unsafe_op_in_unsafe_fn)]
 #![allow(dead_code, unused_extern_crates)]
 #![allow(clippy::only_used_in_recursion)]
-#![feature(asm_experimental_arch, cmpxchg16b_target_feature, core_intrinsics, stdsimd)]
+#![feature(asm_experimental_arch, core_intrinsics, stdsimd)]
 
 use std::{
     hint::black_box,

build.rs

@@ -30,7 +30,6 @@ fn main() {
     let mut target_upper = target.replace(|c: char| c == '-' || c == '.', "_");
     target_upper.make_ascii_uppercase();
     println!("cargo:rerun-if-env-changed=CARGO_TARGET_{}_RUSTFLAGS", target_upper);
-    println!("cargo:rerun-if-env-changed=CARGO_CFG_PORTABLE_ATOMIC_NO_OUTLINE_ATOMICS");
     println!("cargo:rerun-if-env-changed=RUSTC");
 
     let version = match rustc_version() {
@@ -70,7 +69,6 @@ fn main() {
     }
     // asm stabilized in Rust 1.59 (nightly-2021-12-16): https://github.com/rust-lang/rust/pull/91728
     let no_asm = !version.probe(59, 2021, 12, 15);
-    let mut unstable_asm = false;
     if no_asm {
         if version.nightly
             && version.probe(46, 2020, 6, 20)
@@ -85,7 +83,6 @@ fn main() {
             // until it was stabilized in nightly-2021-12-16, so it can be safely enabled in
             // nightly, which is older than nightly-2021-12-16.
             println!("cargo:rustc-cfg=portable_atomic_unstable_asm");
-            unstable_asm = true;
         }
         println!("cargo:rustc-cfg=portable_atomic_no_asm");
     }
@@ -157,19 +154,28 @@ fn main() {
 
     match target_arch {
         "x86_64" => {
+            // cmpxchg16b_target_feature stabilized in Rust 1.69 (nightly-2023-03-01): https://github.com/rust-lang/rust/pull/106774
+            if !version.probe(69, 2023, 2, 28) {
+                if version.nightly && is_allowed_feature("cmpxchg16b_target_feature") {
+                    // This feature has not been changed since 1.33
+                    // (https://github.com/rust-lang/rust/commit/fbb56bcf44d28e65a9495decf091b6d0386e540c)
+                    // until it was stabilized in nightly-2023-03-01, so it can be safely enabled in
+                    // nightly, which is older than nightly-2023-03-01.
+                    println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature");
+                    println!("cargo:rustc-cfg=portable_atomic_cmpxchg16b_target_feature");
+                } else {
+                    // println!("cargo:rustc-cfg=portable_atomic_no_cmpxchg16b_target_feature");
+                }
+            } else {
+                // TODO: invert flag once 1.69 became stable
+                println!("cargo:rustc-cfg=portable_atomic_cmpxchg16b_target_feature");
+            }
+
             // x86_64 macos always support CMPXCHG16B: https://github.com/rust-lang/rust/blob/1.67.0/compiler/rustc_target/src/spec/x86_64_apple_darwin.rs#L8
             let has_cmpxchg16b = target_os == "macos";
             // LLVM recognizes this also as cx16 target feature: https://godbolt.org/z/6dszGeYsf
-            // It is unlikely that rustc will support that name, so we will ignore it for now.
-            target_feature_if("cmpxchg16b", has_cmpxchg16b, &version, None, true);
-            if version.nightly
-                && (!no_asm || unstable_asm)
-                && cfg!(feature = "fallback")
-                && env::var_os("CARGO_CFG_PORTABLE_ATOMIC_NO_OUTLINE_ATOMICS").is_none()
-                && is_allowed_feature("cmpxchg16b_target_feature")
-            {
-                println!("cargo:rustc-cfg=portable_atomic_unstable_cmpxchg16b_target_feature");
-            }
+            // It is unlikely that rustc will support that name, so we ignore it.
+            target_feature_if("cmpxchg16b", has_cmpxchg16b, &version, Some(69), true);
         }
         "aarch64" => {
             // aarch64_target_feature stabilized in Rust 1.61 (nightly-2022-03-16): https://github.com/rust-lang/rust/pull/90621

src/imp/atomic128/README.md

@@ -4,7 +4,7 @@ The table of targets that support 128-bit atomics and the instructions used:
 
 | target_arch | load  | store | CAS  | RMW  | note |
 | ----------- | ----- | ----- | ---- | ---- | ---- |
-| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.  <br> Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires nightly |
+| x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.  <br> Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires rustc 1.69+ |
 | aarch64 | ldxp/stxp or casp or ldp | ldxp/stxp or casp or stp | ldxp/stxp or casp | ldxp/stxp or casp | casp requires lse target feature, ldp/stp requires lse2 target feature. <br> Both compile-time and run-time detection are supported for lse. lse2 is currently compile-time detection only.  <br> Requires rustc 1.59+ |
 | powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Little endian or target CPU pwr8+. <br> Requires nightly |
 | s390x | lpq | stpq | cdsg | cdsg | Requires nightly |

src/imp/atomic128/detect/x86_64.rs

@@ -2,8 +2,8 @@
 
 #![cfg_attr(
     any(
-        portable_atomic_no_outline_atomics,
         not(target_feature = "sse"),
+        portable_atomic_no_outline_atomics,
         target_env = "sgx",
         miri,
         portable_atomic_sanitize_thread,
@@ -69,7 +69,7 @@ fn _detect(info: &mut CpuInfo) {
     {
         info.set(CpuInfo::HAS_CMPXCHG16B);
     }
-    // sgx doesn't support `cpuid`: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
+    // SGX doesn't support CPUID: https://github.com/rust-lang/stdarch/blob/a0c30f3e3c75adcd6ee7efc94014ebcead61c507/crates/core_arch/src/x86/cpuid.rs#L102-L105
     #[cfg(not(any(target_env = "sgx", miri)))]
     {
         use core::arch::x86_64::_xgetbv;
@@ -137,8 +137,8 @@ mod tests {
     }
 
     #[test]
-    // Miri doesn't support inline assembly
-    // sgx doesn't support `cpuid`
+    // SGX doesn't support CPUID.
+    // Miri doesn't support inline assembly.
     #[cfg_attr(any(target_env = "sgx", miri), ignore)]
     fn test_cpuid() {
         assert_eq!(std::is_x86_feature_detected!("cmpxchg16b"), has_cmpxchg16b());

src/imp/atomic128/x86_64.rs

@@ -45,10 +45,35 @@ struct Pair {
     hi: u64,
 }
 
-#[inline(always)]
-unsafe fn __cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
+#[cfg_attr(
+    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+    target_feature(enable = "cmpxchg16b")
+)]
+#[cfg_attr(
+    any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
+    inline
+)]
+#[cfg_attr(
+    not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
+    inline(never)
+)]
+unsafe fn _cmpxchg16b(
+    dst: *mut u128,
+    old: u128,
+    new: u128,
+    success: Ordering,
+    failure: Ordering,
+) -> (u128, bool) {
     debug_assert!(dst as usize % 16 == 0);
 
+    // Miri and Sanitizer do not support inline assembly.
+    #[cfg(any(miri, portable_atomic_sanitize_thread))]
+    // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`.
+    unsafe {
+        let res = core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure);
+        (res, res == old)
+    }
+    #[cfg(not(any(miri, portable_atomic_sanitize_thread)))]
     // SAFETY: the caller must guarantee that `dst` is valid for both writes and
     // reads, 16-byte aligned (required by CMPXCHG16B), that there are no
     // concurrent non-atomic operations, and that the CPU supports CMPXCHG16B.
@@ -62,6 +87,8 @@ unsafe fn __cmpxchg16b(dst: *mut u128, old: u128, new: u128) -> (u128, bool) {
     //
     // Refs: https://www.felixcloutier.com/x86/cmpxchg8b:cmpxchg16b
     unsafe {
+        // cmpxchg16b is always SeqCst.
+        let _ = (success, failure);
         let r: u8;
         let old = U128 { whole: old };
         let new = U128 { whole: new };
@@ -101,57 +128,13 @@ unsafe fn cmpxchg16b(
     success: Ordering,
     failure: Ordering,
 ) -> (u128, bool) {
-    #[cfg_attr(
-        all(
-            any(
-                all(test, portable_atomic_nightly),
-                portable_atomic_unstable_cmpxchg16b_target_feature,
-            ),
-            not(any(
-                target_feature = "cmpxchg16b",
-                portable_atomic_target_feature = "cmpxchg16b",
-            )),
-        ),
-        target_feature(enable = "cmpxchg16b")
-    )]
-    #[cfg_attr(
-        any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"),
-        inline
-    )]
-    #[cfg_attr(
-        not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")),
-        inline(never)
-    )]
-    unsafe fn _cmpxchg16b(
-        dst: *mut u128,
-        old: u128,
-        new: u128,
-        success: Ordering,
-        failure: Ordering,
-    ) -> (u128, bool) {
-        // Miri and Sanitizer do not support inline assembly.
-        #[cfg(any(miri, portable_atomic_sanitize_thread))]
-        // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`.
-        unsafe {
-            let res = core::arch::x86_64::cmpxchg16b(dst, old, new, success, failure);
-            (res, res == old)
-        }
-        #[cfg(not(any(miri, portable_atomic_sanitize_thread)))]
-        // SAFETY: the caller must uphold the safety contract for `_cmpxchg16b`.
-        unsafe {
-            let _ = (success, failure);
-            __cmpxchg16b(dst, old, new)
-        }
-    }
-
     #[cfg(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b"))]
     // SAFETY: the caller must guarantee that `dst` is valid for both writes and
     // reads, 16-byte aligned, that there are no concurrent non-atomic operations,
     // and cfg guarantees that CMPXCHG16B is statically available.
     unsafe {
         _cmpxchg16b(dst, old, new, success, failure)
     }
-    #[cfg(portable_atomic_unstable_cmpxchg16b_target_feature)]
     #[cfg(not(any(target_feature = "cmpxchg16b", portable_atomic_target_feature = "cmpxchg16b")))]
     {
         #[cold]
@@ -290,10 +273,12 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
 
     // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
     // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
+    // SGX doesn't support CPUID.
     // Miri and Sanitizer do not support inline assembly.
     #[cfg(any(
-        portable_atomic_no_outline_atomics,
         not(target_feature = "sse"),
+        portable_atomic_no_outline_atomics,
+        target_env = "sgx",
         miri,
         portable_atomic_sanitize_thread,
     ))]
@@ -302,8 +287,9 @@ unsafe fn atomic_load(src: *mut u128, order: Ordering) -> u128 {
         _atomic_load_cmpxchg16b(src, order)
     }
     #[cfg(not(any(
-        portable_atomic_no_outline_atomics,
         not(target_feature = "sse"),
+        portable_atomic_no_outline_atomics,
+        target_env = "sgx",
         miri,
         portable_atomic_sanitize_thread,
     )))]
@@ -333,10 +319,12 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
 
     // Do not use vector registers on targets such as x86_64-unknown-none unless SSE is explicitly enabled.
     // https://doc.rust-lang.org/nightly/rustc/platform-support/x86_64-unknown-none.html
+    // SGX doesn't support CPUID.
     // Miri and Sanitizer do not support inline assembly.
     #[cfg(any(
-        portable_atomic_no_outline_atomics,
         not(target_feature = "sse"),
+        portable_atomic_no_outline_atomics,
+        target_env = "sgx",
         miri,
         portable_atomic_sanitize_thread,
     ))]
@@ -345,8 +333,9 @@ unsafe fn atomic_store(dst: *mut u128, val: u128, order: Ordering) {
         _atomic_store_cmpxchg16b(dst, val, order);
     }
     #[cfg(not(any(
-        portable_atomic_no_outline_atomics,
         not(target_feature = "sse"),
+        portable_atomic_no_outline_atomics,
+        target_env = "sgx",
         miri,
         portable_atomic_sanitize_thread,
     )))]
@@ -444,9 +433,12 @@ mod tests {
     test_atomic_int!(u128);
 
     #[test]
-    #[cfg_attr(miri, ignore)] // Miri doesn't support inline assembly
     fn test() {
-        assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+        // Miri doesn't support inline assembly used in is_x86_feature_detected
+        #[cfg(not(miri))]
+        {
+            assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+        }
         assert!(AtomicI128::is_lock_free());
         assert!(AtomicU128::is_lock_free());
     }
@@ -460,13 +452,15 @@ mod tests {
         use super::super::*;
 
         ::quickcheck::quickcheck! {
-            #[cfg_attr(miri, ignore)] // Miri doesn't support inline assembly
-            #[cfg_attr(portable_atomic_sanitize_thread, ignore)] // TSan doesn't know the semantics of the asm synchronization instructions.
             fn test(x: u128, y: u128, z: u128) -> bool {
-                assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+                // Miri doesn't support inline assembly used in is_x86_feature_detected
+                #[cfg(not(miri))]
+                {
+                    assert!(std::is_x86_feature_detected!("cmpxchg16b"));
+                }
                 unsafe {
                     let a = Align16(UnsafeCell::new(x));
-                    let (res, ok) = __cmpxchg16b(a.get(), y, z);
+                    let (res, ok) = _cmpxchg16b(a.get(), y, z, Ordering::SeqCst, Ordering::SeqCst);
                     if x == y {
                         assert!(ok);
                         assert_eq!(res, x);