powerpc64: Support outline-atomics for 128-bit atomics

This is currently disabled by default mainly for compatibility with
older versions of operating systems
(can be enabled by `--cfg portable_atomic_outline_atomics`).

.github/.cspell/project-dictionary.txt

@@ -30,6 +30,7 @@ cpsid
 cpsie
 CPSR
 cpuid
+cputable
 crand
 crandc
 cror

.github/workflows/ci.yml

@@ -177,20 +177,21 @@ jobs:
           RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_test_outline_atomics_detect_false
         # outline-atomics is disabled by default on aarch64 musl with static linking
         if: (matrix.target == '' || startsWith(matrix.target, 'x86_64')) || startsWith(matrix.target, 'aarch64') && !contains(matrix.target, '-musl') || startsWith(matrix.target, 'armv5te') || matrix.target == 'arm-linux-androideabi'
-      # outline-atomics is disabled by default on aarch64 musl with static linking
+      # outline-atomics is disabled by default on aarch64 musl with static linking and powerpc64
+      # powerpc64le- (little-endian) is skipped because it is pwr8 by default
       - run: tools/test.sh -vv $TARGET $DOCTEST_XCOMPILE $BUILD_STD $REPORT_TIME
         env:
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg portable_atomic_outline_atomics
           RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_outline_atomics
-        if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl')
+        if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl') || startsWith(matrix.target, 'powerpc64-')
       - run: tools/test.sh -vv $TARGET $DOCTEST_XCOMPILE $BUILD_STD $REPORT_TIME
         env:
           # Note: detect_false cfg is intended to make it easy for portable-atomic developers to
           # test cases such as has_cmpxchg16b == false, has_lse == false,
           # __kuser_helper_version < 5, etc., and is not a public API.
           RUSTDOCFLAGS: ${{ env.RUSTDOCFLAGS }} --cfg portable_atomic_outline_atomics --cfg portable_atomic_test_outline_atomics_detect_false
           RUSTFLAGS: ${{ env.RUSTFLAGS }} --cfg portable_atomic_outline_atomics --cfg portable_atomic_test_outline_atomics_detect_false
-        if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl')
+        if: startsWith(matrix.target, 'aarch64') && contains(matrix.target, '-musl') || startsWith(matrix.target, 'powerpc64-')
       # -crt-static
       - run: tools/test.sh -vv $TARGET $DOCTEST_XCOMPILE $BUILD_STD $REPORT_TIME
         env:
@@ -331,6 +332,7 @@ jobs:
           - '' # x86_64-unknown-linux-gnu
           - aarch64-unknown-linux-gnu
           - i686-unknown-linux-gnu
+          - powerpc64-unknown-linux-gnu
           - powerpc64le-unknown-linux-gnu
           - s390x-unknown-linux-gnu
     runs-on: ubuntu-latest

README.md

@@ -157,9 +157,10 @@ RUSTFLAGS="--cfg portable_atomic_unsafe_assume_single_core" cargo ...
   If dynamic dispatching by run-time CPU feature detection is enabled, it allows maintaining support for older CPUs while using features that are not supported on older CPUs, such as CMPXCHG16B (x86_64) and FEAT_LSE (aarch64).
 
   Note:
-  - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, otherwise it works the same as when this cfg is set.
+  - Dynamic detection is currently only enabled in Rust 1.61+ for aarch64, in Rust 1.59+ (AVX) or 1.69+ (CMPXCHG16B) for x86_64, nightly only for powerpc64 (disabled by default), otherwise it works the same as when this cfg is set.
   - If the required target features are enabled at compile-time, the atomic operations are inlined.
   - This is compatible with no-std (as with all features except `std`).
+  - On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)
   - Some aarch64 targets enable LLVM's `outline-atomics` target feature by default, so if you set this cfg, you may want to disable that as well. (portable-atomic's outline-atomics does not depend on the compiler-rt symbols, so even if you need to disable LLVM's outline-atomics, you may not need to disable portable-atomic's outline-atomics.)
 
   See also the [`atomic128` module's readme](https://github.com/taiki-e/portable-atomic/blob/HEAD/src/imp/atomic128/README.md).

src/imp/atomic128/README.md

@@ -8,7 +8,7 @@ Here is the table of targets that support 128-bit atomics and the instructions u
 | ----------- | ---- | ----- | --- | --- | ---- |
 | x86_64 | cmpxchg16b or vmovdqa | cmpxchg16b or vmovdqa | cmpxchg16b | cmpxchg16b | cmpxchg16b target feature required. vmovdqa requires Intel or AMD CPU with AVX. <br> Both compile-time and run-time detection are supported for cmpxchg16b. vmovdqa is currently run-time detection only.  <br> Requires rustc 1.59+ when cmpxchg16b target feature is enabled at compile-time, otherwise requires rustc 1.69+ |
 | aarch64 | ldxp/stxp or casp or ldp | ldxp/stxp or casp or stp | ldxp/stxp or casp | ldxp/stxp or casp | casp requires lse target feature, ldp/stp requires lse2 target feature. <br> Both compile-time and run-time detection are supported for lse. lse2 is currently compile-time detection only.  <br> Requires rustc 1.59+ |
-| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Little endian or target CPU pwr8+. <br> Requires nightly |
+| powerpc64 | lq | stq | lqarx/stqcx. | lqarx/stqcx. | Little endian or target CPU pwr8+. Both compile-time and run-time detection are supported (run-time detection is currently disabled by default). <br> Requires nightly |
 | s390x | lpq | stpq | cdsg | cdsg | Requires nightly |
 
 On compiler versions or platforms where these are not supported, the fallback implementation is used.
@@ -44,11 +44,15 @@ Here is the table of targets that support run-time feature detection and the ins
 | aarch64     | macos                | sysctl          | Currently test-only because FEAT_LSE and FEAT_LSE2 are always available at compile-time. |
 | aarch64     | windows              | IsProcessorFeaturePresent | Enabled by default |
 | aarch64     | fuchsia              | zx_system_get_features | Enabled by default |
+| powerpc64   | linux                | getauxval       | Disabled by default |
+| powerpc64   | freebsd              | elf_aux_info    | Disabled by default |
 
 Run-time detection is enabled by default on most targets and can be disabled with `--cfg portable_atomic_no_outline_atomics`.
 
-For targets such as `*-linux-musl*` with static linking enabled, where `getauxval` is not always available, so run-time detection is disabled by default and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled `*_no_*` cfg is preferred.)
+On some targets, run-time detection is disabled by default mainly for compatibility with older versions of operating systems or incomplete build environments, and can be enabled by `--cfg portable_atomic_outline_atomics`. (When both cfg are enabled, `*_no_*` cfg is preferred.)
 
 For targets not included in the above table, run-time detection is always disabled and works the same as when `--cfg portable_atomic_no_outline_atomics` is set.
 
 See [detect/auxv.rs](detect/auxv.rs) module-level comments for more details on Linux/Android/FreeBSD.
+
+See also [docs on `portable_atomic_no_outline_atomics`](https://github.com/taiki-e/portable-atomic/blob/HEAD/README.md#optional-cfg-no-outline-atomics) in the top-level readme.

src/imp/atomic128/detect/auxv.rs

@@ -47,13 +47,23 @@
 // but FreeBSD 11 (11.4) was EoL on 2021-09-30, and FreeBSD 11.3 was EoL on 2020-09-30:
 // https://www.freebsd.org/security/unsupported
 // See also https://github.com/rust-lang/stdarch/pull/611#issuecomment-445464613
+//
+// # PowerPC64
+//
+// On PowerPC64, outline-atomics is currently disabled by default mainly for
+// compatibility with older versions of operating systems
+// (can be enabled by `--cfg portable_atomic_outline_atomics`).
 
+#[cfg(target_arch = "aarch64")]
 pub(crate) use ffi::AT_HWCAP;
+#[cfg(target_arch = "powerpc64")]
+pub(crate) use ffi::AT_HWCAP2;
 use os::ffi;
 pub(crate) use os::getauxval;
 #[cfg(any(target_os = "linux", target_os = "android"))]
 mod os {
     // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47
+    #[allow(dead_code)]
     pub(super) mod ffi {
         pub(crate) use super::super::super::c_types::c_ulong;
 
@@ -72,6 +82,7 @@ mod os {
 
         // https://github.com/torvalds/linux/blob/v6.1/include/uapi/linux/auxvec.h
         pub(crate) const AT_HWCAP: c_ulong = 16;
+        pub(crate) const AT_HWCAP2: c_ulong = 26;
     }
 
     pub(crate) fn getauxval(type_: ffi::c_ulong) -> ffi::c_ulong {
@@ -82,6 +93,7 @@ mod os {
 #[cfg(target_os = "freebsd")]
 mod os {
     // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47
+    #[allow(dead_code)]
     pub(super) mod ffi {
         pub(crate) use super::super::super::c_types::{c_int, c_ulong, c_void};
 
@@ -95,6 +107,7 @@ mod os {
         // Defined in sys/elf_common.h.
         // https://github.com/freebsd/freebsd-src/blob/deb63adf945d446ed91a9d84124c71f15ae571d1/sys/sys/elf_common.h
         pub(crate) const AT_HWCAP: c_int = 25;
+        pub(crate) const AT_HWCAP2: c_int = 26;
     }
 
     pub(crate) fn getauxval(aux: ffi::c_int) -> ffi::c_ulong {
@@ -136,6 +149,19 @@ pub(crate) mod arch {
     #[cfg(test)]
     pub(crate) const HWCAP_USCAT: c_ulong = 1 << 25;
 }
+#[cfg(target_arch = "powerpc64")]
+pub(crate) mod arch {
+    pub(crate) use super::ffi::c_ulong;
+
+    // Linux
+    // https://github.com/torvalds/linux/blob/v6.1/arch/powerpc/include/uapi/asm/cputable.h
+    // FreeBSD
+    // Defined in machine/cpu.h.
+    // https://github.com/freebsd/freebsd-src/blob/deb63adf945d446ed91a9d84124c71f15ae571d1/sys/powerpc/include/cpu.h
+    // available on FreeBSD 11.0+
+    // https://github.com/freebsd/freebsd-src/commit/b0bf7fcd298133457991b27625bbed766e612730
+    pub(crate) const PPC_FEATURE2_ARCH_2_07: c_ulong = 0x80000000;
+}
 
 #[allow(
     clippy::alloc_instead_of_core,
@@ -187,12 +213,22 @@ mod tests {
         #[cfg(not(target_os = "freebsd"))] // libc doesn't have this on FreeBSD
         static_assert!(ffi::AT_HWCAP == libc::AT_HWCAP);
         static_assert!(ffi::AT_HWCAP == sys::AT_HWCAP as _);
+        #[cfg(not(target_os = "freebsd"))] // libc doesn't have this on FreeBSD
+        static_assert!(ffi::AT_HWCAP2 == libc::AT_HWCAP2);
+        static_assert!(ffi::AT_HWCAP2 == sys::AT_HWCAP2 as _);
         #[cfg(target_arch = "aarch64")]
         {
             // static_assert!(arch::HWCAP_ATOMICS == libc::HWCAP_ATOMICS); // libc doesn't have this
             static_assert!(arch::HWCAP_ATOMICS == sys::HWCAP_ATOMICS as ffi::c_ulong);
             // static_assert!(HWCAP_USCAT == libc::HWCAP_USCAT); // libc doesn't have this
             static_assert!(arch::HWCAP_USCAT == sys::HWCAP_USCAT as ffi::c_ulong);
         }
+        #[cfg(target_arch = "powerpc64")]
+        {
+            // static_assert!(arch::PPC_FEATURE2_ARCH_2_07 == libc::PPC_FEATURE2_ARCH_2_07); // libc doesn't have this
+            static_assert!(
+                arch::PPC_FEATURE2_ARCH_2_07 == sys::PPC_FEATURE2_ARCH_2_07 as ffi::c_ulong
+            );
+        }
     };
 }

src/imp/atomic128/detect/common.rs

@@ -104,8 +104,34 @@ impl CpuInfo {
     }
 }
 
+#[cfg(target_arch = "powerpc64")]
+impl CpuInfo {
+    /// Whether lqarx and stqcx. instructions are available
+    const HAS_QUADWORD_ATOMICS: u32 = 1;
+
+    #[allow(clippy::unused_self)]
+    #[inline]
+    pub(crate) fn has_quadword_atomics(self) -> bool {
+        #[cfg(any(
+            target_feature = "quadword-atomics",
+            portable_atomic_target_feature = "quadword-atomics",
+        ))]
+        {
+            // lqarx and stqcx. instructions are statically available.
+            true
+        }
+        #[cfg(not(any(
+            target_feature = "quadword-atomics",
+            portable_atomic_target_feature = "quadword-atomics",
+        )))]
+        {
+            self.test(CpuInfo::HAS_QUADWORD_ATOMICS)
+        }
+    }
+}
+
 // core::ffi::c_* (except c_void) requires Rust 1.64, libc will soon require Rust 1.47
-#[cfg(target_arch = "aarch64")]
+#[cfg(any(target_arch = "aarch64", target_arch = "powerpc64"))]
 #[cfg(not(windows))]
 #[allow(dead_code, non_camel_case_types)]
 mod c_types {
@@ -222,6 +248,17 @@ mod tests_common {
             assert!(x.test(CpuInfo::HAS_CMPXCHG16B));
             assert!(x.test(CpuInfo::HAS_VMOVDQA_ATOMIC));
         }
+        #[cfg(target_arch = "powerpc64")]
+        {
+            assert!(!x.test(CpuInfo::INIT));
+            assert!(!x.test(CpuInfo::HAS_QUADWORD_ATOMICS));
+            x.set(CpuInfo::INIT);
+            assert!(x.test(CpuInfo::INIT));
+            assert!(!x.test(CpuInfo::HAS_QUADWORD_ATOMICS));
+            x.set(CpuInfo::HAS_QUADWORD_ATOMICS);
+            assert!(x.test(CpuInfo::INIT));
+            assert!(x.test(CpuInfo::HAS_QUADWORD_ATOMICS));
+        }
     }
 
     #[test]
@@ -265,6 +302,19 @@ mod tests_common {
                 )),
             );
         }
+        #[cfg(target_arch = "powerpc64")]
+        {
+            features.push_str("run-time:\n");
+            print_feature!("quadword-atomics", detect().test(CpuInfo::HAS_QUADWORD_ATOMICS));
+            features.push_str("compile-time:\n");
+            print_feature!(
+                "quadword-atomics",
+                cfg!(any(
+                    target_feature = "quadword-atomics",
+                    portable_atomic_target_feature = "quadword-atomics",
+                )),
+            );
+        }
         let stdout = std::io::stderr();
         let mut stdout = stdout.lock();
         let _ = stdout.write_all(features.as_bytes());
@@ -311,4 +361,20 @@ mod tests_common {
             assert!(!detect().test(CpuInfo::HAS_RCPC3));
         }
     }
+    #[cfg(target_arch = "powerpc64")]
+    #[test]
+    fn test_detect() {
+        let proc_cpuinfo = test_helper::cpuinfo::ProcCpuinfo::new();
+        if detect().has_quadword_atomics() {
+            assert!(detect().test(CpuInfo::HAS_QUADWORD_ATOMICS));
+            if let Ok(proc_cpuinfo) = proc_cpuinfo {
+                assert!(proc_cpuinfo.power8);
+            }
+        } else {
+            assert!(!detect().test(CpuInfo::HAS_QUADWORD_ATOMICS));
+            if let Ok(proc_cpuinfo) = proc_cpuinfo {
+                assert!(!proc_cpuinfo.power8);
+            }
+        }
+    }
 }

src/imp/atomic128/detect/powerpc64_auxv.rs

@@ -0,0 +1,19 @@
+// Run-time feature detection on powerpc64 Linux/FreeBSD by parsing ELF auxiliary vectors.
+//
+// See auxv.rs for details.
+
+include!("common.rs");
+
+#[path = "auxv.rs"]
+mod auxv;
+use auxv::arch;
+
+#[cold]
+fn _detect(info: &mut CpuInfo) {
+    let hwcap2 = auxv::getauxval(auxv::AT_HWCAP2);
+
+    // power8
+    if hwcap2 & arch::PPC_FEATURE2_ARCH_2_07 != 0 {
+        info.set(CpuInfo::HAS_QUADWORD_ATOMICS);
+    }
+}