compiler: add simd_ctpop intrinsic

compiler/rustc_codegen_cranelift/src/intrinsics/simd.rs

@@ -348,6 +348,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
         | sym::simd_bswap
         | sym::simd_bitreverse
         | sym::simd_ctlz
+        | sym::simd_ctpop
         | sym::simd_cttz => {
             intrinsic_args!(fx, args => (a); intrinsic);
 
@@ -367,6 +368,7 @@ pub(super) fn codegen_simd_intrinsic_call<'tcx>(
                 (ty::Uint(_) | ty::Int(_), sym::simd_bswap) => fx.bcx.ins().bswap(lane),
                 (ty::Uint(_) | ty::Int(_), sym::simd_bitreverse) => fx.bcx.ins().bitrev(lane),
                 (ty::Uint(_) | ty::Int(_), sym::simd_ctlz) => fx.bcx.ins().clz(lane),
+                (ty::Uint(_) | ty::Int(_), sym::simd_ctpop) => fx.bcx.ins().popcnt(lane),
                 (ty::Uint(_) | ty::Int(_), sym::simd_cttz) => fx.bcx.ins().ctz(lane),
 
                 _ => unreachable!(),

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -2336,7 +2336,10 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
     }
 
     // Unary integer intrinsics
-    if matches!(name, sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_cttz) {
+    if matches!(
+        name,
+        sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctlz | sym::simd_ctpop | sym::simd_cttz
+    ) {
         let vec_ty = bx.cx.type_vector(
             match *in_elem.kind() {
                 ty::Int(i) => bx.cx.type_int_from_ty(i),
@@ -2354,31 +2357,38 @@ fn generic_simd_intrinsic<'ll, 'tcx>(
             sym::simd_bswap => "bswap",
             sym::simd_bitreverse => "bitreverse",
             sym::simd_ctlz => "ctlz",
+            sym::simd_ctpop => "ctpop",
             sym::simd_cttz => "cttz",
             _ => unreachable!(),
         };
         let int_size = in_elem.int_size_and_signed(bx.tcx()).0.bits();
         let llvm_intrinsic = &format!("llvm.{}.v{}i{}", intrinsic_name, in_len, int_size,);
 
-        return if name == sym::simd_bswap && int_size == 8 {
+        return match name {
             // byte swap is no-op for i8/u8
-            Ok(args[0].immediate())
-        } else if matches!(name, sym::simd_ctlz | sym::simd_cttz) {
-            let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
-            let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
-            Ok(bx.call(
-                fn_ty,
-                None,
-                None,
-                f,
-                &[args[0].immediate(), bx.const_int(bx.type_i1(), 0)],
-                None,
-                None,
-            ))
-        } else {
-            let fn_ty = bx.type_func(&[vec_ty], vec_ty);
-            let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
-            Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
+            sym::simd_bswap if int_size == 8 => Ok(args[0].immediate()),
+            sym::simd_ctlz | sym::simd_cttz => {
+                // for the (int, i1 immediate) pair, the second arg adds `(0, true) => poison`
+                let fn_ty = bx.type_func(&[vec_ty, bx.type_i1()], vec_ty);
+                let dont_poison_on_zero = bx.const_int(bx.type_i1(), 0);
+                let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+                Ok(bx.call(
+                    fn_ty,
+                    None,
+                    None,
+                    f,
+                    &[args[0].immediate(), dont_poison_on_zero],
+                    None,
+                    None,
+                ))
+            }
+            sym::simd_bswap | sym::simd_bitreverse | sym::simd_ctpop => {
+                // simple unary argument cases
+                let fn_ty = bx.type_func(&[vec_ty], vec_ty);
+                let f = bx.declare_cfn(llvm_intrinsic, llvm::UnnamedAddr::No, fn_ty);
+                Ok(bx.call(fn_ty, None, None, f, &[args[0].immediate()], None, None))
+            }
+            _ => unreachable!(),
         };
     }
 

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -607,6 +607,7 @@ pub fn check_intrinsic_type(
             | sym::simd_bitreverse
             | sym::simd_ctlz
             | sym::simd_cttz
+            | sym::simd_ctpop
             | sym::simd_fsqrt
             | sym::simd_fsin
             | sym::simd_fcos

compiler/rustc_span/src/symbol.rs

@@ -1681,6 +1681,7 @@ symbols! {
         simd_cast_ptr,
         simd_ceil,
         simd_ctlz,
+        simd_ctpop,
         simd_cttz,
         simd_div,
         simd_eq,

library/core/src/intrinsics/simd.rs

@@ -569,6 +569,13 @@ extern "rust-intrinsic" {
     #[rustc_nounwind]
     pub fn simd_ctlz<T>(x: T) -> T;
 
+    /// Count the number of ones in each element.
+    ///
+    /// `T` must be a vector of integers.
+    #[rustc_nounwind]
+    #[cfg(not(bootstrap))]
+    pub fn simd_ctpop<T>(x: T) -> T;
+
     /// Count the trailing zeros of each element.
     ///
     /// `T` must be a vector of integers.

src/tools/miri/src/intrinsics/simd.rs

@@ -42,6 +42,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
             | "flog2"
             | "flog10"
             | "ctlz"
+            | "ctpop"
             | "cttz"
             | "bswap"
             | "bitreverse"
@@ -68,6 +69,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
                     "round" => Op::Round(rustc_apfloat::Round::NearestTiesToAway),
                     "trunc" => Op::Round(rustc_apfloat::Round::TowardZero),
                     "ctlz" => Op::Numeric(sym::ctlz),
+                    "ctpop" => Op::Numeric(sym::ctpop),
                     "cttz" => Op::Numeric(sym::cttz),
                     "bswap" => Op::Numeric(sym::bswap),
                     "bitreverse" => Op::Numeric(sym::bitreverse),

src/tools/miri/tests/pass/intrinsics/portable-simd.rs

@@ -505,6 +505,21 @@ fn simd_intrinsics() {
         assert!(simd_reduce_all(i32x4::splat(-1)));
         assert!(!simd_reduce_all(i32x2::from_array([0, -1])));
 
+        assert_eq!(
+            simd_ctlz(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])),
+            i32x4::from_array([32, 1, 0, 0])
+        );
+
+        assert_eq!(
+            simd_ctpop(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])),
+            i32x4::from_array([0, 31, 1, 32])
+        );
+
+        assert_eq!(
+            simd_cttz(i32x4::from_array([0, i32::MAX, i32::MIN, -1_i32])),
+            i32x4::from_array([32, 0, 31, 0])
+        );
+
         assert_eq!(
             simd_select(i8x4::from_array([0, -1, -1, 0]), a, b),
             i32x4::from_array([1, 10, 10, 4])

tests/ui/simd/intrinsic/generic-arithmetic-2.rs

@@ -30,6 +30,7 @@ extern "rust-intrinsic" {
     fn simd_bswap<T>(x: T) -> T;
     fn simd_bitreverse<T>(x: T) -> T;
     fn simd_ctlz<T>(x: T) -> T;
+    fn simd_ctpop<T>(x: T) -> T;
     fn simd_cttz<T>(x: T) -> T;
 }
 
@@ -77,7 +78,6 @@ fn main() {
         simd_cttz(x);
         simd_cttz(y);
 
-
         simd_add(0, 0);
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
         simd_sub(0, 0);
@@ -108,24 +108,25 @@ fn main() {
         simd_cttz(0);
         //~^ ERROR expected SIMD input type, found non-SIMD `i32`
 
-
         simd_shl(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_shr(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_and(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_or(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_xor(z, z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_bswap(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_bitreverse(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_ctlz(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
+        simd_ctpop(z);
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
         simd_cttz(z);
-//~^ ERROR unsupported operation on `f32x4` with element `f32`
+        //~^ ERROR unsupported operation on `f32x4` with element `f32`
     }
 }

tests/ui/simd/intrinsic/generic-arithmetic-2.stderr

@@ -83,59 +83,65 @@ LL |         simd_cttz(0);
    |         ^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_shl` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:112:9
+  --> $DIR/generic-arithmetic-2.rs:111:9
    |
 LL |         simd_shl(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_shr` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:114:9
+  --> $DIR/generic-arithmetic-2.rs:113:9
    |
 LL |         simd_shr(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_and` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:116:9
+  --> $DIR/generic-arithmetic-2.rs:115:9
    |
 LL |         simd_and(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_or` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:118:9
+  --> $DIR/generic-arithmetic-2.rs:117:9
    |
 LL |         simd_or(z, z);
    |         ^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_xor` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:120:9
+  --> $DIR/generic-arithmetic-2.rs:119:9
    |
 LL |         simd_xor(z, z);
    |         ^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_bswap` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:122:9
+  --> $DIR/generic-arithmetic-2.rs:121:9
    |
 LL |         simd_bswap(z);
    |         ^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_bitreverse` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:124:9
+  --> $DIR/generic-arithmetic-2.rs:123:9
    |
 LL |         simd_bitreverse(z);
    |         ^^^^^^^^^^^^^^^^^^
 
 error[E0511]: invalid monomorphization of `simd_ctlz` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:126:9
+  --> $DIR/generic-arithmetic-2.rs:125:9
    |
 LL |         simd_ctlz(z);
    |         ^^^^^^^^^^^^
 
+error[E0511]: invalid monomorphization of `simd_ctpop` intrinsic: unsupported operation on `f32x4` with element `f32`
+  --> $DIR/generic-arithmetic-2.rs:127:9
+   |
+LL |         simd_ctpop(z);
+   |         ^^^^^^^^^^^^^
+
 error[E0511]: invalid monomorphization of `simd_cttz` intrinsic: unsupported operation on `f32x4` with element `f32`
-  --> $DIR/generic-arithmetic-2.rs:128:9
+  --> $DIR/generic-arithmetic-2.rs:129:9
    |
 LL |         simd_cttz(z);
    |         ^^^^^^^^^^^^
 
-error: aborting due to 23 previous errors
+error: aborting due to 24 previous errors
 
 For more information about this error, try `rustc --explain E0511`.

tests/ui/simd/intrinsic/generic-arithmetic-pass.rs

@@ -47,6 +47,7 @@ extern "rust-intrinsic" {
     fn simd_bswap<T>(x: T) -> T;
     fn simd_bitreverse<T>(x: T) -> T;
     fn simd_ctlz<T>(x: T) -> T;
+    fn simd_ctpop<T>(x: T) -> T;
     fn simd_cttz<T>(x: T) -> T;
 }
 
@@ -57,6 +58,8 @@ fn main() {
     let x2 = i32x4(2, 3, 4, 5);
     let y2 = U32::<4>([2, 3, 4, 5]);
     let z2 = f32x4(2.0, 3.0, 4.0, 5.0);
+    let x3 = i32x4(0, i32::MAX, i32::MIN, -1_i32);
+    let y3 = U32::<4>([0, i32::MAX as _, i32::MIN as _, -1_i32 as _]);
 
     unsafe {
         all_eq!(simd_add(x1, x2), i32x4(3, 5, 7, 9));
@@ -147,6 +150,13 @@ fn main() {
         all_eq!(simd_ctlz(x1), i32x4(31, 30, 30, 29));
         all_eq_!(simd_ctlz(y1), U32::<4>([31, 30, 30, 29]));
 
+        all_eq!(simd_ctpop(x1), i32x4(1, 1, 2, 1));
+        all_eq_!(simd_ctpop(y1), U32::<4>([1, 1, 2, 1]));
+        all_eq!(simd_ctpop(x2), i32x4(1, 2, 1, 2));
+        all_eq_!(simd_ctpop(y2), U32::<4>([1, 2, 1, 2]));
+        all_eq!(simd_ctpop(x3), i32x4(0, 31, 1, 32));
+        all_eq_!(simd_ctpop(y3), U32::<4>([0, 31, 1, 32]));
+
         all_eq!(simd_cttz(x1), i32x4(0, 1, 0, 2));
         all_eq_!(simd_cttz(y1), U32::<4>([0, 1, 0, 2]));
     }