Implement Integer funnel shifts

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -383,7 +383,9 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
             | sym::rotate_left
             | sym::rotate_right
             | sym::saturating_add
-            | sym::saturating_sub => {
+            | sym::saturating_sub
+            | sym::unchecked_funnel_shl
+            | sym::unchecked_funnel_shr => {
                 let ty = args[0].layout.ty;
                 if !ty.is_integral() {
                     tcx.dcx().emit_err(InvalidMonomorphization::BasicIntegerType {
@@ -424,18 +426,26 @@ impl<'ll, 'tcx> IntrinsicCallBuilderMethods<'tcx> for Builder<'_, 'll, 'tcx> {
                     sym::bitreverse => {
                         self.call_intrinsic("llvm.bitreverse", &[llty], &[args[0].immediate()])
                     }
-                    sym::rotate_left | sym::rotate_right => {
-                        let is_left = name == sym::rotate_left;
-                        let val = args[0].immediate();
-                        let raw_shift = args[1].immediate();
-                        // rotate = funnel shift with first two args the same
+                    sym::rotate_left
+                    | sym::rotate_right
+                    | sym::unchecked_funnel_shl
+                    | sym::unchecked_funnel_shr => {
+                        let is_left = name == sym::rotate_left || name == sym::unchecked_funnel_shl;
+                        let lhs = args[0].immediate();
+                        let (rhs, raw_shift) =
+                            if name == sym::rotate_left || name == sym::rotate_right {
+                                // rotate = funnel shift with first two args the same
+                                (lhs, args[1].immediate())
+                            } else {
+                                (args[1].immediate(), args[2].immediate())
+                            };
                         let llvm_name = format!("llvm.fsh{}", if is_left { 'l' } else { 'r' });
 
                         // llvm expects shift to be the same type as the values, but rust
                         // always uses `u32`.
-                        let raw_shift = self.intcast(raw_shift, self.val_ty(val), false);
+                        let raw_shift = self.intcast(raw_shift, self.val_ty(lhs), false);
 
-                        self.call_intrinsic(llvm_name, &[llty], &[val, val, raw_shift])
+                        self.call_intrinsic(llvm_name, &[llty], &[lhs, rhs, raw_shift])
                     }
                     sym::saturating_add | sym::saturating_sub => {
                         let is_add = name == sym::saturating_add;

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -449,6 +449,9 @@ pub(crate) fn check_intrinsic_type(
         }
         sym::unchecked_shl | sym::unchecked_shr => (2, 0, vec![param(0), param(1)], param(0)),
         sym::rotate_left | sym::rotate_right => (1, 0, vec![param(0), tcx.types.u32], param(0)),
+        sym::unchecked_funnel_shl | sym::unchecked_funnel_shr => {
+            (1, 0, vec![param(0), param(0), tcx.types.u32], param(0))
+        }
         sym::unchecked_add | sym::unchecked_sub | sym::unchecked_mul => {
             (1, 0, vec![param(0), param(0)], param(0))
         }

compiler/rustc_span/src/symbol.rs

@@ -2269,6 +2269,8 @@ symbols! {
         unboxed_closures,
         unchecked_add,
         unchecked_div,
+        unchecked_funnel_shl,
+        unchecked_funnel_shr,
         unchecked_mul,
         unchecked_rem,
         unchecked_shl,

library/core/src/intrinsics/fallback.rs

@@ -148,3 +148,72 @@ impl_disjoint_bitor! {
     u8, u16, u32, u64, u128, usize,
     i8, i16, i32, i64, i128, isize,
 }
+
+#[const_trait]
+#[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")]
+pub trait FunnelShift: Copy + 'static {
+    /// See [`super::unchecked_funnel_shl`]; we just need the trait indirection to handle
+    /// different types since calling intrinsics with generics doesn't work.
+    unsafe fn unchecked_funnel_shl(self, rhs: Self, shift: u32) -> Self;
+
+    /// See [`super::unchecked_funnel_shr`]; we just need the trait indirection to handle
+    /// different types since calling intrinsics with generics doesn't work.
+    unsafe fn unchecked_funnel_shr(self, rhs: Self, shift: u32) -> Self;
+}
+
+macro_rules! impl_funnel_shifts {
+    ($($type:ident),*) => {$(
+        #[rustc_const_unstable(feature = "core_intrinsics_fallbacks", issue = "none")]
+        impl const FunnelShift for $type {
+            #[cfg_attr(miri, track_caller)]
+            #[inline]
+            unsafe fn unchecked_funnel_shl(self, rhs: Self, shift: u32) -> Self {
+                // This implementation is also used by Miri so we have to check the precondition.
+                // SAFETY: this is guaranteed by the caller
+                unsafe { super::assume(shift < $type::BITS); }
+                if shift == 0 {
+                    self
+                } else {
+                    // SAFETY:
+                    //  - `shift < T::BITS`, which satisfies `unchecked_shl`
+                    //  - this also ensures that `T::BITS - shift < T::BITS`, which satisfies `unchecked_shr`
+                    //  - because the types are unsigned, the combination are disjoint bits (this is not true if they're signed,
+                    //    since SHR will fill in the empty space with a sign bit, not zero)
+                    unsafe {
+                        super::disjoint_bitor(
+                            super::unchecked_shl(self, shift),
+                            super::unchecked_shr(rhs, $type::BITS - shift),
+                        )
+                    }
+                }
+            }
+
+            #[cfg_attr(miri, track_caller)]
+            #[inline]
+            unsafe fn unchecked_funnel_shr(self, rhs: Self, shift: u32) -> Self {
+                // This implementation is also used by Miri so we have to check the precondition.
+                // SAFETY: this is guaranteed by the caller
+                unsafe { super::assume(shift < $type::BITS); }
+                if shift == 0 {
+                    rhs
+                } else {
+                    // SAFETY:
+                    //  - `shift < T::BITS`, which satisfies `unchecked_shr`
+                    //  - this also ensures that `T::BITS - shift < T::BITS`, which satisfies `unchecked_shl`
+                    //  - because the types are unsigned, the combination are disjoint bits (this is not true if they're signed,
+                    //    since SHR will fill in the empty space with a sign bit, not zero)
+                    unsafe {
+                        super::disjoint_bitor(
+                            super::unchecked_shl(self, $type::BITS - shift),
+                            super::unchecked_shr(rhs, shift),
+                        )
+                    }
+                }
+            }
+        }
+    )*};
+}
+
+impl_funnel_shifts! {
+    u8, u16, u32, u64, u128, usize
+}

library/core/src/intrinsics/mod.rs

@@ -2102,6 +2102,61 @@ pub const fn saturating_add<T: Copy>(a: T, b: T) -> T;
 #[rustc_intrinsic]
 pub const fn saturating_sub<T: Copy>(a: T, b: T) -> T;
 
+/// Funnel Shift left.
+///
+/// Concatenates `a` and `b` (with `a` in the most significant half),
+/// creating an integer twice as wide. Then shift this integer left
+/// by `shift`), and extract the most significant half. If `a` and `b`
+/// are the same, this is equivalent to a rotate left operation.
+///
+/// It is undefined behavior if `shift` is greater than or equal to the
+/// bit size of `T`.
+///
+/// Safe versions of this intrinsic are available on the integer primitives
+/// via the `funnel_shl` method. For example, [`u32::funnel_shl`].
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")]
+#[unstable(feature = "funnel_shifts", issue = "145686")]
+#[track_caller]
+#[miri::intrinsic_fallback_is_spec]
+pub const unsafe fn unchecked_funnel_shl<T: [const] fallback::FunnelShift>(
+    a: T,
+    b: T,
+    shift: u32,
+) -> T {
+    // SAFETY: caller ensures that `shift` is in-range
+    unsafe { a.unchecked_funnel_shl(b, shift) }
+}
+
+/// Funnel Shift right.
+///
+/// Concatenates `a` and `b` (with `a` in the most significant half),
+/// creating an integer twice as wide. Then shift this integer right
+/// by `shift` (taken modulo the bit size of `T`), and extract the
+/// least significant half. If `a` and `b` are the same, this is equivalent
+/// to a rotate right operation.
+///
+/// It is undefined behavior if `shift` is greater than or equal to the
+/// bit size of `T`.
+///
+/// Safer versions of this intrinsic are available on the integer primitives
+/// via the `funnel_shr` method. For example, [`u32::funnel_shr`]
+#[rustc_intrinsic]
+#[rustc_nounwind]
+#[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")]
+#[unstable(feature = "funnel_shifts", issue = "145686")]
+#[track_caller]
+#[miri::intrinsic_fallback_is_spec]
+pub const unsafe fn unchecked_funnel_shr<T: [const] fallback::FunnelShift>(
+    a: T,
+    b: T,
+    shift: u32,
+) -> T {
+    // SAFETY: caller ensures that `shift` is in-range
+    unsafe { a.unchecked_funnel_shr(b, shift) }
+}
+
 /// This is an implementation detail of [`crate::ptr::read`] and should
 /// not be used anywhere else.  See its comments for why this exists.
 ///

library/core/src/lib.rs

@@ -156,6 +156,7 @@
 #![feature(f128)]
 #![feature(freeze_impls)]
 #![feature(fundamental)]
+#![feature(funnel_shifts)]
 #![feature(if_let_guard)]
 #![feature(intra_doc_pointers)]
 #![feature(intrinsics)]

library/core/src/num/mod.rs

@@ -454,6 +454,9 @@ impl u8 {
         rot = 2,
         rot_op = "0x82",
         rot_result = "0xa",
+        fsh_op = "0x36",
+        fshl_result = "0x8",
+        fshr_result = "0x8d",
         swap_op = "0x12",
         swapped = "0x12",
         reversed = "0x48",
@@ -1088,6 +1091,9 @@ impl u16 {
         rot = 4,
         rot_op = "0xa003",
         rot_result = "0x3a",
+        fsh_op = "0x2de",
+        fshl_result = "0x30",
+        fshr_result = "0x302d",
         swap_op = "0x1234",
         swapped = "0x3412",
         reversed = "0x2c48",
@@ -1135,6 +1141,9 @@ impl u32 {
         rot = 8,
         rot_op = "0x10000b3",
         rot_result = "0xb301",
+        fsh_op = "0x2fe78e45",
+        fshl_result = "0xb32f",
+        fshr_result = "0xb32fe78e",
         swap_op = "0x12345678",
         swapped = "0x78563412",
         reversed = "0x1e6a2c48",
@@ -1158,6 +1167,9 @@ impl u64 {
         rot = 12,
         rot_op = "0xaa00000000006e1",
         rot_result = "0x6e10aa",
+        fsh_op = "0x2fe78e45983acd98",
+        fshl_result = "0x6e12fe",
+        fshr_result = "0x6e12fe78e45983ac",
         swap_op = "0x1234567890123456",
         swapped = "0x5634129078563412",
         reversed = "0x6a2c48091e6a2c48",
@@ -1181,6 +1193,9 @@ impl u128 {
         rot = 16,
         rot_op = "0x13f40000000000000000000000004f76",
         rot_result = "0x4f7613f4",
+        fsh_op = "0x2fe78e45983acd98039000008736273",
+        fshl_result = "0x4f7602fe",
+        fshr_result = "0x4f7602fe78e45983acd9803900000873",
         swap_op = "0x12345678901234567890123456789012",
         swapped = "0x12907856341290785634129078563412",
         reversed = "0x48091e6a2c48091e6a2c48091e6a2c48",
@@ -1207,6 +1222,9 @@ impl usize {
         rot = 4,
         rot_op = "0xa003",
         rot_result = "0x3a",
+        fsh_op = "0x2fe78e45983acd98039000008736273",
+        fshl_result = "0x4f7602fe",
+        fshr_result = "0x4f7602fe78e45983acd9803900000873",
         swap_op = "0x1234",
         swapped = "0x3412",
         reversed = "0x2c48",
@@ -1231,6 +1249,9 @@ impl usize {
         rot = 8,
         rot_op = "0x10000b3",
         rot_result = "0xb301",
+        fsh_op = "0x2fe78e45",
+        fshl_result = "0xb32f",
+        fshr_result = "0xb32fe78e",
         swap_op = "0x12345678",
         swapped = "0x78563412",
         reversed = "0x1e6a2c48",
@@ -1255,6 +1276,9 @@ impl usize {
         rot = 12,
         rot_op = "0xaa00000000006e1",
         rot_result = "0x6e10aa",
+        fsh_op = "0x2fe78e45983acd98",
+        fshl_result = "0x6e12fe",
+        fshr_result = "0x6e12fe78e45983ac",
         swap_op = "0x1234567890123456",
         swapped = "0x5634129078563412",
         reversed = "0x6a2c48091e6a2c48",

library/core/src/num/uint_macros.rs

@@ -14,6 +14,9 @@ macro_rules! uint_impl {
         rot = $rot:literal,
         rot_op = $rot_op:literal,
         rot_result = $rot_result:literal,
+        fsh_op = $fsh_op:literal,
+        fshl_result = $fshl_result:literal,
+        fshr_result = $fshr_result:literal,
         swap_op = $swap_op:literal,
         swapped = $swapped:literal,
         reversed = $reversed:literal,
@@ -375,6 +378,76 @@ macro_rules! uint_impl {
             return intrinsics::rotate_right(self, n);
         }
 
+        /// Performs a left funnel shift (concatenates `self` with `rhs`, with `self`
+        /// making up the most significant half, then shifts the combined value left
+        /// by `n`, and most significant half is extracted to produce the result).
+        ///
+        /// Please note this isn't the same operation as the `<<` shifting operator or
+        /// [`rotate_left`](Self::rotate_left), although `a.funnel_shl(a, n)` is *equivalent*
+        /// to `a.rotate_left(n)`.
+        ///
+        /// # Panics
+        ///
+        /// If `n` is greater than or equal to the number of bits in `self`
+        ///
+        /// # Examples
+        ///
+        /// Basic usage:
+        ///
+        /// ```
+        /// #![feature(funnel_shifts)]
+        #[doc = concat!("let a = ", $rot_op, stringify!($SelfT), ";")]
+        #[doc = concat!("let b = ", $fsh_op, stringify!($SelfT), ";")]
+        #[doc = concat!("let m = ", $fshl_result, ";")]
+        ///
+        #[doc = concat!("assert_eq!(a.funnel_shl(b, ", $rot, "), m);")]
+        /// ```
+        #[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")]
+        #[unstable(feature = "funnel_shifts", issue = "145686")]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline(always)]
+        pub const fn funnel_shl(self, rhs: Self, n: u32) -> Self {
+            assert!(n < Self::BITS, "attempt to funnel shift left with overflow");
+            // SAFETY: just checked that `shift` is in-range
+            unsafe { intrinsics::unchecked_funnel_shl(self, rhs, n) }
+        }
+
+        /// Performs a right funnel shift (concatenates `self` and `rhs`, with `self`
+        /// making up the most significant half, then shifts the combined value right
+        /// by `n`, and least significant half is extracted to produce the result).
+        ///
+        /// Please note this isn't the same operation as the `>>` shifting operator or
+        /// [`rotate_right`](Self::rotate_right), although `a.funnel_shr(a, n)` is *equivalent*
+        /// to `a.rotate_right(n)`.
+        ///
+        /// # Panics
+        ///
+        /// If `n` is greater than or equal to the number of bits in `self`
+        ///
+        /// # Examples
+        ///
+        /// Basic usage:
+        ///
+        /// ```
+        /// #![feature(funnel_shifts)]
+        #[doc = concat!("let a = ", $rot_op, stringify!($SelfT), ";")]
+        #[doc = concat!("let b = ", $fsh_op, stringify!($SelfT), ";")]
+        #[doc = concat!("let m = ", $fshr_result, ";")]
+        ///
+        #[doc = concat!("assert_eq!(a.funnel_shr(b, ", $rot, "), m);")]
+        /// ```
+        #[rustc_const_unstable(feature = "funnel_shifts", issue = "145686")]
+        #[unstable(feature = "funnel_shifts", issue = "145686")]
+        #[must_use = "this returns the result of the operation, \
+                      without modifying the original"]
+        #[inline(always)]
+        pub const fn funnel_shr(self, rhs: Self, n: u32) -> Self {
+            assert!(n < Self::BITS, "attempt to funnel shift right with overflow");
+            // SAFETY: just checked that `shift` is in-range
+            unsafe { intrinsics::unchecked_funnel_shr(self, rhs, n) }
+        }
+
         /// Reverses the byte order of the integer.
         ///
         /// # Examples

library/coretests/tests/lib.rs

@@ -50,6 +50,7 @@
 #![feature(fmt_internals)]
 #![feature(formatting_options)]
 #![feature(freeze)]
+#![feature(funnel_shifts)]
 #![feature(future_join)]
 #![feature(generic_assert_internals)]
 #![feature(hasher_prefixfree_extras)]