Also use mir::Offset for pointer add

Author: scottmcm

compiler/rustc_codegen_ssa/src/mir/rvalue.rs

@@ -819,8 +819,15 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                     .builtin_deref(true)
                     .unwrap_or_else(|| bug!("deref of non-pointer {:?}", input_ty))
                     .ty;
-                let llty = bx.cx().backend_type(bx.cx().layout_of(pointee_type));
-                bx.inbounds_gep(llty, lhs, &[rhs])
+                let pointee_layout = bx.cx().layout_of(pointee_type);
+                if pointee_layout.is_zst() {
+                    // `Offset` works in terms of the size of pointee,
+                    // so offsetting a pointer to ZST is a noop.
+                    lhs
+                } else {
+                    let llty = bx.cx().backend_type(pointee_layout);
+                    bx.inbounds_gep(llty, lhs, &[rhs])
+                }
             }
             mir::BinOp::Shl => common::build_unchecked_lshift(bx, lhs, rhs),
             mir::BinOp::Shr => common::build_unchecked_rshift(bx, input_ty, lhs, rhs),

compiler/rustc_hir_analysis/src/check/intrinsic.rs

@@ -215,7 +215,8 @@ pub fn check_intrinsic_type(tcx: TyCtxt<'_>, it: &hir::ForeignItem<'_>) {
 
             sym::type_name => (1, Vec::new(), tcx.mk_static_str()),
             sym::type_id => (1, Vec::new(), tcx.types.u64),
-            sym::offset | sym::arith_offset => (
+            sym::offset => (2, vec![param(0), param(1)], param(0)),
+            sym::arith_offset => (
                 1,
                 vec![
                     tcx.mk_ptr(ty::TypeAndMut { ty: param(0), mutbl: hir::Mutability::Not }),

library/core/src/intrinsics.rs

@@ -1413,6 +1413,10 @@ extern "rust-intrinsic" {
     /// This is implemented as an intrinsic to avoid converting to and from an
     /// integer, since the conversion would throw away aliasing information.
     ///
+    /// This can only be used with `Ptr` as a raw pointer type (`*mut` or `*const`)
+    /// to a `Sized` pointee and with `Delta` as `usize` or `isize`.  Any other
+    /// instantiations may arbitrarily misbehave, and that's *not* a compiler bug.
+    ///
     /// # Safety
     ///
     /// Both the starting and resulting pointer must be either in bounds or one
@@ -1421,6 +1425,14 @@ extern "rust-intrinsic" {
     /// returned value will result in undefined behavior.
     ///
     /// The stabilized version of this intrinsic is [`pointer::offset`].
+    #[cfg(not(bootstrap))]
+    #[must_use = "returns a new pointer rather than modifying its argument"]
+    #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")]
+    #[rustc_nounwind]
+    pub fn offset<Ptr, Delta>(dst: Ptr, offset: Delta) -> Ptr;
+
+    /// The bootstrap version of this is more restricted.
+    #[cfg(bootstrap)]
     #[must_use = "returns a new pointer rather than modifying its argument"]
     #[rustc_const_stable(feature = "const_ptr_offset", since = "1.61.0")]
     #[rustc_nounwind]

library/core/src/ptr/const_ptr.rs

@@ -916,8 +916,16 @@ impl<T: ?Sized> *const T {
     where
         T: Sized,
     {
+        #[cfg(bootstrap)]
         // SAFETY: the caller must uphold the safety contract for `offset`.
-        unsafe { self.offset(count as isize) }
+        unsafe {
+            self.offset(count as isize)
+        }
+        #[cfg(not(bootstrap))]
+        // SAFETY: the caller must uphold the safety contract for `offset`.
+        unsafe {
+            intrinsics::offset(self, count)
+        }
     }
 
     /// Calculates the offset from a pointer in bytes (convenience for `.byte_offset(count as isize)`).

library/core/src/ptr/mut_ptr.rs

@@ -473,10 +473,20 @@ impl<T: ?Sized> *mut T {
     where
         T: Sized,
     {
+        #[cfg(bootstrap)]
         // SAFETY: the caller must uphold the safety contract for `offset`.
         // The obtained pointer is valid for writes since the caller must
         // guarantee that it points to the same allocated object as `self`.
-        unsafe { intrinsics::offset(self, count) as *mut T }
+        unsafe {
+            intrinsics::offset(self, count) as *mut T
+        }
+        #[cfg(not(bootstrap))]
+        // SAFETY: the caller must uphold the safety contract for `offset`.
+        // The obtained pointer is valid for writes since the caller must
+        // guarantee that it points to the same allocated object as `self`.
+        unsafe {
+            intrinsics::offset(self, count)
+        }
     }
 
     /// Calculates the offset from a pointer in bytes.
@@ -1016,8 +1026,16 @@ impl<T: ?Sized> *mut T {
     where
         T: Sized,
     {
+        #[cfg(bootstrap)]
+        // SAFETY: the caller must uphold the safety contract for `offset`.
+        unsafe {
+            self.offset(count as isize)
+        }
+        #[cfg(not(bootstrap))]
         // SAFETY: the caller must uphold the safety contract for `offset`.
-        unsafe { self.offset(count as isize) }
+        unsafe {
+            intrinsics::offset(self, count)
+        }
     }
 
     /// Calculates the offset from a pointer in bytes (convenience for `.byte_offset(count as isize)`).

src/doc/unstable-book/src/language-features/intrinsics.md

@@ -22,7 +22,7 @@ via a declaration like
 extern "rust-intrinsic" {
     fn transmute<T, U>(x: T) -> U;
 
-    fn offset<T>(dst: *const T, offset: isize) -> *const T;
+    fn arith_offset<T>(dst: *const T, offset: isize) -> *const T;
 }
 ```
 

tests/codegen/intrinsics/offset.rs

@@ -0,0 +1,34 @@
+// compile-flags: -O -C no-prepopulate-passes
+// min-llvm-version: 15.0 (because we're using opaque pointers)
+
+#![crate_type = "lib"]
+#![feature(core_intrinsics)]
+
+use std::intrinsics::offset;
+
+// CHECK-LABEL: ptr @offset_zst
+// CHECK-SAME: (ptr noundef %p, [[SIZE:i[0-9]+]] noundef %d)
+#[no_mangle]
+pub unsafe fn offset_zst(p: *const (), d: usize) -> *const () {
+    // CHECK-NOT: getelementptr
+    // CHECK: ret ptr %p
+    offset(p, d)
+}
+
+// CHECK-LABEL: ptr @offset_isize
+// CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
+#[no_mangle]
+pub unsafe fn offset_isize(p: *const u32, d: isize) -> *const u32 {
+    // CHECK: %[[R:.*]] = getelementptr inbounds i32, ptr %p, [[SIZE]] %d
+    // CHECK-NEXT: ret ptr %[[R]]
+    offset(p, d)
+}
+
+// CHECK-LABEL: ptr @offset_usize
+// CHECK-SAME: (ptr noundef %p, [[SIZE]] noundef %d)
+#[no_mangle]
+pub unsafe fn offset_usize(p: *const u64, d: usize) -> *const u64 {
+    // CHECK: %[[R:.*]] = getelementptr inbounds i64, ptr %p, [[SIZE]] %d
+    // CHECK-NEXT: ret ptr %[[R]]
+    offset(p, d)
+}

tests/mir-opt/lower_intrinsics.ptr_offset.LowerIntrinsics.diff

@@ -13,10 +13,10 @@
           _3 = _1;                         // scope 0 at $DIR/lower_intrinsics.rs:+1:30: +1:31
           StorageLive(_4);                 // scope 0 at $DIR/lower_intrinsics.rs:+1:33: +1:34
           _4 = _2;                         // scope 0 at $DIR/lower_intrinsics.rs:+1:33: +1:34
--         _0 = offset::<i32>(move _3, move _4) -> [return: bb1, unwind unreachable]; // scope 0 at $DIR/lower_intrinsics.rs:+1:5: +1:35
+-         _0 = offset::<*const i32, isize>(move _3, move _4) -> [return: bb1, unwind unreachable]; // scope 0 at $DIR/lower_intrinsics.rs:+1:5: +1:35
 -                                          // mir::Constant
 -                                          // + span: $DIR/lower_intrinsics.rs:140:5: 140:29
--                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(*const i32, isize) -> *const i32 {offset::<i32>}, val: Value(<ZST>) }
+-                                          // + literal: Const { ty: unsafe extern "rust-intrinsic" fn(*const i32, isize) -> *const i32 {offset::<*const i32, isize>}, val: Value(<ZST>) }
 +         _0 = Offset(move _3, move _4);   // scope 0 at $DIR/lower_intrinsics.rs:+1:5: +1:35
 +         goto -> bb1;                     // scope 0 at $DIR/lower_intrinsics.rs:+1:5: +1:35
       }

tests/mir-opt/pre-codegen/slice_index.slice_get_mut_usize.PreCodegen.after.mir

@@ -45,16 +45,7 @@ fn slice_get_mut_usize(_1: &mut [u32], _2: usize) -> Option<&mut u32> {
                             scope 12 (inlined ptr::mut_ptr::<impl *mut u32>::add) { // at $SRC_DIR/core/src/slice/index.rs:LL:COL
                                 debug self => _9; // in scope 12 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
                                 debug count => _2; // in scope 12 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                let mut _13: isize; // in scope 12 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
                                 scope 13 {
-                                    scope 14 (inlined ptr::mut_ptr::<impl *mut u32>::offset) { // at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        debug self => _9; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        debug count => _13; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        let mut _14: *const u32; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        let mut _15: *const u32; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        scope 15 {
-                                        }
-                                    }
                                 }
                             }
                         }
@@ -86,16 +77,7 @@ fn slice_get_mut_usize(_1: &mut [u32], _2: usize) -> Option<&mut u32> {
         StorageLive(_12);                // scope 3 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         StorageLive(_9);                 // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         _9 = _8 as *mut u32 (PtrToPtr);  // scope 11 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageLive(_13);                // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _13 = _2 as isize (IntToInt);    // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageLive(_14);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageLive(_15);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _15 = _9 as *const u32 (Pointer(MutToConstPointer)); // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _14 = Offset(move _15, _13);     // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageDead(_15);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _7 = move _14 as *mut u32 (PtrToPtr); // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageDead(_14);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageDead(_13);                // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
+        _7 = Offset(_9, _2);             // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
         StorageDead(_9);                 // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         StorageDead(_12);                // scope 3 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         StorageDead(_11);                // scope 3 at $SRC_DIR/core/src/slice/index.rs:LL:COL

tests/mir-opt/pre-codegen/slice_index.slice_get_unchecked_mut_range.PreCodegen.after.mir

@@ -34,32 +34,23 @@ fn slice_get_unchecked_mut_range(_1: &mut [u32], _2: std::ops::Range<usize>) ->
                             scope 12 (inlined ptr::mut_ptr::<impl *mut u32>::add) { // at $SRC_DIR/core/src/slice/index.rs:LL:COL
                                 debug self => _10; // in scope 12 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
                                 debug count => _11; // in scope 12 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                let mut _16: isize; // in scope 12 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
                                 scope 13 {
-                                    scope 14 (inlined ptr::mut_ptr::<impl *mut u32>::offset) { // at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        debug self => _10; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        debug count => _16; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        let mut _17: *const u32; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        let mut _18: *const u32; // in scope 14 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-                                        scope 15 {
-                                        }
-                                    }
                                 }
                             }
-                            scope 16 (inlined slice_from_raw_parts_mut::<u32>) { // at $SRC_DIR/core/src/slice/index.rs:LL:COL
-                                debug data => _9; // in scope 16 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
-                                debug len => _12; // in scope 16 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
-                                let mut _19: *mut (); // in scope 16 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
-                                scope 17 (inlined ptr::mut_ptr::<impl *mut u32>::cast::<()>) { // at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
-                                    debug self => _9; // in scope 17 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
+                            scope 14 (inlined slice_from_raw_parts_mut::<u32>) { // at $SRC_DIR/core/src/slice/index.rs:LL:COL
+                                debug data => _9; // in scope 14 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
+                                debug len => _12; // in scope 14 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
+                                let mut _16: *mut (); // in scope 14 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
+                                scope 15 (inlined ptr::mut_ptr::<impl *mut u32>::cast::<()>) { // at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
+                                    debug self => _9; // in scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
                                 }
-                                scope 18 (inlined std::ptr::from_raw_parts_mut::<[u32]>) { // at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
-                                    debug data_address => _19; // in scope 18 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-                                    debug metadata => _12; // in scope 18 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-                                    let mut _20: std::ptr::metadata::PtrRepr<[u32]>; // in scope 18 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-                                    let mut _21: std::ptr::metadata::PtrComponents<[u32]>; // in scope 18 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-                                    let mut _22: *const (); // in scope 18 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-                                    scope 19 {
+                                scope 16 (inlined std::ptr::from_raw_parts_mut::<[u32]>) { // at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
+                                    debug data_address => _16; // in scope 16 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+                                    debug metadata => _12; // in scope 16 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+                                    let mut _17: std::ptr::metadata::PtrRepr<[u32]>; // in scope 16 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+                                    let mut _18: std::ptr::metadata::PtrComponents<[u32]>; // in scope 16 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+                                    let mut _19: *const (); // in scope 16 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+                                    scope 17 {
                                     }
                                 }
                             }
@@ -110,33 +101,24 @@ fn slice_get_unchecked_mut_range(_1: &mut [u32], _2: std::ops::Range<usize>) ->
         _10 = _4 as *mut u32 (PtrToPtr); // scope 11 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
         StorageLive(_11);                // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         _11 = (_2.0: usize);             // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
-        StorageLive(_16);                // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _16 = _11 as isize (IntToInt);   // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageLive(_17);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageLive(_18);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _18 = _10 as *const u32 (Pointer(MutToConstPointer)); // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _17 = Offset(move _18, _16);     // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageDead(_18);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        _9 = move _17 as *mut u32 (PtrToPtr); // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageDead(_17);                // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageDead(_16);                // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
+        _9 = Offset(_10, _11);           // scope 13 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
         StorageDead(_11);                // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         StorageDead(_10);                // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         StorageLive(_12);                // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         _12 = _6;                        // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
-        StorageLive(_19);                // scope 16 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
-        _19 = _9 as *mut () (PtrToPtr);  // scope 17 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
-        StorageLive(_20);                // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        StorageLive(_21);                // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        StorageLive(_22);                // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        _22 = _19 as *const () (Pointer(MutToConstPointer)); // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        _21 = ptr::metadata::PtrComponents::<[u32]> { data_address: move _22, metadata: _12 }; // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        StorageDead(_22);                // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        _20 = ptr::metadata::PtrRepr::<[u32]> { const_ptr: move _21 }; // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        StorageDead(_21);                // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        _3 = (_20.1: *mut [u32]);        // scope 19 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        StorageDead(_20);                // scope 18 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
-        StorageDead(_19);                // scope 16 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
+        StorageLive(_16);                // scope 14 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
+        _16 = _9 as *mut () (PtrToPtr);  // scope 15 at $SRC_DIR/core/src/ptr/mut_ptr.rs:LL:COL
+        StorageLive(_17);                // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        StorageLive(_18);                // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        StorageLive(_19);                // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        _19 = _16 as *const () (Pointer(MutToConstPointer)); // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        _18 = ptr::metadata::PtrComponents::<[u32]> { data_address: move _19, metadata: _12 }; // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        StorageDead(_19);                // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        _17 = ptr::metadata::PtrRepr::<[u32]> { const_ptr: move _18 }; // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        StorageDead(_18);                // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        _3 = (_17.1: *mut [u32]);        // scope 17 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        StorageDead(_17);                // scope 16 at $SRC_DIR/core/src/ptr/metadata.rs:LL:COL
+        StorageDead(_16);                // scope 14 at $SRC_DIR/core/src/ptr/mod.rs:LL:COL
         StorageDead(_12);                // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         StorageDead(_9);                 // scope 6 at $SRC_DIR/core/src/slice/index.rs:LL:COL
         StorageDead(_6);                 // scope 5 at $SRC_DIR/core/src/slice/index.rs:LL:COL