From f13c4f4d6a4f0bb042839613bc92e9d3cfd9c308 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 7 Oct 2022 20:57:34 +0200 Subject: [PATCH 01/20] constify `exact_div` intrinsic --- compiler/rustc_const_eval/src/interpret/intrinsics.rs | 5 +++++ library/core/src/intrinsics.rs | 1 + library/core/src/lib.rs | 1 + src/tools/miri/src/shims/intrinsics/mod.rs | 5 ----- 4 files changed, 7 insertions(+), 5 deletions(-) diff --git a/compiler/rustc_const_eval/src/interpret/intrinsics.rs b/compiler/rustc_const_eval/src/interpret/intrinsics.rs index 6fc2407b77803..7940efcd2b11f 100644 --- a/compiler/rustc_const_eval/src/interpret/intrinsics.rs +++ b/compiler/rustc_const_eval/src/interpret/intrinsics.rs @@ -243,6 +243,11 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> { let discr_val = self.read_discriminant(&place.into())?.0; self.write_scalar(discr_val, dest)?; } + sym::exact_div => { + let l = self.read_immediate(&args[0])?; + let r = self.read_immediate(&args[1])?; + self.exact_div(&l, &r, dest)?; + } sym::unchecked_shl | sym::unchecked_shr | sym::unchecked_add diff --git a/library/core/src/intrinsics.rs b/library/core/src/intrinsics.rs index 819ccf5a3e9e7..7ed7d767f2fb5 100644 --- a/library/core/src/intrinsics.rs +++ b/library/core/src/intrinsics.rs @@ -1851,6 +1851,7 @@ extern "rust-intrinsic" { /// `x % y != 0` or `y == 0` or `x == T::MIN && y == -1` /// /// This intrinsic does not have a stable counterpart. + #[rustc_const_unstable(feature = "const_exact_div", issue = "none")] pub fn exact_div(x: T, y: T) -> T; /// Performs an unchecked division, resulting in undefined behavior diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 7191bbb93447c..33b969c249a2f 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -109,6 +109,7 @@ #![feature(const_cmp)] #![feature(const_discriminant)] #![feature(const_eval_select)] +#![feature(const_exact_div)] #![feature(const_float_bits_conv)] #![feature(const_float_classify)] #![feature(const_fmt_arguments_new)] diff --git a/src/tools/miri/src/shims/intrinsics/mod.rs b/src/tools/miri/src/shims/intrinsics/mod.rs index 6004e2078ad4f..5ea82adb9c69c 100644 --- a/src/tools/miri/src/shims/intrinsics/mod.rs +++ b/src/tools/miri/src/shims/intrinsics/mod.rs @@ -368,11 +368,6 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> { } // Other - "exact_div" => { - let [num, denom] = check_arg_count(args)?; - this.exact_div(&this.read_immediate(num)?, &this.read_immediate(denom)?, dest)?; - } - "breakpoint" => { let [] = check_arg_count(args)?; // normally this would raise a SIGTRAP, which aborts if no debugger is connected From f770fecfe1cf3415675ac6165a200dff564bd00f Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Thu, 20 Oct 2022 20:15:37 +0200 Subject: [PATCH 02/20] unify inherent impls of `CompileTimeEvalContext` --- .../src/const_eval/machine.rs | 90 +++++++++---------- 1 file changed, 44 insertions(+), 46 deletions(-) diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 35d58d2f638bf..66d6014946cec 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -23,52 +23,6 @@ use crate::interpret::{ use super::error::*; -impl<'mir, 'tcx> InterpCx<'mir, 'tcx, CompileTimeInterpreter<'mir, 'tcx>> { - /// "Intercept" a function call to a panic-related function - /// because we have something special to do for it. - /// If this returns successfully (`Ok`), the function should just be evaluated normally. - fn hook_special_const_fn( - &mut self, - instance: ty::Instance<'tcx>, - args: &[OpTy<'tcx>], - ) -> InterpResult<'tcx, Option>> { - // All `#[rustc_do_not_const_check]` functions should be hooked here. - let def_id = instance.def_id(); - - if Some(def_id) == self.tcx.lang_items().panic_display() - || Some(def_id) == self.tcx.lang_items().begin_panic_fn() - { - // &str or &&str - assert!(args.len() == 1); - - let mut msg_place = self.deref_operand(&args[0])?; - while msg_place.layout.ty.is_ref() { - msg_place = self.deref_operand(&msg_place.into())?; - } - - let msg = Symbol::intern(self.read_str(&msg_place)?); - let span = self.find_closest_untracked_caller_location(); - let (file, line, col) = self.location_triple_for_span(span); - return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into()); - } else if Some(def_id) == self.tcx.lang_items().panic_fmt() { - // For panic_fmt, call const_panic_fmt instead. - if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() { - return Ok(Some( - ty::Instance::resolve( - *self.tcx, - ty::ParamEnv::reveal_all(), - const_panic_fmt, - self.tcx.intern_substs(&[]), - ) - .unwrap() - .unwrap(), - )); - } - } - Ok(None) - } -} - /// Extra machine state for CTFE, and the Machine instance pub struct CompileTimeInterpreter<'mir, 'tcx> { /// For now, the number of terminators that can be evaluated before we throw a resource @@ -191,6 +145,50 @@ impl interpret::MayLeak for ! { } impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { + /// "Intercept" a function call to a panic-related function + /// because we have something special to do for it. + /// If this returns successfully (`Ok`), the function should just be evaluated normally. + fn hook_special_const_fn( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx>], + ) -> InterpResult<'tcx, Option>> { + // All `#[rustc_do_not_const_check]` functions should be hooked here. + let def_id = instance.def_id(); + + if Some(def_id) == self.tcx.lang_items().panic_display() + || Some(def_id) == self.tcx.lang_items().begin_panic_fn() + { + // &str or &&str + assert!(args.len() == 1); + + let mut msg_place = self.deref_operand(&args[0])?; + while msg_place.layout.ty.is_ref() { + msg_place = self.deref_operand(&msg_place.into())?; + } + + let msg = Symbol::intern(self.read_str(&msg_place)?); + let span = self.find_closest_untracked_caller_location(); + let (file, line, col) = self.location_triple_for_span(span); + return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into()); + } else if Some(def_id) == self.tcx.lang_items().panic_fmt() { + // For panic_fmt, call const_panic_fmt instead. + if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() { + return Ok(Some( + ty::Instance::resolve( + *self.tcx, + ty::ParamEnv::reveal_all(), + const_panic_fmt, + self.tcx.intern_substs(&[]), + ) + .unwrap() + .unwrap(), + )); + } + } + Ok(None) + } + /// See documentation on the `ptr_guaranteed_cmp` intrinsic. fn guaranteed_cmp(&mut self, a: Scalar, b: Scalar) -> InterpResult<'tcx, u8> { Ok(match (a, b) { From 211743b2c88e40f43b7e2174bc19920c05b08936 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 7 Oct 2022 22:23:34 +0200 Subject: [PATCH 03/20] make const `align_offset` useful --- .../src/const_eval/machine.rs | 129 ++++++++++++++---- library/core/src/ptr/const_ptr.rs | 15 ++ library/core/src/ptr/mod.rs | 21 ++- library/core/src/ptr/mut_ptr.rs | 15 ++ 4 files changed, 152 insertions(+), 28 deletions(-) diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 66d6014946cec..97d168ceb01a6 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -1,8 +1,11 @@ use rustc_hir::def::DefKind; use rustc_middle::mir; +use rustc_middle::mir::interpret::PointerArithmetic; +use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::{self, Ty, TyCtxt}; use std::borrow::Borrow; use std::hash::Hash; +use std::ops::ControlFlow; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::fx::IndexEntry; @@ -17,8 +20,8 @@ use rustc_target::abi::{Align, Size}; use rustc_target::spec::abi::Abi as CallAbi; use crate::interpret::{ - self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult, - OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, + self, compile_time_machine, AllocId, ConstAllocation, FnVal, Frame, ImmTy, InterpCx, + InterpResult, OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, }; use super::error::*; @@ -145,15 +148,19 @@ impl interpret::MayLeak for ! { } impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { - /// "Intercept" a function call to a panic-related function - /// because we have something special to do for it. - /// If this returns successfully (`Ok`), the function should just be evaluated normally. + /// "Intercept" a function call, because we have something special to do for it. + /// All `#[rustc_do_not_const_check]` functions should be hooked here. + /// If this returns `Some` function, which may be `instance` or a different function with + /// compatible arguments, then evaluation should continue with that function. + /// If this returns `None`, the function call has been handled and the function has returned. fn hook_special_const_fn( &mut self, instance: ty::Instance<'tcx>, + _abi: CallAbi, args: &[OpTy<'tcx>], + dest: &PlaceTy<'tcx>, + ret: Option, ) -> InterpResult<'tcx, Option>> { - // All `#[rustc_do_not_const_check]` functions should be hooked here. let def_id = instance.def_id(); if Some(def_id) == self.tcx.lang_items().panic_display() @@ -173,20 +180,91 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into()); } else if Some(def_id) == self.tcx.lang_items().panic_fmt() { // For panic_fmt, call const_panic_fmt instead. - if let Some(const_panic_fmt) = self.tcx.lang_items().const_panic_fmt() { - return Ok(Some( - ty::Instance::resolve( - *self.tcx, - ty::ParamEnv::reveal_all(), - const_panic_fmt, - self.tcx.intern_substs(&[]), - ) - .unwrap() - .unwrap(), - )); + let Some(const_def_id) = self.tcx.lang_items().const_panic_fmt() else { + bug!("`const_panic_fmt` must be defined to call `panic_fmt` in const eval") + }; + let new_instance = ty::Instance::resolve( + *self.tcx, + ty::ParamEnv::reveal_all(), + const_def_id, + instance.substs, + ) + .unwrap() + .unwrap(); + + return Ok(Some(new_instance)); + } else if Some(def_id) == self.tcx.lang_items().align_offset_fn() { + // For align_offset, we replace the function call if the pointer has no address. + match self.align_offset(instance, args, dest, ret)? { + ControlFlow::Continue(()) => return Ok(Some(instance)), + ControlFlow::Break(()) => return Ok(None), + } + } + Ok(Some(instance)) + } + + /// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer + /// may not have an address. + /// + /// If `ptr` does have a known address, then we return `CONTINUE` and the function call should + /// proceed as normal. + /// + /// If `ptr` doesn't have an address, but its underlying allocation's alignment is at most + /// `target_align`, then we call the function again with an dummy address relative to the + /// allocation. + /// + /// If `ptr` doesn't have an address and `target_align` is stricter than the underlying + /// allocation's alignment, then we return `usize::MAX` immediately. + fn align_offset( + &mut self, + instance: ty::Instance<'tcx>, + args: &[OpTy<'tcx>], + dest: &PlaceTy<'tcx>, + ret: Option, + ) -> InterpResult<'tcx, ControlFlow<()>> { + assert_eq!(args.len(), 2); + + let ptr = self.read_pointer(&args[0])?; + let target_align = self.read_scalar(&args[1])?.to_machine_usize(self)?; + + if !target_align.is_power_of_two() { + throw_ub_format!("`align_offset` called with non-power-of-two align: {}", target_align); + } + + match self.ptr_try_get_alloc_id(ptr) { + Ok((alloc_id, offset, _extra)) => { + let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id); + + if target_align <= alloc_align.bytes() { + // Extract the address relative to the allocation base that is definitely + // sufficiently aligned and call `align_offset` again. + let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into(); + let align = ImmTy::from_uint(target_align, args[1].layout).into(); + + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; + self.eval_fn_call( + FnVal::Instance(instance), + (CallAbi::Rust, fn_abi), + &[addr, align], + false, + dest, + ret, + StackPopUnwind::NotAllowed, + )?; + Ok(ControlFlow::BREAK) + } else { + // Not alignable in const, return `usize::MAX`. + let usize_max = Scalar::from_machine_usize(self.machine_usize_max(), self); + self.write_scalar(usize_max, dest)?; + self.return_to_block(ret)?; + Ok(ControlFlow::BREAK) + } + } + Err(_addr) => { + // The pointer has an address, continue with function call. + Ok(ControlFlow::CONTINUE) } } - Ok(None) } /// See documentation on the `ptr_guaranteed_cmp` intrinsic. @@ -269,8 +347,8 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, instance: ty::Instance<'tcx>, _abi: CallAbi, args: &[OpTy<'tcx>], - _dest: &PlaceTy<'tcx>, - _ret: Option, + dest: &PlaceTy<'tcx>, + ret: Option, _unwind: StackPopUnwind, // unwinding is not supported in consts ) -> InterpResult<'tcx, Option<(&'mir mir::Body<'tcx>, ty::Instance<'tcx>)>> { debug!("find_mir_or_eval_fn: {:?}", instance); @@ -289,7 +367,11 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, } } - if let Some(new_instance) = ecx.hook_special_const_fn(instance, args)? { + let Some(new_instance) = ecx.hook_special_const_fn(instance, _abi, args, dest, ret)? else { + return Ok(None); + }; + + if new_instance != instance { // We call another const fn instead. // However, we return the *original* instance to make backtraces work out // (and we hope this does not confuse the FnAbi checks too much). @@ -298,13 +380,14 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, new_instance, _abi, args, - _dest, - _ret, + dest, + ret, _unwind, )? .map(|(body, _instance)| (body, instance))); } } + // This is a const fn. Call it. Ok(Some((ecx.load_mir(instance.def, None)?, instance))) } diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index bff270b787ece..f0cdc3d839998 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1322,6 +1322,21 @@ impl *const T { /// ``` #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] + #[cfg(not(bootstrap))] + pub const fn align_offset(self, align: usize) -> usize + where + T: Sized, + { + assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two"); + + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(self, align) } + } + + #[stable(feature = "align_offset", since = "1.36.0")] + #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] + #[allow(missing_docs)] + #[cfg(bootstrap)] pub const fn align_offset(self, align: usize) -> usize where T: Sized, diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index e0ddb3154de12..1ebd9dca09838 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -1574,10 +1574,14 @@ pub unsafe fn write_volatile(dst: *mut T, src: T) { /// Align pointer `p`. /// -/// Calculate offset (in terms of elements of `stride` stride) that has to be applied +/// Calculate offset (in terms of elements of `size_of::()` stride) that has to be applied /// to pointer `p` so that pointer `p` would get aligned to `a`. /// -/// Note: This implementation has been carefully tailored to not panic. It is UB for this to panic. +/// # Safety +/// `a` must be a power of two. +/// +/// # Notes +/// This implementation has been carefully tailored to not panic. It is UB for this to panic. /// The only real change that can be made here is change of `INV_TABLE_MOD_16` and associated /// constants. /// @@ -1586,8 +1590,10 @@ pub unsafe fn write_volatile(dst: *mut T, src: T) { /// than trying to adapt this to accommodate that change. /// /// Any questions go to @nagisa. +// #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap +// compiler will always cause an error. #[lang = "align_offset"] -pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { +pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usize { // FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <= // 1, where the method versions of these operations are not inlined. use intrinsics::{ @@ -1604,7 +1610,7 @@ pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { /// /// Implementation of this function shall not panic. Ever. #[inline] - unsafe fn mod_inv(x: usize, m: usize) -> usize { + const unsafe fn mod_inv(x: usize, m: usize) -> usize { /// Multiplicative modular inverse table modulo 2⁴ = 16. /// /// Note, that this table does not contain values where inverse does not exist (i.e., for @@ -1646,8 +1652,13 @@ pub(crate) unsafe fn align_offset(p: *const T, a: usize) -> usize { inverse & m_minus_one } - let addr = p.addr(); let stride = mem::size_of::(); + + // SAFETY: At runtime transmuting a pointer to `usize` is always safe, because they have the + // same layout. During const eval we hook this function to ensure that the pointer always has + // an address (only the standard library can do this). + let addr = unsafe { mem::transmute(p) }; + // SAFETY: `a` is a power-of-two, therefore non-zero. let a_minus_one = unsafe { unchecked_sub(a, 1) }; diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 8f4809ec4baa4..eb1a6a07c6b89 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1590,6 +1590,21 @@ impl *mut T { /// ``` #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] + #[cfg(not(bootstrap))] + pub const fn align_offset(self, align: usize) -> usize + where + T: Sized, + { + assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two"); + + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(self, align) } + } + + #[stable(feature = "align_offset", since = "1.36.0")] + #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] + #[allow(missing_docs)] + #[cfg(bootstrap)] pub const fn align_offset(self, align: usize) -> usize where T: Sized, From 8cf6b16185745822b109dfa17c26d2a4ee4184ab Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 7 Oct 2022 20:59:23 +0200 Subject: [PATCH 04/20] add coretests for const `align_offset` --- library/core/tests/lib.rs | 1 + library/core/tests/ptr.rs | 165 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 166 insertions(+) diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs index 61d31b3448734..7447123253adb 100644 --- a/library/core/tests/lib.rs +++ b/library/core/tests/lib.rs @@ -4,6 +4,7 @@ #![feature(array_windows)] #![feature(bigint_helper_methods)] #![feature(cell_update)] +#![feature(const_align_offset)] #![feature(const_assume)] #![feature(const_align_of_val_raw)] #![feature(const_black_box)] diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs index 0977980ba47bf..186e83d50b6ca 100644 --- a/library/core/tests/ptr.rs +++ b/library/core/tests/ptr.rs @@ -358,6 +358,23 @@ fn align_offset_zst() { } } +#[test] +#[cfg(not(bootstrap))] +fn align_offset_zst_const() { + const { + // For pointers of stride = 0, the pointer is already aligned or it cannot be aligned at + // all, because no amount of elements will align the pointer. + let mut p = 1; + while p < 1024 { + assert!(ptr::invalid::<()>(p).align_offset(p) == 0); + if p != 1 { + assert!(ptr::invalid::<()>(p + 1).align_offset(p) == !0); + } + p = (p + 1).next_power_of_two(); + } + } +} + #[test] fn align_offset_stride_one() { // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to @@ -379,6 +396,26 @@ fn align_offset_stride_one() { } } +#[test] +#[cfg(not(bootstrap))] +fn align_offset_stride_one_const() { + const { + // For pointers of stride = 1, the pointer can always be aligned. The offset is equal to + // number of bytes. + let mut align = 1; + while align < 1024 { + let mut ptr = 1; + while ptr < 2 * align { + let expected = ptr % align; + let offset = if expected == 0 { 0 } else { align - expected }; + assert!(ptr::invalid::(ptr).align_offset(align) == offset); + ptr += 1; + } + align = (align + 1).next_power_of_two(); + } + } +} + #[test] fn align_offset_various_strides() { unsafe fn test_stride(ptr: *const T, align: usize) -> bool { @@ -455,6 +492,134 @@ fn align_offset_various_strides() { assert!(!x); } +#[test] +#[cfg(not(bootstrap))] +fn align_offset_various_strides_const() { + const unsafe fn test_stride(ptr: *const T, numptr: usize, align: usize) { + let mut expected = usize::MAX; + // Naive but definitely correct way to find the *first* aligned element of stride::. + let mut el = 0; + while el < align { + if (numptr + el * ::std::mem::size_of::()) % align == 0 { + expected = el; + break; + } + el += 1; + } + let got = ptr.align_offset(align); + assert!(got == expected); + } + + // For pointers of stride != 1, we verify the algorithm against the naivest possible + // implementation + let mut align = 1; + let limit = 1024; + while align < limit { + for ptr in 1usize..4 * align { + unsafe { + #[repr(packed)] + struct A3(u16, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + struct A4(u32); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A5(u32, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A6(u32, u16); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A7(u32, u16, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A8(u32, u32); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A9(u32, u32, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A10(u32, u32, u16); + test_stride::(ptr::invalid::(ptr), ptr, align); + + test_stride::(ptr::invalid::(ptr), ptr, align); + test_stride::(ptr::invalid::(ptr), ptr, align); + } + } + align = (align + 1).next_power_of_two(); + } +} + +#[test] +#[cfg(not(bootstrap))] +fn align_offset_with_provenance_const() { + const { + let data = 42; + + let ptr: *const i32 = &data; + assert!(ptr.align_offset(1) == 0); + assert!(ptr.align_offset(2) == 0); + assert!(ptr.align_offset(4) == 0); + assert!(ptr.align_offset(8) == usize::MAX); + assert!(ptr.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr.wrapping_byte_add(1).align_offset(2) == usize::MAX); + assert!(ptr.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr.wrapping_byte_add(2).align_offset(4) == usize::MAX); + assert!(ptr.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr.wrapping_byte_add(3).align_offset(2) == usize::MAX); + + assert!(ptr.wrapping_add(42).align_offset(4) == 0); + assert!(ptr.wrapping_add(42).align_offset(8) == usize::MAX); + + let ptr1: *const i8 = ptr.cast(); + assert!(ptr1.align_offset(1) == 0); + assert!(ptr1.align_offset(2) == 0); + assert!(ptr1.align_offset(4) == 0); + assert!(ptr1.align_offset(8) == usize::MAX); + assert!(ptr1.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr1.wrapping_byte_add(1).align_offset(2) == 1); + assert!(ptr1.wrapping_byte_add(1).align_offset(4) == 3); + assert!(ptr1.wrapping_byte_add(1).align_offset(8) == usize::MAX); + assert!(ptr1.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr1.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr1.wrapping_byte_add(2).align_offset(4) == 2); + assert!(ptr1.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr1.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr1.wrapping_byte_add(3).align_offset(2) == 1); + assert!(ptr1.wrapping_byte_add(3).align_offset(4) == 1); + assert!(ptr1.wrapping_byte_add(3).align_offset(8) == usize::MAX); + + let ptr2: *const i16 = ptr.cast(); + assert!(ptr2.align_offset(1) == 0); + assert!(ptr2.align_offset(2) == 0); + assert!(ptr2.align_offset(4) == 0); + assert!(ptr2.align_offset(8) == usize::MAX); + assert!(ptr2.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr2.wrapping_byte_add(1).align_offset(2) == usize::MAX); + assert!(ptr2.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr2.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr2.wrapping_byte_add(2).align_offset(4) == 1); + assert!(ptr2.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr2.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr2.wrapping_byte_add(3).align_offset(2) == usize::MAX); + + let ptr3: *const i64 = ptr.cast(); + assert!(ptr3.align_offset(1) == 0); + assert!(ptr3.align_offset(2) == 0); + assert!(ptr3.align_offset(4) == 0); + assert!(ptr3.align_offset(8) == usize::MAX); + assert!(ptr3.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr3.wrapping_byte_add(1).align_offset(2) == usize::MAX); + } +} + #[test] fn align_offset_issue_103361() { #[cfg(target_pointer_width = "64")] From 6f6320a0a916b57dc37568d3ce5f5ed0e749ef61 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 7 Oct 2022 22:24:31 +0200 Subject: [PATCH 05/20] constify `pointer::is_aligned{,_to}` --- library/core/src/lib.rs | 1 + library/core/src/ptr/const_ptr.rs | 25 +++++++++++++++++++------ library/core/src/ptr/mut_ptr.rs | 25 +++++++++++++++++++------ 3 files changed, 39 insertions(+), 12 deletions(-) diff --git a/library/core/src/lib.rs b/library/core/src/lib.rs index 33b969c249a2f..848eccd7f2908 100644 --- a/library/core/src/lib.rs +++ b/library/core/src/lib.rs @@ -130,6 +130,7 @@ #![feature(const_option)] #![feature(const_option_ext)] #![feature(const_pin)] +#![feature(const_pointer_is_aligned)] #![feature(const_ptr_sub_ptr)] #![feature(const_replace)] #![feature(const_result_drop)] diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index f0cdc3d839998..8b96cf06be462 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1363,10 +1363,13 @@ impl *const T { } /// Returns whether the pointer is properly aligned for `T`. + // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap + // compiler will always return false. #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - pub fn is_aligned(self) -> bool + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + pub const fn is_aligned(self) -> bool where T: Sized, { @@ -1381,16 +1384,26 @@ impl *const T { /// # Panics /// /// The function panics if `align` is not a power-of-two (this includes 0). + // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap + // compiler will always return false. #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - pub fn is_aligned_to(self, align: usize) -> bool { - if !align.is_power_of_two() { - panic!("is_aligned_to: align is not a power-of-two"); + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + pub const fn is_aligned_to(self, align: usize) -> bool { + assert!(align.is_power_of_two(), "is_aligned_to: align is not a power-of-two"); + + #[inline] + fn runtime(ptr: *const u8, align: usize) -> bool { + ptr.addr() & (align - 1) == 0 + } + + const fn comptime(ptr: *const u8, align: usize) -> bool { + ptr.align_offset(align) == 0 } - // Cast is needed for `T: !Sized` - self.cast::().addr() & align - 1 == 0 + // SAFETY: `ptr.align_offset(align)` returns 0 if and only if the pointer is already aligned. + unsafe { intrinsics::const_eval_select((self.cast::(), align), comptime, runtime) } } } diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index eb1a6a07c6b89..2d73c24b5b4c8 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1631,10 +1631,13 @@ impl *mut T { } /// Returns whether the pointer is properly aligned for `T`. + // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap + // compiler will always return false. #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - pub fn is_aligned(self) -> bool + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + pub const fn is_aligned(self) -> bool where T: Sized, { @@ -1649,16 +1652,26 @@ impl *mut T { /// # Panics /// /// The function panics if `align` is not a power-of-two (this includes 0). + // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap + // compiler will always return false. #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - pub fn is_aligned_to(self, align: usize) -> bool { - if !align.is_power_of_two() { - panic!("is_aligned_to: align is not a power-of-two"); + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + pub const fn is_aligned_to(self, align: usize) -> bool { + assert!(align.is_power_of_two(), "is_aligned_to: align is not a power-of-two"); + + #[inline] + fn runtime(ptr: *mut u8, align: usize) -> bool { + ptr.addr() & (align - 1) == 0 + } + + const fn comptime(ptr: *mut u8, align: usize) -> bool { + ptr.align_offset(align) == 0 } - // Cast is needed for `T: !Sized` - self.cast::().addr() & align - 1 == 0 + // SAFETY: `ptr.align_offset(align)` returns 0 if and only if the pointer is already aligned. + unsafe { intrinsics::const_eval_select((self.cast::(), align), comptime, runtime) } } } From 2ef9a8ae0fed0a92b86b316f98f3467aef1e547c Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 7 Oct 2022 21:46:28 +0200 Subject: [PATCH 06/20] add coretests for `is_aligned` --- library/core/tests/lib.rs | 2 ++ library/core/tests/ptr.rs | 48 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+) diff --git a/library/core/tests/lib.rs b/library/core/tests/lib.rs index 7447123253adb..66d28770b87f9 100644 --- a/library/core/tests/lib.rs +++ b/library/core/tests/lib.rs @@ -19,6 +19,7 @@ #![feature(const_nonnull_new)] #![feature(const_num_from_num)] #![feature(const_pointer_byte_offsets)] +#![feature(const_pointer_is_aligned)] #![feature(const_ptr_as_ref)] #![feature(const_ptr_read)] #![feature(const_ptr_write)] @@ -82,6 +83,7 @@ #![feature(never_type)] #![feature(unwrap_infallible)] #![feature(pointer_byte_offsets)] +#![feature(pointer_is_aligned)] #![feature(portable_simd)] #![feature(ptr_metadata)] #![feature(once_cell)] diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs index 186e83d50b6ca..9f74b0c04107e 100644 --- a/library/core/tests/ptr.rs +++ b/library/core/tests/ptr.rs @@ -632,6 +632,54 @@ fn align_offset_issue_103361() { let _ = (SIZE as *const HugeSize).align_offset(SIZE); } +#[test] +fn is_aligned() { + let data = 42; + let ptr: *const i32 = &data; + assert!(ptr.is_aligned()); + assert!(ptr.is_aligned_to(1)); + assert!(ptr.is_aligned_to(2)); + assert!(ptr.is_aligned_to(4)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(1)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + + // At runtime either `ptr` or `ptr+1` is aligned to 8. + assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); +} + +#[test] +#[cfg(not(bootstrap))] +fn is_aligned_const() { + const { + let data = 42; + let ptr: *const i32 = &data; + assert!(ptr.is_aligned()); + assert!(ptr.is_aligned_to(1)); + assert!(ptr.is_aligned_to(2)); + assert!(ptr.is_aligned_to(4)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(1)); + assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + + // At comptime neither `ptr` nor `ptr+1` is aligned to 8. + assert!(!ptr.is_aligned_to(8)); + assert!(!ptr.wrapping_add(1).is_aligned_to(8)); + } +} + +#[test] +#[cfg(bootstrap)] +fn is_aligned_const() { + const { + let data = 42; + let ptr: *const i32 = &data; + // The bootstrap compiler always returns false for is_aligned. + assert!(!ptr.is_aligned()); + assert!(!ptr.is_aligned_to(1)); + } +} + #[test] fn offset_from() { let mut a = [0; 5]; From 24e88066dc702da8fbe0381044645a91669bf02f Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Thu, 20 Oct 2022 19:46:31 +0200 Subject: [PATCH 07/20] mark `align_offset` as `#[must_use]` --- library/core/src/ptr/const_ptr.rs | 1 + library/core/src/ptr/mut_ptr.rs | 1 + 2 files changed, 2 insertions(+) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 8b96cf06be462..cedf58c27ffcc 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1320,6 +1320,7 @@ impl *const T { /// } /// # } /// ``` + #[must_use] #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] #[cfg(not(bootstrap))] diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 2d73c24b5b4c8..21c836efd5c46 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1588,6 +1588,7 @@ impl *mut T { /// } /// # } /// ``` + #[must_use] #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] #[cfg(not(bootstrap))] From a906f6cb698df6d29093e14984878446c269082d Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Sat, 22 Oct 2022 17:31:07 +0200 Subject: [PATCH 08/20] don't call `align_offset` during const eval, ever --- .../src/const_eval/machine.rs | 131 ++++++++++++------ library/core/src/ptr/mod.rs | 9 +- 2 files changed, 90 insertions(+), 50 deletions(-) diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 97d168ceb01a6..6cf8a581d71a6 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -1,11 +1,10 @@ use rustc_hir::def::DefKind; use rustc_middle::mir; use rustc_middle::mir::interpret::PointerArithmetic; -use rustc_middle::ty::layout::FnAbiOf; +use rustc_middle::ty::layout::LayoutOf; use rustc_middle::ty::{self, Ty, TyCtxt}; use std::borrow::Borrow; use std::hash::Hash; -use std::ops::ControlFlow; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::fx::IndexEntry; @@ -20,8 +19,8 @@ use rustc_target::abi::{Align, Size}; use rustc_target::spec::abi::Abi as CallAbi; use crate::interpret::{ - self, compile_time_machine, AllocId, ConstAllocation, FnVal, Frame, ImmTy, InterpCx, - InterpResult, OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, + self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult, + OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, }; use super::error::*; @@ -156,7 +155,6 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { fn hook_special_const_fn( &mut self, instance: ty::Instance<'tcx>, - _abi: CallAbi, args: &[OpTy<'tcx>], dest: &PlaceTy<'tcx>, ret: Option, @@ -194,24 +192,21 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { return Ok(Some(new_instance)); } else if Some(def_id) == self.tcx.lang_items().align_offset_fn() { - // For align_offset, we replace the function call if the pointer has no address. - match self.align_offset(instance, args, dest, ret)? { - ControlFlow::Continue(()) => return Ok(Some(instance)), - ControlFlow::Break(()) => return Ok(None), - } + // For align_offset, we replace the function call entirely. + self.align_offset(instance, args, dest, ret)?; + return Ok(None); } Ok(Some(instance)) } - /// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer - /// may not have an address. + /// This function replaces `align_offset(ptr, target_align)` in const eval, because the + /// pointer may not have an address. /// - /// If `ptr` does have a known address, then we return `CONTINUE` and the function call should - /// proceed as normal. + /// If `ptr` does have a known address, we forward it to [`Self::align_offset_impl`]. /// /// If `ptr` doesn't have an address, but its underlying allocation's alignment is at most - /// `target_align`, then we call the function again with an dummy address relative to the - /// allocation. + /// `target_align`, then we call [`Self::align_offset_impl`] with an dummy address relative + /// to the allocation. /// /// If `ptr` doesn't have an address and `target_align` is stricter than the underlying /// allocation's alignment, then we return `usize::MAX` immediately. @@ -221,50 +216,100 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { args: &[OpTy<'tcx>], dest: &PlaceTy<'tcx>, ret: Option, - ) -> InterpResult<'tcx, ControlFlow<()>> { + ) -> InterpResult<'tcx> { assert_eq!(args.len(), 2); let ptr = self.read_pointer(&args[0])?; let target_align = self.read_scalar(&args[1])?.to_machine_usize(self)?; + let pointee_ty = instance.substs.type_at(0); + let stride = self.layout_of(pointee_ty)?.size.bytes(); + if !target_align.is_power_of_two() { throw_ub_format!("`align_offset` called with non-power-of-two align: {}", target_align); } - match self.ptr_try_get_alloc_id(ptr) { + let mut align_offset = match self.ptr_try_get_alloc_id(ptr) { Ok((alloc_id, offset, _extra)) => { + // Extract the address relative to a base that is definitely sufficiently aligned. let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id); if target_align <= alloc_align.bytes() { - // Extract the address relative to the allocation base that is definitely - // sufficiently aligned and call `align_offset` again. - let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into(); - let align = ImmTy::from_uint(target_align, args[1].layout).into(); - - let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; - self.eval_fn_call( - FnVal::Instance(instance), - (CallAbi::Rust, fn_abi), - &[addr, align], - false, - dest, - ret, - StackPopUnwind::NotAllowed, - )?; - Ok(ControlFlow::BREAK) + // The pointer *is* alignable in const. We use an address relative to the + // allocation base that is definitely sufficiently aligned. + let addr = offset.bytes(); + Self::align_offset_impl(addr, stride, target_align) } else { - // Not alignable in const, return `usize::MAX`. - let usize_max = Scalar::from_machine_usize(self.machine_usize_max(), self); - self.write_scalar(usize_max, dest)?; - self.return_to_block(ret)?; - Ok(ControlFlow::BREAK) + // The pointer *is not* alignable in const, return `usize::MAX`. + // (We clamp this to machine `usize` below.) + u64::MAX } } - Err(_addr) => { - // The pointer has an address, continue with function call. - Ok(ControlFlow::CONTINUE) + Err(addr) => { + // The pointer has a known address. + Self::align_offset_impl(addr, stride, target_align) } + }; + + let usize_max = self.machine_usize_max(); + if align_offset > usize_max { + align_offset = usize_max; + } + + self.write_scalar(Scalar::from_machine_usize(align_offset, self), dest)?; + self.return_to_block(ret)?; + + Ok(()) + } + + /// Const eval implementation of `#[lang = "align_offset"]`. + /// See the runtime version for a detailed explanation how this works. + fn align_offset_impl(addr: u64, stride: u64, align: u64) -> u64 { + assert!(align.is_power_of_two()); + + let addr_mod_align = addr % align; + + if addr_mod_align == 0 { + // The address is already sufficiently aligned. + return 0; } + + if stride == 0 { + // The address cannot be aligned. + return u64::MAX; + } + + let byte_offset = align - addr_mod_align; + + if align % stride == 0 { + if byte_offset % stride == 0 { + return byte_offset / stride; + } else { + return u64::MAX; + } + } + + // This only works, because `align` is a power of two. + let gcd = 1u64 << (stride | align).trailing_zeros(); + + if addr % gcd != 0 { + // The address cannot be aligned. + return u64::MAX; + } + + let align2 = align / gcd; + let stride2 = (stride / gcd) % align2; + + let mut stride_inv = 1u64; + let mut mod_gate = 2u64; + let mut overflow = false; + while !overflow && mod_gate < align2 { + stride_inv = + stride_inv.wrapping_mul(2u64.wrapping_sub(stride2.wrapping_mul(stride_inv))); + (mod_gate, overflow) = mod_gate.overflowing_mul(mod_gate); + } + + byte_offset.wrapping_mul(stride_inv) % align2 } /// See documentation on the `ptr_guaranteed_cmp` intrinsic. @@ -367,7 +412,7 @@ impl<'mir, 'tcx> interpret::Machine<'mir, 'tcx> for CompileTimeInterpreter<'mir, } } - let Some(new_instance) = ecx.hook_special_const_fn(instance, _abi, args, dest, ret)? else { + let Some(new_instance) = ecx.hook_special_const_fn(instance, args, dest, ret)? else { return Ok(None); }; diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 1ebd9dca09838..e762837ff90a9 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -1590,9 +1590,8 @@ pub unsafe fn write_volatile(dst: *mut T, src: T) { /// than trying to adapt this to accommodate that change. /// /// Any questions go to @nagisa. -// #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap -// compiler will always cause an error. #[lang = "align_offset"] +#[rustc_do_not_const_check] // hooked by const-eval pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usize { // FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <= // 1, where the method versions of these operations are not inlined. @@ -1652,13 +1651,9 @@ pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usiz inverse & m_minus_one } + let addr = p.addr(); let stride = mem::size_of::(); - // SAFETY: At runtime transmuting a pointer to `usize` is always safe, because they have the - // same layout. During const eval we hook this function to ensure that the pointer always has - // an address (only the standard library can do this). - let addr = unsafe { mem::transmute(p) }; - // SAFETY: `a` is a power-of-two, therefore non-zero. let a_minus_one = unsafe { unchecked_sub(a, 1) }; From 093c02ed460cfe726badc7d7bee2c868f8288e16 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Sat, 22 Oct 2022 19:15:03 +0200 Subject: [PATCH 09/20] document `is_aligned{,_to}` --- library/core/src/ptr/const_ptr.rs | 138 +++++++++++++++++++++++++++++- library/core/src/ptr/mut_ptr.rs | 138 +++++++++++++++++++++++++++++- 2 files changed, 268 insertions(+), 8 deletions(-) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index cedf58c27ffcc..6457e5184b0e4 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1364,8 +1364,72 @@ impl *const T { } /// Returns whether the pointer is properly aligned for `T`. - // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap - // compiler will always return false. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// + /// assert!(ptr.is_aligned()); + /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// is never aligned if cast to a type with a stricter alignment than the reference's + /// underlying allocation. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// assert!(ptr.is_aligned()); + /// + /// // At runtime either `ptr1` or `ptr2` would be aligned, + /// // but at compiletime neither is aligned. + /// let ptr1: *const i64 = ptr.cast(); + /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// assert!(!ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let ptr = 40 as *const i32; + /// assert!(ptr.is_aligned()); + /// + /// // For pointers with a known address, runtime and + /// // compiletime behavior are identical. + /// let ptr1: *const i64 = ptr.cast(); + /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// assert!(ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] @@ -1385,8 +1449,74 @@ impl *const T { /// # Panics /// /// The function panics if `align` is not a power-of-two (this includes 0). - // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap - // compiler will always return false. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + /// assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + /// + /// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// cannot be stricter aligned than the reference's underlying allocation. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// // At compiletime, we know for sure that the pointer isn't aligned to 8. + /// assert!(!ptr.is_aligned_to(8)); + /// assert!(!ptr.wrapping_add(1).is_aligned_to(8)); + /// }; + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let ptr = 40 as *const i32; + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// assert!(ptr.is_aligned_to(8)); + /// assert!(!ptr.is_aligned_to(16)); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 21c836efd5c46..c79f815e35fd4 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1632,8 +1632,72 @@ impl *mut T { } /// Returns whether the pointer is properly aligned for `T`. - // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap - // compiler will always return false. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// + /// assert!(ptr.is_aligned()); + /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// is never aligned if cast to a type with a stricter alignment than the reference's + /// underlying allocation. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// assert!(ptr.is_aligned()); + /// + /// // At runtime either `ptr1` or `ptr2` would be aligned, + /// // but at compiletime neither is aligned. + /// let ptr1: *const i64 = ptr.cast(); + /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// assert!(!ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let ptr = 40 as *const i32; + /// assert!(ptr.is_aligned()); + /// + /// // For pointers with a known address, runtime and + /// // compiletime behavior are identical. + /// let ptr1: *const i64 = ptr.cast(); + /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// assert!(ptr1.is_aligned()); + /// assert!(!ptr2.is_aligned()); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] @@ -1653,8 +1717,74 @@ impl *mut T { /// # Panics /// /// The function panics if `align` is not a power-of-two (this includes 0). - // #[cfg(not(bootstrap))] -- Calling this function in a const context from the bootstrap - // compiler will always return false. + /// + /// # Examples + /// + /// Basic usage: + /// ``` + /// #![feature(pointer_is_aligned)] + /// #![feature(pointer_byte_offsets)] + /// + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// assert!(ptr.wrapping_byte_add(2).is_aligned_to(2)); + /// assert!(!ptr.wrapping_byte_add(2).is_aligned_to(4)); + /// + /// assert_ne!(ptr.is_aligned_to(8), ptr.wrapping_add(1).is_aligned_to(8)); + /// ``` + /// + /// # At compiletime + /// **Note: Alignment at compiletime is experimental and subject to change. See the + /// [tracking issue] for details.** + /// + /// At compiletime, the compiler may not know where a value will end up in memory. + /// Calling this function on a pointer created from a reference at compiletime will only + /// return `true` if the pointer is guaranteed to be aligned. This means that the pointer + /// cannot be stricter aligned than the reference's underlying allocation. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let data: i32 = 42; + /// let ptr: *const i32 = &data; + /// + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// + /// // At compiletime, we know for sure that the pointer isn't aligned to 8. + /// assert!(!ptr.is_aligned_to(8)); + /// assert!(!ptr.wrapping_add(1).is_aligned_to(8)); + /// }; + /// ``` + /// + /// If a pointer is created from a fixed address, this function behaves the same during + /// runtime and compiletime. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// const _: () = { + /// let ptr = 40 as *const i32; + /// assert!(ptr.is_aligned_to(1)); + /// assert!(ptr.is_aligned_to(2)); + /// assert!(ptr.is_aligned_to(4)); + /// assert!(ptr.is_aligned_to(8)); + /// assert!(!ptr.is_aligned_to(16)); + /// }; + /// ``` + /// + /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] From df0bcfe6448793e423bd6b8fd5294f10798dd469 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Sat, 22 Oct 2022 21:20:04 +0200 Subject: [PATCH 10/20] address more review comments * `cfg` only the body of `align_offset` * put explicit panics back * explain why `ptr.align_offset(align) == 0` is slow --- library/core/src/ptr/const_ptr.rs | 54 +++++++++++++++---------------- library/core/src/ptr/mut_ptr.rs | 54 +++++++++++++++---------------- 2 files changed, 52 insertions(+), 56 deletions(-) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 6457e5184b0e4..37679c504b303 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1323,21 +1323,6 @@ impl *const T { #[must_use] #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] - #[cfg(not(bootstrap))] - pub const fn align_offset(self, align: usize) -> usize - where - T: Sized, - { - assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two"); - - // SAFETY: `align` has been checked to be a power of 2 above - unsafe { align_offset(self, align) } - } - - #[stable(feature = "align_offset", since = "1.36.0")] - #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] - #[allow(missing_docs)] - #[cfg(bootstrap)] pub const fn align_offset(self, align: usize) -> usize where T: Sized, @@ -1346,21 +1331,30 @@ impl *const T { panic!("align_offset: align is not a power-of-two"); } - fn rt_impl(p: *const T, align: usize) -> usize { - // SAFETY: `align` has been checked to be a power of 2 above - unsafe { align_offset(p, align) } - } + #[cfg(bootstrap)] + { + fn rt_impl(p: *const T, align: usize) -> usize { + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(p, align) } + } - const fn ctfe_impl(_: *const T, _: usize) -> usize { - usize::MAX + const fn ctfe_impl(_: *const T, _: usize) -> usize { + usize::MAX + } + + // SAFETY: + // It is permissible for `align_offset` to always return `usize::MAX`, + // algorithm correctness can not depend on `align_offset` returning non-max values. + // + // As such the behaviour can't change after replacing `align_offset` with `usize::MAX`, only performance can. + unsafe { intrinsics::const_eval_select((self, align), ctfe_impl, rt_impl) } } - // SAFETY: - // It is permissible for `align_offset` to always return `usize::MAX`, - // algorithm correctness can not depend on `align_offset` returning non-max values. - // - // As such the behaviour can't change after replacing `align_offset` with `usize::MAX`, only performance can. - unsafe { intrinsics::const_eval_select((self, align), ctfe_impl, rt_impl) } + #[cfg(not(bootstrap))] + { + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(self, align) } + } } /// Returns whether the pointer is properly aligned for `T`. @@ -1522,13 +1516,17 @@ impl *const T { #[unstable(feature = "pointer_is_aligned", issue = "96284")] #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] pub const fn is_aligned_to(self, align: usize) -> bool { - assert!(align.is_power_of_two(), "is_aligned_to: align is not a power-of-two"); + if !align.is_power_of_two() { + panic!("is_aligned_to: align is not a power-of-two") + } #[inline] fn runtime(ptr: *const u8, align: usize) -> bool { ptr.addr() & (align - 1) == 0 } + // This optimizes to `(ptr + align - 1) & -align == ptr`, which is slightly + // slower than `ptr & (align - 1) == 0` const fn comptime(ptr: *const u8, align: usize) -> bool { ptr.align_offset(align) == 0 } diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index c79f815e35fd4..9174728868936 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1591,21 +1591,6 @@ impl *mut T { #[must_use] #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] - #[cfg(not(bootstrap))] - pub const fn align_offset(self, align: usize) -> usize - where - T: Sized, - { - assert!(align.is_power_of_two(), "align_offset: align is not a power-of-two"); - - // SAFETY: `align` has been checked to be a power of 2 above - unsafe { align_offset(self, align) } - } - - #[stable(feature = "align_offset", since = "1.36.0")] - #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] - #[allow(missing_docs)] - #[cfg(bootstrap)] pub const fn align_offset(self, align: usize) -> usize where T: Sized, @@ -1614,21 +1599,30 @@ impl *mut T { panic!("align_offset: align is not a power-of-two"); } - fn rt_impl(p: *mut T, align: usize) -> usize { - // SAFETY: `align` has been checked to be a power of 2 above - unsafe { align_offset(p, align) } + #[cfg(bootstrap)] + { + fn rt_impl(p: *mut T, align: usize) -> usize { + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(p, align) } + } + + const fn ctfe_impl(_: *mut T, _: usize) -> usize { + usize::MAX + } + + // SAFETY: + // It is permissible for `align_offset` to always return `usize::MAX`, + // algorithm correctness can not depend on `align_offset` returning non-max values. + // + // As such the behaviour can't change after replacing `align_offset` with `usize::MAX`, only performance can. + unsafe { intrinsics::const_eval_select((self, align), ctfe_impl, rt_impl) } } - const fn ctfe_impl(_: *mut T, _: usize) -> usize { - usize::MAX + #[cfg(not(bootstrap))] + { + // SAFETY: `align` has been checked to be a power of 2 above + unsafe { align_offset(self, align) } } - - // SAFETY: - // It is permissible for `align_offset` to always return `usize::MAX`, - // algorithm correctness can not depend on `align_offset` returning non-max values. - // - // As such the behaviour can't change after replacing `align_offset` with `usize::MAX`, only performance can. - unsafe { intrinsics::const_eval_select((self, align), ctfe_impl, rt_impl) } } /// Returns whether the pointer is properly aligned for `T`. @@ -1790,13 +1784,17 @@ impl *mut T { #[unstable(feature = "pointer_is_aligned", issue = "96284")] #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] pub const fn is_aligned_to(self, align: usize) -> bool { - assert!(align.is_power_of_two(), "is_aligned_to: align is not a power-of-two"); + if !align.is_power_of_two() { + panic!("is_aligned_to: align is not a power-of-two") + } #[inline] fn runtime(ptr: *mut u8, align: usize) -> bool { ptr.addr() & (align - 1) == 0 } + // This optimizes to `(ptr + align - 1) & -align == ptr`, which is slightly + // slower than `ptr & (align - 1) == 0` const fn comptime(ptr: *mut u8, align: usize) -> bool { ptr.align_offset(align) == 0 } From 4696e8906d362c1e10ffe40a5935de0ada45fb48 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Sun, 23 Oct 2022 00:47:21 +0200 Subject: [PATCH 11/20] =?UTF-8?q?Schr=C3=B6dinger's=20pointer?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It's aligned *and* not aligned! --- library/core/src/ptr/const_ptr.rs | 42 +++++++++++++++++++++++++++++++ library/core/src/ptr/mut_ptr.rs | 42 +++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 37679c504b303..d042f22f0266f 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1402,6 +1402,27 @@ impl *const T { /// }; /// ``` /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. + /// const CONST_PTR: *const i32 = &42; + /// const _: () = assert!(!CONST_PTR.cast::().is_aligned()); + /// const _: () = assert!(!CONST_PTR.wrapping_add(1).cast::().is_aligned()); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = CONST_PTR; + /// assert_ne!( + /// runtime_ptr.cast::().is_aligned(), + /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), + /// ); + /// ``` + /// /// If a pointer is created from a fixed address, this function behaves the same during /// runtime and compiletime. /// @@ -1492,6 +1513,27 @@ impl *const T { /// }; /// ``` /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. + /// const CONST_PTR: *const i32 = &42; + /// const _: () = assert!(!CONST_PTR.is_aligned_to(8)); + /// const _: () = assert!(!CONST_PTR.wrapping_add(1).is_aligned_to(8)); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = CONST_PTR; + /// assert_ne!( + /// runtime_ptr.is_aligned_to(8), + /// runtime_ptr.wrapping_add(1).is_aligned_to(8), + /// ); + /// ``` + /// /// If a pointer is created from a fixed address, this function behaves the same during /// runtime and compiletime. /// diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 9174728868936..764fa9d8ba812 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1670,6 +1670,27 @@ impl *mut T { /// }; /// ``` /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. + /// const CONST_PTR: *const i32 = &42; + /// const _: () = assert!(!CONST_PTR.cast::().is_aligned()); + /// const _: () = assert!(!CONST_PTR.wrapping_add(1).cast::().is_aligned()); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = CONST_PTR; + /// assert_ne!( + /// runtime_ptr.cast::().is_aligned(), + /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), + /// ); + /// ``` + /// /// If a pointer is created from a fixed address, this function behaves the same during /// runtime and compiletime. /// @@ -1760,6 +1781,27 @@ impl *mut T { /// }; /// ``` /// + /// Due to this behavior, it is possible that a runtime pointer derived from a compiletime + /// pointer is aligned, even if the compiletime pointer wasn't aligned. + /// + #[cfg_attr(bootstrap, doc = "```ignore")] + #[cfg_attr(not(bootstrap), doc = "```")] + /// #![feature(pointer_is_aligned)] + /// #![feature(const_pointer_is_aligned)] + /// + /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. + /// const CONST_PTR: *const i32 = &42; + /// const _: () = assert!(!CONST_PTR.is_aligned_to(8)); + /// const _: () = assert!(!CONST_PTR.wrapping_add(1).is_aligned_to(8)); + /// + /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. + /// let runtime_ptr = CONST_PTR; + /// assert_ne!( + /// runtime_ptr.is_aligned_to(8), + /// runtime_ptr.wrapping_add(1).is_aligned_to(8), + /// ); + /// ``` + /// /// If a pointer is created from a fixed address, this function behaves the same during /// runtime and compiletime. /// From daccb8c11a464040acd744729231429dcb2e4d4b Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Sun, 23 Oct 2022 12:30:46 +0200 Subject: [PATCH 12/20] always use `align_offset` in `is_aligned_to` + add assembly test --- library/core/src/ptr/const_ptr.rs | 19 ++++------ library/core/src/ptr/mut_ptr.rs | 19 ++++------ src/test/assembly/is_aligned.rs | 58 +++++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 26 deletions(-) create mode 100644 src/test/assembly/is_aligned.rs diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index d042f22f0266f..82a01a70a4113 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1321,6 +1321,7 @@ impl *const T { /// # } /// ``` #[must_use] + #[inline] #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] pub const fn align_offset(self, align: usize) -> usize @@ -1562,19 +1563,11 @@ impl *const T { panic!("is_aligned_to: align is not a power-of-two") } - #[inline] - fn runtime(ptr: *const u8, align: usize) -> bool { - ptr.addr() & (align - 1) == 0 - } - - // This optimizes to `(ptr + align - 1) & -align == ptr`, which is slightly - // slower than `ptr & (align - 1) == 0` - const fn comptime(ptr: *const u8, align: usize) -> bool { - ptr.align_offset(align) == 0 - } - - // SAFETY: `ptr.align_offset(align)` returns 0 if and only if the pointer is already aligned. - unsafe { intrinsics::const_eval_select((self.cast::(), align), comptime, runtime) } + // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. + // The cast to `()` is used to + // 1. deal with fat pointers; and + // 2. ensure that `align_offset` doesn't actually try to compute an offset. + self.cast::<()>().align_offset(align) == 0 } } diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index 764fa9d8ba812..fdb99818ac7f9 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1589,6 +1589,7 @@ impl *mut T { /// # } /// ``` #[must_use] + #[inline] #[stable(feature = "align_offset", since = "1.36.0")] #[rustc_const_unstable(feature = "const_align_offset", issue = "90962")] pub const fn align_offset(self, align: usize) -> usize @@ -1830,19 +1831,11 @@ impl *mut T { panic!("is_aligned_to: align is not a power-of-two") } - #[inline] - fn runtime(ptr: *mut u8, align: usize) -> bool { - ptr.addr() & (align - 1) == 0 - } - - // This optimizes to `(ptr + align - 1) & -align == ptr`, which is slightly - // slower than `ptr & (align - 1) == 0` - const fn comptime(ptr: *mut u8, align: usize) -> bool { - ptr.align_offset(align) == 0 - } - - // SAFETY: `ptr.align_offset(align)` returns 0 if and only if the pointer is already aligned. - unsafe { intrinsics::const_eval_select((self.cast::(), align), comptime, runtime) } + // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. + // The cast to `()` is used to + // 1. deal with fat pointers; and + // 2. ensure that `align_offset` doesn't actually try to compute an offset. + self.cast::<()>().align_offset(align) == 0 } } diff --git a/src/test/assembly/is_aligned.rs b/src/test/assembly/is_aligned.rs new file mode 100644 index 0000000000000..3949f5f6530b0 --- /dev/null +++ b/src/test/assembly/is_aligned.rs @@ -0,0 +1,58 @@ +// assembly-output: emit-asm +// min-llvm-version: 14.0 +// only-x86_64 +// revisions: opt-speed opt-size +// [opt-speed] compile-flags: -Copt-level=1 +// [opt-size] compile-flags: -Copt-level=s +#![crate_type="rlib"] + +#![feature(core_intrinsics)] +#![feature(pointer_is_aligned)] + +// CHECK-LABEL: is_aligned_to_unchecked +// CHECK: decq %rsi +// CHECK-NEXT: testq %rdi, %rsi +// CHECK-NEXT: sete %al +// CHECK-NEXT: retq +#[no_mangle] +pub unsafe fn is_aligned_to_unchecked(ptr: *const u8, align: usize) -> bool { + unsafe { + std::intrinsics::assume(align.is_power_of_two()) + } + ptr.is_aligned_to(align) +} + +// CHECK-LABEL: is_aligned_1 +// CHECK: movb $1, %al +// CHECK-NEXT: retq +#[no_mangle] +pub fn is_aligned_1(ptr: *const u8) -> bool { + ptr.is_aligned() +} + +// CHECK-LABEL: is_aligned_2 +// CHECK: testb $1, %dil +// CHECK-NEXT: sete %al +// CHECK-NEXT: retq +#[no_mangle] +pub fn is_aligned_2(ptr: *const u16) -> bool { + ptr.is_aligned() +} + +// CHECK-LABEL: is_aligned_4 +// CHECK: testb $3, %dil +// CHECK-NEXT: sete %al +// CHECK-NEXT: retq +#[no_mangle] +pub fn is_aligned_4(ptr: *const u32) -> bool { + ptr.is_aligned() +} + +// CHECK-LABEL: is_aligned_8 +// CHECK: testb $7, %dil +// CHECK-NEXT: sete %al +// CHECK-NEXT: retq +#[no_mangle] +pub fn is_aligned_8(ptr: *const u64) -> bool { + ptr.is_aligned() +} From 8a6053618f99852d31b46144043e881f6556ff1c Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Wed, 9 Nov 2022 20:18:00 +0100 Subject: [PATCH 13/20] docs cleanup * Fix doc examples for Platforms with underaligned integer primitives. * Mutable pointer doc examples use mutable pointers. * Fill out tracking issue. * Minor formatting changes. --- library/core/src/ptr/const_ptr.rs | 104 ++++++++++++++++++---------- library/core/src/ptr/mut_ptr.rs | 108 ++++++++++++++++++++---------- 2 files changed, 140 insertions(+), 72 deletions(-) diff --git a/library/core/src/ptr/const_ptr.rs b/library/core/src/ptr/const_ptr.rs index 82a01a70a4113..8a3eee0dc529f 100644 --- a/library/core/src/ptr/const_ptr.rs +++ b/library/core/src/ptr/const_ptr.rs @@ -1367,8 +1367,12 @@ impl *const T { /// #![feature(pointer_is_aligned)] /// #![feature(pointer_byte_offsets)] /// - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; /// /// assert!(ptr.is_aligned()); /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); @@ -1389,15 +1393,20 @@ impl *const T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// /// const _: () = { - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; /// assert!(ptr.is_aligned()); /// - /// // At runtime either `ptr1` or `ptr2` would be aligned, - /// // but at compiletime neither is aligned. - /// let ptr1: *const i64 = ptr.cast(); - /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); /// assert!(!ptr1.is_aligned()); /// assert!(!ptr2.is_aligned()); /// }; @@ -1411,16 +1420,22 @@ impl *const T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// - /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. - /// const CONST_PTR: *const i32 = &42; - /// const _: () = assert!(!CONST_PTR.cast::().is_aligned()); - /// const _: () = assert!(!CONST_PTR.wrapping_add(1).cast::().is_aligned()); + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); + /// const _: () = assert!(!COMPTIME_PTR.cast::().is_aligned()); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::().is_aligned()); /// /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = CONST_PTR; + /// let runtime_ptr = COMPTIME_PTR; /// assert_ne!( - /// runtime_ptr.cast::().is_aligned(), - /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), + /// runtime_ptr.cast::().is_aligned(), + /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), /// ); /// ``` /// @@ -1432,29 +1447,34 @@ impl *const T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// /// const _: () = { - /// let ptr = 40 as *const i32; + /// let ptr = 40 as *const AlignedI32; /// assert!(ptr.is_aligned()); /// - /// // For pointers with a known address, runtime and - /// // compiletime behavior are identical. - /// let ptr1: *const i64 = ptr.cast(); - /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// // For pointers with a known address, runtime and compiletime behavior are identical. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); /// assert!(ptr1.is_aligned()); /// assert!(!ptr2.is_aligned()); /// }; /// ``` /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] pub const fn is_aligned(self) -> bool where T: Sized, { - self.is_aligned_to(core::mem::align_of::()) + self.is_aligned_to(mem::align_of::()) } /// Returns whether the pointer is aligned to `align`. @@ -1473,8 +1493,12 @@ impl *const T { /// #![feature(pointer_is_aligned)] /// #![feature(pointer_byte_offsets)] /// - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; /// /// assert!(ptr.is_aligned_to(1)); /// assert!(ptr.is_aligned_to(2)); @@ -1500,9 +1524,13 @@ impl *const T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// /// const _: () = { - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// let data = AlignedI32(42); + /// let ptr = &data as *const AlignedI32; /// /// assert!(ptr.is_aligned_to(1)); /// assert!(ptr.is_aligned_to(2)); @@ -1522,13 +1550,17 @@ impl *const T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// - /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. - /// const CONST_PTR: *const i32 = &42; - /// const _: () = assert!(!CONST_PTR.is_aligned_to(8)); - /// const _: () = assert!(!CONST_PTR.wrapping_add(1).is_aligned_to(8)); + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// const COMPTIME_PTR: *const AlignedI32 = &AlignedI32(42); + /// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8)); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8)); /// /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = CONST_PTR; + /// let runtime_ptr = COMPTIME_PTR; /// assert_ne!( /// runtime_ptr.is_aligned_to(8), /// runtime_ptr.wrapping_add(1).is_aligned_to(8), @@ -1544,7 +1576,7 @@ impl *const T { /// #![feature(const_pointer_is_aligned)] /// /// const _: () = { - /// let ptr = 40 as *const i32; + /// let ptr = 40 as *const u8; /// assert!(ptr.is_aligned_to(1)); /// assert!(ptr.is_aligned_to(2)); /// assert!(ptr.is_aligned_to(4)); @@ -1553,14 +1585,14 @@ impl *const T { /// }; /// ``` /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] pub const fn is_aligned_to(self, align: usize) -> bool { if !align.is_power_of_two() { - panic!("is_aligned_to: align is not a power-of-two") + panic!("is_aligned_to: align is not a power-of-two"); } // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. diff --git a/library/core/src/ptr/mut_ptr.rs b/library/core/src/ptr/mut_ptr.rs index fdb99818ac7f9..8472b05ddbd40 100644 --- a/library/core/src/ptr/mut_ptr.rs +++ b/library/core/src/ptr/mut_ptr.rs @@ -1635,8 +1635,12 @@ impl *mut T { /// #![feature(pointer_is_aligned)] /// #![feature(pointer_byte_offsets)] /// - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; /// /// assert!(ptr.is_aligned()); /// assert!(!ptr.wrapping_byte_add(1).is_aligned()); @@ -1656,16 +1660,22 @@ impl *mut T { #[cfg_attr(not(bootstrap), doc = "```")] /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] + /// #![feature(const_mut_refs)] + /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); /// /// const _: () = { - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; /// assert!(ptr.is_aligned()); /// - /// // At runtime either `ptr1` or `ptr2` would be aligned, - /// // but at compiletime neither is aligned. - /// let ptr1: *const i64 = ptr.cast(); - /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// // At runtime either `ptr1` or `ptr2` would be aligned, but at compiletime neither is aligned. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); /// assert!(!ptr1.is_aligned()); /// assert!(!ptr2.is_aligned()); /// }; @@ -1679,16 +1689,23 @@ impl *mut T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// - /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. - /// const CONST_PTR: *const i32 = &42; - /// const _: () = assert!(!CONST_PTR.cast::().is_aligned()); - /// const _: () = assert!(!CONST_PTR.wrapping_add(1).cast::().is_aligned()); + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// // Also, note that mutable references are not allowed in the final value of constants. + /// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut(); + /// const _: () = assert!(!COMPTIME_PTR.cast::().is_aligned()); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).cast::().is_aligned()); /// /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = CONST_PTR; + /// let runtime_ptr = COMPTIME_PTR; /// assert_ne!( - /// runtime_ptr.cast::().is_aligned(), - /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), + /// runtime_ptr.cast::().is_aligned(), + /// runtime_ptr.wrapping_add(1).cast::().is_aligned(), /// ); /// ``` /// @@ -1700,29 +1717,34 @@ impl *mut T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// + /// // On some platforms, the alignment of primitives is less than their size. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// #[repr(align(8))] + /// struct AlignedI64(i64); + /// /// const _: () = { - /// let ptr = 40 as *const i32; + /// let ptr = 40 as *mut AlignedI32; /// assert!(ptr.is_aligned()); /// - /// // For pointers with a known address, runtime and - /// // compiletime behavior are identical. - /// let ptr1: *const i64 = ptr.cast(); - /// let ptr2: *const i64 = ptr.wrapping_add(1).cast(); + /// // For pointers with a known address, runtime and compiletime behavior are identical. + /// let ptr1 = ptr.cast::(); + /// let ptr2 = ptr.wrapping_add(1).cast::(); /// assert!(ptr1.is_aligned()); /// assert!(!ptr2.is_aligned()); /// }; /// ``` /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] pub const fn is_aligned(self) -> bool where T: Sized, { - self.is_aligned_to(core::mem::align_of::()) + self.is_aligned_to(mem::align_of::()) } /// Returns whether the pointer is aligned to `align`. @@ -1741,8 +1763,12 @@ impl *mut T { /// #![feature(pointer_is_aligned)] /// #![feature(pointer_byte_offsets)] /// - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; /// /// assert!(ptr.is_aligned_to(1)); /// assert!(ptr.is_aligned_to(2)); @@ -1767,10 +1793,15 @@ impl *mut T { #[cfg_attr(not(bootstrap), doc = "```")] /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] + /// #![feature(const_mut_refs)] + /// + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); /// /// const _: () = { - /// let data: i32 = 42; - /// let ptr: *const i32 = &data; + /// let mut data = AlignedI32(42); + /// let ptr = &mut data as *mut AlignedI32; /// /// assert!(ptr.is_aligned_to(1)); /// assert!(ptr.is_aligned_to(2)); @@ -1790,13 +1821,18 @@ impl *mut T { /// #![feature(pointer_is_aligned)] /// #![feature(const_pointer_is_aligned)] /// - /// // At compiletime, neither `CONST_PTR` nor `CONST_PTR + 1` is aligned. - /// const CONST_PTR: *const i32 = &42; - /// const _: () = assert!(!CONST_PTR.is_aligned_to(8)); - /// const _: () = assert!(!CONST_PTR.wrapping_add(1).is_aligned_to(8)); + /// // On some platforms, the alignment of i32 is less than 4. + /// #[repr(align(4))] + /// struct AlignedI32(i32); + /// + /// // At compiletime, neither `COMPTIME_PTR` nor `COMPTIME_PTR + 1` is aligned. + /// // Also, note that mutable references are not allowed in the final value of constants. + /// const COMPTIME_PTR: *mut AlignedI32 = (&AlignedI32(42) as *const AlignedI32).cast_mut(); + /// const _: () = assert!(!COMPTIME_PTR.is_aligned_to(8)); + /// const _: () = assert!(!COMPTIME_PTR.wrapping_add(1).is_aligned_to(8)); /// /// // At runtime, either `runtime_ptr` or `runtime_ptr + 1` is aligned. - /// let runtime_ptr = CONST_PTR; + /// let runtime_ptr = COMPTIME_PTR; /// assert_ne!( /// runtime_ptr.is_aligned_to(8), /// runtime_ptr.wrapping_add(1).is_aligned_to(8), @@ -1812,7 +1848,7 @@ impl *mut T { /// #![feature(const_pointer_is_aligned)] /// /// const _: () = { - /// let ptr = 40 as *const i32; + /// let ptr = 40 as *mut u8; /// assert!(ptr.is_aligned_to(1)); /// assert!(ptr.is_aligned_to(2)); /// assert!(ptr.is_aligned_to(4)); @@ -1821,14 +1857,14 @@ impl *mut T { /// }; /// ``` /// - /// [tracking issue]: https://github.com/rust-lang/rust/issues/comming-soon + /// [tracking issue]: https://github.com/rust-lang/rust/issues/104203 #[must_use] #[inline] #[unstable(feature = "pointer_is_aligned", issue = "96284")] - #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "none")] + #[rustc_const_unstable(feature = "const_pointer_is_aligned", issue = "104203")] pub const fn is_aligned_to(self, align: usize) -> bool { if !align.is_power_of_two() { - panic!("is_aligned_to: align is not a power-of-two") + panic!("is_aligned_to: align is not a power-of-two"); } // We can't use the address of `self` in a `const fn`, so we use `align_offset` instead. From 60f352fd7d1b3443a07bbfefb955a4e836749ee6 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 11 Nov 2022 10:01:06 +0100 Subject: [PATCH 14/20] replace potential ICE with graceful error (`no_core` only) --- compiler/rustc_const_eval/src/const_eval/machine.rs | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 6cf8a581d71a6..93afbbd27a4e6 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -1,4 +1,5 @@ use rustc_hir::def::DefKind; +use rustc_hir::LangItem; use rustc_middle::mir; use rustc_middle::mir::interpret::PointerArithmetic; use rustc_middle::ty::layout::LayoutOf; @@ -178,9 +179,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { return Err(ConstEvalErrKind::Panic { msg, file, line, col }.into()); } else if Some(def_id) == self.tcx.lang_items().panic_fmt() { // For panic_fmt, call const_panic_fmt instead. - let Some(const_def_id) = self.tcx.lang_items().const_panic_fmt() else { - bug!("`const_panic_fmt` must be defined to call `panic_fmt` in const eval") - }; + let const_def_id = self.tcx.require_lang_item(LangItem::ConstPanicFmt, None); let new_instance = ty::Instance::resolve( *self.tcx, ty::ParamEnv::reveal_all(), From 8717455b9dd5f56d42d605ae7c0b657070cf4774 Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 11 Nov 2022 15:52:49 +0100 Subject: [PATCH 15/20] fix assembly test on windows --- src/test/assembly/is_aligned.rs | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/src/test/assembly/is_aligned.rs b/src/test/assembly/is_aligned.rs index 3949f5f6530b0..e8e6ad5cf112e 100644 --- a/src/test/assembly/is_aligned.rs +++ b/src/test/assembly/is_aligned.rs @@ -10,9 +10,9 @@ #![feature(pointer_is_aligned)] // CHECK-LABEL: is_aligned_to_unchecked -// CHECK: decq %rsi -// CHECK-NEXT: testq %rdi, %rsi -// CHECK-NEXT: sete %al +// CHECK: decq +// CHECK-NEXT: testq +// CHECK-NEXT: sete // CHECK-NEXT: retq #[no_mangle] pub unsafe fn is_aligned_to_unchecked(ptr: *const u8, align: usize) -> bool { @@ -23,7 +23,7 @@ pub unsafe fn is_aligned_to_unchecked(ptr: *const u8, align: usize) -> bool { } // CHECK-LABEL: is_aligned_1 -// CHECK: movb $1, %al +// CHECK: movb $1 // CHECK-NEXT: retq #[no_mangle] pub fn is_aligned_1(ptr: *const u8) -> bool { @@ -31,8 +31,8 @@ pub fn is_aligned_1(ptr: *const u8) -> bool { } // CHECK-LABEL: is_aligned_2 -// CHECK: testb $1, %dil -// CHECK-NEXT: sete %al +// CHECK: testb $1 +// CHECK-NEXT: sete // CHECK-NEXT: retq #[no_mangle] pub fn is_aligned_2(ptr: *const u16) -> bool { @@ -40,8 +40,8 @@ pub fn is_aligned_2(ptr: *const u16) -> bool { } // CHECK-LABEL: is_aligned_4 -// CHECK: testb $3, %dil -// CHECK-NEXT: sete %al +// CHECK: testb $3 +// CHECK-NEXT: sete // CHECK-NEXT: retq #[no_mangle] pub fn is_aligned_4(ptr: *const u32) -> bool { @@ -49,8 +49,8 @@ pub fn is_aligned_4(ptr: *const u32) -> bool { } // CHECK-LABEL: is_aligned_8 -// CHECK: testb $7, %dil -// CHECK-NEXT: sete %al +// CHECK: testb $7 +// CHECK-NEXT: sete // CHECK-NEXT: retq #[no_mangle] pub fn is_aligned_8(ptr: *const u64) -> bool { From 9e5d497b67c7566001bdcfc8a2767f26a23afc5b Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Tue, 15 Nov 2022 15:55:37 +0100 Subject: [PATCH 16/20] fix const `align_offset` implementation --- .../src/const_eval/machine.rs | 9 +- library/core/tests/ptr.rs | 104 +++++++++++------- 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 93afbbd27a4e6..92ccc94e63024 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -278,9 +278,8 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { return u64::MAX; } - let byte_offset = align - addr_mod_align; - if align % stride == 0 { + let byte_offset = align - addr_mod_align; if byte_offset % stride == 0 { return byte_offset / stride; } else { @@ -296,8 +295,11 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { return u64::MAX; } + // Instead of `(addr + offset * stride) % align == 0`, we solve + // `((addr + offset * stride) / gcd) % (align / gcd) == 0`. + let addr2 = addr / gcd; let align2 = align / gcd; - let stride2 = (stride / gcd) % align2; + let stride2 = stride / gcd; let mut stride_inv = 1u64; let mut mod_gate = 2u64; @@ -308,6 +310,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { (mod_gate, overflow) = mod_gate.overflowing_mul(mod_gate); } + let byte_offset = align2 - addr2 % align2; byte_offset.wrapping_mul(stride_inv) % align2 } diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs index 9f74b0c04107e..036acc46daba8 100644 --- a/library/core/tests/ptr.rs +++ b/library/core/tests/ptr.rs @@ -510,49 +510,53 @@ fn align_offset_various_strides_const() { assert!(got == expected); } - // For pointers of stride != 1, we verify the algorithm against the naivest possible - // implementation - let mut align = 1; - let limit = 1024; - while align < limit { - for ptr in 1usize..4 * align { - unsafe { - #[repr(packed)] - struct A3(u16, u8); - test_stride::(ptr::invalid::(ptr), ptr, align); - - struct A4(u32); - test_stride::(ptr::invalid::(ptr), ptr, align); - - #[repr(packed)] - struct A5(u32, u8); - test_stride::(ptr::invalid::(ptr), ptr, align); - - #[repr(packed)] - struct A6(u32, u16); - test_stride::(ptr::invalid::(ptr), ptr, align); - - #[repr(packed)] - struct A7(u32, u16, u8); - test_stride::(ptr::invalid::(ptr), ptr, align); - - #[repr(packed)] - struct A8(u32, u32); - test_stride::(ptr::invalid::(ptr), ptr, align); - - #[repr(packed)] - struct A9(u32, u32, u8); - test_stride::(ptr::invalid::(ptr), ptr, align); - - #[repr(packed)] - struct A10(u32, u32, u16); - test_stride::(ptr::invalid::(ptr), ptr, align); - - test_stride::(ptr::invalid::(ptr), ptr, align); - test_stride::(ptr::invalid::(ptr), ptr, align); + const { + // For pointers of stride != 1, we verify the algorithm against the naivest possible + // implementation + let mut align = 1; + let limit = 32; + while align < limit { + let mut ptr = 1; + while ptr < 4 * align { + unsafe { + #[repr(packed)] + struct A3(u16, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + struct A4(u32); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A5(u32, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A6(u32, u16); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A7(u32, u16, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A8(u32, u32); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A9(u32, u32, u8); + test_stride::(ptr::invalid::(ptr), ptr, align); + + #[repr(packed)] + struct A10(u32, u32, u16); + test_stride::(ptr::invalid::(ptr), ptr, align); + + test_stride::(ptr::invalid::(ptr), ptr, align); + test_stride::(ptr::invalid::(ptr), ptr, align); + } + ptr += 1; } + align = (align + 1).next_power_of_two(); } - align = (align + 1).next_power_of_two(); } } @@ -632,6 +636,24 @@ fn align_offset_issue_103361() { let _ = (SIZE as *const HugeSize).align_offset(SIZE); } +#[test] +#[cfg(not(bootstrap))] +fn align_offset_issue_103361_const() { + #[cfg(target_pointer_width = "64")] + const SIZE: usize = 1 << 47; + #[cfg(target_pointer_width = "32")] + const SIZE: usize = 1 << 30; + #[cfg(target_pointer_width = "16")] + const SIZE: usize = 1 << 13; + struct HugeSize([u8; SIZE - 1]); + + const { + assert!(ptr::invalid::(SIZE - 1).align_offset(SIZE) == SIZE - 1); + assert!(ptr::invalid::(SIZE).align_offset(SIZE) == 0); + assert!(ptr::invalid::(SIZE + 1).align_offset(SIZE) == 1); + } +} + #[test] fn is_aligned() { let data = 42; From 3d7e9c4b7fdce062b2b337f708980ecb3e9f4c9a Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Wed, 16 Nov 2022 11:41:18 +0100 Subject: [PATCH 17/20] Revert "don't call `align_offset` during const eval, ever" This reverts commit f3a577bfae376c0222e934911865ed14cddd1539. --- .../src/const_eval/machine.rs | 134 ++++++------------ library/core/src/ptr/mod.rs | 7 +- 2 files changed, 49 insertions(+), 92 deletions(-) diff --git a/compiler/rustc_const_eval/src/const_eval/machine.rs b/compiler/rustc_const_eval/src/const_eval/machine.rs index 92ccc94e63024..04e68b9645525 100644 --- a/compiler/rustc_const_eval/src/const_eval/machine.rs +++ b/compiler/rustc_const_eval/src/const_eval/machine.rs @@ -2,10 +2,11 @@ use rustc_hir::def::DefKind; use rustc_hir::LangItem; use rustc_middle::mir; use rustc_middle::mir::interpret::PointerArithmetic; -use rustc_middle::ty::layout::LayoutOf; +use rustc_middle::ty::layout::FnAbiOf; use rustc_middle::ty::{self, Ty, TyCtxt}; use std::borrow::Borrow; use std::hash::Hash; +use std::ops::ControlFlow; use rustc_data_structures::fx::FxIndexMap; use rustc_data_structures::fx::IndexEntry; @@ -20,8 +21,8 @@ use rustc_target::abi::{Align, Size}; use rustc_target::spec::abi::Abi as CallAbi; use crate::interpret::{ - self, compile_time_machine, AllocId, ConstAllocation, Frame, ImmTy, InterpCx, InterpResult, - OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, + self, compile_time_machine, AllocId, ConstAllocation, FnVal, Frame, ImmTy, InterpCx, + InterpResult, OpTy, PlaceTy, Pointer, Scalar, StackPopUnwind, }; use super::error::*; @@ -191,21 +192,24 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { return Ok(Some(new_instance)); } else if Some(def_id) == self.tcx.lang_items().align_offset_fn() { - // For align_offset, we replace the function call entirely. - self.align_offset(instance, args, dest, ret)?; - return Ok(None); + // For align_offset, we replace the function call if the pointer has no address. + match self.align_offset(instance, args, dest, ret)? { + ControlFlow::Continue(()) => return Ok(Some(instance)), + ControlFlow::Break(()) => return Ok(None), + } } Ok(Some(instance)) } - /// This function replaces `align_offset(ptr, target_align)` in const eval, because the - /// pointer may not have an address. + /// `align_offset(ptr, target_align)` needs special handling in const eval, because the pointer + /// may not have an address. /// - /// If `ptr` does have a known address, we forward it to [`Self::align_offset_impl`]. + /// If `ptr` does have a known address, then we return `CONTINUE` and the function call should + /// proceed as normal. /// /// If `ptr` doesn't have an address, but its underlying allocation's alignment is at most - /// `target_align`, then we call [`Self::align_offset_impl`] with an dummy address relative - /// to the allocation. + /// `target_align`, then we call the function again with an dummy address relative to the + /// allocation. /// /// If `ptr` doesn't have an address and `target_align` is stricter than the underlying /// allocation's alignment, then we return `usize::MAX` immediately. @@ -215,103 +219,53 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> { args: &[OpTy<'tcx>], dest: &PlaceTy<'tcx>, ret: Option, - ) -> InterpResult<'tcx> { + ) -> InterpResult<'tcx, ControlFlow<()>> { assert_eq!(args.len(), 2); let ptr = self.read_pointer(&args[0])?; let target_align = self.read_scalar(&args[1])?.to_machine_usize(self)?; - let pointee_ty = instance.substs.type_at(0); - let stride = self.layout_of(pointee_ty)?.size.bytes(); - if !target_align.is_power_of_two() { throw_ub_format!("`align_offset` called with non-power-of-two align: {}", target_align); } - let mut align_offset = match self.ptr_try_get_alloc_id(ptr) { + match self.ptr_try_get_alloc_id(ptr) { Ok((alloc_id, offset, _extra)) => { - // Extract the address relative to a base that is definitely sufficiently aligned. let (_size, alloc_align, _kind) = self.get_alloc_info(alloc_id); if target_align <= alloc_align.bytes() { - // The pointer *is* alignable in const. We use an address relative to the - // allocation base that is definitely sufficiently aligned. - let addr = offset.bytes(); - Self::align_offset_impl(addr, stride, target_align) + // Extract the address relative to the allocation base that is definitely + // sufficiently aligned and call `align_offset` again. + let addr = ImmTy::from_uint(offset.bytes(), args[0].layout).into(); + let align = ImmTy::from_uint(target_align, args[1].layout).into(); + let fn_abi = self.fn_abi_of_instance(instance, ty::List::empty())?; + + // We replace the entire entire function call with a "tail call". + // Note that this happens before the frame of the original function + // is pushed on the stack. + self.eval_fn_call( + FnVal::Instance(instance), + (CallAbi::Rust, fn_abi), + &[addr, align], + /* with_caller_location = */ false, + dest, + ret, + StackPopUnwind::NotAllowed, + )?; + Ok(ControlFlow::BREAK) } else { - // The pointer *is not* alignable in const, return `usize::MAX`. - // (We clamp this to machine `usize` below.) - u64::MAX + // Not alignable in const, return `usize::MAX`. + let usize_max = Scalar::from_machine_usize(self.machine_usize_max(), self); + self.write_scalar(usize_max, dest)?; + self.return_to_block(ret)?; + Ok(ControlFlow::BREAK) } } - Err(addr) => { - // The pointer has a known address. - Self::align_offset_impl(addr, stride, target_align) - } - }; - - let usize_max = self.machine_usize_max(); - if align_offset > usize_max { - align_offset = usize_max; - } - - self.write_scalar(Scalar::from_machine_usize(align_offset, self), dest)?; - self.return_to_block(ret)?; - - Ok(()) - } - - /// Const eval implementation of `#[lang = "align_offset"]`. - /// See the runtime version for a detailed explanation how this works. - fn align_offset_impl(addr: u64, stride: u64, align: u64) -> u64 { - assert!(align.is_power_of_two()); - - let addr_mod_align = addr % align; - - if addr_mod_align == 0 { - // The address is already sufficiently aligned. - return 0; - } - - if stride == 0 { - // The address cannot be aligned. - return u64::MAX; - } - - if align % stride == 0 { - let byte_offset = align - addr_mod_align; - if byte_offset % stride == 0 { - return byte_offset / stride; - } else { - return u64::MAX; + Err(_addr) => { + // The pointer has an address, continue with function call. + Ok(ControlFlow::CONTINUE) } } - - // This only works, because `align` is a power of two. - let gcd = 1u64 << (stride | align).trailing_zeros(); - - if addr % gcd != 0 { - // The address cannot be aligned. - return u64::MAX; - } - - // Instead of `(addr + offset * stride) % align == 0`, we solve - // `((addr + offset * stride) / gcd) % (align / gcd) == 0`. - let addr2 = addr / gcd; - let align2 = align / gcd; - let stride2 = stride / gcd; - - let mut stride_inv = 1u64; - let mut mod_gate = 2u64; - let mut overflow = false; - while !overflow && mod_gate < align2 { - stride_inv = - stride_inv.wrapping_mul(2u64.wrapping_sub(stride2.wrapping_mul(stride_inv))); - (mod_gate, overflow) = mod_gate.overflowing_mul(mod_gate); - } - - let byte_offset = align2 - addr2 % align2; - byte_offset.wrapping_mul(stride_inv) % align2 } /// See documentation on the `ptr_guaranteed_cmp` intrinsic. diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index e762837ff90a9..9283b81a84e2b 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -1591,7 +1591,6 @@ pub unsafe fn write_volatile(dst: *mut T, src: T) { /// /// Any questions go to @nagisa. #[lang = "align_offset"] -#[rustc_do_not_const_check] // hooked by const-eval pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usize { // FIXME(#75598): Direct use of these intrinsics improves codegen significantly at opt-level <= // 1, where the method versions of these operations are not inlined. @@ -1651,9 +1650,13 @@ pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usiz inverse & m_minus_one } - let addr = p.addr(); let stride = mem::size_of::(); + // SAFETY: At runtime, transmuting a pointer to `usize` is always safe, because they have the + // same layout. During const eval, we hook this function to ensure that the pointer always has + // an address (only the standard library can do this). + let addr: usize = unsafe { mem::transmute(p) }; + // SAFETY: `a` is a power-of-two, therefore non-zero. let a_minus_one = unsafe { unchecked_sub(a, 1) }; From e90d15b247bdaf12a6d51f492bf94d2e0064d177 Mon Sep 17 00:00:00 2001 From: Lukas Date: Wed, 16 Nov 2022 15:08:35 +0100 Subject: [PATCH 18/20] Update comment on pointer-to-usize transmute Co-authored-by: Ralf Jung --- library/core/src/ptr/mod.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/library/core/src/ptr/mod.rs b/library/core/src/ptr/mod.rs index 9283b81a84e2b..73923753a3020 100644 --- a/library/core/src/ptr/mod.rs +++ b/library/core/src/ptr/mod.rs @@ -1652,9 +1652,10 @@ pub(crate) const unsafe fn align_offset(p: *const T, a: usize) -> usiz let stride = mem::size_of::(); - // SAFETY: At runtime, transmuting a pointer to `usize` is always safe, because they have the - // same layout. During const eval, we hook this function to ensure that the pointer always has - // an address (only the standard library can do this). + // SAFETY: This is just an inlined `p.addr()` (which is not + // a `const fn` so we cannot call it). + // During const eval, we hook this function to ensure that the pointer never + // has provenance, making this sound. let addr: usize = unsafe { mem::transmute(p) }; // SAFETY: `a` is a power-of-two, therefore non-zero. From 53c2ee8e9b815b687a8f203bc5fb99d535a10a0d Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Wed, 16 Nov 2022 21:07:51 +0100 Subject: [PATCH 19/20] fix assembly test on apple --- src/test/assembly/is_aligned.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/test/assembly/is_aligned.rs b/src/test/assembly/is_aligned.rs index e8e6ad5cf112e..04b5de8342370 100644 --- a/src/test/assembly/is_aligned.rs +++ b/src/test/assembly/is_aligned.rs @@ -13,7 +13,7 @@ // CHECK: decq // CHECK-NEXT: testq // CHECK-NEXT: sete -// CHECK-NEXT: retq +// CHECK: retq #[no_mangle] pub unsafe fn is_aligned_to_unchecked(ptr: *const u8, align: usize) -> bool { unsafe { @@ -24,7 +24,7 @@ pub unsafe fn is_aligned_to_unchecked(ptr: *const u8, align: usize) -> bool { // CHECK-LABEL: is_aligned_1 // CHECK: movb $1 -// CHECK-NEXT: retq +// CHECK: retq #[no_mangle] pub fn is_aligned_1(ptr: *const u8) -> bool { ptr.is_aligned() @@ -33,7 +33,7 @@ pub fn is_aligned_1(ptr: *const u8) -> bool { // CHECK-LABEL: is_aligned_2 // CHECK: testb $1 // CHECK-NEXT: sete -// CHECK-NEXT: retq +// CHECK: retq #[no_mangle] pub fn is_aligned_2(ptr: *const u16) -> bool { ptr.is_aligned() @@ -42,7 +42,7 @@ pub fn is_aligned_2(ptr: *const u16) -> bool { // CHECK-LABEL: is_aligned_4 // CHECK: testb $3 // CHECK-NEXT: sete -// CHECK-NEXT: retq +// CHECK: retq #[no_mangle] pub fn is_aligned_4(ptr: *const u32) -> bool { ptr.is_aligned() @@ -51,7 +51,7 @@ pub fn is_aligned_4(ptr: *const u32) -> bool { // CHECK-LABEL: is_aligned_8 // CHECK: testb $7 // CHECK-NEXT: sete -// CHECK-NEXT: retq +// CHECK: retq #[no_mangle] pub fn is_aligned_8(ptr: *const u64) -> bool { ptr.is_aligned() From c9c017dfb55e375800c3e424311939a7ba3d4deb Mon Sep 17 00:00:00 2001 From: Lukas Markeffsky <@> Date: Fri, 18 Nov 2022 13:59:21 +0100 Subject: [PATCH 20/20] update provenance test * fix allocation alignment for 16bit platforms * add edge case where `stride % align != 0` on pointers with provenance --- library/core/tests/ptr.rs | 48 +++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) diff --git a/library/core/tests/ptr.rs b/library/core/tests/ptr.rs index 036acc46daba8..390148550a4b3 100644 --- a/library/core/tests/ptr.rs +++ b/library/core/tests/ptr.rs @@ -564,9 +564,15 @@ fn align_offset_various_strides_const() { #[cfg(not(bootstrap))] fn align_offset_with_provenance_const() { const { - let data = 42; + // On some platforms (e.g. msp430-none-elf), the alignment of `i32` is less than 4. + #[repr(align(4))] + struct AlignedI32(i32); - let ptr: *const i32 = &data; + let data = AlignedI32(42); + + // `stride % align == 0` (usual case) + + let ptr: *const i32 = &data.0; assert!(ptr.align_offset(1) == 0); assert!(ptr.align_offset(2) == 0); assert!(ptr.align_offset(4) == 0); @@ -621,6 +627,44 @@ fn align_offset_with_provenance_const() { assert!(ptr3.align_offset(8) == usize::MAX); assert!(ptr3.wrapping_byte_add(1).align_offset(1) == 0); assert!(ptr3.wrapping_byte_add(1).align_offset(2) == usize::MAX); + + // `stride % align != 0` (edge case) + + let ptr4: *const [u8; 3] = ptr.cast(); + assert!(ptr4.align_offset(1) == 0); + assert!(ptr4.align_offset(2) == 0); + assert!(ptr4.align_offset(4) == 0); + assert!(ptr4.align_offset(8) == usize::MAX); + assert!(ptr4.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr4.wrapping_byte_add(1).align_offset(2) == 1); + assert!(ptr4.wrapping_byte_add(1).align_offset(4) == 1); + assert!(ptr4.wrapping_byte_add(1).align_offset(8) == usize::MAX); + assert!(ptr4.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr4.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr4.wrapping_byte_add(2).align_offset(4) == 2); + assert!(ptr4.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr4.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr4.wrapping_byte_add(3).align_offset(2) == 1); + assert!(ptr4.wrapping_byte_add(3).align_offset(4) == 3); + assert!(ptr4.wrapping_byte_add(3).align_offset(8) == usize::MAX); + + let ptr5: *const [u8; 5] = ptr.cast(); + assert!(ptr5.align_offset(1) == 0); + assert!(ptr5.align_offset(2) == 0); + assert!(ptr5.align_offset(4) == 0); + assert!(ptr5.align_offset(8) == usize::MAX); + assert!(ptr5.wrapping_byte_add(1).align_offset(1) == 0); + assert!(ptr5.wrapping_byte_add(1).align_offset(2) == 1); + assert!(ptr5.wrapping_byte_add(1).align_offset(4) == 3); + assert!(ptr5.wrapping_byte_add(1).align_offset(8) == usize::MAX); + assert!(ptr5.wrapping_byte_add(2).align_offset(1) == 0); + assert!(ptr5.wrapping_byte_add(2).align_offset(2) == 0); + assert!(ptr5.wrapping_byte_add(2).align_offset(4) == 2); + assert!(ptr5.wrapping_byte_add(2).align_offset(8) == usize::MAX); + assert!(ptr5.wrapping_byte_add(3).align_offset(1) == 0); + assert!(ptr5.wrapping_byte_add(3).align_offset(2) == 1); + assert!(ptr5.wrapping_byte_add(3).align_offset(4) == 1); + assert!(ptr5.wrapping_byte_add(3).align_offset(8) == usize::MAX); } }