Auto merge of rust-lang#113166 - moulins:ref-niches-initial, r=oli-obk

Prototype: Add unstable `-Z reference-niches` option

MCP: rust-lang/compiler-team#641
Relevant RFC: rust-lang/rfcs#3204

This prototype adds a new `-Z reference-niches` option, controlling the range of valid bit-patterns for reference types (`&T` and `&mut T`), thereby enabling new enum niching opportunities. Like `-Z randomize-layout`, this setting is crate-local; as such, references to built-in types (primitives, tuples, ...) are not affected.

The possible settings are (here, `MAX` denotes the all-1 bit-pattern):
| `-Z reference-niches=` | Valid range |
|:---:|:---:|
| `null` (the default) | `1..=MAX` |
| `size` | `1..=(MAX- size)` |
| `align` | `align..=MAX.align_down_to(align)` |
| `size,align` | `align..=(MAX-size).align_down_to(align)` |

------

This is very WIP, and I'm not sure the approach I've taken here is the best one, but stage 1 tests pass locally; I believe this is in a good enough state to unleash this upon unsuspecting 3rd-party code, and see what breaks.

compiler/rustc_abi/src/lib.rs

@@ -49,6 +49,14 @@ bitflags! {
  }
 }
 
+/// Which niches (beyond the `null` niche) are available on references.
+#[derive(Default, Copy, Clone, Hash, Debug, Eq, PartialEq)]
+#[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_Generic))]
+pub struct ReferenceNichePolicy {
+ pub size: bool,
+ pub align: bool,
+}
+
 #[derive(Copy, Clone, Debug, Eq, PartialEq)]
 #[cfg_attr(feature = "nightly", derive(Encodable, Decodable, HashStable_Generic))]
 pub enum IntegerType {
@@ -346,6 +354,33 @@ impl TargetDataLayout {
  }
  }
 
+ #[inline]
+ pub fn target_usize_max(&self) -> u64 {
+ self.pointer_size.unsigned_int_max().try_into().unwrap()
+ }
+
+ #[inline]
+ pub fn target_isize_min(&self) -> i64 {
+ self.pointer_size.signed_int_min().try_into().unwrap()
+ }
+
+ #[inline]
+ pub fn target_isize_max(&self) -> i64 {
+ self.pointer_size.signed_int_max().try_into().unwrap()
+ }
+
+ /// Returns the (inclusive) range of possible addresses for an allocation with
+ /// the given size and alignment.
+ ///
+ /// Note that this doesn't take into account target-specific limitations.
+ #[inline]
+ pub fn address_range_for(&self, size: Size, align: Align) -> (u64, u64) {
+ let end = Size::from_bytes(self.target_usize_max());
+ let min = align.bytes();
+ let max = (end - size).align_down_to(align).bytes();
+ (min, max)
+ }
+
  #[inline]
  pub fn vector_align(&self, vec_size: Size) -> AbiAndPrefAlign {
  for &(size, align) in &self.vector_align {
@@ -473,6 +508,12 @@ impl Size {
  Size::from_bytes((self.bytes() + mask) & !mask)
  }
 
+ #[inline]
+ pub fn align_down_to(self, align: Align) -> Size {
+ let mask = align.bytes() - 1;
+ Size::from_bytes(self.bytes() & !mask)
+ }
+
  #[inline]
  pub fn is_aligned(self, align: Align) -> bool {
  let mask = align.bytes() - 1;
@@ -967,6 +1008,43 @@ impl WrappingRange {
  }
  }
 
+ /// Returns `true` if `range` is contained in `self`.
+ #[inline(always)]
+ pub fn contains_range<I: Into<u128> + Ord>(&self, range: RangeInclusive<I>) -> bool {
+ if range.is_empty() {
+ return true;
+ }
+
+ let (vmin, vmax) = range.into_inner();
+ let (vmin, vmax) = (vmin.into(), vmax.into());
+
+ if self.start <= self.end {
+ self.start <= vmin && vmax <= self.end
+ } else {
+ // The last check is needed to cover the following case:
+ // `vmin ... start, end ... vmax`. In this special case there is no gap
+ // between `start` and `end` so we must return true.
+ self.start <= vmin || vmax <= self.end || self.start == self.end + 1
+ }
+ }
+
+ /// Returns `true` if `range` has an overlap with `self`.
+ #[inline(always)]
+ pub fn overlaps_range<I: Into<u128> + Ord>(&self, range: RangeInclusive<I>) -> bool {
+ if range.is_empty() {
+ return false;
+ }
+
+ let (vmin, vmax) = range.into_inner();
+ let (vmin, vmax) = (vmin.into(), vmax.into());
+
+ if self.start <= self.end {
+ self.start <= vmax && vmin <= self.end
+ } else {
+ self.start <= vmax || vmin <= self.end
+ }
+ }
+
  /// Returns `self` with replaced `start`
  #[inline(always)]
  pub fn with_start(mut self, start: u128) -> Self {
@@ -984,9 +1062,15 @@ impl WrappingRange {
  /// Returns `true` if `size` completely fills the range.
  #[inline]
  pub fn is_full_for(&self, size: Size) -> bool {
+ debug_assert!(self.is_in_range_for(size));
+ self.start == (self.end.wrapping_add(1) & size.unsigned_int_max())
+ }
+
+ /// Returns `true` if the range is valid for `size`.
+ #[inline(always)]
+ pub fn is_in_range_for(&self, size: Size) -> bool {
  let max_value = size.unsigned_int_max();
- debug_assert!(self.start <= max_value && self.end <= max_value);
- self.start == (self.end.wrapping_add(1) & max_value)
+ self.start <= max_value && self.end <= max_value
  }
 }
 
@@ -1427,16 +1511,21 @@ impl Niche {
 
  pub fn reserve<C: HasDataLayout>(&self, cx: &C, count: u128) -> Option<(u128, Scalar)> {
  assert!(count > 0);
+ if count > self.available(cx) {
+ return None;
+ }
 
  let Self { value, valid_range: v, .. } = *self;
- let size = value.size(cx);
- assert!(size.bits() <= 128);
- let max_value = size.unsigned_int_max();
+ let max_value = value.size(cx).unsigned_int_max();
+ let distance_end_zero = max_value - v.end;
 
- let niche = v.end.wrapping_add(1)..v.start;
- let available = niche.end.wrapping_sub(niche.start) & max_value;
- if count > available {
- return None;
+ // Null-pointer optimization. This is guaranteed by Rust (at least for `Option<_>`),
+ // and offers better codegen opportunities.
+ if count == 1 && matches!(value, Pointer(_)) && !v.contains(0) {
+ // Select which bound to move to minimize the number of lost niches.
+ let valid_range =
+ if v.start - 1 > distance_end_zero { v.with_end(0) } else { v.with_start(0) };
+ return Some((0, Scalar::Initialized { value, valid_range }));
  }
 
  // Extend the range of valid values being reserved by moving either `v.start` or `v.end` bound.
@@ -1459,7 +1548,6 @@ impl Niche {
  let end = v.end.wrapping_add(count) & max_value;
  Some((start, Scalar::Initialized { value, valid_range: v.with_end(end) }))
  };
- let distance_end_zero = max_value - v.end;
  if v.start > v.end {
  // zero is unavailable because wrapping occurs
  move_end(v)

compiler/rustc_codegen_gcc/src/type_of.rs

@@ -339,7 +339,8 @@ impl<'tcx> LayoutGccExt<'tcx> for TyAndLayout<'tcx> {
  return pointee;
  }
 
- let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset);
+ let assume_valid_ptr = true;
+ let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset, assume_valid_ptr);
 
  cx.pointee_infos.borrow_mut().insert((self.ty, offset), result);
  result

compiler/rustc_codegen_llvm/src/type_of.rs

@@ -411,8 +411,8 @@ impl<'tcx> LayoutLlvmExt<'tcx> for TyAndLayout<'tcx> {
  if let Some(&pointee) = cx.pointee_infos.borrow().get(&(self.ty, offset)) {
  return pointee;
  }
-
- let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset);
+ let assume_valid_ptr = true;
+ let result = Ty::ty_and_layout_pointee_info_at(*self, cx, offset, assume_valid_ptr);
 
  cx.pointee_infos.borrow_mut().insert((self.ty, offset), result);
  result

compiler/rustc_const_eval/messages.ftl

@@ -244,7 +244,6 @@ const_eval_not_enough_caller_args =
 const_eval_null_box = {$front_matter}: encountered a null box
 const_eval_null_fn_ptr = {$front_matter}: encountered a null function pointer
 const_eval_null_ref = {$front_matter}: encountered a null reference
-const_eval_nullable_ptr_out_of_range = {$front_matter}: encountered a potentially null pointer, but expected something that cannot possibly fail to be {$in_range}
 const_eval_nullary_intrinsic_fail =
  could not evaluate nullary intrinsic
 

compiler/rustc_const_eval/src/const_eval/machine.rs

@@ -1,7 +1,6 @@
 use rustc_hir::def::DefKind;
 use rustc_hir::{LangItem, CRATE_HIR_ID};
 use rustc_middle::mir;
-use rustc_middle::mir::interpret::PointerArithmetic;
 use rustc_middle::ty::layout::{FnAbiOf, TyAndLayout};
 use rustc_middle::ty::{self, Ty, TyCtxt};
 use rustc_session::lint::builtin::INVALID_ALIGNMENT;
@@ -17,7 +16,7 @@ use rustc_ast::Mutability;
 use rustc_hir::def_id::DefId;
 use rustc_middle::mir::AssertMessage;
 use rustc_span::symbol::{sym, Symbol};
-use rustc_target::abi::{Align, Size};
+use rustc_target::abi::{Align, HasDataLayout as _, Size};
 use rustc_target::spec::abi::Abi as CallAbi;
 
 use crate::errors::{LongRunning, LongRunningWarn};
@@ -304,8 +303,8 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
  Ok(ControlFlow::Break(()))
  } else {
  // Not alignable in const, return `usize::MAX`.
- let usize_max = Scalar::from_target_usize(self.target_usize_max(), self);
- self.write_scalar(usize_max, dest)?;
+ let usize_max = self.data_layout().target_usize_max();
+ self.write_scalar(Scalar::from_target_usize(usize_max, self), dest)?;
  self.return_to_block(ret)?;
  Ok(ControlFlow::Break(()))
  }
@@ -333,7 +332,7 @@ impl<'mir, 'tcx: 'mir> CompileTimeEvalContext<'mir, 'tcx> {
  // Inequality with integers other than null can never be known for sure.
  (Scalar::Int(int), ptr @ Scalar::Ptr(..))
  | (ptr @ Scalar::Ptr(..), Scalar::Int(int))
- if int.is_null() && !self.scalar_may_be_null(ptr)? =>
+ if int.is_null() && !self.ptr_scalar_range(ptr)?.contains(&0) =>
  {
  0
  }

compiler/rustc_const_eval/src/errors.rs

@@ -617,7 +617,6 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
  MutableRefInConst => const_eval_mutable_ref_in_const,
  NullFnPtr => const_eval_null_fn_ptr,
  NeverVal => const_eval_never_val,
- NullablePtrOutOfRange { .. } => const_eval_nullable_ptr_out_of_range,
  PtrOutOfRange { .. } => const_eval_ptr_out_of_range,
  OutOfRange { .. } => const_eval_out_of_range,
  UnsafeCell => const_eval_unsafe_cell,
@@ -732,9 +731,7 @@ impl<'tcx> ReportErrorExt for ValidationErrorInfo<'tcx> {
  | InvalidFnPtr { value } => {
  err.set_arg("value", value);
  }
- NullablePtrOutOfRange { range, max_value } | PtrOutOfRange { range, max_value } => {
- add_range_arg(range, max_value, handler, err)
- }
+ PtrOutOfRange { range, max_value } => add_range_arg(range, max_value, handler, err),
  OutOfRange { range, max_value, value } => {
  err.set_arg("value", value);
  add_range_arg(range, max_value, handler, err);

compiler/rustc_const_eval/src/interpret/discriminant.rs

@@ -2,8 +2,7 @@
 
 use rustc_middle::ty::layout::{LayoutOf, PrimitiveExt};
 use rustc_middle::{mir, ty};
-use rustc_target::abi::{self, TagEncoding};
-use rustc_target::abi::{VariantIdx, Variants};
+use rustc_target::abi::{self, TagEncoding, VariantIdx, Variants, WrappingRange};
 
 use super::{ImmTy, InterpCx, InterpResult, Machine, OpTy, PlaceTy, Scalar};
 
@@ -180,19 +179,24 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
  // discriminant (encoded in niche/tag) and variant index are the same.
  let variants_start = niche_variants.start().as_u32();
  let variants_end = niche_variants.end().as_u32();
+ let variants_len = u128::from(variants_end - variants_start);
  let variant = match tag_val.try_to_int() {
  Err(dbg_val) => {
  // So this is a pointer then, and casting to an int failed.
  // Can only happen during CTFE.
- // The niche must be just 0, and the ptr not null, then we know this is
- // okay. Everything else, we conservatively reject.
- let ptr_valid = niche_start == 0
- && variants_start == variants_end
- && !self.scalar_may_be_null(tag_val)?;
- if !ptr_valid {
+ // The pointer and niches ranges must be disjoint, then we know
+ // this is the untagged variant (as the value is not in the niche).
+ // Everything else, we conservatively reject.
+ let range = self.ptr_scalar_range(tag_val)?;
+ let niches = WrappingRange {
+ start: niche_start,
+ end: niche_start.wrapping_add(variants_len),
+ };
+ if niches.overlaps_range(range) {
  throw_ub!(InvalidTag(dbg_val))
+ } else {
+ untagged_variant
  }
- untagged_variant
  }
  Ok(tag_bits) => {
  let tag_bits = tag_bits.assert_bits(tag_layout.size);
@@ -205,7 +209,7 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
  let variant_index_relative =
  variant_index_relative_val.to_scalar().assert_bits(tag_val.layout.size);
  // Check if this is in the range that indicates an actual discriminant.
- if variant_index_relative <= u128::from(variants_end - variants_start) {
+ if variant_index_relative <= variants_len {
  let variant_index_relative = u32::try_from(variant_index_relative)
  .expect("we checked that this fits into a u32");
  // Then computing the absolute variant idx should not overflow any more.

compiler/rustc_const_eval/src/interpret/intrinsics.rs

@@ -5,17 +5,15 @@
 use rustc_hir::def_id::DefId;
 use rustc_middle::mir::{
  self,
- interpret::{
- Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, PointerArithmetic, Scalar,
- },
+ interpret::{Allocation, ConstAllocation, ConstValue, GlobalId, InterpResult, Scalar},
  BinOp, NonDivergingIntrinsic,
 };
 use rustc_middle::ty;
 use rustc_middle::ty::layout::{LayoutOf as _, ValidityRequirement};
 use rustc_middle::ty::GenericArgsRef;
 use rustc_middle::ty::{Ty, TyCtxt};
 use rustc_span::symbol::{sym, Symbol};
-use rustc_target::abi::{Abi, Align, Primitive, Size};
+use rustc_target::abi::{Abi, Align, HasDataLayout as _, Primitive, Size};
 
 use super::{
  util::ensure_monomorphic_enough, CheckInAllocMsg, ImmTy, InterpCx, Machine, OpTy, PlaceTy,
@@ -361,11 +359,12 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
  )?;
 
  // Perform division by size to compute return value.
+ let dl = self.data_layout();
  let ret_layout = if intrinsic_name == sym::ptr_offset_from_unsigned {
- assert!(0 <= dist && dist <= self.target_isize_max());
+ assert!(0 <= dist && dist <= dl.target_isize_max());
  usize_layout
  } else {
- assert!(self.target_isize_min() <= dist && dist <= self.target_isize_max());
+ assert!(dl.target_isize_min() <= dist && dist <= dl.target_isize_max());
  isize_layout
  };
  let pointee_layout = self.layout_of(instance_args.type_at(0))?;

compiler/rustc_const_eval/src/interpret/memory.rs

@@ -10,6 +10,7 @@ use std::assert_matches::assert_matches;
 use std::borrow::Cow;
 use std::collections::VecDeque;
 use std::fmt;
+use std::ops::RangeInclusive;
 use std::ptr;
 
 use rustc_ast::Mutability;
@@ -1222,24 +1223,34 @@ impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
 
 /// Machine pointer introspection.
 impl<'mir, 'tcx: 'mir, M: Machine<'mir, 'tcx>> InterpCx<'mir, 'tcx, M> {
- /// Test if this value might be null.
+ /// Turn a pointer-sized scalar into a (non-empty) range of possible values.
  /// If the machine does not support ptr-to-int casts, this is conservative.
- pub fn scalar_may_be_null(&self, scalar: Scalar<M::Provenance>) -> InterpResult<'tcx, bool> {
- Ok(match scalar.try_to_int() {
- Ok(int) => int.is_null(),
- Err(_) => {
- // Can only happen during CTFE.
- let ptr = scalar.to_pointer(self)?;
- match self.ptr_try_get_alloc_id(ptr) {
- Ok((alloc_id, offset, _)) => {
- let (size, _align, _kind) = self.get_alloc_info(alloc_id);
- // If the pointer is out-of-bounds, it may be null.
- // Note that one-past-the-end (offset == size) is still inbounds, and never null.
- offset > size
- }
- Err(_offset) => bug!("a non-int scalar is always a pointer"),
+ pub fn ptr_scalar_range(
+ &self,
+ scalar: Scalar<M::Provenance>,
+ ) -> InterpResult<'tcx, RangeInclusive<u64>> {
+ if let Ok(int) = scalar.to_target_usize(self) {
+ return Ok(int..=int);
+ }
+
+ let ptr = scalar.to_pointer(self)?;
+
+ // Can only happen during CTFE.
+ Ok(match self.ptr_try_get_alloc_id(ptr) {
+ Ok((alloc_id, offset, _)) => {
+ let offset = offset.bytes();
+ let (size, align, _) = self.get_alloc_info(alloc_id);
+ let dl = self.data_layout();
+ if offset > size.bytes() {
+ // If the pointer is out-of-bounds, we do not have a
+ // meaningful range to return.
+ 0..=dl.target_usize_max()
+ } else {
+ let (min, max) = dl.address_range_for(size, align);
+ (min + offset)..=(max + offset)
  }
  }
+ Err(_offset) => bug!("a non-int scalar is always a pointer"),
  })
  }