CoAlloc: Big Squash; const_trait TODO and ICE on Dec 4, 2023

compiler/rustc_arena/src/lib.rs

@@ -11,12 +11,16 @@
     html_root_url = "https://doc.rust-lang.org/nightly/nightly-rustc/",
     test(no_crate_inject, attr(deny(warnings)))
 )]
+#![allow(incomplete_features)]
 #![cfg_attr(not(bootstrap), doc(rust_logo))]
 #![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![feature(core_intrinsics)]
 #![feature(dropck_eyepatch)]
 #![feature(new_uninit)]
 #![feature(maybe_uninit_slice)]
+//#![feature(min_specialization)]
+// TODO CoAlloc specialization!
+#![feature(specialization)]
 #![feature(decl_macro)]
 #![feature(rustc_attrs)]
 #![cfg_attr(test, feature(test))]

compiler/rustc_ast/src/ast.rs

@@ -38,7 +38,6 @@ pub use rustc_type_ir::{Movability, Mutability};
 use std::fmt;
 use std::mem;
 use thin_vec::{thin_vec, ThinVec};
-
 /// A "Label" is an identifier of some point in sources,
 /// e.g. in the following code:
 ///
@@ -3144,30 +3143,31 @@ pub type ForeignItem = Item<ForeignItemKind>;
 mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
+    use std::alloc::{Allocator, Global};
     // tidy-alphabetical-start
     static_assert_size!(AssocItem, 88);
     static_assert_size!(AssocItemKind, 16);
     static_assert_size!(Attribute, 32);
-    static_assert_size!(Block, 32);
-    static_assert_size!(Expr, 72);
-    static_assert_size!(ExprKind, 40);
-    static_assert_size!(Fn, 152);
+    static_assert_size!(Block, 32 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Expr, 72 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(ExprKind, 40 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Fn, 152 + 2 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(ForeignItem, 96);
     static_assert_size!(ForeignItemKind, 24);
     static_assert_size!(GenericArg, 24);
-    static_assert_size!(GenericBound, 56);
-    static_assert_size!(Generics, 40);
-    static_assert_size!(Impl, 136);
-    static_assert_size!(Item, 136);
-    static_assert_size!(ItemKind, 64);
+    static_assert_size!(GenericBound, 56 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Generics, 40 + 2 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Impl, 136 + 3 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(Item, 136 + 3 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
+    static_assert_size!(ItemKind, 64 + 3 * mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(LitKind, 24);
     static_assert_size!(Local, 72);
     static_assert_size!(MetaItemLit, 40);
     static_assert_size!(Param, 40);
-    static_assert_size!(Pat, 72);
+    static_assert_size!(Pat, 72 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(Path, 24);
     static_assert_size!(PathSegment, 24);
-    static_assert_size!(PatKind, 48);
+    static_assert_size!(PatKind, 48 + mem::size_of::<<Global as Allocator>::CoAllocMeta>());
     static_assert_size!(Stmt, 32);
     static_assert_size!(StmtKind, 16);
     static_assert_size!(Ty, 64);

compiler/rustc_ast/src/lib.rs

@@ -11,9 +11,11 @@
 #![cfg_attr(not(bootstrap), doc(rust_logo))]
 #![cfg_attr(not(bootstrap), allow(internal_features))]
 #![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
+#![feature(allocator_api)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(const_trait_impl)]
+#![feature(global_co_alloc_meta)]
 #![feature(if_let_guard)]
 #![feature(let_chains)]
 #![feature(min_specialization)]

compiler/rustc_hir_typeck/src/fn_ctxt/suggestions.rs

@@ -390,7 +390,7 @@ impl<'a, 'tcx> FnCtxt<'a, 'tcx> {
                         return None; // do not suggest code that is already there (#53348)
                     }
 
-                    let method_call_list = [sym::to_vec, sym::to_string];
+                    let method_call_list = [sym::to_vec, sym::to_vec_co, sym::to_string];
                     let mut sugg = if let ExprKind::MethodCall(receiver_method, ..) = expr.kind
                         && receiver_method.ident.name == sym::clone
                         && method_call_list.contains(&conversion_method.name)

compiler/rustc_middle/src/lib.rs

@@ -35,6 +35,7 @@
 #![cfg_attr(bootstrap, feature(generators))]
 #![cfg_attr(not(bootstrap), feature(coroutines))]
 #![feature(get_mut_unchecked)]
+#![feature(global_co_alloc_meta)]
 #![feature(if_let_guard)]
 #![feature(inline_const)]
 #![feature(iter_from_coroutine)]

compiler/rustc_middle/src/mir/mod.rs

@@ -1629,7 +1629,10 @@ mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
     // tidy-alphabetical-start
-    static_assert_size!(BasicBlockData<'_>, 136);
+    static_assert_size!(
+        BasicBlockData<'_>,
+        136 + mem::size_of::<<std::alloc::Global as std::alloc::Allocator>::CoAllocMeta>()
+    );
     static_assert_size!(LocalDecl<'_>, 40);
     static_assert_size!(SourceScopeData<'_>, 72);
     static_assert_size!(Statement<'_>, 32);

compiler/rustc_middle/src/mir/syntax.rs

@@ -1442,6 +1442,9 @@ mod size_asserts {
     static_assert_size!(Operand<'_>, 24);
     static_assert_size!(Place<'_>, 16);
     static_assert_size!(PlaceElem<'_>, 24);
-    static_assert_size!(Rvalue<'_>, 40);
+    static_assert_size!(
+        Rvalue<'_>,
+        40 + std::mem::size_of::<<std::alloc::Global as std::alloc::Allocator>::CoAllocMeta>()
+    );
     // tidy-alphabetical-end
 }

compiler/rustc_parse/src/lib.rs

@@ -1,7 +1,9 @@
 //! The main parser interface.
 
+#![feature(allocator_api)]
 #![feature(array_windows)]
 #![feature(box_patterns)]
+#![feature(global_co_alloc_meta)]
 #![feature(if_let_guard)]
 #![feature(iter_intersperse)]
 #![feature(let_chains)]

compiler/rustc_parse/src/parser/attr_wrapper.rs

@@ -458,8 +458,12 @@ fn make_token_stream(
 mod size_asserts {
     use super::*;
     use rustc_data_structures::static_assert_size;
+    use std::alloc::{Allocator, Global};
     // tidy-alphabetical-start
     static_assert_size!(AttrWrapper, 16);
-    static_assert_size!(LazyAttrTokenStreamImpl, 104);
+    static_assert_size!(
+        LazyAttrTokenStreamImpl,
+        104 + std::mem::size_of::<<Global as Allocator>::CoAllocMeta>()
+    );
     // tidy-alphabetical-end
 }

compiler/rustc_parse/src/parser/mod.rs

@@ -179,7 +179,10 @@ pub struct Parser<'a> {
 // This type is used a lot, e.g. it's cloned when matching many declarative macro rules with nonterminals. Make sure
 // it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-rustc_data_structures::static_assert_size!(Parser<'_>, 264);
+rustc_data_structures::static_assert_size!(
+    Parser<'_>,
+    264 + 4 * mem::size_of::<<std::alloc::Global as std::alloc::Allocator>::CoAllocMeta>()
+);
 
 /// Stores span information about a closure.
 #[derive(Clone)]

compiler/rustc_span/src/symbol.rs

@@ -1628,6 +1628,7 @@ symbols! {
         to_string,
         to_string_method,
         to_vec,
+        to_vec_co,
         todo_macro,
         tool_attributes,
         tool_lints,

compiler/rustc_trait_selection/src/lib.rs

@@ -14,10 +14,14 @@
 #![cfg_attr(not(bootstrap), doc(rust_logo))]
 #![cfg_attr(not(bootstrap), feature(rustdoc_internals))]
 #![cfg_attr(not(bootstrap), allow(internal_features))]
+#![feature(allocator_api)]
 #![feature(associated_type_bounds)]
 #![feature(box_patterns)]
 #![feature(control_flow_enum)]
-#![feature(extract_if)]
+// TODO CoAlloc #![feature(drain_filter)]
+// TODO CoAlloc #![feature(extract_if)]
+#![feature(global_co_alloc_meta)]
+// TODO CoAlloc #![feature(hash_drain_filter)]
 #![feature(let_chains)]
 #![feature(if_let_guard)]
 #![feature(never_type)]

compiler/rustc_trait_selection/src/traits/fulfill.rs

@@ -74,7 +74,10 @@ pub struct PendingPredicateObligation<'tcx> {
 
 // `PendingPredicateObligation` is used a lot. Make sure it doesn't unintentionally get bigger.
 #[cfg(all(target_arch = "x86_64", target_pointer_width = "64"))]
-static_assert_size!(PendingPredicateObligation<'_>, 72);
+static_assert_size!(
+    PendingPredicateObligation<'_>,
+    72 + std::mem::size_of::<<std::alloc::Global as std::alloc::Allocator>::CoAllocMeta>()
+);
 
 impl<'tcx> FulfillmentContext<'tcx> {
     /// Creates a new fulfillment context.

library/alloc/src/boxed.rs

@@ -146,6 +146,8 @@
 
 #![stable(feature = "rust1", since = "1.0.0")]
 
+#[cfg(not(no_global_oom_handling))]
+use crate::co_alloc::CoAllocPref;
 use core::any::Any;
 use core::async_iter::AsyncIterator;
 use core::borrow;
@@ -632,7 +634,10 @@ impl<T> Box<[T]> {
     #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     pub fn new_uninit_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
-        unsafe { RawVec::with_capacity(len).into_box(len) }
+        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
+        unsafe {
+            RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::with_capacity(len).into_box(len)
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents, with the memory
@@ -657,7 +662,11 @@ impl<T> Box<[T]> {
     #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
     pub fn new_zeroed_slice(len: usize) -> Box<[mem::MaybeUninit<T>]> {
-        unsafe { RawVec::with_capacity_zeroed(len).into_box(len) }
+        // false = no need for co-alloc metadata, since it would get lost once converted to Box.
+        unsafe {
+            RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::with_capacity_zeroed(len)
+                .into_box(len)
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents. Returns an error if
@@ -692,7 +701,14 @@ impl<T> Box<[T]> {
             };
             Global.allocate(layout)?.cast()
         };
-        unsafe { Ok(RawVec::from_raw_parts_in(ptr.as_ptr(), len, Global).into_box(len)) }
+        unsafe {
+            Ok(RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::from_raw_parts_in(
+                ptr.as_ptr(),
+                len,
+                Global,
+            )
+            .into_box(len))
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents, with the memory
@@ -726,11 +742,22 @@ impl<T> Box<[T]> {
             };
             Global.allocate_zeroed(layout)?.cast()
         };
-        unsafe { Ok(RawVec::from_raw_parts_in(ptr.as_ptr(), len, Global).into_box(len)) }
+        unsafe {
+            Ok(RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::from_raw_parts_in(
+                ptr.as_ptr(),
+                len,
+                Global,
+            )
+            .into_box(len))
+        }
     }
 }
 
-impl<T, A: Allocator> Box<[T], A> {
+#[allow(unused_braces)]
+impl<T, A: Allocator> Box<[T], A>
+where
+    [(); { crate::meta_num_slots!(A, crate::CO_ALLOC_PREF_META_NO!()) }]:,
+{
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator.
     ///
     /// # Examples
@@ -757,8 +784,11 @@ impl<T, A: Allocator> Box<[T], A> {
     #[unstable(feature = "allocator_api", issue = "32838")]
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
+    #[allow(unused_braces)]
     pub fn new_uninit_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
-        unsafe { RawVec::with_capacity_in(len, alloc).into_box(len) }
+        unsafe {
+            RawVec::<T, A, { CO_ALLOC_PREF_META_NO!() }>::with_capacity_in(len, alloc).into_box(len)
+        }
     }
 
     /// Constructs a new boxed slice with uninitialized contents in the provided allocator,
@@ -785,8 +815,12 @@ impl<T, A: Allocator> Box<[T], A> {
     #[unstable(feature = "allocator_api", issue = "32838")]
     // #[unstable(feature = "new_uninit", issue = "63291")]
     #[must_use]
+    #[allow(unused_braces)]
     pub fn new_zeroed_slice_in(len: usize, alloc: A) -> Box<[mem::MaybeUninit<T>], A> {
-        unsafe { RawVec::with_capacity_zeroed_in(len, alloc).into_box(len) }
+        unsafe {
+            RawVec::<T, A, { CO_ALLOC_PREF_META_NO!() }>::with_capacity_zeroed_in(len, alloc)
+                .into_box(len)
+        }
     }
 }
 
@@ -1487,7 +1521,7 @@ trait BoxFromSlice<T> {
 impl<T: Clone> BoxFromSlice<T> for Box<[T]> {
     #[inline]
     default fn from_slice(slice: &[T]) -> Self {
-        slice.to_vec().into_boxed_slice()
+        slice.to_vec_co::<{ CO_ALLOC_PREF_META_NO!() }>().into_boxed_slice()
     }
 }
 
@@ -1496,7 +1530,7 @@ impl<T: Copy> BoxFromSlice<T> for Box<[T]> {
     #[inline]
     fn from_slice(slice: &[T]) -> Self {
         let len = slice.len();
-        let buf = RawVec::with_capacity(len);
+        let buf = RawVec::<T, Global, { CO_ALLOC_PREF_META_NO!() }>::with_capacity(len);
         unsafe {
             ptr::copy_nonoverlapping(slice.as_ptr(), buf.ptr(), len);
             buf.into_box(slice.len()).assume_init()
@@ -1682,8 +1716,13 @@ impl<T, const N: usize> TryFrom<Box<[T]>> for Box<[T; N]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "boxed_array_try_from_vec", since = "1.66.0")]
-impl<T, const N: usize> TryFrom<Vec<T>> for Box<[T; N]> {
-    type Error = Vec<T>;
+#[allow(unused_braces)]
+impl<T, const N: usize, const CO_ALLOC_PREF: CoAllocPref> TryFrom<Vec<T, Global, CO_ALLOC_PREF>>
+    for Box<[T; N]>
+where
+    [(); { meta_num_slots_global!(CO_ALLOC_PREF) }]:,
+{
+    type Error = Vec<T, Global, CO_ALLOC_PREF>;
 
     /// Attempts to convert a `Vec<T>` into a `Box<[T; N]>`.
     ///
@@ -1703,7 +1742,7 @@ impl<T, const N: usize> TryFrom<Vec<T>> for Box<[T; N]> {
     /// let state: Box<[f32; 100]> = vec![1.0; 100].try_into().unwrap();
     /// assert_eq!(state.len(), 100);
     /// ```
-    fn try_from(vec: Vec<T>) -> Result<Self, Self::Error> {
+    fn try_from(vec: Vec<T, Global, CO_ALLOC_PREF>) -> Result<Self, Self::Error> {
         if vec.len() == N {
             let boxed_slice = vec.into_boxed_slice();
             Ok(unsafe { boxed_slice_as_array_unchecked(boxed_slice) })
@@ -2038,10 +2077,15 @@ impl<I> FromIterator<I> for Box<[I]> {
 
 #[cfg(not(no_global_oom_handling))]
 #[stable(feature = "box_slice_clone", since = "1.3.0")]
-impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A> {
+#[allow(unused_braces)]
+impl<T: Clone, A: Allocator + Clone> Clone for Box<[T], A>
+where
+    [(); { crate::meta_num_slots!(A, crate::CO_ALLOC_PREF_META_NO!()) }]:,
+{
     fn clone(&self) -> Self {
         let alloc = Box::allocator(self).clone();
-        self.to_vec_in(alloc).into_boxed_slice()
+        // false = no need for co-alloc metadata, since it would get lost once converted to the boxed slice.
+        self.to_vec_in_co::<A, { CO_ALLOC_PREF_META_NO!() }>(alloc).into_boxed_slice()
     }
 
     fn clone_from(&mut self, other: &Self) {