Rollup of 7 pull requests

compiler/rustc_arena/src/lib.rs

@@ -19,6 +19,7 @@
 #![feature(rustc_attrs)]
 #![cfg_attr(test, feature(test))]
 #![feature(strict_provenance)]
+#![feature(ptr_const_cast)]
 
 use smallvec::SmallVec;
 
@@ -27,7 +28,7 @@ use std::cell::{Cell, RefCell};
 use std::cmp;
 use std::marker::{PhantomData, Send};
 use std::mem::{self, MaybeUninit};
-use std::ptr;
+use std::ptr::{self, NonNull};
 use std::slice;
 
 #[inline(never)]
@@ -55,15 +56,24 @@ pub struct TypedArena<T> {
 
 struct ArenaChunk<T = u8> {
     /// The raw storage for the arena chunk.
-    storage: Box<[MaybeUninit<T>]>,
+    storage: NonNull<[MaybeUninit<T>]>,
     /// The number of valid entries in the chunk.
     entries: usize,
 }
 
+unsafe impl<#[may_dangle] T> Drop for ArenaChunk<T> {
+    fn drop(&mut self) {
+        unsafe { Box::from_raw(self.storage.as_mut()) };
+    }
+}
+
 impl<T> ArenaChunk<T> {
     #[inline]
     unsafe fn new(capacity: usize) -> ArenaChunk<T> {
-        ArenaChunk { storage: Box::new_uninit_slice(capacity), entries: 0 }
+        ArenaChunk {
+            storage: NonNull::new(Box::into_raw(Box::new_uninit_slice(capacity))).unwrap(),
+            entries: 0,
+        }
     }
 
     /// Destroys this arena chunk.
@@ -72,14 +82,15 @@ impl<T> ArenaChunk<T> {
         // The branch on needs_drop() is an -O1 performance optimization.
         // Without the branch, dropping TypedArena<u8> takes linear time.
         if mem::needs_drop::<T>() {
-            ptr::drop_in_place(MaybeUninit::slice_assume_init_mut(&mut self.storage[..len]));
+            let slice = &mut *(self.storage.as_mut());
+            ptr::drop_in_place(MaybeUninit::slice_assume_init_mut(&mut slice[..len]));
         }
     }
 
     // Returns a pointer to the first allocated object.
     #[inline]
     fn start(&mut self) -> *mut T {
-        MaybeUninit::slice_as_mut_ptr(&mut self.storage)
+        self.storage.as_ptr() as *mut T
     }
 
     // Returns a pointer to the end of the allocated space.
@@ -90,7 +101,7 @@ impl<T> ArenaChunk<T> {
                 // A pointer as large as possible for zero-sized elements.
                 ptr::invalid_mut(!0)
             } else {
-                self.start().add(self.storage.len())
+                self.start().add((*self.storage.as_ptr()).len())
             }
         }
     }
@@ -274,7 +285,7 @@ impl<T> TypedArena<T> {
                 // If the previous chunk's len is less than HUGE_PAGE
                 // bytes, then this chunk will be least double the previous
                 // chunk's size.
-                new_cap = last_chunk.storage.len().min(HUGE_PAGE / elem_size / 2);
+                new_cap = (*last_chunk.storage.as_ptr()).len().min(HUGE_PAGE / elem_size / 2);
                 new_cap *= 2;
             } else {
                 new_cap = PAGE / elem_size;
@@ -382,7 +393,7 @@ impl DroplessArena {
                 // If the previous chunk's len is less than HUGE_PAGE
                 // bytes, then this chunk will be least double the previous
                 // chunk's size.
-                new_cap = last_chunk.storage.len().min(HUGE_PAGE / 2);
+                new_cap = (*last_chunk.storage.as_ptr()).len().min(HUGE_PAGE / 2);
                 new_cap *= 2;
             } else {
                 new_cap = PAGE;

compiler/rustc_arena/src/tests.rs

@@ -79,7 +79,11 @@ fn test_arena_alloc_nested() {
 #[test]
 pub fn test_copy() {
     let arena = TypedArena::default();
-    for _ in 0..100000 {
+    #[cfg(not(miri))]
+    const N: usize = 100000;
+    #[cfg(miri)]
+    const N: usize = 1000;
+    for _ in 0..N {
         arena.alloc(Point { x: 1, y: 2, z: 3 });
     }
 }
@@ -106,15 +110,23 @@ struct Noncopy {
 #[test]
 pub fn test_noncopy() {
     let arena = TypedArena::default();
-    for _ in 0..100000 {
+    #[cfg(not(miri))]
+    const N: usize = 100000;
+    #[cfg(miri)]
+    const N: usize = 1000;
+    for _ in 0..N {
         arena.alloc(Noncopy { string: "hello world".to_string(), array: vec![1, 2, 3, 4, 5] });
     }
 }
 
 #[test]
 pub fn test_typed_arena_zero_sized() {
     let arena = TypedArena::default();
-    for _ in 0..100000 {
+    #[cfg(not(miri))]
+    const N: usize = 100000;
+    #[cfg(miri)]
+    const N: usize = 1000;
+    for _ in 0..N {
         arena.alloc(());
     }
 }
@@ -124,7 +136,11 @@ pub fn test_typed_arena_clear() {
     let mut arena = TypedArena::default();
     for _ in 0..10 {
         arena.clear();
-        for _ in 0..10000 {
+        #[cfg(not(miri))]
+        const N: usize = 10000;
+        #[cfg(miri)]
+        const N: usize = 100;
+        for _ in 0..N {
             arena.alloc(Point { x: 1, y: 2, z: 3 });
         }
     }

compiler/rustc_borrowck/src/diagnostics/conflict_errors.rs

@@ -1628,7 +1628,7 @@ impl<'cx, 'tcx> MirBorrowckCtxt<'cx, 'tcx> {
             location: Location,
         ) -> impl Iterator<Item = Location> + Captures<'tcx> + 'a {
             if location.statement_index == 0 {
-                let predecessors = body.predecessors()[location.block].to_vec();
+                let predecessors = body.basic_blocks.predecessors()[location.block].to_vec();
                 Either::Left(predecessors.into_iter().map(move |bb| body.terminator_loc(bb)))
             } else {
                 Either::Right(std::iter::once(Location {

compiler/rustc_borrowck/src/invalidation.rs

@@ -26,7 +26,7 @@ pub(super) fn generate_invalidates<'tcx>(
 
     if let Some(all_facts) = all_facts {
         let _prof_timer = tcx.prof.generic_activity("polonius_fact_generation");
-        let dominators = body.dominators();
+        let dominators = body.basic_blocks.dominators();
         let mut ig = InvalidationGenerator {
             all_facts,
             borrow_set,

compiler/rustc_borrowck/src/lib.rs

@@ -334,7 +334,7 @@ fn do_mir_borrowck<'a, 'tcx>(
         };
     }
 
-    let dominators = body.dominators();
+    let dominators = body.basic_blocks.dominators();
 
     let mut mbcx = MirBorrowckCtxt {
         infcx,

compiler/rustc_borrowck/src/type_check/liveness/trace.rs

@@ -258,7 +258,7 @@ impl<'me, 'typeck, 'flow, 'tcx> LivenessResults<'me, 'typeck, 'flow, 'tcx> {
 
                 let block = self.cx.elements.to_location(block_start).block;
                 self.stack.extend(
-                    self.cx.body.predecessors()[block]
+                    self.cx.body.basic_blocks.predecessors()[block]
                         .iter()
                         .map(|&pred_bb| self.cx.body.terminator_loc(pred_bb))
                         .map(|pred_loc| self.cx.elements.point_from_location(pred_loc)),
@@ -354,7 +354,7 @@ impl<'me, 'typeck, 'flow, 'tcx> LivenessResults<'me, 'typeck, 'flow, 'tcx> {
         }
 
         let body = self.cx.body;
-        for &pred_block in body.predecessors()[block].iter() {
+        for &pred_block in body.basic_blocks.predecessors()[block].iter() {
             debug!("compute_drop_live_points_for_block: pred_block = {:?}", pred_block,);
 
             // Check whether the variable is (at least partially)

compiler/rustc_codegen_cranelift/src/driver/aot.rs

@@ -66,7 +66,11 @@ fn emit_module(
     let work_product = if backend_config.disable_incr_cache {
         None
     } else {
-        rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(tcx.sess, &name, &tmp_file)
+        rustc_incremental::copy_cgu_workproduct_to_incr_comp_cache_dir(
+            tcx.sess,
+            &name,
+            &[("o", &tmp_file)],
+        )
     };
 
     ModuleCodegenResult(
@@ -82,7 +86,10 @@ fn reuse_workproduct_for_cgu(
 ) -> CompiledModule {
     let work_product = cgu.previous_work_product(tcx);
     let obj_out = tcx.output_filenames(()).temp_path(OutputType::Object, Some(cgu.name().as_str()));
-    let source_file = rustc_incremental::in_incr_comp_dir_sess(&tcx.sess, &work_product.saved_file);
+    let source_file = rustc_incremental::in_incr_comp_dir_sess(
+        &tcx.sess,
+        &work_product.saved_files.get("o").expect("no saved object file in work product"),
+    );
     if let Err(err) = rustc_fs_util::link_or_copy(&source_file, &obj_out) {
         tcx.sess.err(&format!(
             "unable to copy {} to {}: {}",

compiler/rustc_codegen_ssa/src/back/link.rs

@@ -151,11 +151,23 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(
             return;
         }
 
-        let remove_temps_from_module = |module: &CompiledModule| {
-            if let Some(ref obj) = module.object {
-                ensure_removed(sess.diagnostic(), obj);
-            }
-        };
+        let maybe_remove_temps_from_module =
+            |preserve_objects: bool, preserve_dwarf_objects: bool, module: &CompiledModule| {
+                if !preserve_objects {
+                    if let Some(ref obj) = module.object {
+                        ensure_removed(sess.diagnostic(), obj);
+                    }
+                }
+
+                if !preserve_dwarf_objects {
+                    if let Some(ref dwo_obj) = module.dwarf_object {
+                        ensure_removed(sess.diagnostic(), dwo_obj);
+                    }
+                }
+            };
+
+        let remove_temps_from_module =
+            |module: &CompiledModule| maybe_remove_temps_from_module(false, false, module);
 
         // Otherwise, always remove the metadata and allocator module temporaries.
         if let Some(ref metadata_module) = codegen_results.metadata_module {
@@ -177,15 +189,7 @@ pub fn link_binary<'a, B: ArchiveBuilder<'a>>(
         debug!(?preserve_objects, ?preserve_dwarf_objects);
 
         for module in &codegen_results.modules {
-            if !preserve_objects {
-                remove_temps_from_module(module);
-            }
-
-            if !preserve_dwarf_objects {
-                if let Some(ref obj) = module.dwarf_object {
-                    ensure_removed(sess.diagnostic(), obj);
-                }
-            }
+            maybe_remove_temps_from_module(preserve_objects, preserve_dwarf_objects, module);
         }
     });
 
@@ -649,6 +653,7 @@ fn link_dwarf_object<'a>(
             sess.struct_err("linking dwarf objects with thorin failed")
                 .note(&format!("{:?}", e))
                 .emit();
+            sess.abort_if_errors();
         }
     }
 }

compiler/rustc_codegen_ssa/src/back/write.rs

@@ -494,12 +494,18 @@ fn copy_all_cgu_workproducts_to_incr_comp_cache_dir(
     let _timer = sess.timer("copy_all_cgu_workproducts_to_incr_comp_cache_dir");
 
     for module in compiled_modules.modules.iter().filter(|m| m.kind == ModuleKind::Regular) {
-        if let Some(path) = &module.object {
-            if let Some((id, product)) =
-                copy_cgu_workproduct_to_incr_comp_cache_dir(sess, &module.name, path)
-            {
-                work_products.insert(id, product);
-            }
+        let mut files = Vec::new();
+        if let Some(object_file_path) = &module.object {
+            files.push(("o", object_file_path.as_path()));
+        }
+        if let Some(dwarf_object_file_path) = &module.dwarf_object {
+            files.push(("dwo", dwarf_object_file_path.as_path()));
+        }
+
+        if let Some((id, product)) =
+            copy_cgu_workproduct_to_incr_comp_cache_dir(sess, &module.name, files.as_slice())
+        {
+            work_products.insert(id, product);
         }
     }
 
@@ -856,29 +862,50 @@ fn execute_copy_from_cache_work_item<B: ExtraBackendMethods>(
     assert!(module_config.emit_obj != EmitObj::None);
 
     let incr_comp_session_dir = cgcx.incr_comp_session_dir.as_ref().unwrap();
-    let obj_out = cgcx.output_filenames.temp_path(OutputType::Object, Some(&module.name));
-    let source_file = in_incr_comp_dir(&incr_comp_session_dir, &module.source.saved_file);
-    debug!(
-        "copying pre-existing module `{}` from {:?} to {}",
-        module.name,
-        source_file,
-        obj_out.display()
+
+    let load_from_incr_comp_dir = |output_path: PathBuf, saved_path: &str| {
+        let source_file = in_incr_comp_dir(&incr_comp_session_dir, saved_path);
+        debug!(
+            "copying pre-existing module `{}` from {:?} to {}",
+            module.name,
+            source_file,
+            output_path.display()
+        );
+        match link_or_copy(&source_file, &output_path) {
+            Ok(_) => Some(output_path),
+            Err(err) => {
+                let diag_handler = cgcx.create_diag_handler();
+                diag_handler.err(&format!(
+                    "unable to copy {} to {}: {}",
+                    source_file.display(),
+                    output_path.display(),
+                    err
+                ));
+                None
+            }
+        }
+    };
+
+    let object = load_from_incr_comp_dir(
+        cgcx.output_filenames.temp_path(OutputType::Object, Some(&module.name)),
+        &module.source.saved_files.get("o").expect("no saved object file in work product"),
     );
-    if let Err(err) = link_or_copy(&source_file, &obj_out) {
-        let diag_handler = cgcx.create_diag_handler();
-        diag_handler.err(&format!(
-            "unable to copy {} to {}: {}",
-            source_file.display(),
-            obj_out.display(),
-            err
-        ));
-    }
+    let dwarf_object =
+        module.source.saved_files.get("dwo").as_ref().and_then(|saved_dwarf_object_file| {
+            let dwarf_obj_out = cgcx
+                .output_filenames
+                .split_dwarf_path(cgcx.split_debuginfo, cgcx.split_dwarf_kind, Some(&module.name))
+                .expect(
+                    "saved dwarf object in work product but `split_dwarf_path` returned `None`",
+                );
+            load_from_incr_comp_dir(dwarf_obj_out, &saved_dwarf_object_file)
+        });
 
     WorkItemResult::Compiled(CompiledModule {
         name: module.name,
         kind: ModuleKind::Regular,
-        object: Some(obj_out),
-        dwarf_object: None,
+        object,
+        dwarf_object,
         bytecode: None,
     })
 }

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -15,7 +15,7 @@ pub fn non_ssa_locals<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>>(
     fx: &FunctionCx<'a, 'tcx, Bx>,
 ) -> BitSet<mir::Local> {
     let mir = fx.mir;
-    let dominators = mir.dominators();
+    let dominators = mir.basic_blocks.dominators();
     let locals = mir
         .local_decls
         .iter()

compiler/rustc_const_eval/src/transform/promote_consts.rs

@@ -856,7 +856,8 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                     literal: ConstantKind::from_const(_const, tcx),
                 }))
             };
-            let (blocks, local_decls) = self.source.basic_blocks_and_local_decls_mut();
+            let blocks = self.source.basic_blocks.as_mut();
+            let local_decls = &mut self.source.local_decls;
             let loc = candidate.location;
             let statement = &mut blocks[loc.block].statements[loc.statement_index];
             match statement.kind {
@@ -865,7 +866,7 @@ impl<'a, 'tcx> Promoter<'a, 'tcx> {
                     Rvalue::Ref(ref mut region, borrow_kind, ref mut place),
                 )) => {
                     // Use the underlying local for this (necessarily interior) borrow.
-                    let ty = local_decls.local_decls()[place.local].ty;
+                    let ty = local_decls[place.local].ty;
                     let span = statement.source_info.span;
 
                     let ref_ty = tcx.mk_ref(