Auto merge of #67502 - Mark-Simulacrum:opt-catch, r=<try>

Optimize catch_unwind to match C++ try/catch

This refactors the implementation of catching unwinds to allow LLVM to inline the "try" closure directly into the happy path, avoiding indirection. This means that the catch_unwind implementation is (after this PR) zero-cost unless a panic is thrown.

https://rust.godbolt.org/z/cZcUSB is an example of the current codegen in a simple case. Notably, the codegen is *exactly the same* if `-Cpanic=abort` is passed, which is clearly not great.

This PR, on the other hand, generates the following assembly:

```asm
# -Cpanic=unwind:
	push   rbx
	mov    ebx,0x2a
	call   QWORD PTR [rip+0x1c53c]        # <happy>
	mov    eax,ebx
	pop    rbx
	ret
	mov    rdi,rax
	call   QWORD PTR [rip+0x1c537]        # cleanup function call
	call   QWORD PTR [rip+0x1c539]        # <unfortunate>
	mov    ebx,0xd
	mov    eax,ebx
	pop    rbx
	ret

# -Cpanic=abort:
	push   rax
	call   QWORD PTR [rip+0x20a1]        # <happy>
	mov    eax,0x2a
	pop    rcx
	ret
```

Fixes #64224, and resolves #64222.

Author: bors

Cargo.lock

@@ -2309,6 +2309,7 @@ dependencies = [
 name = "panic_abort"
 version = "0.0.0"
 dependencies = [
+ "cfg-if",
  "compiler_builtins",
  "core",
  "libc",

src/ci/azure-pipelines/try.yml

@@ -6,17 +6,17 @@ variables:
 - group: prod-credentials
 
 jobs:
-- job: Linux
-  timeoutInMinutes: 600
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-  - template: steps/run.yml
-  strategy:
-    matrix:
-      dist-x86_64-linux: {}
-      dist-x86_64-linux-alt:
-        IMAGE: dist-x86_64-linux
+    # - job: Linux
+    #   timeoutInMinutes: 600
+    #   pool:
+    #     vmImage: ubuntu-16.04
+    #   steps:
+    #   - template: steps/run.yml
+    #   strategy:
+    #     matrix:
+    #       dist-x86_64-linux: {}
+    #       dist-x86_64-linux-alt:
+    #         IMAGE: dist-x86_64-linux
 
 # The macOS and Windows builds here are currently disabled due to them not being
 # overly necessary on `try` builds. We also don't actually have anything that
@@ -49,25 +49,38 @@ jobs:
 #         NO_LLVM_ASSERTIONS: 1
 #         NO_DEBUG_ASSERTIONS: 1
 #
-# - job: Windows
-#   timeoutInMinutes: 600
-#   pool:
-#     vmImage: 'vs2017-win2016'
-#   steps:
-#   - template: steps/run.yml
-#   strategy:
-#     matrix:
-#       dist-x86_64-msvc:
-#         RUST_CONFIGURE_ARGS: >
-#           --build=x86_64-pc-windows-msvc
-#           --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
-#           --enable-full-tools
-#           --enable-profiler
-#         SCRIPT: python x.py dist
-#         DIST_REQUIRE_ALL_TOOLS: 1
-#         DEPLOY: 1
-#
-#       dist-x86_64-msvc-alt:
-#         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
-#         SCRIPT: python x.py dist
-#         DEPLOY_ALT: 1
+- job: Windows
+  timeoutInMinutes: 600
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - template: steps/run.yml
+  strategy:
+    matrix:
+      x86_64-mingw-1:
+        SCRIPT: make ci-mingw-subset-1
+        RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+        CUSTOM_MINGW: 1
+        # FIXME(#59637)
+        NO_DEBUG_ASSERTIONS: 1
+        NO_LLVM_ASSERTIONS: 1
+#     x86_64-mingw-2:
+#       SCRIPT: make ci-mingw-subset-2
+#       RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-gnu
+#       CUSTOM_MINGW: 1
+#     i686-mingw-1:
+#       RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+#       SCRIPT: make ci-mingw-subset-1
+#       CUSTOM_MINGW: 1
+#       # FIXME(#59637)
+#       NO_DEBUG_ASSERTIONS: 1
+#       NO_LLVM_ASSERTIONS: 1
+      i686-mingw-2:
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+        SCRIPT: make ci-mingw-subset-2
+        CUSTOM_MINGW: 1
+#      dist-i686-mingw:
+#        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-full-tools --enable-profiler
+#        SCRIPT: python x.py dist
+#        CUSTOM_MINGW: 1
+#        DIST_REQUIRE_ALL_TOOLS: 1

src/doc/unstable-book/src/language-features/lang-items.md

@@ -52,7 +52,11 @@ fn main(_argc: isize, _argv: *const *const u8) -> isize {
 
 #[lang = "eh_personality"] extern fn rust_eh_personality() {}
 #[lang = "panic_impl"] extern fn rust_begin_panic(info: &PanicInfo) -> ! { unsafe { intrinsics::abort() } }
-#[lang = "eh_unwind_resume"] extern fn rust_eh_unwind_resume() {}
+extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    loop {}
+}
+#[lang = "eh_unwind_resume"]
+static _RESUME: fn(*mut u8) -> ! = eh_unwind_resume;
 #[no_mangle] pub extern fn rust_eh_register_frames () {}
 #[no_mangle] pub extern fn rust_eh_unregister_frames () {}
 ```
@@ -131,10 +135,12 @@ pub extern fn rust_eh_personality() {
 }
 
 // This function may be needed based on the compilation target.
-#[lang = "eh_unwind_resume"]
-#[no_mangle]
-pub extern fn rust_eh_unwind_resume() {
+extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    loop {}
 }
+#[lang = "eh_unwind_resume"]
+static _RESUME: fn(*mut u8) -> ! = eh_unwind_resume;
+
 
 #[lang = "panic_impl"]
 #[no_mangle]
@@ -248,7 +254,6 @@ the source code.
   - `eh_personality`: `libpanic_unwind/gcc.rs` (GNU)
   - `eh_personality`: `libpanic_unwind/seh.rs` (SEH)
   - `eh_unwind_resume`: `libpanic_unwind/gcc.rs` (GCC)
-  - `eh_catch_typeinfo`: `libpanic_unwind/seh.rs` (SEH)
   - `eh_catch_typeinfo`: `libpanic_unwind/emcc.rs` (EMCC)
   - `panic`: `libcore/panicking.rs`
   - `panic_bounds_check`: `libcore/panicking.rs`

src/libpanic_abort/Cargo.toml

@@ -14,3 +14,4 @@ doc = false
 core = { path = "../libcore" }
 libc = { version = "0.2", default-features = false }
 compiler_builtins = "0.1.0"
+cfg-if = "0.1.8"

src/libpanic_abort/lib.rs

@@ -18,17 +18,14 @@
 #![feature(staged_api)]
 #![feature(rustc_attrs)]
 
-// Rust's "try" function, but if we're aborting on panics we just call the
-// function as there's nothing else we need to do here.
+use core::any::Any;
+
+// We need the definition of TryPayload for __rust_panic_cleanup.
+include!("../libpanic_unwind/payload.rs");
+
 #[rustc_std_internal_symbol]
-pub unsafe extern "C" fn __rust_maybe_catch_panic(
-    f: fn(*mut u8),
-    data: *mut u8,
-    _data_ptr: *mut usize,
-    _vtable_ptr: *mut usize,
-) -> u32 {
-    f(data);
-    0
+pub unsafe extern "C" fn __rust_panic_cleanup(_: TryPayload) -> *mut (dyn Any + Send + 'static) {
+    unreachable!()
 }
 
 // "Leak" the payload and shim to the relevant abort on the platform in

src/libpanic_unwind/dummy.rs

@@ -6,10 +6,6 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::intrinsics;
 
-pub fn payload() -> *mut u8 {
-    core::ptr::null_mut()
-}
-
 pub unsafe fn cleanup(_ptr: *mut u8) -> Box<dyn Any + Send> {
     intrinsics::abort()
 }

src/libpanic_unwind/emcc.rs

@@ -48,10 +48,6 @@ static EXCEPTION_TYPE_INFO: TypeInfo = TypeInfo {
     name: b"rust_panic\0".as_ptr(),
 };
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 struct Exception {
     // This needs to be an Option because the object's lifetime follows C++
     // semantics: when catch_unwind moves the Box out of the exception it must

src/libpanic_unwind/gcc.rs

@@ -48,7 +48,6 @@
 
 use alloc::boxed::Box;
 use core::any::Any;
-use core::ptr;
 
 use crate::dwarf::eh::{self, EHAction, EHContext};
 use libc::{c_int, uintptr_t};
@@ -83,10 +82,6 @@ pub unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
     }
 }
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 pub unsafe fn cleanup(ptr: *mut u8) -> Box<dyn Any + Send> {
     let exception = Box::from_raw(ptr as *mut Exception);
     exception.cause
@@ -329,16 +324,25 @@ unsafe fn find_eh_action(
     eh::find_eh_action(lsda, &eh_context, foreign_exception)
 }
 
-// See docs in the `unwind` module.
+#[cfg(all(
+    target_os = "windows",
+    any(target_arch = "x86", target_arch = "x86_64"),
+    target_env = "gnu"
+))]
+#[cfg_attr(not(bootstrap), lang = "eh_unwind_resume")]
+#[used]
+pub static RESUME: unsafe extern "C" fn(*mut uw::_Unwind_Exception) -> ! =
+    uw::_Unwind_Resume as unsafe extern "C" fn(_) -> !;
+
 #[cfg(all(
     target_os = "windows",
     any(target_arch = "x86", target_arch = "x86_64"),
     target_env = "gnu"
 ))]
 #[lang = "eh_unwind_resume"]
-#[unwind(allowed)]
-unsafe extern "C" fn rust_eh_unwind_resume(panic_ctx: *mut u8) -> ! {
-    uw::_Unwind_Resume(panic_ctx as *mut uw::_Unwind_Exception);
+#[cfg(bootstrap)]
+pub unsafe extern "C" fn rust_eh_unwind_resume(p: *mut u8) -> ! {
+    uw::_Unwind_Resume(p as *mut uw::_Unwind_Exception)
 }
 
 // Frame unwind info registration

src/libpanic_unwind/hermit.rs

@@ -6,10 +6,6 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::ptr;
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 pub unsafe fn cleanup(_ptr: *mut u8) -> Box<dyn Any + Send> {
     extern "C" {
         pub fn __rust_abort() -> !;

src/libpanic_unwind/lib.rs

@@ -22,20 +22,22 @@
 #![feature(libc)]
 #![feature(nll)]
 #![feature(panic_unwind)]
-#![feature(raw)]
 #![feature(staged_api)]
 #![feature(std_internals)]
 #![feature(unwind_attributes)]
 #![feature(abi_thiscall)]
+#![feature(rustc_attrs)]
+#![feature(raw)]
 #![panic_runtime]
 #![feature(panic_runtime)]
+#![allow(dead_code)]
 
 use alloc::boxed::Box;
-use core::intrinsics;
-use core::mem;
+use core::any::Any;
 use core::panic::BoxMeUp;
-use core::raw;
 
+// If adding to this list, you should also look at the list of TryPayload types
+// defined in payload.rs and likely add to there as well.
 cfg_if::cfg_if! {
     if #[cfg(target_os = "emscripten")] {
         #[path = "emcc.rs"]
@@ -61,6 +63,8 @@ cfg_if::cfg_if! {
     }
 }
 
+include!("payload.rs");
+
 extern "C" {
     /// Handler in libstd called when a panic object is dropped outside of
     /// `catch_unwind`.
@@ -69,28 +73,11 @@ extern "C" {
 
 mod dwarf;
 
-// Entry point for catching an exception, implemented using the `try` intrinsic
-// in the compiler.
-//
-// The interaction between the `payload` function and the compiler is pretty
-// hairy and tightly coupled, for more information see the compiler's
-// implementation of this.
 #[no_mangle]
-pub unsafe extern "C" fn __rust_maybe_catch_panic(
-    f: fn(*mut u8),
-    data: *mut u8,
-    data_ptr: *mut usize,
-    vtable_ptr: *mut usize,
-) -> u32 {
-    let mut payload = imp::payload();
-    if intrinsics::r#try(f, data, &mut payload as *mut _ as *mut _) == 0 {
-        0
-    } else {
-        let obj = mem::transmute::<_, raw::TraitObject>(imp::cleanup(payload));
-        *data_ptr = obj.data as usize;
-        *vtable_ptr = obj.vtable as usize;
-        1
-    }
+pub unsafe extern "C" fn __rust_panic_cleanup(
+    payload: TryPayload,
+) -> *mut (dyn Any + Send + 'static) {
+    Box::into_raw(imp::cleanup(payload))
 }
 
 // Entry point for raising an exception, just delegates to the platform-specific

src/libpanic_unwind/payload.rs

@@ -0,0 +1,21 @@
+// Type definition for the payload argument of the try intrinsic.
+//
+// This must be kept in sync with the implementations of the try intrinsic.
+//
+// This file is included by both panic runtimes and libstd. It is part of the
+// panic runtime ABI.
+cfg_if::cfg_if! {
+    if #[cfg(target_os = "emscripten")] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(target_arch = "wasm32")] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(target_os = "hermit")] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(all(target_env = "msvc", target_arch = "aarch64"))] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(target_env = "msvc")] {
+        type TryPayload = [u64; 2];
+    } else {
+        type TryPayload = *mut u8;
+    }
+}

src/libpanic_unwind/seh.rs

@@ -167,6 +167,9 @@ pub struct _TypeDescriptor {
 
 // Note that we intentionally ignore name mangling rules here: we don't want C++
 // to be able to catch Rust panics by simply declaring a `struct rust_panic`.
+//
+// When modifying, make sure that the type name string exactly matches
+// the one used in src/librustc_codegen_llvm/intrinsic.rs.
 const TYPE_NAME: [u8; 11] = *b"rust_panic\0";
 
 static mut THROW_INFO: _ThrowInfo = _ThrowInfo {
@@ -199,12 +202,12 @@ extern "C" {
     static TYPE_INFO_VTABLE: *const u8;
 }
 
-// We use #[lang = "eh_catch_typeinfo"] here as this is the type descriptor which
-// we'll use in LLVM's `catchpad` instruction which ends up also being passed as
-// an argument to the C++ personality function.
+// This type descriptor is only used when throwing an exception. The catch part
+// is handled by the try intrinsic, which generates its own TypeDescriptor.
 //
-// Again, I'm not entirely sure what this is describing, it just seems to work.
-#[cfg_attr(not(test), lang = "eh_catch_typeinfo")]
+// This is fine since the MSVC runtime uses string comparison on the type name
+// to match TypeDescriptors rather than pointer equality.
+#[cfg_attr(bootstrap, lang = "eh_catch_typeinfo")]
 static mut TYPE_DESCRIPTOR: _TypeDescriptor = _TypeDescriptor {
     pVFTable: unsafe { &TYPE_INFO_VTABLE } as *const _ as *const _,
     spare: core::ptr::null_mut(),
@@ -315,10 +318,6 @@ pub unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
     _CxxThrowException(throw_ptr, &mut THROW_INFO as *mut _ as *mut _);
 }
 
-pub fn payload() -> [u64; 2] {
-    [0; 2]
-}
-
 pub unsafe fn cleanup(payload: [u64; 2]) -> Box<dyn Any + Send> {
     mem::transmute(raw::TraitObject { data: payload[0] as *mut _, vtable: payload[1] as *mut _ })
 }

src/librustc/middle/lang_items.rs

@@ -385,7 +385,7 @@ language_item_table! {
     StartFnLangItem,             "start",              start_fn,                Target::Fn;
 
     EhPersonalityLangItem,       "eh_personality",     eh_personality,          Target::Fn;
-    EhUnwindResumeLangItem,      "eh_unwind_resume",   eh_unwind_resume,        Target::Fn;
+    EhUnwindResumeLangItem,      "eh_unwind_resume",   eh_unwind_resume,        Target::Static;
     EhCatchTypeinfoLangItem,     "eh_catch_typeinfo",  eh_catch_typeinfo,       Target::Static;
 
     OwnedBoxLangItem,            "owned_box",          owned_box,               Target::Struct;

src/librustc_codegen_llvm/context.rs

@@ -407,17 +407,9 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         let tcx = self.tcx;
         assert!(self.sess().target.target.options.custom_unwind_resume);
         if let Some(def_id) = tcx.lang_items().eh_unwind_resume() {
-            let llfn = self.get_fn_addr(
-                ty::Instance::resolve(
-                    tcx,
-                    ty::ParamEnv::reveal_all(),
-                    def_id,
-                    tcx.intern_substs(&[]),
-                )
-                .unwrap(),
-            );
-            unwresume.set(Some(llfn));
-            return llfn;
+            let static_ptr = self.get_static(def_id);
+            unwresume.set(Some(static_ptr));
+            return static_ptr;
         }
 
         let sig = ty::Binder::bind(tcx.mk_fn_sig(
@@ -431,8 +423,9 @@ impl MiscMethods<'tcx> for CodegenCx<'ll, 'tcx> {
         let fn_abi = FnAbi::of_fn_ptr(self, sig, &[]);
         let llfn = self.declare_fn("rust_eh_unwind_resume", &fn_abi);
         attributes::apply_target_cpu_attr(self, llfn);
-        unwresume.set(Some(llfn));
-        llfn
+        let static_ptr = self.static_addr_of(llfn, tcx.data_layout.pointer_align.abi, None);
+        unwresume.set(Some(static_ptr));
+        static_ptr
     }
 
     fn sess(&self) -> &Session {

src/librustc_codegen_llvm/intrinsic.rs

@@ -858,8 +858,10 @@ fn try_intrinsic(
 ) {
     if bx.sess().no_landing_pads() {
         bx.call(func, &[data], None);
-        let ptr_align = bx.tcx().data_layout.pointer_align.abi;
-        bx.store(bx.const_null(bx.type_i8p()), dest, ptr_align);
+        // Return 0 unconditionally from the intrinsic call;
+        // we can never unwind.
+        let ret_align = bx.tcx().data_layout.i32_align.abi;
+        bx.store(bx.const_i32(0), dest, ret_align);
     } else if wants_msvc_seh(bx.sess()) {
         codegen_msvc_try(bx, func, data, local_ptr, dest);
     } else {
@@ -952,19 +954,39 @@ fn codegen_msvc_try(
         let cs = catchswitch.catch_switch(None, None, 1);
         catchswitch.add_handler(cs, catchpad.llbb());
 
+        // We can't use the TypeDescriptor defined in libpanic_unwind because it
+        // might be in another DLL and the SEH encoding only supports specifying
+        // a TypeDescriptor from the current module.
+        //
+        // However this isn't an issue since the MSVC runtime uses string
+        // comparison on the type name to match TypeDescriptors rather than
+        // pointer equality.
+        //
+        // So instead we generate a new TypeDescriptor in each module that uses
+        // `try` and let the linker merge duplicate definitions in the same
+        // module.
+        //
+        // When modifying, make sure that the type_name string exactly matches
+        // the one used in src/libpanic_unwind/seh.rs.
+        let type_info_vtable = bx.declare_global("??_7type_info@@6B@", bx.type_i8p());
+        let type_name = bx.const_bytes(b"rust_panic\0");
+        let type_info =
+            bx.const_struct(&[type_info_vtable, bx.const_null(bx.type_i8p()), type_name], false);
+        let tydesc = bx.declare_global("__rust_panic_type_info", bx.val_ty(type_info));
+        unsafe {
+            llvm::LLVMRustSetLinkage(tydesc, llvm::Linkage::LinkOnceODRLinkage);
+            llvm::SetUniqueComdat(bx.llmod, tydesc);
+            llvm::LLVMSetInitializer(tydesc, type_info);
+        }
+
         // The flag value of 8 indicates that we are catching the exception by
         // reference instead of by value. We can't use catch by value because
         // that requires copying the exception object, which we don't support
         // since our exception object effectively contains a Box.
         //
         // Source: MicrosoftCXXABI::getAddrOfCXXCatchHandlerType in clang
         let flags = bx.const_i32(8);
-        let tydesc = match bx.tcx().lang_items().eh_catch_typeinfo() {
-            Some(did) => bx.get_static(did),
-            None => bug!("eh_catch_typeinfo not defined, but needed for SEH unwinding"),
-        };
         let funclet = catchpad.catch_pad(cs, &[tydesc, flags, slot]);
-
         let i64_align = bx.tcx().data_layout.i64_align.abi;
         let payload_ptr = catchpad.load(slot, ptr_align);
         let payload = catchpad.load(payload_ptr, i64_align);
@@ -1084,6 +1106,8 @@ fn gen_fn<'ll, 'tcx>(
     ));
     let fn_abi = FnAbi::of_fn_ptr(cx, rust_fn_sig, &[]);
     let llfn = cx.declare_fn(name, &fn_abi);
+    cx.set_frame_pointer_elimination(llfn);
+    cx.apply_target_cpu_attr(llfn);
     // FIXME(eddyb) find a nicer way to do this.
     unsafe { llvm::LLVMRustSetLinkage(llfn, llvm::Linkage::InternalLinkage) };
     let bx = Builder::new_block(cx, llfn, "entry-block");

src/librustc_codegen_ssa/mir/block.rs

@@ -184,7 +184,8 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> {
                 lp = bx.insert_value(lp, lp1, 1);
                 bx.resume(lp);
             } else {
-                bx.call(bx.eh_unwind_resume(), &[lp0], helper.funclet(self));
+                let llfn = bx.load(bx.eh_unwind_resume(), bx.tcx().data_layout.pointer_align.abi);
+                bx.call(llfn, &[lp0], helper.funclet(self));
                 bx.unreachable();
             }
         }

src/libstd/panicking.rs

@@ -12,9 +12,8 @@ use core::panic::{BoxMeUp, Location, PanicInfo};
 use crate::any::Any;
 use crate::fmt;
 use crate::intrinsics;
-use crate::mem::{self, ManuallyDrop};
+use crate::mem::{self, ManuallyDrop, MaybeUninit};
 use crate::process;
-use crate::raw;
 use crate::sync::atomic::{AtomicBool, Ordering};
 use crate::sys::stdio::panic_output;
 use crate::sys_common::backtrace::{self, RustBacktrace};
@@ -29,6 +28,9 @@ use crate::io::set_panic;
 #[cfg(test)]
 use realstd::io::set_panic;
 
+// Include the definition of UnwindPayload from libpanic_unwind.
+include!("../libpanic_unwind/payload.rs");
+
 // Binary interface to the panic runtime that the standard library depends on.
 //
 // The standard library is tagged with `#![needs_panic_runtime]` (introduced in
@@ -41,12 +43,9 @@ use realstd::io::set_panic;
 // hook up these functions, but it is not this day!
 #[allow(improper_ctypes)]
 extern "C" {
-    fn __rust_maybe_catch_panic(
-        f: fn(*mut u8),
-        data: *mut u8,
-        data_ptr: *mut usize,
-        vtable_ptr: *mut usize,
-    ) -> u32;
+    /// The payload ptr here is actually the same as the payload ptr for the try
+    /// intrinsic (i.e., is really `*mut [u64; 2]` or `*mut *mut u8`).
+    fn __rust_panic_cleanup(payload: TryPayload) -> *mut (dyn Any + Send + 'static);
 
     /// `payload` is actually a `*mut &mut dyn BoxMeUp` but that would cause FFI warnings.
     /// It cannot be `Box<dyn BoxMeUp>` because the other end of this call does not depend
@@ -250,9 +249,9 @@ pub unsafe fn r#try<R, F: FnOnce() -> R>(f: F) -> Result<R, Box<dyn Any + Send>>
     }
 
     // We do some sketchy operations with ownership here for the sake of
-    // performance. We can only  pass pointers down to
-    // `__rust_maybe_catch_panic` (can't pass objects by value), so we do all
-    // the ownership tracking here manually using a union.
+    // performance. We can only pass pointers down to `do_call` (can't pass
+    // objects by value), so we do all the ownership tracking here manually
+    // using a union.
     //
     // We go through a transition where:
     //
@@ -263,7 +262,7 @@ pub unsafe fn r#try<R, F: FnOnce() -> R>(f: F) -> Result<R, Box<dyn Any + Send>>
     // * If the closure successfully returns, we write the return value into the
     //   data's return slot. Note that `ptr::write` is used as it's overwriting
     //   uninitialized data.
-    // * Finally, when we come back out of the `__rust_maybe_catch_panic` we're
+    // * Finally, when we come back out of the `try` intrinsic we're
     //   in one of two states:
     //
     //      1. The closure didn't panic, in which case the return value was
@@ -274,28 +273,30 @@ pub unsafe fn r#try<R, F: FnOnce() -> R>(f: F) -> Result<R, Box<dyn Any + Send>>
     //
     // Once we stack all that together we should have the "most efficient'
     // method of calling a catch panic whilst juggling ownership.
-    let mut any_data = 0;
-    let mut any_vtable = 0;
     let mut data = Data { f: ManuallyDrop::new(f) };
 
-    let r = __rust_maybe_catch_panic(
-        do_call::<F, R>,
-        &mut data as *mut _ as *mut u8,
-        &mut any_data,
-        &mut any_vtable,
-    );
+    let mut payload: MaybeUninit<TryPayload> = MaybeUninit::uninit();
 
-    return if r == 0 {
+    let data_ptr = &mut data as *mut _ as *mut u8;
+    let payload_ptr = payload.as_mut_ptr() as *mut _;
+    return if intrinsics::r#try(do_call::<F, R>, data_ptr, payload_ptr) == 0 {
         debug_assert!(update_panic_count(0) == 0);
         Ok(ManuallyDrop::into_inner(data.r))
     } else {
+        Err(cleanup(payload.assume_init()))
+    };
+
+    // We consider unwinding to be rare, so mark this function as cold. However,
+    // do not mark it no-inline -- that decision is best to leave to the
+    // optimizer (in most cases this function is not inlined even as a normal,
+    // non-cold function, though, as of the writing of this comment).
+    #[cold]
+    unsafe fn cleanup(payload: TryPayload) -> Box<dyn Any + Send + 'static> {
+        let obj = Box::from_raw(__rust_panic_cleanup(payload));
         update_panic_count(-1);
         debug_assert!(update_panic_count(0) == 0);
-        Err(mem::transmute(raw::TraitObject {
-            data: any_data as *mut _,
-            vtable: any_vtable as *mut _,
-        }))
-    };
+        obj
+    }
 
     fn do_call<F: FnOnce() -> R, R>(data: *mut u8) {
         unsafe {

src/libunwind/lib.rs

@@ -4,6 +4,7 @@
 #![feature(nll)]
 #![feature(staged_api)]
 #![feature(unwind_attributes)]
+#![feature(lang_items)]
 #![feature(static_nobundle)]
 #![cfg_attr(not(target_env = "msvc"), feature(libc))]
 

src/test/codegen/catch-unwind.rs

@@ -0,0 +1,19 @@
+// compile-flags: -O
+
+#![crate_type = "lib"]
+
+extern "C" {
+    fn bar();
+}
+
+// CHECK-LABEL: @foo
+#[no_mangle]
+pub unsafe fn foo() -> i32 {
+    // CHECK: call void @bar
+    // CHECK: ret i32 0
+    std::panic::catch_unwind(|| {
+        bar();
+        0
+    })
+    .unwrap()
+}

src/test/codegen/try-panic-abort.rs

@@ -0,0 +1,17 @@
+// compile-flags: -C panic=abort -O
+
+#![crate_type = "lib"]
+#![feature(unwind_attributes, core_intrinsics)]
+
+extern "C" {
+    #[unwind(allow)]
+    fn bar(data: *mut u8);
+}
+
+// CHECK-LABEL: @foo
+#[no_mangle]
+pub unsafe fn foo() -> i32 {
+    // CHECK: call void @bar
+    // CHECK: ret i32 0
+    std::intrinsics::r#try(|x| bar(x), 0 as *mut u8, 0 as *mut u8)
+}

src/test/compile-fail/auxiliary/panic-runtime-lang-items.rs

@@ -1,15 +1,20 @@
 // no-prefer-dynamic
 
 #![crate_type = "rlib"]
-
 #![no_std]
-#![feature(lang_items)]
+#![feature(lang_items, core_intrinsics)]
 
 use core::panic::PanicInfo;
 
 #[lang = "panic_impl"]
-fn panic_impl(info: &PanicInfo) -> ! { loop {} }
+fn panic_impl(info: &PanicInfo) -> ! {
+    loop {}
+}
 #[lang = "eh_personality"]
 fn eh_personality() {}
+
+unsafe extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    core::intrinsics::abort();
+}
 #[lang = "eh_unwind_resume"]
-fn eh_unwind_resume() {}
+static _RESUME: unsafe extern "C" fn(*mut u8) -> ! = eh_unwind_resume;

src/test/ui/consts/const-eval/const_panic_libcore_main.rs

@@ -1,5 +1,5 @@
 #![crate_type = "bin"]
-#![feature(lang_items)]
+#![feature(lang_items, core_intrinsics)]
 #![feature(const_panic)]
 #![no_main]
 #![no_std]
@@ -17,8 +17,12 @@ const X: () = unimplemented!();
 
 #[lang = "eh_personality"]
 fn eh() {}
+
+unsafe extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    core::intrinsics::abort();
+}
 #[lang = "eh_unwind_resume"]
-fn eh_unwind_resume() {}
+static _RESUME: unsafe extern "C" fn(*mut u8) -> ! = eh_unwind_resume;
 
 #[panic_handler]
 fn panic(_info: &PanicInfo) -> ! {

src/test/ui/macros/macro-comma-behavior.core.stderr

@@ -1,41 +1,41 @@
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:21:23
+  --> $DIR/macro-comma-behavior.rs:36:23
    |
 LL |     assert_eq!(1, 1, "{}",);
    |                       ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:24:23
+  --> $DIR/macro-comma-behavior.rs:39:23
    |
 LL |     assert_ne!(1, 2, "{}",);
    |                       ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:30:29
+  --> $DIR/macro-comma-behavior.rs:45:29
    |
 LL |     debug_assert_eq!(1, 1, "{}",);
    |                             ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:33:29
+  --> $DIR/macro-comma-behavior.rs:48:29
    |
 LL |     debug_assert_ne!(1, 2, "{}",);
    |                             ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:54:19
+  --> $DIR/macro-comma-behavior.rs:72:19
    |
 LL |     format_args!("{}",);
    |                   ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:72:21
+  --> $DIR/macro-comma-behavior.rs:92:21
    |
 LL |     unimplemented!("{}",);
    |                     ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:81:24
+  --> $DIR/macro-comma-behavior.rs:101:24
    |
 LL |             write!(f, "{}",)?;
    |                        ^^

src/test/ui/macros/macro-comma-behavior.rs

@@ -3,14 +3,29 @@
 // compile-flags: -C debug_assertions=yes
 // revisions: std core
 
-#![feature(lang_items)]
+#![feature(lang_items, core_intrinsics)]
 #![cfg_attr(core, no_std)]
 
-#[cfg(std)] use std::fmt;
-#[cfg(core)] use core::fmt;
-#[cfg(core)] #[lang = "eh_personality"] fn eh_personality() {}
-#[cfg(core)] #[lang = "eh_unwind_resume"] fn eh_unwind_resume() {}
-#[cfg(core)] #[lang = "panic_impl"] fn panic_impl(panic: &core::panic::PanicInfo) -> ! { loop {} }
+#[cfg(core)]
+use core::fmt;
+#[cfg(std)]
+use std::fmt;
+#[cfg(core)]
+#[lang = "eh_personality"]
+fn eh_personality() {}
+#[cfg(core)]
+#[lang = "panic_impl"]
+fn panic_impl(panic: &core::panic::PanicInfo) -> ! {
+    loop {}
+}
+
+#[cfg(core)]
+unsafe extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    core::intrinsics::abort();
+}
+#[cfg(core)]
+#[lang = "eh_unwind_resume"]
+static _RESUME: unsafe extern "C" fn(*mut u8) -> ! = eh_unwind_resume;
 
 // (see documentation of the similarly-named test in run-pass)
 fn to_format_or_not_to_format() {
@@ -34,19 +49,22 @@ fn to_format_or_not_to_format() {
     //[core]~^ ERROR no arguments
     //[std]~^^ ERROR no arguments
 
-    #[cfg(std)] {
+    #[cfg(std)]
+    {
         eprint!("{}",);
         //[std]~^ ERROR no arguments
     }
 
-    #[cfg(std)] {
+    #[cfg(std)]
+    {
         // FIXME: compile-fail says "expected error not found" even though
         //        rustc does emit an error
         // eprintln!("{}",);
         // <DISABLED> [std]~^ ERROR no arguments
     }
 
-    #[cfg(std)] {
+    #[cfg(std)]
+    {
         format!("{}",);
         //[std]~^ ERROR no arguments
     }
@@ -57,12 +75,14 @@ fn to_format_or_not_to_format() {
 
     // if falsum() { panic!("{}",); } // see run-pass
 
-    #[cfg(std)] {
+    #[cfg(std)]
+    {
         print!("{}",);
         //[std]~^ ERROR no arguments
     }
 
-    #[cfg(std)] {
+    #[cfg(std)]
+    {
         // FIXME: compile-fail says "expected error not found" even though
         //        rustc does emit an error
         // println!("{}",);

src/test/ui/macros/macro-comma-behavior.std.stderr

@@ -1,59 +1,59 @@
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:21:23
+  --> $DIR/macro-comma-behavior.rs:36:23
    |
 LL |     assert_eq!(1, 1, "{}",);
    |                       ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:24:23
+  --> $DIR/macro-comma-behavior.rs:39:23
    |
 LL |     assert_ne!(1, 2, "{}",);
    |                       ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:30:29
+  --> $DIR/macro-comma-behavior.rs:45:29
    |
 LL |     debug_assert_eq!(1, 1, "{}",);
    |                             ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:33:29
+  --> $DIR/macro-comma-behavior.rs:48:29
    |
 LL |     debug_assert_ne!(1, 2, "{}",);
    |                             ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:38:18
+  --> $DIR/macro-comma-behavior.rs:54:18
    |
 LL |         eprint!("{}",);
    |                  ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:50:18
+  --> $DIR/macro-comma-behavior.rs:68:18
    |
 LL |         format!("{}",);
    |                  ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:54:19
+  --> $DIR/macro-comma-behavior.rs:72:19
    |
 LL |     format_args!("{}",);
    |                   ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:61:17
+  --> $DIR/macro-comma-behavior.rs:80:17
    |
 LL |         print!("{}",);
    |                 ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:72:21
+  --> $DIR/macro-comma-behavior.rs:92:21
    |
 LL |     unimplemented!("{}",);
    |                     ^^
 
 error: 1 positional argument in format string, but no arguments were given
-  --> $DIR/macro-comma-behavior.rs:81:24
+  --> $DIR/macro-comma-behavior.rs:101:24
    |
 LL |             write!(f, "{}",)?;
    |                        ^^

src/test/ui/no-landing-pads.rs

@@ -2,7 +2,7 @@
 
 // error-pattern: requires `owned_box` lang_item
 
-#![feature(lang_items, box_syntax)]
+#![feature(lang_items, box_syntax, core_intrinsics)]
 #![no_std]
 
 use core::panic::PanicInfo;
@@ -11,6 +11,14 @@ fn main() {
     let x = box 1i32;
 }
 
-#[lang = "eh_personality"] extern fn eh_personality() {}
-#[lang = "eh_unwind_resume"] extern fn eh_unwind_resume() {}
-#[lang = "panic_impl"] fn panic_impl(panic: &PanicInfo) -> ! { loop {} }
+#[lang = "eh_personality"]
+extern "C" fn eh_personality() {}
+unsafe extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    core::intrinsics::abort();
+}
+#[lang = "eh_unwind_resume"]
+static _RESUME: unsafe extern "C" fn(*mut u8) -> ! = eh_unwind_resume;
+#[lang = "panic_impl"]
+fn panic_impl(panic: &PanicInfo) -> ! {
+    loop {}
+}

src/test/ui/no_owned_box_lang_item.rs

@@ -1,15 +1,20 @@
 // no-prefer-dynamic
 
 #![crate_type = "rlib"]
-
 #![no_std]
-#![feature(lang_items)]
+#![feature(lang_items, core_intrinsics)]
 
 use core::panic::PanicInfo;
 
 #[lang = "panic_impl"]
-fn panic_impl(info: &PanicInfo) -> ! { loop {} }
+fn panic_impl(info: &PanicInfo) -> ! {
+    loop {}
+}
 #[lang = "eh_personality"]
 fn eh_personality() {}
+
+unsafe extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    core::intrinsics::abort();
+}
 #[lang = "eh_unwind_resume"]
-fn eh_unwind_resume() {}
+static _RESUME: unsafe extern "C" fn(*mut u8) -> ! = eh_unwind_resume;

src/test/ui/panic-runtime/auxiliary/panic-runtime-lang-items.rs

@@ -7,23 +7,25 @@
 // (so all Ranges resolve to core::ops::Range...)
 
 #![no_std]
-#![feature(lang_items)]
+#![feature(core_intrinsics, lang_items)]
 
 use core::ops::RangeBounds;
 
 #[cfg(any(not(target_arch = "wasm32"), target_os = "emscripten"))]
 #[lang = "eh_personality"]
-extern fn eh_personality() {}
+extern "C" fn eh_personality() {}
 
+#[cfg(target_os = "windows")]
+extern "C" fn eh_unwind_resume(_: *mut u8) -> ! {
+    core::intrinsics::abort();
+}
 #[cfg(target_os = "windows")]
 #[lang = "eh_unwind_resume"]
-extern fn eh_unwind_resume() {}
-
+static _RESUME: fn(*mut u8) -> ! = eh_unwind_resume;
 
 // take a reference to any built-in range
 fn take_range(_r: &impl RangeBounds<i8>) {}
 
-
 fn main() {
     take_range(0..1);
     //~^ ERROR mismatched types [E0308]

src/test/ui/range/issue-54505-no-std.rs

@@ -1,7 +1,7 @@
 error: `#[panic_handler]` function required, but not found
 
 error[E0308]: mismatched types
-  --> $DIR/issue-54505-no-std.rs:28:16
+  --> $DIR/issue-54505-no-std.rs:30:16
    |
 LL |     take_range(0..1);
    |                ^^^^
@@ -13,7 +13,7 @@ LL |     take_range(0..1);
                  found struct `core::ops::Range<{integer}>`
 
 error[E0308]: mismatched types
-  --> $DIR/issue-54505-no-std.rs:33:16
+  --> $DIR/issue-54505-no-std.rs:35:16
    |
 LL |     take_range(1..);
    |                ^^^
@@ -25,7 +25,7 @@ LL |     take_range(1..);
                  found struct `core::ops::RangeFrom<{integer}>`
 
 error[E0308]: mismatched types
-  --> $DIR/issue-54505-no-std.rs:38:16
+  --> $DIR/issue-54505-no-std.rs:40:16
    |
 LL |     take_range(..);
    |                ^^
@@ -37,7 +37,7 @@ LL |     take_range(..);
                  found struct `core::ops::RangeFull`
 
 error[E0308]: mismatched types
-  --> $DIR/issue-54505-no-std.rs:43:16
+  --> $DIR/issue-54505-no-std.rs:45:16
    |
 LL |     take_range(0..=1);
    |                ^^^^^
@@ -49,7 +49,7 @@ LL |     take_range(0..=1);
                  found struct `core::ops::RangeInclusive<{integer}>`
 
 error[E0308]: mismatched types
-  --> $DIR/issue-54505-no-std.rs:48:16
+  --> $DIR/issue-54505-no-std.rs:50:16
    |
 LL |     take_range(..5);
    |                ^^^
@@ -61,7 +61,7 @@ LL |     take_range(..5);
                  found struct `core::ops::RangeTo<{integer}>`
 
 error[E0308]: mismatched types
-  --> $DIR/issue-54505-no-std.rs:53:16
+  --> $DIR/issue-54505-no-std.rs:55:16
    |
 LL |     take_range(..=42);
    |                ^^^^^

src/test/ui/range/issue-54505-no-std.stderr

@@ -59,6 +59,8 @@ const EXCEPTION_PATHS: &[&str] = &[
     "src/libstd/sys_common/mod.rs",
     "src/libstd/sys_common/net.rs",
     "src/libstd/sys_common/backtrace.rs",
+    // panic_unwind shims
+    "src/libstd/panicking.rs",
     "src/libterm", // Not sure how to make this crate portable, but test crate needs it.
     "src/libtest", // Probably should defer to unstable `std::sys` APIs.
     "src/libstd/sync/mpsc", // some tests are only run on non-emscripten