Auto merge of #67502 - Mark-Simulacrum:opt-catch, r=<try>

Optimize catch_unwind to match C++ try/catch

This refactors the implementation of catching unwinds to allow LLVM to inline the "try" closure directly into the happy path, avoiding indirection. This means that the catch_unwind implementation is (after this PR) zero-cost unless a panic is thrown.

https://rust.godbolt.org/z/cZcUSB is an example of the current codegen in a simple case. Notably, the codegen is *exactly the same* if `-Cpanic=abort` is passed, which is clearly not great.

This PR, on the other hand, generates the following assembly:

```asm
# -Cpanic=unwind:
	push   rbx
	mov    ebx,0x2a
	call   QWORD PTR [rip+0x1c53c]        # <happy>
	mov    eax,ebx
	pop    rbx
	ret
	mov    rdi,rax
	call   QWORD PTR [rip+0x1c537]        # cleanup function call
	call   QWORD PTR [rip+0x1c539]        # <unfortunate>
	mov    ebx,0xd
	mov    eax,ebx
	pop    rbx
	ret

# -Cpanic=abort:
	push   rax
	call   QWORD PTR [rip+0x20a1]        # <happy>
	mov    eax,0x2a
	pop    rcx
	ret
```

Fixes #64224, and resolves #64222.

Author: bors

.gitmodules

@@ -39,8 +39,8 @@
 	url = https://github.com/rust-lang/edition-guide.git
 [submodule "src/llvm-project"]
 	path = src/llvm-project
-	url = https://github.com/rust-lang/llvm-project.git
-	branch = rustc/9.0-2019-12-19
+	url = https://github.com/Amanieu/llvm-project.git
+	branch = opt-catch
 [submodule "src/doc/embedded-book"]
 	path = src/doc/embedded-book
 	url = https://github.com/rust-embedded/book.git

Cargo.lock

@@ -2308,6 +2308,7 @@ dependencies = [
 name = "panic_abort"
 version = "0.0.0"
 dependencies = [
+ "cfg-if",
  "compiler_builtins",
  "core",
  "libc",

src/ci/azure-pipelines/try.yml

@@ -6,17 +6,17 @@ variables:
 - group: prod-credentials
 
 jobs:
-- job: Linux
-  timeoutInMinutes: 600
-  pool:
-    vmImage: ubuntu-16.04
-  steps:
-  - template: steps/run.yml
-  strategy:
-    matrix:
-      dist-x86_64-linux: {}
-      dist-x86_64-linux-alt:
-        IMAGE: dist-x86_64-linux
+    # - job: Linux
+    #   timeoutInMinutes: 600
+    #   pool:
+    #     vmImage: ubuntu-16.04
+    #   steps:
+    #   - template: steps/run.yml
+    #   strategy:
+    #     matrix:
+    #       dist-x86_64-linux: {}
+    #       dist-x86_64-linux-alt:
+    #         IMAGE: dist-x86_64-linux
 
 # The macOS and Windows builds here are currently disabled due to them not being
 # overly necessary on `try` builds. We also don't actually have anything that
@@ -49,25 +49,20 @@ jobs:
 #         NO_LLVM_ASSERTIONS: 1
 #         NO_DEBUG_ASSERTIONS: 1
 #
-# - job: Windows
-#   timeoutInMinutes: 600
-#   pool:
-#     vmImage: 'vs2017-win2016'
-#   steps:
-#   - template: steps/run.yml
-#   strategy:
-#     matrix:
-#       dist-x86_64-msvc:
-#         RUST_CONFIGURE_ARGS: >
-#           --build=x86_64-pc-windows-msvc
-#           --target=x86_64-pc-windows-msvc,aarch64-pc-windows-msvc
-#           --enable-full-tools
-#           --enable-profiler
-#         SCRIPT: python x.py dist
-#         DIST_REQUIRE_ALL_TOOLS: 1
-#         DEPLOY: 1
-#
-#       dist-x86_64-msvc-alt:
-#         RUST_CONFIGURE_ARGS: --build=x86_64-pc-windows-msvc --enable-extended --enable-profiler
-#         SCRIPT: python x.py dist
-#         DEPLOY_ALT: 1
+- job: Windows
+  timeoutInMinutes: 600
+  pool:
+    vmImage: 'vs2017-win2016'
+  steps:
+  - template: steps/run.yml
+  strategy:
+    matrix:
+      i686-mingw-2:
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu
+        SCRIPT: make ci-mingw-subset-2
+        CUSTOM_MINGW: 1
+      dist-i686-mingw:
+        RUST_CONFIGURE_ARGS: --build=i686-pc-windows-gnu --enable-debug-assertions --enable-optimize --debuginfo-level=1
+        SCRIPT: python x.py dist
+        CUSTOM_MINGW: 1
+        N_DIST_REQUIRE_ALL_TOOLS: 1

src/doc/unstable-book/src/language-features/lang-items.md

@@ -248,7 +248,6 @@ the source code.
   - `eh_personality`: `libpanic_unwind/gcc.rs` (GNU)
   - `eh_personality`: `libpanic_unwind/seh.rs` (SEH)
   - `eh_unwind_resume`: `libpanic_unwind/gcc.rs` (GCC)
-  - `eh_catch_typeinfo`: `libpanic_unwind/seh.rs` (SEH)
   - `eh_catch_typeinfo`: `libpanic_unwind/emcc.rs` (EMCC)
   - `panic`: `libcore/panicking.rs`
   - `panic_bounds_check`: `libcore/panicking.rs`

src/libpanic_abort/Cargo.toml

@@ -14,3 +14,4 @@ doc = false
 core = { path = "../libcore" }
 libc = { version = "0.2", default-features = false }
 compiler_builtins = "0.1.0"
+cfg-if = "0.1.8"

src/libpanic_abort/lib.rs

@@ -18,17 +18,14 @@
 #![feature(staged_api)]
 #![feature(rustc_attrs)]
 
-// Rust's "try" function, but if we're aborting on panics we just call the
-// function as there's nothing else we need to do here.
+use core::any::Any;
+
+// We need the definition of TryPayload for __rust_panic_cleanup.
+include!("../libpanic_unwind/payload.rs");
+
 #[rustc_std_internal_symbol]
-pub unsafe extern "C" fn __rust_maybe_catch_panic(
-    f: fn(*mut u8),
-    data: *mut u8,
-    _data_ptr: *mut usize,
-    _vtable_ptr: *mut usize,
-) -> u32 {
-    f(data);
-    0
+pub unsafe extern "C" fn __rust_panic_cleanup(_: TryPayload) -> *mut (dyn Any + Send + 'static) {
+    unreachable!()
 }
 
 // "Leak" the payload and shim to the relevant abort on the platform in

src/libpanic_unwind/dummy.rs

@@ -6,10 +6,6 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::intrinsics;
 
-pub fn payload() -> *mut u8 {
-    core::ptr::null_mut()
-}
-
 pub unsafe fn cleanup(_ptr: *mut u8) -> Box<dyn Any + Send> {
     intrinsics::abort()
 }

src/libpanic_unwind/emcc.rs

@@ -48,10 +48,6 @@ static EXCEPTION_TYPE_INFO: TypeInfo = TypeInfo {
     name: b"rust_panic\0".as_ptr(),
 };
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 struct Exception {
     // This needs to be an Option because the object's lifetime follows C++
     // semantics: when catch_unwind moves the Box out of the exception it must

src/libpanic_unwind/gcc.rs

@@ -48,7 +48,6 @@
 
 use alloc::boxed::Box;
 use core::any::Any;
-use core::ptr;
 
 use crate::dwarf::eh::{self, EHAction, EHContext};
 use libc::{c_int, uintptr_t};
@@ -83,10 +82,6 @@ pub unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
     }
 }
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 pub unsafe fn cleanup(ptr: *mut u8) -> Box<dyn Any + Send> {
     let exception = Box::from_raw(ptr as *mut Exception);
     exception.cause
@@ -337,8 +332,14 @@ unsafe fn find_eh_action(
 ))]
 #[lang = "eh_unwind_resume"]
 #[unwind(allowed)]
-unsafe extern "C" fn rust_eh_unwind_resume(panic_ctx: *mut u8) -> ! {
-    uw::_Unwind_Resume(panic_ctx as *mut uw::_Unwind_Exception);
+#[naked]
+#[inline(never)]
+unsafe extern "C" fn rust_eh_unwind_resume(_panic_ctx: *mut u8) -> ! {
+    // This needs to be a naked function because _Unwind_Resume expects to be
+    // called directly from the landing pad. This means that we need to force
+    // a tail call here.
+    asm!("jmp ${0:P}" :: "s" (uw::_Unwind_Resume as usize) :: "volatile");
+    core::hint::unreachable_unchecked();
 }
 
 // Frame unwind info registration

src/libpanic_unwind/hermit.rs

@@ -6,10 +6,6 @@ use alloc::boxed::Box;
 use core::any::Any;
 use core::ptr;
 
-pub fn payload() -> *mut u8 {
-    ptr::null_mut()
-}
-
 pub unsafe fn cleanup(_ptr: *mut u8) -> Box<dyn Any + Send> {
     extern "C" {
         pub fn __rust_abort() -> !;

src/libpanic_unwind/lib.rs

@@ -22,20 +22,23 @@
 #![feature(libc)]
 #![feature(nll)]
 #![feature(panic_unwind)]
-#![feature(raw)]
 #![feature(staged_api)]
 #![feature(std_internals)]
 #![feature(unwind_attributes)]
 #![feature(abi_thiscall)]
+#![feature(rustc_attrs)]
+#![feature(raw)]
+#![feature(asm)]
+#![feature(naked_functions)]
 #![panic_runtime]
 #![feature(panic_runtime)]
 
 use alloc::boxed::Box;
-use core::intrinsics;
-use core::mem;
+use core::any::Any;
 use core::panic::BoxMeUp;
-use core::raw;
 
+// If adding to this list, you should also look at the list of TryPayload types
+// defined in payload.rs and likely add to there as well.
 cfg_if::cfg_if! {
     if #[cfg(target_os = "emscripten")] {
         #[path = "emcc.rs"]
@@ -61,6 +64,8 @@ cfg_if::cfg_if! {
     }
 }
 
+include!("payload.rs");
+
 extern "C" {
     /// Handler in libstd called when a panic object is dropped outside of
     /// `catch_unwind`.
@@ -69,28 +74,11 @@ extern "C" {
 
 mod dwarf;
 
-// Entry point for catching an exception, implemented using the `try` intrinsic
-// in the compiler.
-//
-// The interaction between the `payload` function and the compiler is pretty
-// hairy and tightly coupled, for more information see the compiler's
-// implementation of this.
 #[no_mangle]
-pub unsafe extern "C" fn __rust_maybe_catch_panic(
-    f: fn(*mut u8),
-    data: *mut u8,
-    data_ptr: *mut usize,
-    vtable_ptr: *mut usize,
-) -> u32 {
-    let mut payload = imp::payload();
-    if intrinsics::r#try(f, data, &mut payload as *mut _ as *mut _) == 0 {
-        0
-    } else {
-        let obj = mem::transmute::<_, raw::TraitObject>(imp::cleanup(payload));
-        *data_ptr = obj.data as usize;
-        *vtable_ptr = obj.vtable as usize;
-        1
-    }
+pub unsafe extern "C" fn __rust_panic_cleanup(
+    payload: TryPayload,
+) -> *mut (dyn Any + Send + 'static) {
+    Box::into_raw(imp::cleanup(payload))
 }
 
 // Entry point for raising an exception, just delegates to the platform-specific

src/libpanic_unwind/payload.rs

@@ -0,0 +1,21 @@
+// Type definition for the payload argument of the try intrinsic.
+//
+// This must be kept in sync with the implementations of the try intrinsic.
+//
+// This file is included by both panic runtimes and libstd. It is part of the
+// panic runtime ABI.
+cfg_if::cfg_if! {
+    if #[cfg(target_os = "emscripten")] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(target_arch = "wasm32")] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(target_os = "hermit")] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(all(target_env = "msvc", target_arch = "aarch64"))] {
+        type TryPayload = *mut u8;
+    } else if #[cfg(target_env = "msvc")] {
+        type TryPayload = [u64; 2];
+    } else {
+        type TryPayload = *mut u8;
+    }
+}

src/libpanic_unwind/seh.rs

@@ -167,6 +167,9 @@ pub struct _TypeDescriptor {
 
 // Note that we intentionally ignore name mangling rules here: we don't want C++
 // to be able to catch Rust panics by simply declaring a `struct rust_panic`.
+//
+// When modifying, make sure that the type name string exactly matches
+// the one used in src/librustc_codegen_llvm/intrinsic.rs.
 const TYPE_NAME: [u8; 11] = *b"rust_panic\0";
 
 static mut THROW_INFO: _ThrowInfo = _ThrowInfo {
@@ -199,12 +202,12 @@ extern "C" {
     static TYPE_INFO_VTABLE: *const u8;
 }
 
-// We use #[lang = "eh_catch_typeinfo"] here as this is the type descriptor which
-// we'll use in LLVM's `catchpad` instruction which ends up also being passed as
-// an argument to the C++ personality function.
+// This type descriptor is only used when throwing an exception. The catch part
+// is handled by the try intrinsic, which generates its own TypeDescriptor.
 //
-// Again, I'm not entirely sure what this is describing, it just seems to work.
-#[cfg_attr(not(test), lang = "eh_catch_typeinfo")]
+// This is fine since the MSVC runtime uses string comparison on the type name
+// to match TypeDescriptors rather than pointer equality.
+#[cfg_attr(bootstrap, lang = "eh_catch_typeinfo")]
 static mut TYPE_DESCRIPTOR: _TypeDescriptor = _TypeDescriptor {
     pVFTable: unsafe { &TYPE_INFO_VTABLE } as *const _ as *const _,
     spare: core::ptr::null_mut(),
@@ -308,10 +311,6 @@ pub unsafe fn panic(data: Box<dyn Any + Send>) -> u32 {
     _CxxThrowException(throw_ptr, &mut THROW_INFO as *mut _ as *mut _);
 }
 
-pub fn payload() -> [u64; 2] {
-    [0; 2]
-}
-
 pub unsafe fn cleanup(payload: [u64; 2]) -> Box<dyn Any + Send> {
     mem::transmute(raw::TraitObject { data: payload[0] as *mut _, vtable: payload[1] as *mut _ })
 }

src/librustc_codegen_llvm/attributes.rs

@@ -270,12 +270,23 @@ pub fn from_fn_attrs(
         }
     }
 
-    // FIXME(eddyb) consolidate these two `inline` calls (and avoid overwrites).
-    if instance.def.requires_inline(cx.tcx) {
-        inline(cx, llfn, attributes::InlineAttr::Hint);
-    }
+    // Naked functions must never be inlined and must not contain any prologue
+    // or epilogue.
+    if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) {
+        naked(llfn, true);
+        inline(cx, llfn, attributes::InlineAttr::Never);
+    } else {
+        set_frame_pointer_elimination(cx, llfn);
+        set_instrument_function(cx, llfn);
+        set_probestack(cx, llfn);
 
-    inline(cx, llfn, codegen_fn_attrs.inline);
+        // FIXME(eddyb) consolidate these two `inline` calls (and avoid overwrites).
+        if instance.def.requires_inline(cx.tcx) {
+            inline(cx, llfn, attributes::InlineAttr::Hint);
+        }
+
+        inline(cx, llfn, codegen_fn_attrs.inline);
+    }
 
     // The `uwtable` attribute according to LLVM is:
     //
@@ -297,19 +308,12 @@ pub fn from_fn_attrs(
         attributes::emit_uwtable(llfn, true);
     }
 
-    set_frame_pointer_elimination(cx, llfn);
-    set_instrument_function(cx, llfn);
-    set_probestack(cx, llfn);
-
     if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::COLD) {
         Attribute::Cold.apply_llfn(Function, llfn);
     }
     if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::FFI_RETURNS_TWICE) {
         Attribute::ReturnsTwice.apply_llfn(Function, llfn);
     }
-    if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::NAKED) {
-        naked(llfn, true);
-    }
     if codegen_fn_attrs.flags.contains(CodegenFnAttrFlags::ALLOCATOR) {
         Attribute::NoAlias.apply_llfn(llvm::AttributePlace::ReturnValue, llfn);
     }