Unwinding support for inline assembly

compiler/rustc_ast/src/ast.rs

@@ -1981,7 +1981,7 @@ pub enum InlineAsmRegOrRegClass {
 
 bitflags::bitflags! {
     #[derive(Encodable, Decodable, HashStable_Generic)]
-    pub struct InlineAsmOptions: u8 {
+    pub struct InlineAsmOptions: u16 {
         const PURE = 1 << 0;
         const NOMEM = 1 << 1;
         const READONLY = 1 << 2;
@@ -1990,6 +1990,7 @@ bitflags::bitflags! {
         const NOSTACK = 1 << 5;
         const ATT_SYNTAX = 1 << 6;
         const RAW = 1 << 7;
+        const MAY_UNWIND = 1 << 8;
     }
 }
 

compiler/rustc_ast_lowering/src/asm.rs

@@ -49,6 +49,17 @@ impl<'a, 'hir> LoweringContext<'a, 'hir> {
                 .struct_span_err(sp, "the `att_syntax` option is only supported on x86")
                 .emit();
         }
+        if asm.options.contains(InlineAsmOptions::MAY_UNWIND)
+            && !self.sess.features_untracked().asm_unwind
+        {
+            feature_err(
+                &self.sess.parse_sess,
+                sym::asm_unwind,
+                sp,
+                "the `may_unwind` option is unstable",
+            )
+            .emit();
+        }
 
         let mut clobber_abis = FxHashMap::default();
         if let Some(asm_arch) = asm_arch {

compiler/rustc_ast_pretty/src/pprust/state.rs

@@ -2338,6 +2338,9 @@ impl<'a> State<'a> {
                 if opts.contains(InlineAsmOptions::RAW) {
                     options.push("raw");
                 }
+                if opts.contains(InlineAsmOptions::MAY_UNWIND) {
+                    options.push("may_unwind");
+                }
                 s.commasep(Inconsistent, &options, |s, &opt| {
                     s.word(opt);
                 });

compiler/rustc_borrowck/src/dataflow.rs

@@ -5,7 +5,7 @@ use rustc_middle::ty::RegionVid;
 use rustc_middle::ty::TyCtxt;
 use rustc_mir_dataflow::impls::{EverInitializedPlaces, MaybeUninitializedPlaces};
 use rustc_mir_dataflow::ResultsVisitable;
-use rustc_mir_dataflow::{self, fmt::DebugWithContext, GenKill};
+use rustc_mir_dataflow::{self, fmt::DebugWithContext, CallReturnPlaces, GenKill};
 use rustc_mir_dataflow::{Analysis, Direction, Results};
 use std::fmt;
 use std::iter;
@@ -434,9 +434,7 @@ impl<'tcx> rustc_mir_dataflow::GenKillAnalysis<'tcx> for Borrows<'_, 'tcx> {
         &self,
         _trans: &mut impl GenKill<Self::Idx>,
         _block: mir::BasicBlock,
-        _func: &mir::Operand<'tcx>,
-        _args: &[mir::Operand<'tcx>],
-        _dest_place: mir::Place<'tcx>,
+        _return_places: CallReturnPlaces<'_, 'tcx>,
     ) {
     }
 }

compiler/rustc_borrowck/src/def_use.rs

@@ -17,7 +17,7 @@ pub fn categorize(context: PlaceContext) -> Option<DefUse> {
         PlaceContext::MutatingUse(MutatingUseContext::Store) |
 
         // This is potentially both a def and a use...
-        PlaceContext::MutatingUse(MutatingUseContext::AsmOutput) |
+        PlaceContext::MutatingUse(MutatingUseContext::LlvmAsmOutput) |
 
         // We let Call define the result in both the success and
         // unwind cases. This is not really correct, however it
@@ -26,6 +26,7 @@ pub fn categorize(context: PlaceContext) -> Option<DefUse> {
         // the def in call only to the input from the success
         // path and not the unwind path. -nmatsakis
         PlaceContext::MutatingUse(MutatingUseContext::Call) |
+        PlaceContext::MutatingUse(MutatingUseContext::AsmOutput) |
         PlaceContext::MutatingUse(MutatingUseContext::Yield) |
 
         // Storage live and storage dead aren't proper defines, but we can ignore

compiler/rustc_borrowck/src/invalidation.rs

@@ -199,6 +199,7 @@ impl<'cx, 'tcx> Visitor<'tcx> for InvalidationGenerator<'cx, 'tcx> {
                 options: _,
                 line_spans: _,
                 destination: _,
+                cleanup: _,
             } => {
                 for op in operands {
                     match *op {

compiler/rustc_borrowck/src/lib.rs

@@ -791,6 +791,7 @@ impl<'cx, 'tcx> rustc_mir_dataflow::ResultsVisitor<'cx, 'tcx> for MirBorrowckCtx
                 options: _,
                 line_spans: _,
                 destination: _,
+                cleanup: _,
             } => {
                 for op in operands {
                     match *op {

compiler/rustc_borrowck/src/type_check/mod.rs

@@ -1828,10 +1828,16 @@ impl<'a, 'tcx> TypeChecker<'a, 'tcx> {
                     self.assert_iscleanup(body, block_data, unwind, true);
                 }
             }
-            TerminatorKind::InlineAsm { destination, .. } => {
+            TerminatorKind::InlineAsm { destination, cleanup, .. } => {
                 if let Some(target) = destination {
                     self.assert_iscleanup(body, block_data, target, is_cleanup);
                 }
+                if let Some(cleanup) = cleanup {
+                    if is_cleanup {
+                        span_mirbug!(self, block_data, "cleanup on cleanup block")
+                    }
+                    self.assert_iscleanup(body, block_data, cleanup, true);
+                }
             }
         }
     }

compiler/rustc_builtin_macros/src/asm.rs

@@ -420,6 +420,8 @@ fn parse_options<'a>(
             try_set_option(p, args, sym::att_syntax, ast::InlineAsmOptions::ATT_SYNTAX);
         } else if p.eat_keyword(kw::Raw) {
             try_set_option(p, args, kw::Raw, ast::InlineAsmOptions::RAW);
+        } else if p.eat_keyword(sym::may_unwind) {
+            try_set_option(p, args, kw::Raw, ast::InlineAsmOptions::MAY_UNWIND);
         } else {
             return p.unexpected();
         }

compiler/rustc_codegen_cranelift/src/base.rs

@@ -1,6 +1,7 @@
 //! Codegen of a single function
 
 use cranelift_codegen::binemit::{NullStackMapSink, NullTrapSink};
+use rustc_ast::InlineAsmOptions;
 use rustc_index::vec::IndexVec;
 use rustc_middle::ty::adjustment::PointerCast;
 use rustc_middle::ty::layout::FnAbiOf;
@@ -239,7 +240,8 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             fx.add_comment(inst, terminator_head);
         }
 
-        fx.set_debug_loc(bb_data.terminator().source_info);
+        let source_info = bb_data.terminator().source_info;
+        fx.set_debug_loc(source_info);
 
         match &bb_data.terminator().kind {
             TerminatorKind::Goto { target } => {
@@ -295,19 +297,19 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                         let len = codegen_operand(fx, len).load_scalar(fx);
                         let index = codegen_operand(fx, index).load_scalar(fx);
                         let location = fx
-                            .get_caller_location(bb_data.terminator().source_info.span)
+                            .get_caller_location(source_info.span)
                             .load_scalar(fx);
 
                         codegen_panic_inner(
                             fx,
                             rustc_hir::LangItem::PanicBoundsCheck,
                             &[index, len, location],
-                            bb_data.terminator().source_info.span,
+                            source_info.span,
                         );
                     }
                     _ => {
                         let msg_str = msg.description();
-                        codegen_panic(fx, msg_str, bb_data.terminator().source_info.span);
+                        codegen_panic(fx, msg_str, source_info.span);
                     }
                 }
             }
@@ -378,10 +380,18 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
                 options,
                 destination,
                 line_spans: _,
+                cleanup: _,
             } => {
+                if options.contains(InlineAsmOptions::MAY_UNWIND) {
+                    fx.tcx.sess.span_fatal(
+                        source_info.span,
+                        "cranelift doesn't support unwinding from inline assembly.",
+                    );
+                }
+
                 crate::inline_asm::codegen_inline_asm(
                     fx,
-                    bb_data.terminator().source_info.span,
+                    source_info.span,
                     template,
                     operands,
                     *options,
@@ -415,7 +425,7 @@ fn codegen_fn_content(fx: &mut FunctionCx<'_, '_, '_>) {
             }
             TerminatorKind::Drop { place, target, unwind: _ } => {
                 let drop_place = codegen_place(fx, *place);
-                crate::abi::codegen_drop(fx, bb_data.terminator().source_info.span, drop_place);
+                crate::abi::codegen_drop(fx, source_info.span, drop_place);
 
                 let target_block = fx.get_block(*target);
                 fx.bcx.ins().jump(target_block, &[]);

compiler/rustc_codegen_gcc/src/asm.rs

@@ -118,7 +118,14 @@ impl<'a, 'gcc, 'tcx> AsmBuilderMethods<'tcx> for Builder<'a, 'gcc, 'tcx> {
         true
     }
 
-    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, _span: &[Span], _instance: Instance<'_>) {
+    fn codegen_inline_asm(&mut self, template: &[InlineAsmTemplatePiece], rust_operands: &[InlineAsmOperandRef<'tcx, Self>], options: InlineAsmOptions, span: &[Span], _instance: Instance<'_>, _dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>) {
+        if options.contains(InlineAsmOptions::MAY_UNWIND) {
+            self.sess()
+                .struct_span_err(span[0], "GCC backend does not support unwinding from inline asm")
+                .emit();
+            return;
+        }
+
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
         let is_x86 = matches!(asm_arch, InlineAsmArch::X86 | InlineAsmArch::X86_64);
         let att_dialect = is_x86 && options.contains(InlineAsmOptions::ATT_SYNTAX);

compiler/rustc_codegen_llvm/src/asm.rs

@@ -1,6 +1,8 @@
 use crate::builder::Builder;
+use crate::common::Funclet;
 use crate::context::CodegenCx;
 use crate::llvm;
+use crate::llvm_util;
 use crate::type_::Type;
 use crate::type_of::LayoutLlvmExt;
 use crate::value::Value;
@@ -98,6 +100,8 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             ia.alignstack,
             ia.dialect,
             &[span],
+            false,
+            None,
         );
         if r.is_none() {
             return false;
@@ -121,6 +125,7 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
         options: InlineAsmOptions,
         line_spans: &[Span],
         instance: Instance<'_>,
+        dest_catch_funclet: Option<(Self::BasicBlock, Self::BasicBlock, Option<&Self::Funclet>)>,
     ) {
         let asm_arch = self.tcx.sess.asm_arch.unwrap();
 
@@ -355,6 +360,8 @@ impl AsmBuilderMethods<'tcx> for Builder<'a, 'll, 'tcx> {
             alignstack,
             dialect,
             line_spans,
+            options.contains(InlineAsmOptions::MAY_UNWIND),
+            dest_catch_funclet,
         )
         .unwrap_or_else(|| span_bug!(line_spans[0], "LLVM asm constraint validation failed"));
 
@@ -447,9 +454,16 @@ pub(crate) fn inline_asm_call(
     alignstack: bool,
     dia: LlvmAsmDialect,
     line_spans: &[Span],
+    unwind: bool,
+    dest_catch_funclet: Option<(
+        &'ll llvm::BasicBlock,
+        &'ll llvm::BasicBlock,
+        Option<&Funclet<'ll>>,
+    )>,
 ) -> Option<&'ll Value> {
     let volatile = if volatile { llvm::True } else { llvm::False };
     let alignstack = if alignstack { llvm::True } else { llvm::False };
+    let can_throw = if unwind { llvm::True } else { llvm::False };
 
     let argtys = inputs
         .iter()
@@ -466,6 +480,13 @@ pub(crate) fn inline_asm_call(
         let constraints_ok = llvm::LLVMRustInlineAsmVerify(fty, cons.as_ptr().cast(), cons.len());
         debug!("constraint verification result: {:?}", constraints_ok);
         if constraints_ok {
+            if unwind && llvm_util::get_version() < (13, 0, 0) {
+                bx.cx.sess().span_fatal(
+                    line_spans[0],
+                    "unwinding from inline assembly is only supported on llvm >= 13.",
+                );
+            }
+
             let v = llvm::LLVMRustInlineAsm(
                 fty,
                 asm.as_ptr().cast(),
@@ -475,8 +496,14 @@ pub(crate) fn inline_asm_call(
                 volatile,
                 alignstack,
                 llvm::AsmDialect::from_generic(dia),
+                can_throw,
             );
-            let call = bx.call(fty, v, inputs, None);
+
+            let call = if let Some((dest, catch, funclet)) = dest_catch_funclet {
+                bx.invoke(fty, v, inputs, dest, catch, funclet)
+            } else {
+                bx.call(fty, v, inputs, None)
+            };
 
             // Store mark in a metadata node so we can map LLVM errors
             // back to source locations.  See #17552.

compiler/rustc_codegen_llvm/src/intrinsic.rs

@@ -350,6 +350,8 @@ impl IntrinsicCallMethods<'tcx> for Builder<'a, 'll, 'tcx> {
                     false,
                     ast::LlvmAsmDialect::Att,
                     &[span],
+                    false,
+                    None,
                 )
                 .unwrap_or_else(|| bug!("failed to generate inline asm call for `black_box`"));
 

compiler/rustc_codegen_llvm/src/llvm/ffi.rs

@@ -1847,6 +1847,7 @@ extern "C" {
         SideEffects: Bool,
         AlignStack: Bool,
         Dialect: AsmDialect,
+        CanThrow: Bool,
     ) -> &Value;
     pub fn LLVMRustInlineAsmVerify(
         Ty: &Type,

compiler/rustc_codegen_ssa/src/mir/analyze.rs

@@ -211,6 +211,7 @@ impl<'mir, 'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> Visitor<'tcx>
 
             PlaceContext::MutatingUse(
                 MutatingUseContext::Store
+                | MutatingUseContext::LlvmAsmOutput
                 | MutatingUseContext::AsmOutput
                 | MutatingUseContext::Borrow
                 | MutatingUseContext::AddressOf
@@ -275,9 +276,9 @@ pub fn cleanup_kinds(mir: &mir::Body<'_>) -> IndexVec<mir::BasicBlock, CleanupKi
                 | TerminatorKind::SwitchInt { .. }
                 | TerminatorKind::Yield { .. }
                 | TerminatorKind::FalseEdge { .. }
-                | TerminatorKind::FalseUnwind { .. }
-                | TerminatorKind::InlineAsm { .. } => { /* nothing to do */ }
+                | TerminatorKind::FalseUnwind { .. } => { /* nothing to do */ }
                 TerminatorKind::Call { cleanup: unwind, .. }
+                | TerminatorKind::InlineAsm { cleanup: unwind, .. }
                 | TerminatorKind::Assert { cleanup: unwind, .. }
                 | TerminatorKind::DropAndReplace { unwind, .. }
                 | TerminatorKind::Drop { unwind, .. } => {