cmse: don't use BackendRepr when checking return type

compiler/rustc_abi/src/layout/ty.rs

@@ -290,7 +290,19 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
     /// function call isn't allowed (a.k.a. `va_list`).
     ///
     /// This function handles transparent types automatically.
-    pub fn pass_indirectly_in_non_rustic_abis<C>(mut self, cx: &C) -> bool
+    pub fn pass_indirectly_in_non_rustic_abis<C>(self, cx: &C) -> bool
+    where
+        Ty: TyAbiInterface<'a, C> + Copy,
+    {
+        let base = self.peel_transparent_wrappers(cx);
+        Ty::is_pass_indirectly_in_non_rustic_abis_flag_set(base)
+    }
+
+    /// Recursively peel away transparent wrappers, returning the inner value.
+    ///
+    /// The return value is not `repr(transparent)` and/or does
+    /// not have a non-1zst field.
+    pub fn peel_transparent_wrappers<C>(mut self, cx: &C) -> Self
     where
         Ty: TyAbiInterface<'a, C> + Copy,
     {
@@ -300,7 +312,7 @@ impl<'a, Ty> TyAndLayout<'a, Ty> {
             self = field;
         }
 
-        Ty::is_pass_indirectly_in_non_rustic_abis_flag_set(self)
+        self
     }
 
     /// Finds the one field that is not a 1-ZST.

compiler/rustc_hir_analysis/src/hir_ty_lowering/cmse.rs

@@ -1,8 +1,8 @@
-use rustc_abi::{BackendRepr, ExternAbi, Float, Integer, Primitive, Scalar};
+use rustc_abi::ExternAbi;
 use rustc_errors::{DiagCtxtHandle, E0781, struct_span_code_err};
 use rustc_hir::{self as hir, HirId};
 use rustc_middle::bug;
-use rustc_middle::ty::layout::{LayoutError, TyAndLayout};
+use rustc_middle::ty::layout::{LayoutCx, LayoutError, TyAndLayout};
 use rustc_middle::ty::{self, TyCtxt, TypeVisitableExt};
 use rustc_span::Span;
 
@@ -150,34 +150,30 @@ fn is_valid_cmse_output<'tcx>(
 
     let typing_env = ty::TypingEnv::fully_monomorphized();
     let layout = tcx.layout_of(typing_env.as_query_input(return_type))?;
+    let layout_cx = LayoutCx::new(tcx, typing_env);
 
-    if !is_valid_cmse_output_layout(layout) {
+    if !is_valid_cmse_output_layout(layout_cx, layout) {
         dcx.emit_err(errors::CmseOutputStackSpill { span: fn_decl.output.span(), abi });
     }
 
     Ok(())
 }
 
 /// Returns whether the output will fit into the available registers
-fn is_valid_cmse_output_layout<'tcx>(layout: TyAndLayout<'tcx>) -> bool {
+fn is_valid_cmse_output_layout<'tcx>(cx: LayoutCx<'tcx>, layout: TyAndLayout<'tcx>) -> bool {
     let size = layout.layout.size().bytes();
 
     if size <= 4 {
         return true;
-    } else if size > 8 {
+    } else if size != 8 {
         return false;
     }
 
-    // Accept scalar 64-bit types.
-    let BackendRepr::Scalar(scalar) = layout.layout.backend_repr else {
-        return false;
-    };
-
-    let Scalar::Initialized { value, .. } = scalar else {
-        return false;
-    };
-
-    matches!(value, Primitive::Int(Integer::I64, _) | Primitive::Float(Float::F64))
+    // Accept (transparently wrapped) scalar 64-bit primitives.
+    matches!(
+        layout.peel_transparent_wrappers(&cx).ty.kind(),
+        ty::Int(ty::IntTy::I64) | ty::Uint(ty::UintTy::U64) | ty::Float(ty::FloatTy::F64)
+    )
 }
 
 fn should_emit_layout_error<'tcx>(abi: ExternAbi, layout_err: &'tcx LayoutError<'tcx>) -> bool {

tests/ui/cmse-nonsecure/cmse-nonsecure-call/return-via-stack.rs

@@ -13,6 +13,9 @@ use minicore::*;
 #[repr(C)]
 pub struct ReprCU64(u64);
 
+#[repr(Rust)]
+pub struct ReprRustU64(u64);
+
 #[repr(C)]
 pub struct ReprCBytes(u8, u8, u8, u8, u8);
 
@@ -25,10 +28,11 @@ pub struct ReprCAlign16(u16);
 #[no_mangle]
 pub fn test(
     f1: extern "cmse-nonsecure-call" fn() -> ReprCU64, //~ ERROR [E0798]
-    f2: extern "cmse-nonsecure-call" fn() -> ReprCBytes, //~ ERROR [E0798]
-    f3: extern "cmse-nonsecure-call" fn() -> U64Compound, //~ ERROR [E0798]
-    f4: extern "cmse-nonsecure-call" fn() -> ReprCAlign16, //~ ERROR [E0798]
-    f5: extern "cmse-nonsecure-call" fn() -> [u8; 5],  //~ ERROR [E0798]
+    f2: extern "cmse-nonsecure-call" fn() -> ReprRustU64, //~ ERROR [E0798]
+    f3: extern "cmse-nonsecure-call" fn() -> ReprCBytes, //~ ERROR [E0798]
+    f4: extern "cmse-nonsecure-call" fn() -> U64Compound, //~ ERROR [E0798]
+    f5: extern "cmse-nonsecure-call" fn() -> ReprCAlign16, //~ ERROR [E0798]
+    f6: extern "cmse-nonsecure-call" fn() -> [u8; 5],  //~ ERROR [E0798]
 ) {
 }
 

tests/ui/cmse-nonsecure/cmse-nonsecure-call/return-via-stack.stderr

@@ -1,5 +1,5 @@
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:37:48
+  --> $DIR/return-via-stack.rs:41:48
    |
 LL |     u128: extern "cmse-nonsecure-call" fn() -> u128,
    |                                                ^^^^ this type doesn't fit in the available registers
@@ -8,7 +8,7 @@ LL |     u128: extern "cmse-nonsecure-call" fn() -> u128,
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:38:48
+  --> $DIR/return-via-stack.rs:42:48
    |
 LL |     i128: extern "cmse-nonsecure-call" fn() -> i128,
    |                                                ^^^^ this type doesn't fit in the available registers
@@ -17,7 +17,7 @@ LL |     i128: extern "cmse-nonsecure-call" fn() -> i128,
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:27:46
+  --> $DIR/return-via-stack.rs:30:46
    |
 LL |     f1: extern "cmse-nonsecure-call" fn() -> ReprCU64,
    |                                              ^^^^^^^^ this type doesn't fit in the available registers
@@ -26,43 +26,52 @@ LL |     f1: extern "cmse-nonsecure-call" fn() -> ReprCU64,
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:28:46
+  --> $DIR/return-via-stack.rs:31:46
+   |
+LL |     f2: extern "cmse-nonsecure-call" fn() -> ReprRustU64,
+   |                                              ^^^^^^^^^^^ this type doesn't fit in the available registers
    |
-LL |     f2: extern "cmse-nonsecure-call" fn() -> ReprCBytes,
+   = note: functions with the `"cmse-nonsecure-call"` ABI must pass their result via the available return registers
+   = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
+
+error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
+  --> $DIR/return-via-stack.rs:32:46
+   |
+LL |     f3: extern "cmse-nonsecure-call" fn() -> ReprCBytes,
    |                                              ^^^^^^^^^^ this type doesn't fit in the available registers
    |
    = note: functions with the `"cmse-nonsecure-call"` ABI must pass their result via the available return registers
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:29:46
+  --> $DIR/return-via-stack.rs:33:46
    |
-LL |     f3: extern "cmse-nonsecure-call" fn() -> U64Compound,
+LL |     f4: extern "cmse-nonsecure-call" fn() -> U64Compound,
    |                                              ^^^^^^^^^^^ this type doesn't fit in the available registers
    |
    = note: functions with the `"cmse-nonsecure-call"` ABI must pass their result via the available return registers
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:30:46
+  --> $DIR/return-via-stack.rs:34:46
    |
-LL |     f4: extern "cmse-nonsecure-call" fn() -> ReprCAlign16,
+LL |     f5: extern "cmse-nonsecure-call" fn() -> ReprCAlign16,
    |                                              ^^^^^^^^^^^^ this type doesn't fit in the available registers
    |
    = note: functions with the `"cmse-nonsecure-call"` ABI must pass their result via the available return registers
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:31:46
+  --> $DIR/return-via-stack.rs:35:46
    |
-LL |     f5: extern "cmse-nonsecure-call" fn() -> [u8; 5],
+LL |     f6: extern "cmse-nonsecure-call" fn() -> [u8; 5],
    |                                              ^^^^^^^ this type doesn't fit in the available registers
    |
    = note: functions with the `"cmse-nonsecure-call"` ABI must pass their result via the available return registers
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:53:46
+  --> $DIR/return-via-stack.rs:57:46
    |
 LL |     f1: extern "cmse-nonsecure-call" fn() -> ReprRustUnionU64,
    |                                              ^^^^^^^^^^^^^^^^ this type doesn't fit in the available registers
@@ -71,14 +80,14 @@ LL |     f1: extern "cmse-nonsecure-call" fn() -> ReprRustUnionU64,
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
 error[E0798]: return value of `"cmse-nonsecure-call"` function too large to pass via registers
-  --> $DIR/return-via-stack.rs:54:46
+  --> $DIR/return-via-stack.rs:58:46
    |
 LL |     f2: extern "cmse-nonsecure-call" fn() -> ReprCUnionU64,
    |                                              ^^^^^^^^^^^^^ this type doesn't fit in the available registers
    |
    = note: functions with the `"cmse-nonsecure-call"` ABI must pass their result via the available return registers
    = note: the result must either be a (transparently wrapped) i64, u64 or f64, or be at most 4 bytes in size
 
-error: aborting due to 9 previous errors
+error: aborting due to 10 previous errors
 
 For more information about this error, try `rustc --explain E0798`.

tests/ui/cmse-nonsecure/cmse-nonsecure-entry/params-via-stack.rs

@@ -24,3 +24,24 @@ pub extern "cmse-nonsecure-entry" fn f4(_: AlignRelevant, _: u32) {} //~ ERROR [
 #[no_mangle]
 #[allow(improper_ctypes_definitions)]
 pub extern "cmse-nonsecure-entry" fn f5(_: [u32; 5]) {} //~ ERROR [E0798]
+
+struct Four {
+    a: u32,
+    b: u32,
+    c: u32,
+    d: u32,
+}
+
+struct Five {
+    a: u32,
+    b: u32,
+    c: u32,
+    d: u32,
+    e: u32,
+}
+
+#[no_mangle]
+pub extern "cmse-nonsecure-entry" fn four(_: Four) {}
+
+#[no_mangle]
+pub extern "cmse-nonsecure-entry" fn five(_: Five) {} //~ ERROR [E0798]

tests/ui/cmse-nonsecure/cmse-nonsecure-entry/params-via-stack.stderr

@@ -40,6 +40,14 @@ LL | pub extern "cmse-nonsecure-entry" fn f5(_: [u32; 5]) {}
    |
    = note: functions with the `"cmse-nonsecure-entry"` ABI must pass all their arguments via the 4 32-bit argument registers
 
-error: aborting due to 5 previous errors
+error[E0798]: arguments for `"cmse-nonsecure-entry"` function too large to pass via registers
+  --> $DIR/params-via-stack.rs:47:46
+   |
+LL | pub extern "cmse-nonsecure-entry" fn five(_: Five) {}
+   |                                              ^^^^ does not fit in the available registers
+   |
+   = note: functions with the `"cmse-nonsecure-entry"` ABI must pass all their arguments via the 4 32-bit argument registers
+
+error: aborting due to 6 previous errors
 
 For more information about this error, try `rustc --explain E0798`.

tests/ui/cmse-nonsecure/cmse-nonsecure-entry/via-registers.rs

@@ -40,9 +40,15 @@ pub extern "cmse-nonsecure-entry" fn inputs5(_: f64, _: f32, _: f32) {}
 #[no_mangle]
 pub extern "cmse-nonsecure-entry" fn inputs6(_: ReprTransparentStruct<u64>, _: U32Compound) {}
 #[no_mangle]
-#[allow(improper_ctypes_definitions)]
+#[expect(improper_ctypes_definitions)]
 pub extern "cmse-nonsecure-entry" fn inputs7(_: [u32; 4]) {}
 
+// With zero-sized types we can actually have more than 4 arguments.
+#[expect(improper_ctypes_definitions)]
+pub extern "cmse-nonsecure-entry" fn inputs8(_: (), _: (), _: (), _: (), _: ()) {}
+#[expect(improper_ctypes_definitions)]
+pub extern "cmse-nonsecure-entry" fn inputs9(_: (), _: (), _: (), _: (), _: ()) {}
+
 #[no_mangle]
 pub extern "cmse-nonsecure-entry" fn outputs1() -> u32 {
     0
@@ -69,8 +75,8 @@ pub extern "cmse-nonsecure-entry" fn outputs6() -> ReprTransparentStruct<u64> {
     ReprTransparentStruct { _marker1: (), _marker2: (), field: 0xAA, _marker3: () }
 }
 #[no_mangle]
-pub extern "cmse-nonsecure-entry" fn outputs7(
-) -> ReprTransparentStruct<ReprTransparentStruct<u64>> {
+pub extern "cmse-nonsecure-entry" fn outputs7() -> ReprTransparentStruct<ReprTransparentStruct<u64>>
+{
     ReprTransparentStruct {
         _marker1: (),
         _marker2: (),