Correctly handle recursive #[repr(transparent)] layouts

Author: jieyouxu

compiler/rustc_ty_utils/src/abi.rs

@@ -787,18 +787,10 @@ fn fn_abi_adjust_for_abi<'tcx>(
 
                 // Let's see if we can add a `noundef`. This is only legal for arrays, definitely
                 // not for unions. This is also legal for `#[repr(transparent)] struct` or
-                // `#[repr(transparent)] enum` containing array.
-                let is_transparent_array =
-                    if arg.layout.is_transparent::<LayoutCx<'tcx, TyCtxt<'tcx>>>()
-                        && let Some((_, layout)) = arg.layout.non_1zst_field(cx)
-                        && layout.ty.is_array()
-                    {
-                        true
-                    } else {
-                        false
-                    };
-
-                if arg.layout.ty.is_array() || is_transparent_array {
+                // `#[repr(transparent)] enum` containing array. Note that `#[repr(transparent)]`
+                // can contain other `#[repr(transparent)]` structs or enums, which can eventually
+                // contain an array!
+                if arg.layout.ty.is_array() || is_transparent_array(cx, arg.layout) {
                     // Fixup arg attribute with `noundef`.
                     let PassMode::Cast { ref mut cast, .. } = &mut arg.mode else {
                         bug!("this cannot fail because of the previous cast_to `Reg`");
@@ -842,6 +834,20 @@ fn fn_abi_adjust_for_abi<'tcx>(
     Ok(())
 }
 
+fn is_transparent_array<'tcx>(
+    cx: &LayoutCx<'tcx, TyCtxt<'tcx>>,
+    outermost_layout: TyAndLayout<'tcx>,
+) -> bool {
+    let mut adt_layout = outermost_layout;
+    // Recursively walk a layout, seeing through all `#[repr(transparent)]` layers.
+    while adt_layout.is_transparent::<LayoutCx<'tcx, TyCtxt<'tcx>>>() {
+        if let Some((_, layout)) = adt_layout.non_1zst_field(cx) {
+            adt_layout = layout;
+        }
+    }
+    adt_layout.ty.is_array()
+}
+
 #[tracing::instrument(level = "debug", skip(cx))]
 fn make_thin_self_ptr<'tcx>(
     cx: &(impl HasTyCtxt<'tcx> + HasParamEnv<'tcx>),

tests/codegen/array-immediate-param-noundef.rs

@@ -8,67 +8,67 @@
 //@ compile-flags: -C no-prepopulate-passes -O
 #![crate_type = "lib"]
 
-// CHECK: define noundef i64 @replace_short_array_u64x1(ptr noalias noundef align 8 dereferenceable(8) %r, i64 noundef %0)
+// CHECK: define noundef i64 @replace_short_array_u64x1(ptr {{.*}}, i64 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u64x1(r: &mut [u64; 1], v: [u64; 1]) -> [u64; 1] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i32 @replace_short_array_u32x1(ptr noalias noundef align 4 dereferenceable(4) %r, i32 noundef %0)
+// CHECK: define noundef i32 @replace_short_array_u32x1(ptr {{.*}}, i32 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u32x1(r: &mut [u32; 1], v: [u32; 1]) -> [u32; 1] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i64 @replace_short_array_u32x2(ptr noalias noundef align 4 dereferenceable(8) %r, i64 noundef %0)
+// CHECK: define noundef i64 @replace_short_array_u32x2(ptr {{.*}}, i64 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u32x2(r: &mut [u32; 2], v: [u32; 2]) -> [u32; 2] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i16 @replace_short_array_u16x1(ptr noalias noundef align 2 dereferenceable(2) %r, i16 noundef %0)
+// CHECK: define noundef i16 @replace_short_array_u16x1(ptr {{.*}}, i16 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u16x1(r: &mut [u16; 1], v: [u16; 1]) -> [u16; 1] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i32 @replace_short_array_u16x2(ptr noalias noundef align 2 dereferenceable(4) %r, i32 noundef %0)
+// CHECK: define noundef i32 @replace_short_array_u16x2(ptr {{.*}}, i32 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u16x2(r: &mut [u16; 2], v: [u16; 2]) -> [u16; 2] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i48 @replace_short_array_u16x3(ptr noalias noundef align 2 dereferenceable(6) %r, i48 noundef %0)
+// CHECK: define noundef i48 @replace_short_array_u16x3(ptr {{.*}}, i48 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u16x3(r: &mut [u16; 3], v: [u16; 3]) -> [u16; 3] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i64 @replace_short_array_u16x4(ptr noalias noundef align 2 dereferenceable(8) %r, i64 noundef %0)
+// CHECK: define noundef i64 @replace_short_array_u16x4(ptr {{.*}}, i64 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u16x4(r: &mut [u16; 4], v: [u16; 4]) -> [u16; 4] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i8 @replace_short_array_u8x1(ptr noalias noundef align 1 dereferenceable(1) %r, i8 noundef %0)
+// CHECK: define noundef i8 @replace_short_array_u8x1(ptr {{.*}}, i8 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u8x1(r: &mut [u8; 1], v: [u8; 1]) -> [u8; 1] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i16 @replace_short_array_u8x2(ptr noalias noundef align 1 dereferenceable(2) %r, i16 noundef %0)
+// CHECK: define noundef i16 @replace_short_array_u8x2(ptr {{.*}}, i16 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u8x2(r: &mut [u8; 2], v: [u8; 2]) -> [u8; 2] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i24 @replace_short_array_u8x3(ptr noalias noundef align 1 dereferenceable(3) %r, i24 noundef %0)
+// CHECK: define noundef i24 @replace_short_array_u8x3(ptr {{.*}}, i24 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u8x3(r: &mut [u8; 3], v: [u8; 3]) -> [u8; 3] {
     std::mem::replace(r, v)
 }
 
-// CHECK: define noundef i64 @replace_short_array_u8x8(ptr noalias noundef align 1 dereferenceable(8) %r, i64 noundef %0)
+// CHECK: define noundef i64 @replace_short_array_u8x8(ptr {{.*}}, i64 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_short_array_u8x8(r: &mut [u8; 8], v: [u8; 8]) -> [u8; 8] {
     std::mem::replace(r, v)
@@ -77,7 +77,7 @@ pub fn replace_short_array_u8x8(r: &mut [u8; 8], v: [u8; 8]) -> [u8; 8] {
 #[repr(transparent)]
 pub struct Foo([u8; 4]);
 
-// CHECK: define noundef i32 @replace_repr_transparent_struct_short_array(ptr noalias noundef align 1 dereferenceable(4) %r, i32 noundef %0)
+// CHECK: define noundef i32 @replace_repr_transparent_struct_short_array(ptr {{.*}}, i32 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_repr_transparent_struct_short_array(r: &mut Foo, v: Foo) -> Foo {
     std::mem::replace(r, v)
@@ -88,8 +88,23 @@ pub enum Bar {
     Default([u8; 4])
 }
 
-// CHECK: define noundef i32 @replace_repr_transparent_enum_short_array(ptr noalias noundef align 1 dereferenceable(4) %r, i32 noundef %0)
+// CHECK: define noundef i32 @replace_repr_transparent_enum_short_array(ptr {{.*}}, i32 noundef %{{.*}})
 #[no_mangle]
 pub fn replace_repr_transparent_enum_short_array(r: &mut Bar, v: Bar) -> Bar {
     std::mem::replace(r, v)
 }
+
+#[repr(transparent)]
+pub struct Owo([u8; 4]);
+
+#[repr(transparent)]
+pub struct Uwu(Owo);
+
+#[repr(transparent)]
+pub struct Oowoo(Uwu);
+
+// CHECK: define noundef i32 @replace_repr_transparent_nested_struct_short_array(ptr {{.*}}, i32 noundef %{{.*}})
+#[no_mangle]
+pub fn replace_repr_transparent_nested_struct_short_array(r: &mut Oowoo, v: Oowoo) -> Oowoo {
+    std::mem::replace(r, v)
+}