diff --git a/compiler/rustc_codegen_ssa/src/mir/block.rs b/compiler/rustc_codegen_ssa/src/mir/block.rs index 03ef6d50d44cd..6c3f1790f854b 100644 --- a/compiler/rustc_codegen_ssa/src/mir/block.rs +++ b/compiler/rustc_codegen_ssa/src/mir/block.rs @@ -295,6 +295,28 @@ impl<'a, 'tcx, Bx: BuilderMethods<'a, 'tcx>> FunctionCx<'a, 'tcx, Bx> { } fn codegen_return_terminator(&mut self, mut bx: Bx) { + // Mark storage dead for arguments passed indirectly. + for (arg_index, local) in self.mir.args_iter().enumerate() { + if Some(local) == self.mir.spread_arg { + // Bail out on spread args. This may result in reduced optimization in "rust-call" ABI functions. + // FIXME: handle spread args. This is subtle, see `mir::arg_local_refs`. + break; + } + if self.fn_abi.c_variadic && arg_index == self.fn_abi.args.len() { + // Ignore C variadic args (always the last argument, if present). + continue; + } + let abi = &self.fn_abi.args[arg_index]; + if !abi.is_indirect() { + // Only indirect arguments need storage markers. + continue; + } + match self.locals[local] { + LocalRef::Place(place) => place.storage_dead(&mut bx), + LocalRef::UnsizedPlace(place) => place.storage_dead(&mut bx), + _ => bug!("Unexpected non-place argument local: {:?}", local), + } + } // Call `va_end` if this is the definition of a C-variadic function. if self.fn_abi.c_variadic { // The `VaList` "spoofed" argument is just after all the real arguments. diff --git a/src/test/codegen/array-equality.rs b/src/test/codegen/array-equality.rs index cd5e82a9205c1..cf46e2c8813a2 100644 --- a/src/test/codegen/array-equality.rs +++ b/src/test/codegen/array-equality.rs @@ -29,6 +29,8 @@ pub fn array_eq_value_still_passed_by_pointer(a: [u16; 9], b: [u16; 9]) -> bool // CHECK-NEXT: start: // CHECK: %[[CMP:.+]] = tail call i32 @{{bcmp|memcmp}}({{i8\*|ptr}} {{.*}} dereferenceable(18) %{{.+}}, {{i8\*|ptr}} {{.*}} dereferenceable(18) %{{.+}}, i64 18) // CHECK-NEXT: %[[EQ:.+]] = icmp eq i32 %[[CMP]], 0 + // CHECK-NEXT: call void @llvm.lifetime.end + // CHECK-NEXT: call void @llvm.lifetime.end // CHECK-NEXT: ret i1 %[[EQ]] a == b } @@ -58,6 +60,8 @@ pub fn array_eq_zero_mid(x: [u16; 8]) -> bool { // CHECK-NEXT: start: // CHECK: %[[LOAD:.+]] = load i128, // CHECK-NEXT: %[[EQ:.+]] = icmp eq i128 %[[LOAD]], 0 + // CHECK-NEXT: bitcast + // CHECK-NEXT: call void @llvm.lifetime.end // CHECK-NEXT: ret i1 %[[EQ]] x == [0; 8] } @@ -69,6 +73,7 @@ pub fn array_eq_zero_long(x: [u16; 1234]) -> bool { // CHECK-NOT: alloca // CHECK: %[[CMP:.+]] = tail call i32 @{{bcmp|memcmp}}( // CHECK-NEXT: %[[EQ:.+]] = icmp eq i32 %[[CMP]], 0 + // CHECK-NEXT: call void @llvm.lifetime.end // CHECK-NEXT: ret i1 %[[EQ]] x == [0; 1234] } diff --git a/src/test/codegen/issue-96497-array-indirect-writes.rs b/src/test/codegen/issue-96497-array-indirect-writes.rs new file mode 100644 index 0000000000000..fa5d9f1345b1b --- /dev/null +++ b/src/test/codegen/issue-96497-array-indirect-writes.rs @@ -0,0 +1,13 @@ +// Check that LLVM can see that stores to a by-move array (passed indirectly) are dead. + +// compile-flags: -O -Zmir-opt-level=0 +// min-llvm-version: 14.0 + +#![crate_type="lib"] + +// CHECK-LABEL: @array_dead_store +#[no_mangle] +pub fn array_dead_store(mut x: [u8; 1234]) { + // CHECK-NOT: store + x[10] = 42; +} diff --git a/src/test/codegen/lifetime_start_end.rs b/src/test/codegen/lifetime-start-end-basic-locals.rs similarity index 100% rename from src/test/codegen/lifetime_start_end.rs rename to src/test/codegen/lifetime-start-end-basic-locals.rs diff --git a/src/test/codegen/lifetime-start-end-indirect-args.rs b/src/test/codegen/lifetime-start-end-indirect-args.rs new file mode 100644 index 0000000000000..89190b62bf596 --- /dev/null +++ b/src/test/codegen/lifetime-start-end-indirect-args.rs @@ -0,0 +1,73 @@ +// This test checks that moved arguments passed indirectly get lifetime markers. + +// compile-flags: -O -C no-prepopulate-passes -Zmir-opt-level=0 + +#![crate_type = "lib"] + +// CHECK-LABEL: @arg_indirect +#[no_mangle] +pub fn arg_indirect(a: [u8; 1234]) { + // Arguments passed indirectly should get lifetime markers. + + // CHECK: [[A:%[0-9]+]] = bitcast{{.*}} %a + // CHECK: call void @llvm.lifetime.end{{.*}} [[A]]) +} + +// CHECK-LABEL: @arg_by_val +#[no_mangle] +pub fn arg_by_val(a: u8) { + // Arguments passed by value should not get lifetime markers. + + // CHECK-NOT: call void @llvm.lifetime.end +} + +// CHECK-LABEL: @arg_by_ref +#[no_mangle] +pub fn arg_by_ref(a: &[u8; 1234]) { + // Arguments passed by ref should not get lifetime markers. + + // CHECK-NOT: call void @llvm.lifetime.end +} + +// CHECK-LABEL: @with_other_args +#[no_mangle] +pub fn with_other_args(x: (), y: (), a: [u8; 1234], z: ()) { + // Lifetime markers should be applied to the correct argument, + // even in the presence of ignored ZST arguments. + + // CHECK-NOT: call void @llvm.lifetime.end + // CHECK: [[A:%[0-9]+]] = bitcast{{.*}} %a + // CHECK: call void @llvm.lifetime.end{{.*}} [[A]]) + // CHECK-NOT: call void @llvm.lifetime.end +} + +// CHECK-LABEL: @forward_directly_to_ret +#[no_mangle] +pub fn forward_directly_to_ret(a: [u8; 1234]) -> [u8; 1234] { + // Ensure that lifetime markers appear after the argument is copied to the return place. + // (Since reading from `a` after `lifetime.end` would return poison.) + + // CHECK: memcpy + // CHECK: [[A:%[0-9]+]] = bitcast{{.*}} %a + // CHECK: call void @llvm.lifetime.end{{.*}} [[A]]) + // CHECK-NEXT: ret void + a +} + +pub struct LargeWithField { + x: u8, + _rest: [u8; 1234], +} + +// CHECK-LABEL: @read_from_arg +#[no_mangle] +pub fn read_from_arg(a: LargeWithField) -> u8 { + // Ensure that lifetime markers appear after reading from the argument. + // (Since reading from `a` after `lifetime.end` would return poison.) + + // CHECK: [[LOAD:%[0-9]+]] = load i8 + // CHECK: [[A:%[0-9]+]] = bitcast{{.*}} %a + // CHECK: call void @llvm.lifetime.end{{.*}} [[A]]) + // CHECK-NEXT: ret i8 [[LOAD]] + a.x +} diff --git a/src/test/codegen/vec-in-place.rs b/src/test/codegen/vec-in-place.rs index 13c41f7d4a906..421de8dbb0219 100644 --- a/src/test/codegen/vec-in-place.rs +++ b/src/test/codegen/vec-in-place.rs @@ -31,6 +31,8 @@ pub struct Bar { pub fn vec_iterator_cast_primitive(vec: Vec) -> Vec { // CHECK-NOT: loop // CHECK-NOT: call + // CHECK: call void @llvm.lifetime.end + // CHECK-NEXT: ret vec.into_iter().map(|e| e as u8).collect() } @@ -39,6 +41,8 @@ pub fn vec_iterator_cast_primitive(vec: Vec) -> Vec { pub fn vec_iterator_cast_wrapper(vec: Vec) -> Vec> { // CHECK-NOT: loop // CHECK-NOT: call + // CHECK: call void @llvm.lifetime.end + // CHECK-NEXT: ret vec.into_iter().map(|e| Wrapper(e)).collect() } @@ -47,6 +51,8 @@ pub fn vec_iterator_cast_wrapper(vec: Vec) -> Vec> { pub fn vec_iterator_cast_unwrap(vec: Vec>) -> Vec { // CHECK-NOT: loop // CHECK-NOT: call + // CHECK: call void @llvm.lifetime.end + // CHECK-NEXT: ret vec.into_iter().map(|e| e.0).collect() } @@ -55,6 +61,8 @@ pub fn vec_iterator_cast_unwrap(vec: Vec>) -> Vec { pub fn vec_iterator_cast_aggregate(vec: Vec<[u64; 4]>) -> Vec { // CHECK-NOT: loop // CHECK-NOT: call + // CHECK: call void @llvm.lifetime.end + // CHECK-NEXT: ret vec.into_iter().map(|e| unsafe { std::mem::transmute(e) }).collect() } @@ -63,6 +71,8 @@ pub fn vec_iterator_cast_aggregate(vec: Vec<[u64; 4]>) -> Vec { pub fn vec_iterator_cast_deaggregate(vec: Vec) -> Vec<[u64; 4]> { // CHECK-NOT: loop // CHECK-NOT: call + // CHECK: call void @llvm.lifetime.end + // CHECK-NEXT: ret // Safety: For the purpose of this test we assume that Bar layout matches [u64; 4]. // This currently is not guaranteed for repr(Rust) types, but it happens to work here and