Auto merge of #53436 - cuviper:trace_fn-stop, r=alexcrichton

std: stop backtracing when the frames are full

This is a defensive measure to mitigate the infinite unwind loop seen in #53372.  That case will still repeatedly unwind `__rust_try`, but now it will at least stop when `cx.frames` is full.

r? @alexcrichton

Author: bors

src/libstd/sys/cloudabi/backtrace.rs

@@ -64,6 +64,10 @@ extern "C" fn trace_fn(
     arg: *mut libc::c_void,
 ) -> uw::_Unwind_Reason_Code {
     let cx = unsafe { &mut *(arg as *mut Context) };
+    if cx.idx >= cx.frames.len() {
+        return uw::_URC_NORMAL_STOP;
+    }
+
     let mut ip_before_insn = 0;
     let mut ip = unsafe { uw::_Unwind_GetIPInfo(ctx, &mut ip_before_insn) as *mut libc::c_void };
     if !ip.is_null() && ip_before_insn == 0 {
@@ -73,14 +77,12 @@ extern "C" fn trace_fn(
     }
 
     let symaddr = unsafe { uw::_Unwind_FindEnclosingFunction(ip) };
-    if cx.idx < cx.frames.len() {
-        cx.frames[cx.idx] = Frame {
-            symbol_addr: symaddr as *mut u8,
-            exact_position: ip as *mut u8,
-            inline_context: 0,
-        };
-        cx.idx += 1;
-    }
+    cx.frames[cx.idx] = Frame {
+        symbol_addr: symaddr as *mut u8,
+        exact_position: ip as *mut u8,
+        inline_context: 0,
+    };
+    cx.idx += 1;
 
     uw::_URC_NO_REASON
 }

src/libstd/sys/redox/backtrace/tracing.rs

@@ -68,6 +68,10 @@ pub fn unwind_backtrace(frames: &mut [Frame])
 extern fn trace_fn(ctx: *mut uw::_Unwind_Context,
                    arg: *mut libc::c_void) -> uw::_Unwind_Reason_Code {
     let cx = unsafe { &mut *(arg as *mut Context) };
+    if cx.idx >= cx.frames.len() {
+        return uw::_URC_NORMAL_STOP;
+    }
+
     let mut ip_before_insn = 0;
     let mut ip = unsafe {
         uw::_Unwind_GetIPInfo(ctx, &mut ip_before_insn) as *mut libc::c_void
@@ -94,14 +98,12 @@ extern fn trace_fn(ctx: *mut uw::_Unwind_Context,
         unsafe { uw::_Unwind_FindEnclosingFunction(ip) }
     };
 
-    if cx.idx < cx.frames.len() {
-        cx.frames[cx.idx] = Frame {
-            symbol_addr: symaddr as *mut u8,
-            exact_position: ip as *mut u8,
-            inline_context: 0,
-        };
-        cx.idx += 1;
-    }
+    cx.frames[cx.idx] = Frame {
+        symbol_addr: symaddr as *mut u8,
+        exact_position: ip as *mut u8,
+        inline_context: 0,
+    };
+    cx.idx += 1;
 
     uw::_URC_NO_REASON
 }

src/libstd/sys/unix/backtrace/tracing/gcc_s.rs

@@ -68,6 +68,10 @@ pub fn unwind_backtrace(frames: &mut [Frame])
 extern fn trace_fn(ctx: *mut uw::_Unwind_Context,
                    arg: *mut libc::c_void) -> uw::_Unwind_Reason_Code {
     let cx = unsafe { &mut *(arg as *mut Context) };
+    if cx.idx >= cx.frames.len() {
+        return uw::_URC_NORMAL_STOP;
+    }
+
     let mut ip_before_insn = 0;
     let mut ip = unsafe {
         uw::_Unwind_GetIPInfo(ctx, &mut ip_before_insn) as *mut libc::c_void
@@ -94,14 +98,12 @@ extern fn trace_fn(ctx: *mut uw::_Unwind_Context,
         unsafe { uw::_Unwind_FindEnclosingFunction(ip) }
     };
 
-    if cx.idx < cx.frames.len() {
-        cx.frames[cx.idx] = Frame {
-            symbol_addr: symaddr as *mut u8,
-            exact_position: ip as *mut u8,
-            inline_context: 0,
-        };
-        cx.idx += 1;
-    }
+    cx.frames[cx.idx] = Frame {
+        symbol_addr: symaddr as *mut u8,
+        exact_position: ip as *mut u8,
+        inline_context: 0,
+    };
+    cx.idx += 1;
 
     uw::_URC_NO_REASON
 }