Skip to content

Commit

Permalink
arm64: stacktrace: unwind exception boundaries
Browse files Browse the repository at this point in the history
When arm64's stack unwinder encounters an exception boundary, it uses
the pt_regs::stackframe created by the entry code, which has a copy of
the PC and FP at the time the exception was taken. The unwinder doesn't
know anything about pt_regs, and reports the PC from the stackframe, but
does not report the LR.

The LR is only guaranteed to contain the return address at function call
boundaries, and can be used as a scratch register at other times, so the
LR at an exception boundary may or may not be a legitimate return
address. It would be useful to report the LR value regardless, as it can
be helpful when debugging, and in future it will be helpful for reliable
stacktrace support.

This patch changes the way we unwind across exception boundaries,
allowing both the PC and LR to be reported. The entry code creates a
frame_record_meta structure embedded within pt_regs, which the unwinder
uses to find the pt_regs. The unwinder can then extract pt_regs::pc and
pt_regs::lr as two separate unwind steps before continuing with a
regular walk of frame records.

When a PC is unwound from pt_regs::lr, dump_backtrace() will log this
with an "L" marker so that it can be identified easily. For example,
an unwind across an exception boundary will appear as follows:

|  el1h_64_irq+0x6c/0x70
|  _raw_spin_unlock_irqrestore+0x10/0x60 (P)
|  __aarch64_insn_write+0x6c/0x90 (L)
|  aarch64_insn_patch_text_nosync+0x28/0x80

... with a (P) entry for pt_regs::pc, and an (L) entry for pt_regs:lr.

Note that the LR may be stale at the point of the exception, for example,
shortly after a return:

|  el1h_64_irq+0x6c/0x70
|  default_idle_call+0x34/0x180 (P)
|  default_idle_call+0x28/0x180 (L)
|  do_idle+0x204/0x268

... where the LR points a few instructions before the current PC.

This plays nicely with all the other unwind metadata tracking. With the
ftrace_graph profiler enabled globally, and kretprobes installed on
generic_handle_domain_irq() and do_interrupt_handler(), a backtrace triggered
by magic-sysrq + L reports:

| Call trace:
|  show_stack+0x20/0x40 (CF)
|  dump_stack_lvl+0x60/0x80 (F)
|  dump_stack+0x18/0x28
|  nmi_cpu_backtrace+0xfc/0x140
|  nmi_trigger_cpumask_backtrace+0x1c8/0x200
|  arch_trigger_cpumask_backtrace+0x20/0x40
|  sysrq_handle_showallcpus+0x24/0x38 (F)
|  __handle_sysrq+0xa8/0x1b0 (F)
|  handle_sysrq+0x38/0x50 (F)
|  pl011_int+0x460/0x5a8 (F)
|  __handle_irq_event_percpu+0x60/0x220 (F)
|  handle_irq_event+0x54/0xc0 (F)
|  handle_fasteoi_irq+0xa8/0x1d0 (F)
|  generic_handle_domain_irq+0x34/0x58 (F)
|  gic_handle_irq+0x54/0x140 (FK)
|  call_on_irq_stack+0x24/0x58 (F)
|  do_interrupt_handler+0x88/0xa0
|  el1_interrupt+0x34/0x68 (FK)
|  el1h_64_irq_handler+0x18/0x28
|  el1h_64_irq+0x6c/0x70
|  default_idle_call+0x34/0x180 (P)
|  default_idle_call+0x28/0x180 (L)
|  do_idle+0x204/0x268
|  cpu_startup_entry+0x3c/0x50 (F)
|  rest_init+0xe4/0xf0
|  start_kernel+0x744/0x750
|  __primary_switched+0x88/0x98

Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Reviewed-by: Mark Brown <broonie@kernel.org>
Reviewed-by: Miroslav Benes <mbenes@suse.cz>
Reviewed-by: Puranjay Mohan <puranjay12@gmail.com>
Cc: Ard Biesheuvel <ardb@kernel.org>
Cc: Josh Poimboeuf <jpoimboe@kernel.org>
Cc: Kalesh Singh <kaleshsingh@google.com>
Cc: Madhavan T. Venkataraman <madvenka@linux.microsoft.com>
Cc: Marc Zyngier <maz@kernel.org>
Cc: Will Deacon <will@kernel.org>
Link: https://lore.kernel.org/r/20241017092538.1859841-11-mark.rutland@arm.com
Signed-off-by: Catalin Marinas <catalin.marinas@arm.com>
  • Loading branch information
Mark Rutland authored and ctmarinas committed Oct 17, 2024
1 parent f05a4a4 commit c2c6b27
Show file tree
Hide file tree
Showing 7 changed files with 158 additions and 19 deletions.
4 changes: 1 addition & 3 deletions arch/arm64/include/asm/ptrace.h
Original file line number Diff line number Diff line change
Expand Up @@ -168,9 +168,7 @@ struct pt_regs {
u32 pmr;

u64 sdei_ttbr1;
u64 unused;

struct frame_record stackframe;
struct frame_record_meta stackframe;

/* Only valid for some EL1 exceptions. */
u64 lockdep_hardirqs;
Expand Down
35 changes: 35 additions & 0 deletions arch/arm64/include/asm/stacktrace/frame.h
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,46 @@
#define __ASM_STACKTRACE_FRAME_H

/*
* - FRAME_META_TYPE_NONE
*
* This value is reserved.
*
* - FRAME_META_TYPE_FINAL
*
* The record is the last entry on the stack.
* Unwinding should terminate successfully.
*
* - FRAME_META_TYPE_PT_REGS
*
* The record is embedded within a struct pt_regs, recording the registers at
* an arbitrary point in time.
* Unwinding should consume pt_regs::pc, followed by pt_regs::lr.
*
* Note: all other values are reserved and should result in unwinding
* terminating with an error.
*/
#define FRAME_META_TYPE_NONE 0
#define FRAME_META_TYPE_FINAL 1
#define FRAME_META_TYPE_PT_REGS 2

#ifndef __ASSEMBLY__
/*
* A standard AAPCS64 frame record.
*/
struct frame_record {
u64 fp;
u64 lr;
};

/*
* A metadata frame record indicating a special unwind.
* The record::{fp,lr} fields must be zero to indicate the presence of
* metadata.
*/
struct frame_record_meta {
struct frame_record record;
u64 type;
};
#endif /* __ASSEMBLY */

#endif /* __ASM_STACKTRACE_FRAME_H */
1 change: 1 addition & 0 deletions arch/arm64/kernel/asm-offsets.c
Original file line number Diff line number Diff line change
Expand Up @@ -81,6 +81,7 @@ int main(void)
DEFINE(S_SDEI_TTBR1, offsetof(struct pt_regs, sdei_ttbr1));
DEFINE(S_PMR, offsetof(struct pt_regs, pmr));
DEFINE(S_STACKFRAME, offsetof(struct pt_regs, stackframe));
DEFINE(S_STACKFRAME_TYPE, offsetof(struct pt_regs, stackframe.type));
DEFINE(PT_REGS_SIZE, sizeof(struct pt_regs));
BLANK();
#ifdef CONFIG_DYNAMIC_FTRACE_WITH_ARGS
Expand Down
12 changes: 7 additions & 5 deletions arch/arm64/kernel/entry.S
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
#include <asm/processor.h>
#include <asm/ptrace.h>
#include <asm/scs.h>
#include <asm/stacktrace/frame.h>
#include <asm/thread_info.h>
#include <asm/asm-uaccess.h>
#include <asm/unistd.h>
Expand Down Expand Up @@ -284,15 +285,16 @@ alternative_else_nop_endif
stp lr, x21, [sp, #S_LR]

/*
* For exceptions from EL0, create a final frame record.
* For exceptions from EL1, create a synthetic frame record so the
* interrupted code shows up in the backtrace.
* Create a metadata frame record. The unwinder will use this to
* identify and unwind exception boundaries.
*/
.if \el == 0
stp xzr, xzr, [sp, #S_STACKFRAME]
.if \el == 0
mov x0, #FRAME_META_TYPE_FINAL
.else
stp x29, x22, [sp, #S_STACKFRAME]
mov x0, #FRAME_META_TYPE_PT_REGS
.endif
str x0, [sp, #S_STACKFRAME_TYPE]
add x29, sp, #S_STACKFRAME

#ifdef CONFIG_ARM64_SW_TTBR0_PAN
Expand Down
3 changes: 3 additions & 0 deletions arch/arm64/kernel/head.S
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
#include <asm/scs.h>
#include <asm/smp.h>
#include <asm/sysreg.h>
#include <asm/stacktrace/frame.h>
#include <asm/thread_info.h>
#include <asm/virt.h>

Expand Down Expand Up @@ -199,6 +200,8 @@ SYM_CODE_END(preserve_boot_args)
sub sp, sp, #PT_REGS_SIZE

stp xzr, xzr, [sp, #S_STACKFRAME]
mov \tmp1, #FRAME_META_TYPE_FINAL
str \tmp1, [sp, #S_STACKFRAME_TYPE]
add x29, sp, #S_STACKFRAME

scs_load_current
Expand Down
1 change: 1 addition & 0 deletions arch/arm64/kernel/process.c
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,7 @@ int copy_thread(struct task_struct *p, const struct kernel_clone_args *args)
*/
memset(childregs, 0, sizeof(struct pt_regs));
childregs->pstate = PSR_MODE_EL1h | PSR_IL_BIT;
childregs->stackframe.type = FRAME_META_TYPE_FINAL;

p->thread.cpu_context.x19 = (unsigned long)args->fn;
p->thread.cpu_context.x20 = (unsigned long)args->fn_arg;
Expand Down
121 changes: 110 additions & 11 deletions arch/arm64/kernel/stacktrace.c
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ enum kunwind_source {
KUNWIND_SOURCE_CALLER,
KUNWIND_SOURCE_TASK,
KUNWIND_SOURCE_REGS_PC,
KUNWIND_SOURCE_REGS_LR,
};

union unwind_flags {
Expand Down Expand Up @@ -55,6 +56,7 @@ struct kunwind_state {
#endif
enum kunwind_source source;
union unwind_flags flags;
struct pt_regs *regs;
};

static __always_inline void
Expand All @@ -65,6 +67,7 @@ kunwind_init(struct kunwind_state *state,
state->task = task;
state->source = KUNWIND_SOURCE_UNKNOWN;
state->flags.all = 0;
state->regs = NULL;
}

/*
Expand All @@ -80,6 +83,7 @@ kunwind_init_from_regs(struct kunwind_state *state,
{
kunwind_init(state, current);

state->regs = regs;
state->common.fp = regs->regs[29];
state->common.pc = regs->pc;
state->source = KUNWIND_SOURCE_REGS_PC;
Expand Down Expand Up @@ -155,6 +159,103 @@ kunwind_recover_return_address(struct kunwind_state *state)
return 0;
}

static __always_inline
int kunwind_next_regs_pc(struct kunwind_state *state)
{
struct stack_info *info;
unsigned long fp = state->common.fp;
struct pt_regs *regs;

regs = container_of((u64 *)fp, struct pt_regs, stackframe.record.fp);

info = unwind_find_stack(&state->common, (unsigned long)regs, sizeof(*regs));
if (!info)
return -EINVAL;

unwind_consume_stack(&state->common, info, (unsigned long)regs,
sizeof(*regs));

state->regs = regs;
state->common.pc = regs->pc;
state->common.fp = regs->regs[29];
state->source = KUNWIND_SOURCE_REGS_PC;
return 0;
}

static __always_inline int
kunwind_next_regs_lr(struct kunwind_state *state)
{
/*
* The stack for the regs was consumed by kunwind_next_regs_pc(), so we
* cannot consume that again here, but we know the regs are safe to
* access.
*/
state->common.pc = state->regs->regs[30];
state->common.fp = state->regs->regs[29];
state->regs = NULL;
state->source = KUNWIND_SOURCE_REGS_LR;

return 0;
}

static __always_inline int
kunwind_next_frame_record_meta(struct kunwind_state *state)
{
struct task_struct *tsk = state->task;
unsigned long fp = state->common.fp;
struct frame_record_meta *meta;
struct stack_info *info;

info = unwind_find_stack(&state->common, fp, sizeof(*meta));
if (!info)
return -EINVAL;

meta = (struct frame_record_meta *)fp;
switch (READ_ONCE(meta->type)) {
case FRAME_META_TYPE_FINAL:
if (meta == &task_pt_regs(tsk)->stackframe)
return -ENOENT;
WARN_ON_ONCE(1);
return -EINVAL;
case FRAME_META_TYPE_PT_REGS:
return kunwind_next_regs_pc(state);
default:
WARN_ON_ONCE(1);
return -EINVAL;
}
}

static __always_inline int
kunwind_next_frame_record(struct kunwind_state *state)
{
unsigned long fp = state->common.fp;
struct frame_record *record;
struct stack_info *info;
unsigned long new_fp, new_pc;

if (fp & 0x7)
return -EINVAL;

info = unwind_find_stack(&state->common, fp, sizeof(*record));
if (!info)
return -EINVAL;

record = (struct frame_record *)fp;
new_fp = READ_ONCE(record->fp);
new_pc = READ_ONCE(record->lr);

if (!new_fp && !new_pc)
return kunwind_next_frame_record_meta(state);

unwind_consume_stack(&state->common, info, fp, sizeof(*record));

state->common.fp = new_fp;
state->common.pc = new_pc;
state->source = KUNWIND_SOURCE_FRAME;

return 0;
}

/*
* Unwind from one frame record (A) to the next frame record (B).
*
Expand All @@ -165,30 +266,27 @@ kunwind_recover_return_address(struct kunwind_state *state)
static __always_inline int
kunwind_next(struct kunwind_state *state)
{
struct task_struct *tsk = state->task;
unsigned long fp = state->common.fp;
int err;

state->flags.all = 0;

/* Final frame; nothing to unwind */
if (fp == (unsigned long)&task_pt_regs(tsk)->stackframe)
return -ENOENT;

switch (state->source) {
case KUNWIND_SOURCE_FRAME:
case KUNWIND_SOURCE_CALLER:
case KUNWIND_SOURCE_TASK:
case KUNWIND_SOURCE_REGS_LR:
err = kunwind_next_frame_record(state);
break;
case KUNWIND_SOURCE_REGS_PC:
err = unwind_next_frame_record(&state->common);
if (err)
return err;
state->source = KUNWIND_SOURCE_FRAME;
err = kunwind_next_regs_lr(state);
break;
default:
return -EINVAL;
err = -EINVAL;
}

if (err)
return err;

state->common.pc = ptrauth_strip_kernel_insn_pac(state->common.pc);

return kunwind_recover_return_address(state);
Expand Down Expand Up @@ -338,6 +436,7 @@ static const char *state_source_string(const struct kunwind_state *state)
case KUNWIND_SOURCE_CALLER: return "C";
case KUNWIND_SOURCE_TASK: return "T";
case KUNWIND_SOURCE_REGS_PC: return "P";
case KUNWIND_SOURCE_REGS_LR: return "L";
default: return "U";
}
}
Expand Down

0 comments on commit c2c6b27

Please sign in to comment.