[hal] Refactor trap handler with linkme::distributed_slice

It allows trap handlers to be defined in any crate

modules/axhal/Cargo.toml

@@ -22,6 +22,7 @@ default = []
 [dependencies]
 log = "0.4"
 cfg-if = "1.0"
+linkme = "0.3"
 bitflags = "2.6"
 static_assertions = "1.1.0"
 kernel_guard = "0.1"

modules/axhal/linker.lds.S

@@ -49,17 +49,14 @@ SECTIONS
 
  . = ALIGN(4K);
  _percpu_start = .;
+ _percpu_end = _percpu_start + SIZEOF(.percpu);
  .percpu 0x0 : AT(_percpu_start) {
  _percpu_load_start = .;
  *(.percpu .percpu.*)
  _percpu_load_end = .;
- . = ALIGN(64);
- _percpu_size_aligned = .;
-
- . = _percpu_load_start + _percpu_size_aligned * %SMP%;
+ . = _percpu_load_start + ALIGN(64) * %SMP%;
  }
- . = _percpu_start + SIZEOF(.percpu);
- _percpu_end = .;
+ . = _percpu_end;
 
  . = ALIGN(4K);
  _edata = .;
@@ -84,3 +81,11 @@ SECTIONS
  *(.comment) *(.gnu*) *(.note*) *(.eh_frame*)
  }
 }
+
+SECTIONS {
+ linkme_IRQ : { *(linkme_IRQ) }
+ linkm2_IRQ : { *(linkm2_IRQ) }
+ linkme_PAGE_FAULT : { *(linkme_PAGE_FAULT) }
+ linkm2_PAGE_FAULT : { *(linkm2_PAGE_FAULT) }
+}
+INSERT AFTER .tbss;

modules/axhal/src/arch/aarch64/trap.rs

@@ -1,6 +1,8 @@
 use core::arch::global_asm;
 
 use aarch64_cpu::registers::{ESR_EL1, FAR_EL1};
+use memory_addr::VirtAddr;
+use page_table_entry::MappingFlags;
 use tock_registers::interfaces::Readable;
 
 use super::TrapFrame;
@@ -35,39 +37,79 @@ fn invalid_exception(tf: &TrapFrame, kind: TrapKind, source: TrapSource) {
  );
 }
 
+#[no_mangle]
+fn handle_irq_exception(_tf: &TrapFrame) {
+ handle_trap!(IRQ, 0);
+}
+
+fn handle_instruction_abort(tf: &TrapFrame, iss: u64, is_user: bool) {
+ let mut access_flags = MappingFlags::EXECUTE;
+ if is_user {
+ access_flags |= MappingFlags::USER;
+ }
+ let vaddr = VirtAddr::from(FAR_EL1.get() as usize);
+
+ // Only handle Translation fault and Permission fault
+ if !matches!(iss & 0b111100, 0b0100 | 0b1100) // IFSC or DFSC bits
+ || !handle_trap!(PAGE_FAULT, vaddr, access_flags, is_user)
+ {
+ panic!(
+ "Unhandled {} Instruction Abort @ {:#x}, fault_vaddr={:#x}, ISS={:#x} ({:?}):\n{:#x?}",
+ if is_user { "EL0" } else { "EL1" },
+ tf.elr,
+ vaddr,
+ iss,
+ access_flags,
+ tf,
+ );
+ }
+}
+
+fn handle_data_abort(tf: &TrapFrame, iss: u64, is_user: bool) {
+ let wnr = (iss & (1 << 6)) != 0; // WnR: Write not Read
+ let cm = (iss & (1 << 8)) != 0; // CM: Cache maintenance
+ let mut access_flags = if wnr & !cm {
+ MappingFlags::WRITE
+ } else {
+ MappingFlags::READ
+ };
+ if is_user {
+ access_flags |= MappingFlags::USER;
+ }
+ let vaddr = VirtAddr::from(FAR_EL1.get() as usize);
+
+ // Only handle Translation fault and Permission fault
+ if !matches!(iss & 0b111100, 0b0100 | 0b1100) // IFSC or DFSC bits
+ || !handle_trap!(PAGE_FAULT, vaddr, access_flags, is_user)
+ {
+ panic!(
+ "Unhandled {} Data Abort @ {:#x}, fault_vaddr={:#x}, ISS=0b{:08b} ({:?}):\n{:#x?}",
+ if is_user { "EL0" } else { "EL1" },
+ tf.elr,
+ vaddr,
+ iss,
+ access_flags,
+ tf,
+ );
+ }
+}
+
 #[no_mangle]
 fn handle_sync_exception(tf: &mut TrapFrame) {
  let esr = ESR_EL1.extract();
+ let iss = esr.read(ESR_EL1::ISS);
  match esr.read_as_enum(ESR_EL1::EC) {
+ Some(ESR_EL1::EC::Value::SVC64) => {
+ warn!("No syscall is supported currently!");
+ }
+ Some(ESR_EL1::EC::Value::InstrAbortLowerEL) => handle_instruction_abort(tf, iss, true),
+ Some(ESR_EL1::EC::Value::InstrAbortCurrentEL) => handle_instruction_abort(tf, iss, false),
+ Some(ESR_EL1::EC::Value::DataAbortLowerEL) => handle_data_abort(tf, iss, true),
+ Some(ESR_EL1::EC::Value::DataAbortCurrentEL) => handle_data_abort(tf, iss, false),
  Some(ESR_EL1::EC::Value::Brk64) => {
- let iss = esr.read(ESR_EL1::ISS);
  debug!("BRK #{:#x} @ {:#x} ", iss, tf.elr);
  tf.elr += 4;
  }
- Some(ESR_EL1::EC::Value::SVC64) => {
- warn!("No supervisor call is supported currently!");
- }
- Some(ESR_EL1::EC::Value::DataAbortLowerEL)
- | Some(ESR_EL1::EC::Value::InstrAbortLowerEL) => {
- let iss = esr.read(ESR_EL1::ISS);
- warn!(
- "EL0 Page Fault @ {:#x}, FAR={:#x}, ISS={:#x}",
- tf.elr,
- FAR_EL1.get(),
- iss
- );
- }
- Some(ESR_EL1::EC::Value::DataAbortCurrentEL)
- | Some(ESR_EL1::EC::Value::InstrAbortCurrentEL) => {
- let iss = esr.read(ESR_EL1::ISS);
- panic!(
- "EL1 Page Fault @ {:#x}, FAR={:#x}, ISS={:#x}:\n{:#x?}",
- tf.elr,
- FAR_EL1.get(),
- iss,
- tf,
- );
- }
  _ => {
  panic!(
  "Unhandled synchronous exception @ {:#x}: ESR={:#x} (EC {:#08b}, ISS {:#x})",
@@ -79,8 +121,3 @@ fn handle_sync_exception(tf: &mut TrapFrame) {
  }
  }
 }
-
-#[no_mangle]
-fn handle_irq_exception(_tf: &TrapFrame) {
- crate::trap::handle_irq_extern(0)
-}

modules/axhal/src/arch/riscv/trap.rs

@@ -1,4 +1,7 @@
+use memory_addr::VirtAddr;
+use page_table_entry::MappingFlags;
 use riscv::register::scause::{self, Exception as E, Trap};
+use riscv::register::stval;
 
 use super::TrapFrame;
 
@@ -14,12 +17,36 @@ fn handle_breakpoint(sepc: &mut usize) {
  *sepc += 2
 }
 
+fn handle_page_fault(tf: &TrapFrame, mut access_flags: MappingFlags, is_user: bool) {
+ if is_user {
+ access_flags |= MappingFlags::USER;
+ }
+ let vaddr = VirtAddr::from(stval::read());
+ if !handle_trap!(PAGE_FAULT, vaddr, access_flags, is_user) {
+ panic!(
+ "Unhandled {} Page Fault @ {:#x}, fault_vaddr={:#x} ({:?}):\n{:#x?}",
+ if is_user { "User" } else { "Supervisor" },
+ tf.sepc,
+ vaddr,
+ access_flags,
+ tf,
+ );
+ }
+}
+
 #[no_mangle]
-fn riscv_trap_handler(tf: &mut TrapFrame, _from_user: bool) {
+fn riscv_trap_handler(tf: &mut TrapFrame, from_user: bool) {
  let scause = scause::read();
  match scause.cause() {
+ Trap::Exception(E::LoadPageFault) => handle_page_fault(tf, MappingFlags::READ, from_user),
+ Trap::Exception(E::StorePageFault) => handle_page_fault(tf, MappingFlags::WRITE, from_user),
+ Trap::Exception(E::InstructionPageFault) => {
+ handle_page_fault(tf, MappingFlags::EXECUTE, from_user)
+ }
  Trap::Exception(E::Breakpoint) => handle_breakpoint(&mut tf.sepc),
- Trap::Interrupt(_) => crate::trap::handle_irq_extern(scause.bits()),
+ Trap::Interrupt(_) => {
+ handle_trap!(IRQ, scause.bits());
+ }
  _ => {
  panic!(
  "Unhandled trap {:?} @ {:#x}:\n{:#x?}",

modules/axhal/src/arch/x86_64/trap.rs

@@ -1,4 +1,7 @@
+use memory_addr::VirtAddr;
+use page_table_entry::MappingFlags;
 use x86::{controlregs::cr2, irq::*};
+use x86_64::structures::idt::PageFaultErrorCode;
 
 use super::context::TrapFrame;
 
@@ -7,40 +10,79 @@ core::arch::global_asm!(include_str!("trap.S"));
 const IRQ_VECTOR_START: u8 = 0x20;
 const IRQ_VECTOR_END: u8 = 0xff;
 
+fn handle_page_fault(tf: &TrapFrame) {
+ let access_flags = err_code_to_flags(tf.error_code)
+ .unwrap_or_else(|e| panic!("Invalid #PF error code: {:#x}", e));
+ let vaddr = VirtAddr::from(unsafe { cr2() });
+ if !handle_trap!(PAGE_FAULT, vaddr, access_flags, tf.is_user()) {
+ panic!(
+ "Unhandled {} #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x} ({:?}):\n{:#x?}",
+ if tf.is_user() { "user" } else { "kernel" },
+ tf.rip,
+ vaddr,
+ tf.error_code,
+ access_flags,
+ tf,
+ );
+ }
+}
+
 #[no_mangle]
 fn x86_trap_handler(tf: &TrapFrame) {
  match tf.vector as u8 {
- PAGE_FAULT_VECTOR => {
- if tf.is_user() {
- warn!(
- "User #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}",
- tf.rip,
- unsafe { cr2() },
- tf.error_code,
- );
- } else {
- panic!(
- "Kernel #PF @ {:#x}, fault_vaddr={:#x}, error_code={:#x}:\n{:#x?}",
- tf.rip,
- unsafe { cr2() },
- tf.error_code,
- tf,
- );
- }
- }
+ PAGE_FAULT_VECTOR => handle_page_fault(tf),
  BREAKPOINT_VECTOR => debug!("#BP @ {:#x} ", tf.rip),
  GENERAL_PROTECTION_FAULT_VECTOR => {
  panic!(
  "#GP @ {:#x}, error_code={:#x}:\n{:#x?}",
  tf.rip, tf.error_code, tf
  );
  }
- IRQ_VECTOR_START..=IRQ_VECTOR_END => crate::trap::handle_irq_extern(tf.vector as _),
+ IRQ_VECTOR_START..=IRQ_VECTOR_END => {
+ handle_trap!(IRQ, tf.vector as _);
+ }
  _ => {
  panic!(
- "Unhandled exception {} (error_code = {:#x}) @ {:#x}:\n{:#x?}",
- tf.vector, tf.error_code, tf.rip, tf
+ "Unhandled exception {} ({}, error_code={:#x}) @ {:#x}:\n{:#x?}",
+ tf.vector,
+ vec_to_str(tf.vector),
+ tf.error_code,
+ tf.rip,
+ tf
  );
  }
  }
 }
+
+fn vec_to_str(vec: u64) -> &'static str {
+ if vec < 32 {
+ EXCEPTIONS[vec as usize].mnemonic
+ } else {
+ "Unknown"
+ }
+}
+
+fn err_code_to_flags(err_code: u64) -> Result<MappingFlags, u64> {
+ let code = PageFaultErrorCode::from_bits_truncate(err_code);
+ let reserved_bits = (PageFaultErrorCode::CAUSED_BY_WRITE
+ | PageFaultErrorCode::USER_MODE
+ | PageFaultErrorCode::INSTRUCTION_FETCH)
+ .complement();
+ if code.intersects(reserved_bits) {
+ Err(err_code)
+ } else {
+ let mut flags = MappingFlags::empty();
+ if code.contains(PageFaultErrorCode::CAUSED_BY_WRITE) {
+ flags |= MappingFlags::WRITE;
+ } else {
+ flags |= MappingFlags::READ;
+ }
+ if code.contains(PageFaultErrorCode::USER_MODE) {
+ flags |= MappingFlags::USER;
+ }
+ if code.contains(PageFaultErrorCode::INSTRUCTION_FETCH) {
+ flags |= MappingFlags::EXECUTE;
+ }
+ Ok(flags)
+ }
+}

modules/axhal/src/irq.rs

@@ -2,9 +2,10 @@
 
 use handler_table::HandlerTable;
 
-use crate::platform::irq::MAX_IRQ_COUNT;
+use crate::platform::irq::{dispatch_irq, MAX_IRQ_COUNT};
+use crate::trap::{reg_trap_handler, IRQ};
 
-pub use crate::platform::irq::{dispatch_irq, register_handler, set_enable};
+pub use crate::platform::irq::{register_handler, set_enable};
 
 /// The type if an IRQ handler.
 pub type IrqHandler = handler_table::Handler;
@@ -33,3 +34,11 @@ pub(crate) fn register_handler_common(irq_num: usize, handler: IrqHandler) -> bo
  warn!("register handler for IRQ {} failed", irq_num);
  false
 }
+
+#[reg_trap_handler(IRQ)]
+fn handler_irq(irq_num: usize) -> bool {
+ let guard = kernel_guard::NoPreempt::new();
+ dispatch_irq(irq_num);
+ drop(guard); // rescheduling may occur when preemption is re-enabled.
+ true
+}

modules/axhal/src/lib.rs

@@ -36,11 +36,13 @@ extern crate log;
 
 mod platform;
 
+#[macro_use]
+pub mod trap;
+
 pub mod arch;
 pub mod cpu;
 pub mod mem;
 pub mod time;
-pub mod trap;
 
 #[cfg(feature = "tls")]
 pub mod tls;