From 48003ecf89ddfefc5149a759ce695b5ad5186696 Mon Sep 17 00:00:00 2001 From: Jack Thomson Date: Thu, 12 Sep 2024 12:19:19 +0000 Subject: [PATCH 1/3] feat: Enable gdb debugging on x86 Enabling GDB support for debugging the guest kernel. This allows us to connect a gdb server to firecracker and debug the guest. Signed-off-by: Jack Thomson --- Cargo.lock | 39 ++ src/firecracker/Cargo.toml | 1 + src/vmm/Cargo.toml | 5 + src/vmm/src/builder.rs | 56 ++- src/vmm/src/gdb/arch/aarch64.rs | 62 ++++ src/vmm/src/gdb/arch/mod.rs | 13 + src/vmm/src/gdb/arch/x86.rs | 160 ++++++++ src/vmm/src/gdb/event_loop.rs | 159 ++++++++ src/vmm/src/gdb/mod.rs | 66 ++++ src/vmm/src/gdb/target.rs | 622 ++++++++++++++++++++++++++++++++ src/vmm/src/lib.rs | 9 + src/vmm/src/resources.rs | 12 + src/vmm/src/vstate/vcpu/mod.rs | 61 ++++ 13 files changed, 1255 insertions(+), 10 deletions(-) create mode 100644 src/vmm/src/gdb/arch/aarch64.rs create mode 100644 src/vmm/src/gdb/arch/mod.rs create mode 100644 src/vmm/src/gdb/arch/x86.rs create mode 100644 src/vmm/src/gdb/event_loop.rs create mode 100644 src/vmm/src/gdb/mod.rs create mode 100644 src/vmm/src/gdb/target.rs diff --git a/Cargo.lock b/Cargo.lock index 1733e9ac749..45324e6a5c3 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -111,6 +111,12 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + [[package]] name = "autocfg" version = "1.4.0" @@ -627,6 +633,30 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" +[[package]] +name = "gdbstub" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fbcc892208d6998fb57e7c3e05883def66f8130924bba066beb0cfe71566a9f6" +dependencies = [ + "bitflags 2.6.0", + "cfg-if", + "log", + "managed", + "num-traits", + "paste", +] + +[[package]] +name = "gdbstub_arch" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e3b1357bd3203fc09a6601327ae0ab38865d14231d0b65d3143f5762cc7977d" +dependencies = [ + "gdbstub", + "num-traits", +] + [[package]] name = "generic-array" version = "0.14.7" @@ -888,6 +918,12 @@ dependencies = [ "syn", ] +[[package]] +name = "managed" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ca88d725a0a943b096803bd34e73a4437208b6077654cc4ecb2947a5f91618d" + [[package]] name = "memchr" version = "2.7.4" @@ -1534,6 +1570,7 @@ version = "0.1.0" dependencies = [ "acpi_tables", "aes-gcm", + "arrayvec", "aws-lc-rs", "base64", "bincode", @@ -1544,6 +1581,8 @@ dependencies = [ "device_tree", "displaydoc", "event-manager", + "gdbstub", + "gdbstub_arch", "itertools 0.13.0", "kvm-bindings", "kvm-ioctls", diff --git a/src/firecracker/Cargo.toml b/src/firecracker/Cargo.toml index e3e5bca6ded..0d678afc28e 100644 --- a/src/firecracker/Cargo.toml +++ b/src/firecracker/Cargo.toml @@ -49,6 +49,7 @@ serde_json = "1.0.128" [features] tracing = ["log-instrument", "seccompiler/tracing", "utils/tracing", "vmm/tracing"] +gdb = ["vmm/gdb"] [lints] workspace = true diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index 88a421136ad..7be2200b8b2 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -11,6 +11,7 @@ bench = false [dependencies] acpi_tables = { path = "../acpi-tables" } aes-gcm = { version = "0.10.1", default-features = false, features = ["aes"] } +arrayvec = { version = "0.7.6", optional = true } aws-lc-rs = { version = "1.10.0", features = ["bindgen"] } base64 = "0.22.1" bincode = "1.2.1" @@ -19,6 +20,8 @@ crc64 = "2.0.0" derive_more = { version = "1.0.0", default-features = false, features = ["from", "display"] } displaydoc = "0.2.5" event-manager = "0.4.0" +gdbstub = { version = "0.7.2", optional = true } +gdbstub_arch = { version = "0.3.0", optional = true } kvm-bindings = { version = "0.9.1", features = ["fam-wrappers", "serde"] } kvm-ioctls = "0.18.0" lazy_static = "1.5.0" @@ -55,7 +58,9 @@ itertools = "0.13.0" proptest = { version = "1.5.0", default-features = false, features = ["std"] } [features] +default = [] tracing = ["log-instrument"] +gdb = ["arrayvec", "gdbstub", "gdbstub_arch"] [[bench]] name = "cpu_templates" diff --git a/src/vmm/src/builder.rs b/src/vmm/src/builder.rs index 6bbfec6fabb..760c22a27b5 100644 --- a/src/vmm/src/builder.rs +++ b/src/vmm/src/builder.rs @@ -7,6 +7,8 @@ use std::convert::TryFrom; use std::fmt::Debug; use std::io::{self, Seek, SeekFrom}; +#[cfg(feature = "gdb")] +use std::sync::mpsc; use std::sync::{Arc, Mutex}; use event_manager::{MutEventSubscriber, SubscriberOps}; @@ -26,6 +28,9 @@ use vm_superio::Rtc; use vm_superio::Serial; use vmm_sys_util::eventfd::EventFd; +#[cfg(all(feature = "gdb", target_arch = "aarch64"))] +compile_error!("GDB feature not supported on ARM"); + #[cfg(target_arch = "x86_64")] use crate::acpi; use crate::arch::InitrdConfig; @@ -56,6 +61,8 @@ use crate::devices::virtio::net::Net; use crate::devices::virtio::rng::Entropy; use crate::devices::virtio::vsock::{Vsock, VsockUnixBackend}; use crate::devices::BusDevice; +#[cfg(feature = "gdb")] +use crate::gdb; use crate::logger::{debug, error}; use crate::persist::{MicrovmState, MicrovmStateError}; use crate::resources::VmResources; @@ -128,6 +135,12 @@ pub enum StartMicrovmError { /// Error configuring ACPI: {0} #[cfg(target_arch = "x86_64")] Acpi(#[from] crate::acpi::AcpiError), + /// Error starting GDB debug session + #[cfg(feature = "gdb")] + GdbServer(gdb::target::GdbTargetError), + /// Error cloning Vcpu fds + #[cfg(feature = "gdb")] + VcpuFdCloneError(#[from] crate::vstate::vcpu::CopyKvmFdError), } /// It's convenient to automatically convert `linux_loader::cmdline::Error`s @@ -274,6 +287,18 @@ pub fn build_microvm_for_boot( cpu_template.kvm_capabilities.clone(), )?; + #[cfg(feature = "gdb")] + let (gdb_tx, gdb_rx) = mpsc::channel(); + #[cfg(feature = "gdb")] + vcpus + .iter_mut() + .for_each(|vcpu| vcpu.attach_debug_info(gdb_tx.clone())); + #[cfg(feature = "gdb")] + let vcpu_fds = vcpus + .iter() + .map(|vcpu| vcpu.copy_kvm_vcpu_fd(vmm.vm())) + .collect::, _>>()?; + // The boot timer device needs to be the first device attached in order // to maintain the same MMIO address referenced in the documentation // and tests. @@ -321,16 +346,28 @@ pub fn build_microvm_for_boot( boot_cmdline, )?; + let vmm = Arc::new(Mutex::new(vmm)); + + #[cfg(feature = "gdb")] + if let Some(gdb_socket_addr) = &vm_resources.gdb_socket_addr { + gdb::gdb_thread(vmm.clone(), vcpu_fds, gdb_rx, entry_addr, gdb_socket_addr) + .map_err(GdbServer)?; + } else { + debug!("No GDB socket provided not starting gdb server."); + } + // Move vcpus to their own threads and start their state machine in the 'Paused' state. - vmm.start_vcpus( - vcpus, - seccomp_filters - .get("vcpu") - .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))? - .clone(), - ) - .map_err(VmmError::VcpuStart) - .map_err(Internal)?; + vmm.lock() + .unwrap() + .start_vcpus( + vcpus, + seccomp_filters + .get("vcpu") + .ok_or_else(|| MissingSeccompFilters("vcpu".to_string()))? + .clone(), + ) + .map_err(VmmError::VcpuStart) + .map_err(Internal)?; // Load seccomp filters for the VMM thread. // Execution panics if filters cannot be loaded, use --no-seccomp if skipping filters @@ -344,7 +381,6 @@ pub fn build_microvm_for_boot( .map_err(VmmError::SeccompFilters) .map_err(Internal)?; - let vmm = Arc::new(Mutex::new(vmm)); event_manager.add_subscriber(vmm.clone()); Ok(vmm) diff --git a/src/vmm/src/gdb/arch/aarch64.rs b/src/vmm/src/gdb/arch/aarch64.rs new file mode 100644 index 00000000000..d6e667f9fcb --- /dev/null +++ b/src/vmm/src/gdb/arch/aarch64.rs @@ -0,0 +1,62 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use gdbstub_arch::aarch64::reg::AArch64CoreRegs as CoreRegs; +use kvm_ioctls::VcpuFd; +use vm_memory::GuestAddress; + +use crate::gdb::target::GdbTargetError; + +/// Configures the number of bytes required for a software breakpoint +pub const SW_BP_SIZE: usize = 1; + +/// The bytes stored for a software breakpoint +pub const SW_BP: [u8; SW_BP_SIZE] = [0]; + +/// Gets the RIP value for a Vcpu +pub fn get_instruction_pointer(_vcpu_fd: &VcpuFd) -> Result { + unimplemented!() +} + +/// Translates a virtual address according to the vCPU's current address translation mode. +pub fn translate_gva(_vcpu_fd: &VcpuFd, _gva: u64) -> Result { + unimplemented!() +} + +/// Configures the kvm guest debug regs to register the hardware breakpoints +fn set_kvm_debug( + _control: u32, + _vcpu_fd: &VcpuFd, + _addrs: &[GuestAddress], +) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Configures the Vcpu for debugging and sets the hardware breakpoints on the Vcpu +pub fn vcpu_set_debug( + _vcpu_fd: &VcpuFd, + _addrs: &[GuestAddress], + _step: bool, +) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Injects a BP back into the guest kernel for it to handle, this is particularly useful for the +/// kernels selftesting which can happen during boot. +pub fn vcpu_inject_bp( + _vcpu_fd: &VcpuFd, + _addrs: &[GuestAddress], + _step: bool, +) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Reads the registers for the Vcpu +pub fn read_registers(_vcpu_fd: &VcpuFd, _regs: &mut CoreRegs) -> Result<(), GdbTargetError> { + unimplemented!() +} + +/// Writes to the registers for the Vcpu +pub fn write_registers(_vcpu_fd: &VcpuFd, _regs: &CoreRegs) -> Result<(), GdbTargetError> { + unimplemented!() +} diff --git a/src/vmm/src/gdb/arch/mod.rs b/src/vmm/src/gdb/arch/mod.rs new file mode 100644 index 00000000000..424b6626ba1 --- /dev/null +++ b/src/vmm/src/gdb/arch/mod.rs @@ -0,0 +1,13 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 +// + +#[cfg(target_arch = "aarch64")] +mod aarch64; +#[cfg(target_arch = "aarch64")] +pub use aarch64::*; + +#[cfg(target_arch = "x86_64")] +mod x86; +#[cfg(target_arch = "x86_64")] +pub use x86::*; diff --git a/src/vmm/src/gdb/arch/x86.rs b/src/vmm/src/gdb/arch/x86.rs new file mode 100644 index 00000000000..6671b83443f --- /dev/null +++ b/src/vmm/src/gdb/arch/x86.rs @@ -0,0 +1,160 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use gdbstub_arch::x86::reg::X86_64CoreRegs as CoreRegs; +use kvm_bindings::*; +use kvm_ioctls::VcpuFd; +use vm_memory::GuestAddress; + +use crate::gdb::target::GdbTargetError; +use crate::logger::error; + +/// Sets the 9th (Global Exact Breakpoint enable) and the 10th (always 1) bits for the DR7 debug +/// control register +const X86_GLOBAL_DEBUG_ENABLE: u64 = 0b11 << 9; + +/// Op code to trigger a software breakpoint in x86 +const X86_SW_BP_OP: u8 = 0xCC; + +/// Configures the number of bytes required for a software breakpoint +pub const SW_BP_SIZE: usize = 1; + +/// The bytes stored for an x86 software breakpoint +pub const SW_BP: [u8; SW_BP_SIZE] = [X86_SW_BP_OP]; + +/// Gets the RIP value for a Vcpu +pub fn get_instruction_pointer(vcpu_fd: &VcpuFd) -> Result { + let regs = vcpu_fd.get_regs()?; + + Ok(regs.rip) +} + +/// Translates a virtual address according to the vCPU's current address translation mode. +pub fn translate_gva(vcpu_fd: &VcpuFd, gva: u64) -> Result { + let tr = vcpu_fd.translate_gva(gva)?; + + if tr.valid == 0 { + return Err(GdbTargetError::KvmGvaTranslateError); + } + + Ok(tr.physical_address) +} + +/// Configures the kvm guest debug regs to register the hardware breakpoints, the `arch.debugreg` +/// attribute is used to store the location of the hardware breakpoints, with the 8th slot being +/// used as a bitfield to track which registers are enabled and setting the +/// `X86_GLOBAL_DEBUG_ENABLE` flags. Further reading on the DR7 register can be found here: +/// https://en.wikipedia.org/wiki/X86_debug_register#DR7_-_Debug_control +fn set_kvm_debug( + control: u32, + vcpu_fd: &VcpuFd, + addrs: &[GuestAddress], +) -> Result<(), GdbTargetError> { + let mut dbg = kvm_guest_debug { + control, + ..Default::default() + }; + + dbg.arch.debugreg[7] = X86_GLOBAL_DEBUG_ENABLE; + + for (i, addr) in addrs.iter().enumerate() { + dbg.arch.debugreg[i] = addr.0; + // Set global breakpoint enable flag for the specific breakpoint number by setting the bit + dbg.arch.debugreg[7] |= 2 << (i * 2); + } + + vcpu_fd.set_guest_debug(&dbg)?; + + Ok(()) +} + +/// Configures the Vcpu for debugging and sets the hardware breakpoints on the Vcpu +pub fn vcpu_set_debug( + vcpu_fd: &VcpuFd, + addrs: &[GuestAddress], + step: bool, +) -> Result<(), GdbTargetError> { + let mut control = KVM_GUESTDBG_ENABLE | KVM_GUESTDBG_USE_HW_BP | KVM_GUESTDBG_USE_SW_BP; + if step { + control |= KVM_GUESTDBG_SINGLESTEP; + } + + set_kvm_debug(control, vcpu_fd, addrs) +} + +/// Injects a BP back into the guest kernel for it to handle, this is particularly useful for the +/// kernels selftesting which can happen during boot. +pub fn vcpu_inject_bp( + vcpu_fd: &VcpuFd, + addrs: &[GuestAddress], + step: bool, +) -> Result<(), GdbTargetError> { + let mut control = KVM_GUESTDBG_ENABLE + | KVM_GUESTDBG_USE_HW_BP + | KVM_GUESTDBG_USE_SW_BP + | KVM_GUESTDBG_INJECT_BP; + + if step { + control |= KVM_GUESTDBG_SINGLESTEP; + } + + set_kvm_debug(control, vcpu_fd, addrs) +} + +/// Reads the registers for the Vcpu +pub fn read_registers(vcpu_fd: &VcpuFd, regs: &mut CoreRegs) -> Result<(), GdbTargetError> { + let cpu_regs = vcpu_fd.get_regs()?; + + regs.regs[0] = cpu_regs.rax; + regs.regs[1] = cpu_regs.rbx; + regs.regs[2] = cpu_regs.rcx; + regs.regs[3] = cpu_regs.rdx; + regs.regs[4] = cpu_regs.rsi; + regs.regs[5] = cpu_regs.rdi; + regs.regs[6] = cpu_regs.rbp; + regs.regs[7] = cpu_regs.rsp; + + regs.regs[8] = cpu_regs.r8; + regs.regs[9] = cpu_regs.r9; + regs.regs[10] = cpu_regs.r10; + regs.regs[11] = cpu_regs.r11; + regs.regs[12] = cpu_regs.r12; + regs.regs[13] = cpu_regs.r13; + regs.regs[14] = cpu_regs.r14; + regs.regs[15] = cpu_regs.r15; + + regs.rip = cpu_regs.rip; + regs.eflags = u32::try_from(cpu_regs.rflags).map_err(|e| { + error!("Error {e:?} converting rflags to u32"); + GdbTargetError::RegFlagConversionError + })?; + + Ok(()) +} +/// Writes to the registers for the Vcpu +pub fn write_registers(vcpu_fd: &VcpuFd, regs: &CoreRegs) -> Result<(), GdbTargetError> { + let new_regs = kvm_regs { + rax: regs.regs[0], + rbx: regs.regs[1], + rcx: regs.regs[2], + rdx: regs.regs[3], + rsi: regs.regs[4], + rdi: regs.regs[5], + rbp: regs.regs[6], + rsp: regs.regs[7], + + r8: regs.regs[8], + r9: regs.regs[9], + r10: regs.regs[10], + r11: regs.regs[11], + r12: regs.regs[12], + r13: regs.regs[13], + r14: regs.regs[14], + r15: regs.regs[15], + + rip: regs.rip, + rflags: regs.eflags as u64, + }; + + Ok(vcpu_fd.set_regs(&new_regs)?) +} diff --git a/src/vmm/src/gdb/event_loop.rs b/src/vmm/src/gdb/event_loop.rs new file mode 100644 index 00000000000..ae4de0e64d7 --- /dev/null +++ b/src/vmm/src/gdb/event_loop.rs @@ -0,0 +1,159 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::os::unix::net::UnixStream; +use std::sync::mpsc::Receiver; +use std::sync::mpsc::TryRecvError::Empty; +use std::sync::{Arc, Mutex}; + +use gdbstub::common::{Signal, Tid}; +use gdbstub::conn::{Connection, ConnectionExt}; +use gdbstub::stub::run_blocking::{self, WaitForStopReasonError}; +use gdbstub::stub::{DisconnectReason, GdbStub, MultiThreadStopReason}; +use gdbstub::target::Target; +use kvm_ioctls::VcpuFd; +use vm_memory::GuestAddress; + +use super::target::{vcpuid_to_tid, FirecrackerTarget, GdbTargetError}; +use crate::logger::{error, trace}; +use crate::Vmm; + +/// Starts the GDB event loop which acts as a proxy between the Vcpus and GDB +pub fn event_loop( + connection: UnixStream, + vmm: Arc>, + vcpu_fds: Vec, + gdb_event_receiver: Receiver, + entry_addr: GuestAddress, +) { + let target = FirecrackerTarget::new(vmm, vcpu_fds, gdb_event_receiver, entry_addr); + let connection: Box> = { Box::new(connection) }; + let debugger = GdbStub::new(connection); + + // We wait for the VM to reach the inital breakpoint we inserted before starting the event loop + target + .gdb_event + .recv() + .expect("Error getting initial gdb event"); + + gdb_event_loop_thread(debugger, target); +} + +struct GdbBlockingEventLoop {} + +impl run_blocking::BlockingEventLoop for GdbBlockingEventLoop { + type Target = FirecrackerTarget; + type Connection = Box>; + + type StopReason = MultiThreadStopReason; + + /// Poll for events from either Vcpu's or packets from the GDB connection + fn wait_for_stop_reason( + target: &mut FirecrackerTarget, + conn: &mut Self::Connection, + ) -> Result< + run_blocking::Event>, + run_blocking::WaitForStopReasonError< + ::Error, + ::Error, + >, + > { + loop { + match target.gdb_event.try_recv() { + Ok(cpu_id) => { + // The Vcpu reports it's id from raw_id so we straight convert here + let tid = Tid::new(cpu_id).expect("Error converting cpu id to Tid"); + // If notify paused returns false this means we were already debugging a single + // core, the target will track this for us to pick up later + target.set_paused_vcpu(tid); + trace!("Vcpu: {tid:?} paused from debug exit"); + + let stop_reason = target + .get_stop_reason(tid) + .map_err(WaitForStopReasonError::Target)?; + + let Some(stop_response) = stop_reason else { + // If we returned None this is a break which should be handled by + // the guest kernel (e.g. kernel int3 self testing) so we won't notify + // GDB and instead inject this back into the guest + target + .inject_bp_to_guest(tid) + .map_err(WaitForStopReasonError::Target)?; + target + .resume_vcpu(tid) + .map_err(WaitForStopReasonError::Target)?; + + trace!("Injected BP into guest early exit"); + continue; + }; + + trace!("Returned stop reason to gdb: {stop_response:?}"); + return Ok(run_blocking::Event::TargetStopped(stop_response)); + } + Err(Empty) => (), + Err(_) => { + return Err(WaitForStopReasonError::Target( + GdbTargetError::GdbQueueError, + )); + } + } + + if conn.peek().map(|b| b.is_some()).unwrap_or(false) { + let byte = conn + .read() + .map_err(run_blocking::WaitForStopReasonError::Connection)?; + return Ok(run_blocking::Event::IncomingData(byte)); + } + } + } + + /// Invoked when the GDB client sends a Ctrl-C interrupt. + fn on_interrupt( + target: &mut FirecrackerTarget, + ) -> Result>, ::Error> { + // notify the target that a ctrl-c interrupt has occurred. + let main_core = vcpuid_to_tid(0)?; + + target.pause_vcpu(main_core)?; + target.set_paused_vcpu(main_core); + + let exit_reason = MultiThreadStopReason::SignalWithThread { + tid: main_core, + signal: Signal::SIGINT, + }; + Ok(Some(exit_reason)) + } +} + +/// Runs while communication with GDB is in progress, after GDB disconnects we +/// shutdown firecracker +fn gdb_event_loop_thread( + debugger: GdbStub>>, + mut target: FirecrackerTarget, +) { + match debugger.run_blocking::(&mut target) { + Ok(disconnect_reason) => match disconnect_reason { + DisconnectReason::Disconnect => { + trace!("Client disconnected") + } + DisconnectReason::TargetExited(code) => { + trace!("Target exited with code {}", code) + } + DisconnectReason::TargetTerminated(sig) => { + trace!("Target terminated with signal {}", sig) + } + DisconnectReason::Kill => trace!("GDB sent a kill command"), + }, + Err(e) => { + if e.is_target_error() { + error!("target encountered a fatal error: {e:?}") + } else if e.is_connection_error() { + error!("connection error: {e:?}") + } else { + error!("gdbstub encountered a fatal error {e:?}") + } + } + } + + target.shutdown_vmm(); +} diff --git a/src/vmm/src/gdb/mod.rs b/src/vmm/src/gdb/mod.rs new file mode 100644 index 00000000000..9d209541425 --- /dev/null +++ b/src/vmm/src/gdb/mod.rs @@ -0,0 +1,66 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +/// Arch specific implementations +mod arch; +/// Event loop for connection to GDB server +mod event_loop; +/// Target for gdb +pub mod target; + +use std::os::unix::net::UnixListener; +use std::path::Path; +use std::sync::mpsc::Receiver; +use std::sync::{Arc, Mutex}; + +use arch::vcpu_set_debug; +use event_loop::event_loop; +use kvm_ioctls::VcpuFd; +use target::GdbTargetError; +use vm_memory::GuestAddress; + +use crate::logger::trace; +use crate::Vmm; + +/// Kickstarts the GDB debugging process, it takes in the VMM object, a slice of +/// the paused Vcpu's, the GDB event queue which is used as a mechanism for the Vcpu's to notify +/// our GDB thread that they've been paused, then finally the entry address of the kernel. +/// +/// Firstly the function will start by configuring the Vcpus with KVM for debugging +/// +/// This will then create the GDB socket which will be used for communication to the GDB process. +/// After creating this, the function will block while waiting for GDB to connect. +/// +/// After the connection has been established the function will start a new thread for handling +/// communcation to the GDB server +pub fn gdb_thread( + vmm: Arc>, + vcpu_fds: Vec, + gdb_event_receiver: Receiver, + entry_addr: GuestAddress, + socket_addr: &str, +) -> Result<(), GdbTargetError> { + // We register a hw breakpoint at the entry point as GDB expects the application + // to be stopped as it connects. This also allows us to set breakpoints before kernel starts. + // This entry adddress is automatically used as it is not tracked inside the target state, so + // when resumed will be removed + vcpu_set_debug(&vcpu_fds[0], &[entry_addr], false)?; + + for vcpu_fd in &vcpu_fds[1..] { + vcpu_set_debug(vcpu_fd, &[], false)?; + } + + let path = Path::new(socket_addr); + let listener = UnixListener::bind(path).map_err(|_| GdbTargetError::ServerSocketError)?; + trace!("Waiting for GDB server connection on {}...", path.display()); + let (connection, _addr) = listener + .accept() + .map_err(|_| GdbTargetError::ServerSocketError)?; + + std::thread::Builder::new() + .name("gdb".into()) + .spawn(move || event_loop(connection, vmm, vcpu_fds, gdb_event_receiver, entry_addr)) + .map_err(|_| GdbTargetError::GdbThreadError)?; + + Ok(()) +} diff --git a/src/vmm/src/gdb/target.rs b/src/vmm/src/gdb/target.rs new file mode 100644 index 00000000000..c3234241379 --- /dev/null +++ b/src/vmm/src/gdb/target.rs @@ -0,0 +1,622 @@ +// Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::collections::HashMap; +use std::sync::mpsc::{Receiver, RecvError}; +use std::sync::{Arc, Mutex, PoisonError}; + +use arrayvec::ArrayVec; +use gdbstub::arch::Arch; +use gdbstub::common::{Signal, Tid}; +use gdbstub::stub::{BaseStopReason, MultiThreadStopReason}; +use gdbstub::target::ext::base::multithread::{ + MultiThreadBase, MultiThreadResume, MultiThreadResumeOps, MultiThreadSingleStep, + MultiThreadSingleStepOps, +}; +use gdbstub::target::ext::base::BaseOps; +use gdbstub::target::ext::breakpoints::{ + Breakpoints, BreakpointsOps, HwBreakpoint, HwBreakpointOps, SwBreakpoint, SwBreakpointOps, +}; +use gdbstub::target::ext::thread_extra_info::{ThreadExtraInfo, ThreadExtraInfoOps}; +use gdbstub::target::{Target, TargetError, TargetResult}; +#[cfg(target_arch = "aarch64")] +use gdbstub_arch::aarch64::reg::AArch64CoreRegs as CoreRegs; +#[cfg(target_arch = "aarch64")] +use gdbstub_arch::aarch64::AArch64 as GdbArch; +#[cfg(target_arch = "x86_64")] +use gdbstub_arch::x86::reg::X86_64CoreRegs as CoreRegs; +#[cfg(target_arch = "x86_64")] +use gdbstub_arch::x86::X86_64_SSE as GdbArch; +use kvm_ioctls::VcpuFd; +use vm_memory::{Bytes, GuestAddress}; + +use super::arch; +use crate::arch::PAGE_SIZE; +use crate::logger::{error, info}; +use crate::utils::u64_to_usize; +use crate::vstate::vcpu::VcpuSendEventError; +use crate::{FcExitCode, VcpuEvent, VcpuResponse, Vmm}; + +#[derive(Debug)] +/// Stores the current state of a Vcpu with a copy of the Vcpu file descriptor +struct VcpuState { + single_step: bool, + paused: bool, + vcpu_fd: VcpuFd, +} + +impl VcpuState { + /// Constructs a new instance of a VcpuState from a VcpuFd + fn from_vcpu_fd(vcpu_fd: VcpuFd) -> Self { + Self { + single_step: false, + paused: false, + vcpu_fd, + } + } + + /// Disables single stepping on the Vcpu state + fn reset_vcpu_state(&mut self) { + self.single_step = false; + } + + /// Updates the kvm debug flags set against the Vcpu with a check + fn update_kvm_debug(&self, hw_breakpoints: &[GuestAddress]) -> Result<(), GdbTargetError> { + if !self.paused { + info!("Attempted to update kvm debug on a non paused Vcpu"); + return Ok(()); + } + + arch::vcpu_set_debug(&self.vcpu_fd, hw_breakpoints, self.single_step) + } +} + +/// Errors from interactions between GDB and the VMM +#[derive(Debug, thiserror::Error, displaydoc::Display)] +pub enum GdbTargetError { + /// An error during a GDB request + GdbRequest, + /// An error with the queue between the target and the Vcpus + GdbQueueError, + /// The response from the Vcpu was not allowed + VcuRequestError, + /// No currently paused Vcpu error + NoPausedVcpu, + /// Error when setting Vcpu debug flags + VcpuKvmError, + /// Server socket Error + ServerSocketError, + /// Error with creating GDB thread + GdbThreadError, + /// VMM locking error + VmmLockError, + /// Vcpu send event error + VcpuSendEventError(#[from] VcpuSendEventError), + /// Recieve error from Vcpu channel + VcpuRecvError(#[from] RecvError), + /// TID Conversion error + TidConversionError, + /// KVM set guest debug error + KvmIoctlsError(#[from] kvm_ioctls::Error), + /// Gva no translation available + KvmGvaTranslateError, + /// Conversion error with cpu rflags + RegFlagConversionError, +} + +impl From for TargetError { + fn from(error: GdbTargetError) -> Self { + match error { + GdbTargetError::VmmLockError => TargetError::Fatal(GdbTargetError::VmmLockError), + _ => TargetError::NonFatal, + } + } +} + +impl From> for GdbTargetError { + fn from(_value: PoisonError) -> Self { + GdbTargetError::VmmLockError + } +} + +/// Debug Target for firecracker. +/// +/// This is used the manage the debug implementation and handle requests sent via GDB +#[derive(Debug)] +pub struct FirecrackerTarget { + /// A mutex around the VMM to allow communicataion to the Vcpus + vmm: Arc>, + /// Store the guest entry point + entry_addr: GuestAddress, + + /// Listener for events sent from the Vcpu + pub gdb_event: Receiver, + + /// Used to track the currently configured hardware breakpoints. + /// Limited to 4 in x86 see: + /// https://elixir.bootlin.com/linux/v6.1/source/arch/x86/include/asm/kvm_host.h#L210 + hw_breakpoints: ArrayVec, + /// Used to track the currently configured software breakpoints and store the op-code + /// which was swapped out + sw_breakpoints: HashMap<::Usize, [u8; arch::SW_BP_SIZE]>, + + /// Stores the current state of each Vcpu + vcpu_state: Vec, + + /// Stores the current paused thread id, GDB can inact commands without providing us a Tid to + /// run on and expects us to use the last paused thread. + paused_vcpu: Option, +} + +/// Convert the 1 indexed Tid to the 0 indexed Vcpuid +fn tid_to_vcpuid(tid: Tid) -> usize { + tid.get() - 1 +} + +/// Converts the inernal index of a Vcpu to +/// the Tid required by GDB +pub fn vcpuid_to_tid(cpu_id: usize) -> Result { + Tid::new(get_raw_tid(cpu_id)).ok_or(GdbTargetError::TidConversionError) +} + +/// Converts the inernal index of a Vcpu to +/// the 1 indexed value for GDB +pub fn get_raw_tid(cpu_id: usize) -> usize { + cpu_id + 1 +} + +impl FirecrackerTarget { + /// Creates a new Target for GDB stub. This is used as the layer between GDB and the VMM it + /// will handle requests from GDB and perform the appropriate actions, while also updating GDB + /// with the state of the VMM / Vcpu's as we hit debug events + pub fn new( + vmm: Arc>, + vcpu_fds: Vec, + gdb_event: Receiver, + entry_addr: GuestAddress, + ) -> Self { + let mut vcpu_state: Vec = + vcpu_fds.into_iter().map(VcpuState::from_vcpu_fd).collect(); + // By default vcpu 1 will be paused at the entry point + vcpu_state[0].paused = true; + + Self { + vmm, + entry_addr, + gdb_event, + // We only support 4 hw breakpoints on x86 this will need to be configurable on arm + hw_breakpoints: Default::default(), + sw_breakpoints: HashMap::new(), + vcpu_state, + + paused_vcpu: Tid::new(1), + } + } + + /// Retrieves the currently paused Vcpu id returns an error if there is no currently paused Vcpu + fn get_paused_vcpu_id(&self) -> Result { + self.paused_vcpu.ok_or(GdbTargetError::NoPausedVcpu) + } + + /// Retrieves the currently paused Vcpu state returns an error if there is no currently paused + /// Vcpu + fn get_paused_vcpu(&self) -> Result<&VcpuState, GdbTargetError> { + let vcpu_index = tid_to_vcpuid(self.get_paused_vcpu_id()?); + Ok(&self.vcpu_state[vcpu_index]) + } + + /// Updates state to reference the currently paused Vcpu and store that the cpu is currently + /// paused + pub fn set_paused_vcpu(&mut self, tid: Tid) { + self.vcpu_state[tid_to_vcpuid(tid)].paused = true; + self.paused_vcpu = Some(tid); + } + + /// Resumes execution of all paused Vcpus, update them with current kvm debug info + /// and resumes + fn resume_all_vcpus(&mut self) -> Result<(), GdbTargetError> { + self.vcpu_state + .iter() + .try_for_each(|state| state.update_kvm_debug(&self.hw_breakpoints))?; + + for cpu_id in 0..self.vcpu_state.len() { + let tid = vcpuid_to_tid(cpu_id)?; + self.resume_vcpu(tid)?; + } + + self.paused_vcpu = None; + + Ok(()) + } + + /// Resets all Vcpus to their base state + fn reset_all_vcpu_states(&mut self) { + for value in self.vcpu_state.iter_mut() { + value.reset_vcpu_state(); + } + } + + /// Shuts down the VMM + pub fn shutdown_vmm(&self) { + self.vmm + .lock() + .expect("error unlocking vmm") + .stop(FcExitCode::Ok) + } + + /// Pauses the requested Vcpu + pub fn pause_vcpu(&mut self, tid: Tid) -> Result<(), GdbTargetError> { + let vcpu_state = &mut self.vcpu_state[tid_to_vcpuid(tid)]; + + if vcpu_state.paused { + info!("Attempted to pause a vcpu already paused."); + // Pausing an already paused vcpu is not considered an error case from GDB + return Ok(()); + } + + let cpu_handle = &self.vmm.lock()?.vcpus_handles[tid_to_vcpuid(tid)]; + + cpu_handle.send_event(VcpuEvent::Pause)?; + let _ = cpu_handle.response_receiver().recv()?; + + vcpu_state.paused = true; + Ok(()) + } + + /// A helper function to allow the event loop to inject this breakpoint back into the Vcpu + pub fn inject_bp_to_guest(&mut self, tid: Tid) -> Result<(), GdbTargetError> { + let vcpu_state = &mut self.vcpu_state[tid_to_vcpuid(tid)]; + arch::vcpu_inject_bp(&vcpu_state.vcpu_fd, &self.hw_breakpoints, false) + } + + /// Resumes the Vcpu, will return early if the Vcpu is already running + pub fn resume_vcpu(&mut self, tid: Tid) -> Result<(), GdbTargetError> { + let vcpu_state = &mut self.vcpu_state[tid_to_vcpuid(tid)]; + + if !vcpu_state.paused { + info!("Attempted to resume a vcpu already running."); + // Resuming an already running Vcpu is not considered an error case from GDB + return Ok(()); + } + + let cpu_handle = &self.vmm.lock()?.vcpus_handles[tid_to_vcpuid(tid)]; + cpu_handle.send_event(VcpuEvent::Resume)?; + + let response = cpu_handle.response_receiver().recv()?; + if let VcpuResponse::NotAllowed(message) = response { + error!("Response resume : {message}"); + return Err(GdbTargetError::VcuRequestError); + } + + vcpu_state.paused = false; + Ok(()) + } + + /// Identifies why the specific core was paused to be returned to GDB if None is returned this + /// indicates to handle this internally and don't notify GDB + pub fn get_stop_reason( + &self, + tid: Tid, + ) -> Result>, GdbTargetError> { + let vcpu_state = &self.vcpu_state[tid_to_vcpuid(tid)]; + if vcpu_state.single_step { + return Ok(Some(MultiThreadStopReason::SignalWithThread { + tid, + signal: Signal::SIGTRAP, + })); + } + + let Ok(ip) = arch::get_instruction_pointer(&vcpu_state.vcpu_fd) else { + // If we error here we return an arbitrary Software Breakpoint, GDB will handle + // this gracefully + return Ok(Some(MultiThreadStopReason::SwBreak(tid))); + }; + + let gpa = arch::translate_gva(&vcpu_state.vcpu_fd, ip)?; + if self.sw_breakpoints.contains_key(&gpa) { + return Ok(Some(MultiThreadStopReason::SwBreak(tid))); + } + + if self.hw_breakpoints.contains(&GuestAddress(ip)) { + return Ok(Some(MultiThreadStopReason::HwBreak(tid))); + } + + if ip == self.entry_addr.0 { + return Ok(Some(MultiThreadStopReason::HwBreak(tid))); + } + + // This is not a breakpoint we've set, likely one set by the guest + Ok(None) + } +} + +impl Target for FirecrackerTarget { + type Error = GdbTargetError; + type Arch = GdbArch; + + #[inline(always)] + fn base_ops(&mut self) -> BaseOps { + BaseOps::MultiThread(self) + } + + #[inline(always)] + fn support_breakpoints(&mut self) -> Option> { + Some(self) + } + + /// We disable implicit sw breakpoints as we want to manage these internally so we can inject + /// breakpoints back into the guest if we didn't create them + #[inline(always)] + fn guard_rail_implicit_sw_breakpoints(&self) -> bool { + false + } +} + +impl MultiThreadBase for FirecrackerTarget { + /// Reads the registers for the Vcpu + fn read_registers(&mut self, regs: &mut CoreRegs, tid: Tid) -> TargetResult<(), Self> { + arch::read_registers(&self.vcpu_state[tid_to_vcpuid(tid)].vcpu_fd, regs)?; + + Ok(()) + } + + /// Writes to the registers for the Vcpu + fn write_registers(&mut self, regs: &CoreRegs, tid: Tid) -> TargetResult<(), Self> { + arch::write_registers(&self.vcpu_state[tid_to_vcpuid(tid)].vcpu_fd, regs)?; + + Ok(()) + } + + /// Writes data to a guest virtual address for the Vcpu + fn read_addrs( + &mut self, + mut gva: ::Usize, + mut data: &mut [u8], + tid: Tid, + ) -> TargetResult { + let data_len = data.len(); + let vcpu_state = &self.vcpu_state[tid_to_vcpuid(tid)]; + + while !data.is_empty() { + let gpa = arch::translate_gva(&vcpu_state.vcpu_fd, gva).map_err(|e| { + error!("Error {e:?} translating gva on read address: {gva:X}"); + })?; + + // Compute the amount space left in the page after the gpa + let read_len = std::cmp::min( + data.len(), + PAGE_SIZE - (u64_to_usize(gpa) & (PAGE_SIZE - 1)), + ); + + let vmm = &self.vmm.lock().map_err(|_| { + error!("Error locking vmm in read addr"); + TargetError::Fatal(GdbTargetError::VmmLockError) + })?; + vmm.guest_memory() + .read(&mut data[..read_len], GuestAddress(gpa as u64)) + .map_err(|e| { + error!("Error reading memory {e:?}"); + })?; + + data = &mut data[read_len..]; + gva += read_len as u64; + } + + Ok(data_len) + } + + /// Writes data at a guest virtual address for the Vcpu + fn write_addrs( + &mut self, + mut gva: ::Usize, + mut data: &[u8], + tid: Tid, + ) -> TargetResult<(), Self> { + let vcpu_state = &self.vcpu_state[tid_to_vcpuid(tid)]; + while !data.is_empty() { + let gpa = arch::translate_gva(&vcpu_state.vcpu_fd, gva).map_err(|e| { + error!("Error {e:?} translating gva on read address: {gva:X}"); + })?; + + // Compute the amount space left in the page after the gpa + let write_len = std::cmp::min( + data.len(), + PAGE_SIZE - (u64_to_usize(gpa) & (PAGE_SIZE - 1)), + ); + + let vmm = &self.vmm.lock().map_err(|_| { + error!("Error locking vmm in write addr"); + TargetError::Fatal(GdbTargetError::VmmLockError) + })?; + + vmm.guest_memory() + .write(&data[..write_len], GuestAddress(gpa)) + .map_err(|e| { + error!("Error {e:?} writing memory at {gpa:X}"); + })?; + + data = &data[write_len..]; + gva += write_len as u64; + } + + Ok(()) + } + + #[inline(always)] + /// Makes the callback provided with each Vcpu + /// GDB expects us to return all threads currently running with this command, for firecracker + /// this is all Vcpus + fn list_active_threads( + &mut self, + thread_is_active: &mut dyn FnMut(Tid), + ) -> Result<(), Self::Error> { + for id in 0..self.vcpu_state.len() { + thread_is_active(vcpuid_to_tid(id)?) + } + + Ok(()) + } + + #[inline(always)] + fn support_resume(&mut self) -> Option> { + Some(self) + } + + #[inline(always)] + fn support_thread_extra_info(&mut self) -> Option> { + Some(self) + } +} + +impl MultiThreadResume for FirecrackerTarget { + /// Disables single step on the Vcpu + fn set_resume_action_continue( + &mut self, + tid: Tid, + _signal: Option, + ) -> Result<(), Self::Error> { + self.vcpu_state[tid_to_vcpuid(tid)].single_step = false; + + Ok(()) + } + + /// Resumes the execution of all currently paused Vcpus + fn resume(&mut self) -> Result<(), Self::Error> { + self.resume_all_vcpus() + } + + /// Clears the state of all Vcpus setting it back to base config + fn clear_resume_actions(&mut self) -> Result<(), Self::Error> { + self.reset_all_vcpu_states(); + + Ok(()) + } + + #[inline(always)] + fn support_single_step(&mut self) -> Option> { + Some(self) + } +} + +impl MultiThreadSingleStep for FirecrackerTarget { + /// Enabled single step on the Vcpu + fn set_resume_action_step( + &mut self, + tid: Tid, + _signal: Option, + ) -> Result<(), Self::Error> { + self.vcpu_state[tid_to_vcpuid(tid)].single_step = true; + + Ok(()) + } +} + +impl Breakpoints for FirecrackerTarget { + #[inline(always)] + fn support_hw_breakpoint(&mut self) -> Option> { + Some(self) + } + + #[inline(always)] + fn support_sw_breakpoint(&mut self) -> Option> { + Some(self) + } +} + +impl HwBreakpoint for FirecrackerTarget { + /// Adds a hardware breakpoint The breakpoint addresses are + /// stored in state so we can track the reason for an exit. + fn add_hw_breakpoint( + &mut self, + gva: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + let ga = GuestAddress(gva); + if self.hw_breakpoints.contains(&ga) { + return Ok(true); + } + + if self.hw_breakpoints.try_push(ga).is_err() { + return Ok(false); + } + + let state = self.get_paused_vcpu()?; + state.update_kvm_debug(&self.hw_breakpoints)?; + + Ok(true) + } + + /// Removes a hardware breakpoint. + fn remove_hw_breakpoint( + &mut self, + gva: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + match self.hw_breakpoints.iter().position(|&b| b.0 == gva) { + None => return Ok(false), + Some(pos) => self.hw_breakpoints.remove(pos), + }; + + let state = self.get_paused_vcpu()?; + state.update_kvm_debug(&self.hw_breakpoints)?; + + Ok(true) + } +} + +impl SwBreakpoint for FirecrackerTarget { + /// Inserts a software breakpoint. + /// We initially translate the guest virtual address to a guest physical address and then check + /// if this is already present, if so we return early. Otherwise we store the opcode at the + /// specified guest physical address in our store and replace it with the `X86_SW_BP_OP` + fn add_sw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + let gpa = arch::translate_gva(&self.get_paused_vcpu()?.vcpu_fd, addr)?; + + if self.sw_breakpoints.contains_key(&gpa) { + return Ok(true); + } + + let paused_vcpu_id = self.get_paused_vcpu_id()?; + + let mut saved_register = [0; arch::SW_BP_SIZE]; + self.read_addrs(addr, &mut saved_register, paused_vcpu_id)?; + self.sw_breakpoints.insert(gpa, saved_register); + + self.write_addrs(addr, &arch::SW_BP, paused_vcpu_id)?; + Ok(true) + } + + /// Removes a software breakpoint. + /// We firstly translate the guest virtual address to a guest physical address, we then check if + /// the resulting gpa is in our store, if so we load the stored opcode and write this back + fn remove_sw_breakpoint( + &mut self, + addr: ::Usize, + _kind: ::BreakpointKind, + ) -> TargetResult { + let gpa = arch::translate_gva(&self.get_paused_vcpu()?.vcpu_fd, addr)?; + + if let Some(removed) = self.sw_breakpoints.remove(&gpa) { + self.write_addrs(addr, &removed, self.get_paused_vcpu_id()?)?; + return Ok(true); + } + + Ok(false) + } +} + +impl ThreadExtraInfo for FirecrackerTarget { + /// Allows us to configure the formatting of the thread information, we just return the ID of + /// the Vcpu + fn thread_extra_info(&self, tid: Tid, buf: &mut [u8]) -> Result { + let info = format!("Vcpu ID: {}", tid_to_vcpuid(tid)); + let size = buf.len().min(info.len()); + + buf[..size].copy_from_slice(&info.as_bytes()[..size]); + Ok(size) + } +} diff --git a/src/vmm/src/lib.rs b/src/vmm/src/lib.rs index 94dfcfbf409..c80f004e789 100644 --- a/src/vmm/src/lib.rs +++ b/src/vmm/src/lib.rs @@ -83,6 +83,9 @@ pub(crate) mod device_manager; pub mod devices; /// minimalist HTTP/TCP/IPv4 stack named DUMBO pub mod dumbo; +/// Support for GDB debugging the guest +#[cfg(feature = "gdb")] +pub mod gdb; /// Logger pub mod logger; /// microVM Metadata Service MMDS @@ -846,6 +849,12 @@ impl Vmm { // Break the main event loop, propagating the Vmm exit-code. self.shutdown_exit_code = Some(exit_code); } + + /// Gets a reference to kvm-ioctls Vm + #[cfg(feature = "gdb")] + pub fn vm(&self) -> &Vm { + &self.vm + } } /// Process the content of the MPIDR_EL1 register in order to be able to pass it to KVM diff --git a/src/vmm/src/resources.rs b/src/vmm/src/resources.rs index a4d15641975..923225c6a8a 100644 --- a/src/vmm/src/resources.rs +++ b/src/vmm/src/resources.rs @@ -86,6 +86,9 @@ pub struct VmmConfig { vsock_device: Option, #[serde(rename = "entropy")] entropy_device: Option, + #[cfg(feature = "gdb")] + #[serde(rename = "gdb-socket")] + gdb_socket_addr: Option, } /// A data structure that encapsulates the device configurations @@ -114,6 +117,9 @@ pub struct VmResources { pub mmds_size_limit: usize, /// Whether or not to load boot timer device. pub boot_timer: bool, + #[cfg(feature = "gdb")] + /// Configures the location of the GDB socket + pub gdb_socket_addr: Option, } impl VmResources { @@ -136,6 +142,8 @@ impl VmResources { let mut resources: Self = Self { mmds_size_limit, + #[cfg(feature = "gdb")] + gdb_socket_addr: vmm_config.gdb_socket_addr, ..Default::default() }; if let Some(machine_config) = vmm_config.machine_config { @@ -521,6 +529,8 @@ impl From<&VmResources> for VmmConfig { net_devices: resources.net_builder.configs(), vsock_device: resources.vsock.config(), entropy_device: resources.entropy.config(), + #[cfg(feature = "gdb")] + gdb_socket_addr: resources.gdb_socket_addr.clone(), } } } @@ -630,6 +640,8 @@ mod tests { boot_timer: false, mmds_size_limit: HTTP_MAX_PAYLOAD_SIZE, entropy: Default::default(), + #[cfg(feature = "gdb")] + gdb_socket_addr: None, } } diff --git a/src/vmm/src/vstate/vcpu/mod.rs b/src/vmm/src/vstate/vcpu/mod.rs index 89b167bbd5b..43a0946931e 100644 --- a/src/vmm/src/vstate/vcpu/mod.rs +++ b/src/vmm/src/vstate/vcpu/mod.rs @@ -6,6 +6,8 @@ // found in the THIRD-PARTY file. use std::cell::Cell; +#[cfg(feature = "gdb")] +use std::os::fd::AsRawFd; use std::sync::atomic::{fence, Ordering}; use std::sync::mpsc::{channel, Receiver, Sender, TryRecvError}; use std::sync::{Arc, Barrier}; @@ -13,6 +15,8 @@ use std::{fmt, io, thread}; use kvm_bindings::{KVM_SYSTEM_EVENT_RESET, KVM_SYSTEM_EVENT_SHUTDOWN}; use kvm_ioctls::VcpuExit; +#[cfg(feature = "gdb")] +use kvm_ioctls::VcpuFd; use libc::{c_int, c_void, siginfo_t}; use log::{error, info, warn}; use seccompiler::{BpfProgram, BpfProgramRef}; @@ -20,6 +24,8 @@ use vmm_sys_util::errno; use vmm_sys_util::eventfd::EventFd; use crate::cpu_config::templates::{CpuConfiguration, GuestConfigError}; +#[cfg(feature = "gdb")] +use crate::gdb::target::{get_raw_tid, GdbTargetError}; use crate::logger::{IncMetric, METRICS}; use crate::utils::signal::{register_signal_handler, sigrtmin, Killable}; use crate::utils::sm::StateMachine; @@ -60,6 +66,9 @@ pub enum VcpuError { VcpuTlsInit, /// Vcpu not present in TLS VcpuTlsNotPresent, + /// Error with gdb request sent + #[cfg(feature = "gdb")] + GdbRequest(GdbTargetError), } /// Encapsulates configuration parameters for the guest vCPUS. @@ -81,6 +90,17 @@ type VcpuCell = Cell>; #[error("Failed to spawn vCPU thread: {0}")] pub struct StartThreadedError(std::io::Error); +/// Error type for [`Vcpu::copy_kvm_vcpu_fd`]. +#[cfg(feature = "gdb")] +#[derive(Debug, thiserror::Error)] +#[error("Failed to clone kvm Vcpu fd: {0}")] +pub enum CopyKvmFdError { + /// Error with libc dup of kvm Vcpu fd + DupError(#[from] std::io::Error), + /// Error creating the Vcpu from the duplicated Vcpu fd + CreateVcpuError(#[from] kvm_ioctls::Error), +} + /// A wrapper around creating and using a vcpu. #[derive(Debug)] pub struct Vcpu { @@ -89,6 +109,9 @@ pub struct Vcpu { /// File descriptor for vcpu to trigger exit event on vmm. exit_evt: EventFd, + /// Debugger emitter for gdb events + #[cfg(feature = "gdb")] + gdb_event: Option>, /// The receiving end of events channel owned by the vcpu side. event_receiver: Receiver, /// The transmitting end of the events channel which will be given to the handler. @@ -200,6 +223,8 @@ impl Vcpu { event_sender: Some(event_sender), response_receiver: Some(response_receiver), response_sender, + #[cfg(feature = "gdb")] + gdb_event: None, kvm_vcpu, }) } @@ -209,6 +234,24 @@ impl Vcpu { self.kvm_vcpu.peripherals.mmio_bus = Some(mmio_bus); } + /// Attaches the fields required for debugging + #[cfg(feature = "gdb")] + pub fn attach_debug_info(&mut self, gdb_event: Sender) { + self.gdb_event = Some(gdb_event); + } + + /// Obtains a copy of the VcpuFd + #[cfg(feature = "gdb")] + pub fn copy_kvm_vcpu_fd(&self, vm: &Vm) -> Result { + // SAFETY: We own this fd so it is considered safe to clone + let r = unsafe { libc::dup(self.kvm_vcpu.fd.as_raw_fd()) }; + if r < 0 { + return Err(std::io::Error::last_os_error().into()); + } + // SAFETY: We assert this is a valid fd by checking the result from the dup + unsafe { Ok(vm.fd().create_vcpu_from_rawfd(r)?) } + } + /// Moves the vcpu to its own thread and constructs a VcpuHandle. /// The handle can be used to control the remote vcpu. pub fn start_threaded( @@ -271,6 +314,11 @@ impl Vcpu { // - the other vCPUs won't ever exit out of `KVM_RUN`, but they won't consume CPU. // So we pause vCPU0 and send a signal to the emulation thread to stop the VMM. Ok(VcpuEmulation::Stopped) => return self.exit(FcExitCode::Ok), + // If the emulation requests a pause lets do this + #[cfg(feature = "gdb")] + Ok(VcpuEmulation::Paused) => { + return StateMachine::next(Self::paused); + } // Emulation errors lead to vCPU exit. Err(_) => return self.exit(FcExitCode::GenericError), } @@ -448,6 +496,16 @@ impl Vcpu { // Notify that this KVM_RUN was interrupted. Ok(VcpuEmulation::Interrupted) } + #[cfg(feature = "gdb")] + Ok(VcpuExit::Debug(_)) => { + if let Some(gdb_event) = &self.gdb_event { + gdb_event + .send(get_raw_tid(self.kvm_vcpu.index.into())) + .expect("Unable to notify gdb event"); + } + + Ok(VcpuEmulation::Paused) + } emulation_result => handle_kvm_exit(&mut self.kvm_vcpu.peripherals, emulation_result), } } @@ -688,6 +746,9 @@ pub enum VcpuEmulation { Interrupted, /// Stopped. Stopped, + /// Pause request + #[cfg(feature = "gdb")] + Paused, } #[cfg(test)] From 082db606b952e54dafdd35f0c9006b8d42fbbf8b Mon Sep 17 00:00:00 2001 From: Jack Thomson Date: Tue, 17 Sep 2024 13:58:27 +0000 Subject: [PATCH 2/3] docs: Add documentation on how to use GDB Add documentation on how to use gdb with firecracker with examples on how to use the basic functionality to debug the guest kernel Signed-off-by: Jack Thomson --- docs/gdb-debugging.md | 110 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 110 insertions(+) create mode 100644 docs/gdb-debugging.md diff --git a/docs/gdb-debugging.md b/docs/gdb-debugging.md new file mode 100644 index 00000000000..6e96983a309 --- /dev/null +++ b/docs/gdb-debugging.md @@ -0,0 +1,110 @@ +# GDB Debugging with Firecracker + +Firecracker supports debugging the guest kernel via GDB remote serial protocol. +This allows us to connect GDB to the firecracker process and step through debug +the guest kernel. Currently only debugging on x86 is supported. + +The GDB feature requires Firecracker to be booted with a config file. + +## Prerequisites + +Firstly, to enable GDB debugging we need to compile Firecracker with the `debug` +feature enabled, this will enable the necessary components for the debugging +process. + +To build firecracker with the `gdb` feature enabled we run: + +```bash +cargo build --features "gdb" +``` + +Secondly, we need to compile a kernel with specific features enabled for +debugging to work. The key config options to enable are: + +``` +CONFIG_FRAME_POINTER=y +CONFIG_KGDB=y +CONFIG_KGDB_SERIAL_CONSOLE=y +CONFIG_DEBUG_INFO=y +``` + +For GDB debugging the `gdb-socket` option should be set in your config file. In +this example we set it to `/tmp/gdb.socket` + +``` +{ + ... + "gdb-socket": "/tmp/gdb.socket" + ... +} +``` + +## Starting Firecracker with GDB + +With all the prerequisites in place you can now start firecracker ready to +connect to GDB. When you start the firecracker binary now you'll notice it'll be +blocked waiting for the GDB connection. This is done to allow us to set +breakpoints before the boot process begins. + +With Firecracker running and waiting for GDB we are now able to start GDB and +connect to Firecracker. You may need to set the permissions of your GDB socket +E.g. `/tmp/gdb.socket` to `0666` before connecting. + +An example of the steps taken to start GDB, load the symbols and connect to +Firecracker: + +1. Start the GDB process, you can attach the symbols by appending the kernel + blob, for example here `vmlinux` + + ```bash + gdb vmlinux + ``` + +1. When GDB has started set the target remote to `/tmp/gdb.socket` to connect to + Firecracker + + ```bash + (gdb) target remote /tmp/gdb.socket + ``` + +With these steps completed you'll now see GDB has stopped at the entry point +ready for us to start inserting breakpoints and debugging. + +## Notes + +### Software Breakpoints not working on start + +When at the initial paused state you'll notice software breakpoints won't work +and only hardware breakpoints will until memory virtualisation is enabled. To +circumvent this one solution is to set a hardware breakpoint at `start_kernel` +and continue. Once you've hit the `start_kernel` set the regular breakpoints as +you would do normally. E.g. + +```bash +> hbreak start_kernel +> c +``` + +### Pausing Firecracker while it's running + +While Firecracker is running you can pause vcpu 1 by pressing `Ctrl+C` which +will stop the vcpu and allow you to set breakpoints or inspect the current +location. + +### Halting execution of GDB and Firecracker + +To end the debugging session and shut down Firecracker you can run the `exit` +command in the GDB session which will terminate both. + +## Known limitations + +- The multi-core scheduler can in some cases cause issues with GDB, this can be + mitigated by setting these kernel config values: + + ``` + CONFIG_SCHED_MC=y + CONFIG_SCHED_MC_PRIO=y + ``` + +- Currently we support a limited subset of cpu registers for get and set + operations, if more are required feel free to contribute. From 2ab155f441205d0b690db77a5f997451b82ecbef Mon Sep 17 00:00:00 2001 From: Jack Thomson Date: Tue, 1 Oct 2024 15:46:09 +0000 Subject: [PATCH 3/3] tests: Add build test for GDB Adding a test to ensure that firecracker will build with the gdb flag enabled Signed-off-by: Jack Thomson --- tests/integration_tests/build/test_gdb.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) create mode 100644 tests/integration_tests/build/test_gdb.py diff --git a/tests/integration_tests/build/test_gdb.py b/tests/integration_tests/build/test_gdb.py new file mode 100644 index 00000000000..541807dd5c1 --- /dev/null +++ b/tests/integration_tests/build/test_gdb.py @@ -0,0 +1,19 @@ +# Copyright 2024 Amazon.com, Inc. or its affiliates. All Rights Reserved. +# SPDX-License-Identifier: Apache-2.0 +"""A test that ensures that firecracker builds with GDB feature enabled at integration time.""" + +import platform + +import pytest + +import host_tools.cargo_build as host + +MACHINE = platform.machine() +TARGET = "{}-unknown-linux-musl".format(MACHINE) + + +@pytest.mark.skipif(MACHINE != "x86_64", reason="GDB runs only on x86_64.") +def test_gdb_compiles(): + """Checks that Firecracker compiles with GDB enabled""" + + host.cargo("build", f"--features gdb --target {TARGET}")