diff --git a/Cargo.lock b/Cargo.lock index 83d35409f1ba..1b516a2bd10e 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2272,6 +2272,7 @@ dependencies = [ "cranelift-frontend", "cranelift-native", "cranelift-wasm", + "gimli", "log", "more-asserts", "region", diff --git a/cranelift/codegen/src/binemit/mod.rs b/cranelift/codegen/src/binemit/mod.rs index 33655a26bd44..4cec5f801d85 100644 --- a/cranelift/codegen/src/binemit/mod.rs +++ b/cranelift/codegen/src/binemit/mod.rs @@ -173,36 +173,6 @@ pub trait CodeSink { } } -/// Type of the frame unwind information. -#[derive(Debug, Copy, Clone, PartialEq, Eq)] -pub enum FrameUnwindKind { - /// Windows fastcall unwinding (as in .pdata). - Fastcall, - /// FDE entry for libunwind (similar to .eh_frame format). - Libunwind, -} - -/// Offset in frame unwind information buffer. -pub type FrameUnwindOffset = usize; - -/// Sink for frame unwind information. -pub trait FrameUnwindSink { - /// Get the current position. - fn len(&self) -> FrameUnwindOffset; - - /// Add bytes to the code section. - fn bytes(&mut self, _: &[u8]); - - /// Reserves bytes in the buffer. - fn reserve(&mut self, _len: usize) {} - - /// Add a relocation entry. - fn reloc(&mut self, _: Reloc, _: FrameUnwindOffset); - - /// Specified offset to main structure. - fn set_entry_offset(&mut self, _: FrameUnwindOffset); -} - /// Report a bad encoding error. #[cold] pub fn bad_encoding(func: &Function, inst: Inst) -> ! { diff --git a/cranelift/codegen/src/context.rs b/cranelift/codegen/src/context.rs index faf8c23d4e39..df7df0da174d 100644 --- a/cranelift/codegen/src/context.rs +++ b/cranelift/codegen/src/context.rs @@ -10,8 +10,8 @@ //! single ISA instance. use crate::binemit::{ - relax_branches, shrink_instructions, CodeInfo, FrameUnwindKind, FrameUnwindSink, - MemoryCodeSink, RelocSink, StackmapSink, TrapSink, + relax_branches, shrink_instructions, CodeInfo, MemoryCodeSink, RelocSink, StackmapSink, + TrapSink, }; use crate::dce::do_dce; use crate::dominator_tree::DominatorTree; @@ -231,19 +231,15 @@ impl Context { sink.info } - /// Emit unwind information. + /// Creates unwind information for the function. /// - /// Requires that the function layout be calculated (see `relax_branches`). - /// - /// Only some calling conventions (e.g. Windows fastcall) will have unwind information. - /// This is a no-op if the function has no unwind information. - pub fn emit_unwind_info( + /// Returns `None` if the function has no unwind information. + #[cfg(feature = "unwind")] + pub fn create_unwind_info( &self, isa: &dyn TargetIsa, - kind: FrameUnwindKind, - sink: &mut dyn FrameUnwindSink, - ) -> CodegenResult<()> { - isa.emit_unwind_info(&self.func, kind, sink) + ) -> CodegenResult> { + isa.create_unwind_info(&self.func) } /// Run the verifier on the function. diff --git a/cranelift/codegen/src/ir/framelayout.rs b/cranelift/codegen/src/ir/framelayout.rs deleted file mode 100644 index f4cb5bb909f3..000000000000 --- a/cranelift/codegen/src/ir/framelayout.rs +++ /dev/null @@ -1,74 +0,0 @@ -//! Frame layout item changes. - -use crate::ir::entities::Inst; -use crate::isa::RegUnit; -use std::boxed::Box; - -use crate::HashMap; - -#[cfg(feature = "enable-serde")] -use serde::{Deserialize, Serialize}; - -/// Change in the frame layout information. -#[derive(Clone, Copy, Debug, Eq, PartialEq, Hash)] -#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] -pub enum FrameLayoutChange { - /// Base CallFrameAddress (CFA) pointer moved to different register/offset. - CallFrameAddressAt { - /// CFA register. - reg: RegUnit, - /// CFA offset. - offset: isize, - }, - /// Register saved at. - RegAt { - /// Saved register. - reg: RegUnit, - /// Offset in the frame (offset from CFA). - cfa_offset: isize, - }, - /// Return address saved at. - ReturnAddressAt { - /// Offset in the frame (offset from CFA). - cfa_offset: isize, - }, - /// The entire frame layout must be preserved somewhere to be restored at a corresponding - /// `Restore` change. - /// - /// This likely maps to the DWARF call frame instruction `.cfa_remember_state`. - Preserve, - /// Restore the entire frame layout from a corresponding prior `Preserve` frame change. - /// - /// This likely maps to the DWARF call frame instruction `.cfa_restore_state`. - Restore, -} - -/// Set of frame layout changes. -pub type FrameLayoutChanges = Box<[FrameLayoutChange]>; - -/// Frame items layout for (prologue/epilogue) instructions. -#[derive(Debug, Clone)] -pub struct FrameLayout { - /// Initial frame layout. - pub initial: FrameLayoutChanges, - - /// Instruction frame layout (changes). Because the map will not be dense, - /// a HashMap is used instead of a SecondaryMap. - pub instructions: HashMap, -} - -impl FrameLayout { - /// Create instance of FrameLayout. - pub fn new() -> Self { - Self { - initial: vec![].into_boxed_slice(), - instructions: HashMap::new(), - } - } - - /// Clear the structure. - pub fn clear(&mut self) { - self.initial = vec![].into_boxed_slice(); - self.instructions.clear(); - } -} diff --git a/cranelift/codegen/src/ir/function.rs b/cranelift/codegen/src/ir/function.rs index 4a3829780bb0..db989818f34a 100644 --- a/cranelift/codegen/src/ir/function.rs +++ b/cranelift/codegen/src/ir/function.rs @@ -10,13 +10,14 @@ use crate::ir::{ Block, ExtFuncData, FuncRef, GlobalValue, GlobalValueData, Heap, HeapData, Inst, JumpTable, JumpTableData, Opcode, SigRef, StackSlot, StackSlotData, Table, TableData, }; -use crate::ir::{BlockOffsets, FrameLayout, InstEncodings, SourceLocs, StackSlots, ValueLocations}; +use crate::ir::{BlockOffsets, InstEncodings, SourceLocs, StackSlots, ValueLocations}; use crate::ir::{DataFlowGraph, ExternalName, Layout, Signature}; use crate::ir::{JumpTableOffsets, JumpTables}; use crate::isa::{CallConv, EncInfo, Encoding, Legalize, TargetIsa}; use crate::regalloc::{EntryRegDiversions, RegDiversions}; use crate::value_label::ValueLabelsRanges; use crate::write::write_function; +use alloc::vec::Vec; use core::fmt; /// A function. @@ -87,15 +88,13 @@ pub struct Function { /// Instruction that marks the end (inclusive) of the function's prologue. /// - /// This is used for some calling conventions to track the end of unwind information. + /// This is used for some ABIs to generate unwind information. pub prologue_end: Option, - /// Frame layout for the instructions. + /// The instructions that mark the start (inclusive) of an epilogue in the function. /// - /// The stack unwinding requires to have information about which registers and where they - /// are saved in the frame. This information is created during the prologue and epilogue - /// passes. - pub frame_layout: Option, + /// This is used for some ABIs to generate unwind information. + pub epilogues_start: Vec, } impl Function { @@ -119,7 +118,7 @@ impl Function { jt_offsets: SecondaryMap::new(), srclocs: SecondaryMap::new(), prologue_end: None, - frame_layout: None, + epilogues_start: Vec::new(), } } @@ -140,7 +139,7 @@ impl Function { self.jt_offsets.clear(); self.srclocs.clear(); self.prologue_end = None; - self.frame_layout = None; + self.epilogues_start.clear(); } /// Create a new empty, anonymous function with a Fast calling convention. @@ -258,12 +257,6 @@ impl Function { /// Starts collection of debug information. pub fn collect_debug_info(&mut self) { self.dfg.collect_debug_info(); - self.collect_frame_layout_info(); - } - - /// Starts collection of frame layout information. - pub fn collect_frame_layout_info(&mut self) { - self.frame_layout = Some(FrameLayout::new()); } /// Changes the destination of a jump or branch instruction. diff --git a/cranelift/codegen/src/ir/mod.rs b/cranelift/codegen/src/ir/mod.rs index 3c222ca9f525..7f3c36b7be0f 100644 --- a/cranelift/codegen/src/ir/mod.rs +++ b/cranelift/codegen/src/ir/mod.rs @@ -6,7 +6,6 @@ pub mod dfg; pub mod entities; mod extfunc; mod extname; -mod framelayout; pub mod function; mod globalvalue; mod heap; @@ -40,7 +39,6 @@ pub use crate::ir::extfunc::{ AbiParam, ArgumentExtension, ArgumentPurpose, ExtFuncData, Signature, }; pub use crate::ir::extname::ExternalName; -pub use crate::ir::framelayout::{FrameLayout, FrameLayoutChange, FrameLayoutChanges}; pub use crate::ir::function::{DisplayFunctionAnnotations, Function}; pub use crate::ir::globalvalue::GlobalValueData; pub use crate::ir::heap::{HeapData, HeapStyle}; diff --git a/cranelift/codegen/src/isa/fde.rs b/cranelift/codegen/src/isa/fde.rs deleted file mode 100644 index acb7a01d43a6..000000000000 --- a/cranelift/codegen/src/isa/fde.rs +++ /dev/null @@ -1,14 +0,0 @@ -//! Support for FDE data generation. -use thiserror::Error; - -/// Enumerate the errors possible in mapping Cranelift registers to their DWARF equivalent. -#[allow(missing_docs)] -#[derive(Error, Debug)] -pub enum RegisterMappingError { - #[error("unable to find bank for register info")] - MissingBank, - #[error("register mapping is currently only implemented for x86_64")] - UnsupportedArchitecture, - #[error("unsupported register bank: {0}")] - UnsupportedRegisterBank(&'static str), -} diff --git a/cranelift/codegen/src/isa/mod.rs b/cranelift/codegen/src/isa/mod.rs index 71c540dd91c5..d72b46981fc6 100644 --- a/cranelift/codegen/src/isa/mod.rs +++ b/cranelift/codegen/src/isa/mod.rs @@ -56,8 +56,8 @@ pub use crate::isa::stack::{StackBase, StackBaseMask, StackRef}; use crate::binemit; use crate::flowgraph; use crate::ir; -use crate::isa::fde::RegisterMappingError; #[cfg(feature = "unwind")] +use crate::isa::unwind::systemv::RegisterMappingError; use crate::machinst::MachBackend; use crate::regalloc; use crate::result::CodegenResult; @@ -77,15 +77,15 @@ mod riscv; #[cfg(feature = "x86")] mod x86; -#[cfg(feature = "unwind")] -pub mod fde; - #[cfg(feature = "arm32")] mod arm32; #[cfg(feature = "arm64")] mod aarch64; +#[cfg(feature = "unwind")] +pub mod unwind; + mod call_conv; mod constraints; mod enc_tables; @@ -394,17 +394,25 @@ pub trait TargetIsa: fmt::Display + Send + Sync { /// IntCC condition for Unsigned Subtraction Overflow (Borrow/Carry). fn unsigned_sub_overflow_condition(&self) -> ir::condcodes::IntCC; - /// Emit unwind information for the given function. + /// Creates unwind information for the function. /// - /// Only some calling conventions (e.g. Windows fastcall) will have unwind information. - fn emit_unwind_info( + /// Returns `None` if there is no unwind information for the function. + #[cfg(feature = "unwind")] + fn create_unwind_info( &self, _func: &ir::Function, - _kind: binemit::FrameUnwindKind, - _sink: &mut dyn binemit::FrameUnwindSink, - ) -> CodegenResult<()> { - // No-op by default - Ok(()) + ) -> CodegenResult> { + // By default, an ISA has no unwind information + Ok(None) + } + + /// Creates a new System V Common Information Entry for the ISA. + /// + /// Returns `None` if the ISA does not support System V unwind information. + #[cfg(feature = "unwind")] + fn create_systemv_cie(&self) -> Option { + // By default, an ISA cannot create a System V CIE + None } /// Get the new-style MachBackend, if this is an adapter around one. diff --git a/cranelift/codegen/src/isa/unwind.rs b/cranelift/codegen/src/isa/unwind.rs new file mode 100644 index 000000000000..b594720a5d00 --- /dev/null +++ b/cranelift/codegen/src/isa/unwind.rs @@ -0,0 +1,16 @@ +//! Represents information relating to function unwinding. +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +pub mod systemv; +pub mod winx64; + +/// Represents unwind information for a single function. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub enum UnwindInfo { + /// Windows x64 ABI unwind information. + WindowsX64(winx64::UnwindInfo), + /// System V ABI unwind information. + SystemV(systemv::UnwindInfo), +} diff --git a/cranelift/codegen/src/isa/unwind/systemv.rs b/cranelift/codegen/src/isa/unwind/systemv.rs new file mode 100644 index 000000000000..5985082f7195 --- /dev/null +++ b/cranelift/codegen/src/isa/unwind/systemv.rs @@ -0,0 +1,129 @@ +//! System V ABI unwind information. + +use alloc::vec::Vec; +use gimli::write::{Address, FrameDescriptionEntry}; +use thiserror::Error; + +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +type Register = u16; +type Expression = Vec; + +/// Enumerate the errors possible in mapping Cranelift registers to their DWARF equivalent. +#[allow(missing_docs)] +#[derive(Error, Debug, PartialEq, Eq)] +pub enum RegisterMappingError { + #[error("unable to find bank for register info")] + MissingBank, + #[error("register mapping is currently only implemented for x86_64")] + UnsupportedArchitecture, + #[error("unsupported register bank: {0}")] + UnsupportedRegisterBank(&'static str), +} + +// This mirrors gimli's CallFrameInstruction, but is serializable +// TODO: if gimli ever adds serialization support, remove this type +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub(crate) enum CallFrameInstruction { + Cfa(Register, i32), + CfaRegister(Register), + CfaOffset(i32), + CfaExpression(Expression), + Restore(Register), + Undefined(Register), + SameValue(Register), + Offset(Register, i32), + ValOffset(Register, i32), + Register(Register, Register), + Expression(Register, Expression), + ValExpression(Register, Expression), + RememberState, + RestoreState, + ArgsSize(u32), +} + +impl From for CallFrameInstruction { + fn from(cfi: gimli::write::CallFrameInstruction) -> Self { + use gimli::write::CallFrameInstruction; + + match cfi { + CallFrameInstruction::Cfa(reg, offset) => Self::Cfa(reg.0, offset), + CallFrameInstruction::CfaRegister(reg) => Self::CfaRegister(reg.0), + CallFrameInstruction::CfaOffset(offset) => Self::CfaOffset(offset), + CallFrameInstruction::CfaExpression(expr) => Self::CfaExpression(expr.0), + CallFrameInstruction::Restore(reg) => Self::Restore(reg.0), + CallFrameInstruction::Undefined(reg) => Self::Undefined(reg.0), + CallFrameInstruction::SameValue(reg) => Self::SameValue(reg.0), + CallFrameInstruction::Offset(reg, offset) => Self::Offset(reg.0, offset), + CallFrameInstruction::ValOffset(reg, offset) => Self::ValOffset(reg.0, offset), + CallFrameInstruction::Register(reg1, reg2) => Self::Register(reg1.0, reg2.0), + CallFrameInstruction::Expression(reg, expr) => Self::Expression(reg.0, expr.0), + CallFrameInstruction::ValExpression(reg, expr) => Self::ValExpression(reg.0, expr.0), + CallFrameInstruction::RememberState => Self::RememberState, + CallFrameInstruction::RestoreState => Self::RestoreState, + CallFrameInstruction::ArgsSize(size) => Self::ArgsSize(size), + } + } +} + +impl Into for CallFrameInstruction { + fn into(self) -> gimli::write::CallFrameInstruction { + use gimli::{ + write::{CallFrameInstruction, Expression}, + Register, + }; + + match self { + Self::Cfa(reg, offset) => CallFrameInstruction::Cfa(Register(reg), offset), + Self::CfaRegister(reg) => CallFrameInstruction::CfaRegister(Register(reg)), + Self::CfaOffset(offset) => CallFrameInstruction::CfaOffset(offset), + Self::CfaExpression(expr) => CallFrameInstruction::CfaExpression(Expression(expr)), + Self::Restore(reg) => CallFrameInstruction::Restore(Register(reg)), + Self::Undefined(reg) => CallFrameInstruction::Undefined(Register(reg)), + Self::SameValue(reg) => CallFrameInstruction::SameValue(Register(reg)), + Self::Offset(reg, offset) => CallFrameInstruction::Offset(Register(reg), offset), + Self::ValOffset(reg, offset) => CallFrameInstruction::ValOffset(Register(reg), offset), + Self::Register(reg1, reg2) => { + CallFrameInstruction::Register(Register(reg1), Register(reg2)) + } + Self::Expression(reg, expr) => { + CallFrameInstruction::Expression(Register(reg), Expression(expr)) + } + Self::ValExpression(reg, expr) => { + CallFrameInstruction::ValExpression(Register(reg), Expression(expr)) + } + Self::RememberState => CallFrameInstruction::RememberState, + Self::RestoreState => CallFrameInstruction::RestoreState, + Self::ArgsSize(size) => CallFrameInstruction::ArgsSize(size), + } + } +} + +/// Represents unwind information for a single System V ABI function. +/// +/// This representation is not ISA specific. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UnwindInfo { + instructions: Vec<(u32, CallFrameInstruction)>, + len: u32, +} + +impl UnwindInfo { + pub(crate) fn new(instructions: Vec<(u32, CallFrameInstruction)>, len: u32) -> Self { + Self { instructions, len } + } + + /// Converts the unwind information into a `FrameDescriptionEntry`. + pub fn to_fde(&self, address: Address) -> gimli::write::FrameDescriptionEntry { + let mut fde = FrameDescriptionEntry::new(address, self.len); + + for (offset, inst) in &self.instructions { + fde.add_instruction(*offset, inst.clone().into()); + } + + fde + } +} diff --git a/cranelift/codegen/src/isa/unwind/winx64.rs b/cranelift/codegen/src/isa/unwind/winx64.rs new file mode 100644 index 000000000000..9219d293e2bc --- /dev/null +++ b/cranelift/codegen/src/isa/unwind/winx64.rs @@ -0,0 +1,216 @@ +//! System V ABI unwind information. + +use alloc::vec::Vec; +use byteorder::{ByteOrder, LittleEndian}; +#[cfg(feature = "enable-serde")] +use serde::{Deserialize, Serialize}; + +/// Maximum (inclusive) size of a "small" stack allocation +const SMALL_ALLOC_MAX_SIZE: u32 = 128; +/// Maximum (inclusive) size of a "large" stack allocation that can represented in 16-bits +const LARGE_ALLOC_16BIT_MAX_SIZE: u32 = 524280; + +struct Writer<'a> { + buf: &'a mut [u8], + offset: usize, +} + +impl<'a> Writer<'a> { + pub fn new(buf: &'a mut [u8]) -> Self { + Self { buf, offset: 0 } + } + + fn write_u8(&mut self, v: u8) { + self.buf[self.offset] = v; + self.offset += 1; + } + + fn write_u16(&mut self, v: u16) { + T::write_u16(&mut self.buf[self.offset..(self.offset + 2)], v); + self.offset += 2; + } + + fn write_u32(&mut self, v: u32) { + T::write_u32(&mut self.buf[self.offset..(self.offset + 4)], v); + self.offset += 4; + } +} + +/// The supported unwind codes for the x64 Windows ABI. +/// +/// See: https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 +/// Only what is needed to describe the prologues generated by the Cranelift x86 ISA are represented here. +/// Note: the Cranelift x86 ISA RU enum matches the Windows unwind GPR encoding values. +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub(crate) enum UnwindCode { + PushRegister { + offset: u8, + reg: u8, + }, + SaveXmm { + offset: u8, + reg: u8, + stack_offset: u32, + }, + StackAlloc { + offset: u8, + size: u32, + }, + SetFramePointer { + offset: u8, + sp_offset: u8, + }, +} + +impl UnwindCode { + fn emit(&self, writer: &mut Writer) { + enum UnwindOperation { + PushNonvolatileRegister = 0, + LargeStackAlloc = 1, + SmallStackAlloc = 2, + SetFramePointer = 3, + SaveXmm128 = 8, + SaveXmm128Far = 9, + } + + match self { + Self::PushRegister { offset, reg } => { + writer.write_u8(*offset); + writer.write_u8((*reg << 4) | (UnwindOperation::PushNonvolatileRegister as u8)); + } + Self::SaveXmm { + offset, + reg, + stack_offset, + } => { + writer.write_u8(*offset); + let stack_offset = stack_offset / 16; + if stack_offset <= core::u16::MAX as u32 { + writer.write_u8((*reg << 4) | (UnwindOperation::SaveXmm128 as u8)); + writer.write_u16::(stack_offset as u16); + } else { + writer.write_u8((*reg << 4) | (UnwindOperation::SaveXmm128Far as u8)); + writer.write_u16::(stack_offset as u16); + writer.write_u16::((stack_offset >> 16) as u16); + } + } + Self::StackAlloc { offset, size } => { + // Stack allocations on Windows must be a multiple of 8 and be at least 1 slot + assert!(*size >= 8); + assert!((*size % 8) == 0); + + writer.write_u8(*offset); + if *size <= SMALL_ALLOC_MAX_SIZE { + writer.write_u8( + ((((*size - 8) / 8) as u8) << 4) | UnwindOperation::SmallStackAlloc as u8, + ); + } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { + writer.write_u8(UnwindOperation::LargeStackAlloc as u8); + writer.write_u16::((*size / 8) as u16); + } else { + writer.write_u8((1 << 4) | (UnwindOperation::LargeStackAlloc as u8)); + writer.write_u32::(*size); + } + } + Self::SetFramePointer { offset, sp_offset } => { + writer.write_u8(*offset); + writer.write_u8((*sp_offset << 4) | (UnwindOperation::SetFramePointer as u8)); + } + }; + } + + fn node_count(&self) -> usize { + match self { + Self::StackAlloc { size, .. } => { + if *size <= SMALL_ALLOC_MAX_SIZE { + 1 + } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { + 2 + } else { + 3 + } + } + Self::SaveXmm { stack_offset, .. } => { + if *stack_offset <= core::u16::MAX as u32 { + 2 + } else { + 3 + } + } + _ => 1, + } + } +} + +/// Represents Windows x64 unwind information. +/// +/// For information about Windows x64 unwind info, see: +/// https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 +#[derive(Clone, Debug, PartialEq, Eq)] +#[cfg_attr(feature = "enable-serde", derive(Serialize, Deserialize))] +pub struct UnwindInfo { + pub(crate) flags: u8, + pub(crate) prologue_size: u8, + pub(crate) frame_register: Option, + pub(crate) frame_register_offset: u8, + pub(crate) unwind_codes: Vec, +} + +impl UnwindInfo { + /// Gets the emit size of the unwind information, in bytes. + pub fn emit_size(&self) -> usize { + let node_count = self.node_count(); + + // Calculation of the size requires no SEH handler or chained info + assert!(self.flags == 0); + + // Size of fixed part of UNWIND_INFO is 4 bytes + // Then comes the UNWIND_CODE nodes (2 bytes each) + // Then comes 2 bytes of padding for the unwind codes if necessary + // Next would come the SEH data, but we assert above that the function doesn't have SEH data + + 4 + (node_count * 2) + if (node_count & 1) == 1 { 2 } else { 0 } + } + + /// Emits the unwind information into the given mutable byte slice. + /// + /// This function will panic if the slice is not at least `emit_size` in length. + pub fn emit(&self, buf: &mut [u8]) { + const UNWIND_INFO_VERSION: u8 = 1; + + let node_count = self.node_count(); + assert!(node_count <= 256); + + let mut writer = Writer::new(buf); + + writer.write_u8((self.flags << 3) | UNWIND_INFO_VERSION); + writer.write_u8(self.prologue_size); + writer.write_u8(node_count as u8); + + if let Some(reg) = self.frame_register { + writer.write_u8((self.frame_register_offset << 4) | reg); + } else { + writer.write_u8(0); + } + + // Unwind codes are written in reverse order (prologue offset descending) + for code in self.unwind_codes.iter().rev() { + code.emit(&mut writer); + } + + // To keep a 32-bit alignment, emit 2 bytes of padding if there's an odd number of 16-bit nodes + if (node_count & 1) == 1 { + writer.write_u16::(0); + } + + // Ensure the correct number of bytes was emitted + assert_eq!(writer.offset, self.emit_size()); + } + + fn node_count(&self) -> usize { + self.unwind_codes + .iter() + .fold(0, |nodes, c| nodes + c.node_count()) + } +} diff --git a/cranelift/codegen/src/isa/x86/abi.rs b/cranelift/codegen/src/isa/x86/abi.rs index 71fdddd39378..cfa698698d39 100644 --- a/cranelift/codegen/src/isa/x86/abi.rs +++ b/cranelift/codegen/src/isa/x86/abi.rs @@ -1,15 +1,9 @@ //! x86 ABI implementation. use super::super::settings as shared_settings; -#[cfg(feature = "unwind")] -use super::fde::emit_fde; use super::registers::{FPR, GPR, RU}; use super::settings as isa_settings; -#[cfg(feature = "unwind")] -use super::unwind::UnwindInfo; use crate::abi::{legalize_args, ArgAction, ArgAssigner, ValueConversion}; -#[cfg(feature = "unwind")] -use crate::binemit::{FrameUnwindKind, FrameUnwindSink}; use crate::cursor::{Cursor, CursorPosition, EncCursor}; use crate::ir; use crate::ir::entities::StackSlot; @@ -17,8 +11,8 @@ use crate::ir::immediates::Imm64; use crate::ir::stackslot::{StackOffset, StackSize}; use crate::ir::types; use crate::ir::{ - get_probestack_funcref, AbiParam, ArgumentExtension, ArgumentLoc, ArgumentPurpose, - FrameLayoutChange, InstBuilder, ValueLoc, + get_probestack_funcref, AbiParam, ArgumentExtension, ArgumentLoc, ArgumentPurpose, InstBuilder, + ValueLoc, }; use crate::isa::{CallConv, RegClass, RegUnit, TargetIsa}; use crate::regalloc::RegisterSet; @@ -27,7 +21,6 @@ use crate::stack_layout::layout_stack; use alloc::borrow::Cow; use alloc::vec::Vec; use core::i32; -use std::boxed::Box; use target_lexicon::{PointerWidth, Triple}; /// Argument registers for x86-64 @@ -525,32 +518,6 @@ fn baldrdash_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> Ok(()) } -/// CFAState is cranelift's model of the call frame layout at any particular point in a function. -/// It describes the call frame's layout in terms of a call frame address, where it is with respect -/// to the start of the call frame, and the where the top of the stack is with respect to it. -/// -/// Changes in this layout are used to derive appropriate `ir::FrameLayoutChange` to record for -/// relevant instructions. -#[derive(Clone)] -struct CFAState { - /// The register from which we can derive the call frame address. On x86_64, this is typically - /// `rbp`, but at function entry and exit may be `rsp` while the call frame is being - /// established. - cf_ptr_reg: RegUnit, - /// Given that `cf_ptr_reg` is a register containing a pointer to some memory, `cf_ptr_offset` - /// is the offset from that pointer to the address of the start of this function's call frame. - /// - /// For a concrete x86_64 example, we will start this at 8 - the call frame begins immediately - /// before the return address. This will typically then be set to 16, after pushing `rbp` to - /// preserve the parent call frame. It is very unlikely the offset should be anything other - /// than one or two pointer widths. - cf_ptr_offset: isize, - /// The offset between the start of the call frame and the current stack pointer. This is - /// primarily useful to point to where on the stack preserved registers are, but is maintained - /// through the whole function for consistency. - current_depth: isize, -} - /// Implementation of the fastcall-based Win64 calling convention described at [1] /// [1] https://docs.microsoft.com/en-us/cpp/build/x64-calling-convention fn fastcall_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> CodegenResult<()> { @@ -629,7 +596,7 @@ fn fastcall_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> C // Set up the cursor and insert the prologue let entry_block = func.layout.entry_block().expect("missing entry block"); let mut pos = EncCursor::new(func, isa).at_first_insertion_point(entry_block); - let prologue_cfa_state = insert_common_prologue( + insert_common_prologue( &mut pos, local_stack_size, reg_type, @@ -646,8 +613,6 @@ fn fastcall_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> C reg_type, &csrs, fpr_slot.as_ref(), - isa, - prologue_cfa_state, ); Ok(()) @@ -701,20 +666,11 @@ fn system_v_prologue_epilogue(func: &mut ir::Function, isa: &dyn TargetIsa) -> C // Set up the cursor and insert the prologue let entry_block = func.layout.entry_block().expect("missing entry block"); let mut pos = EncCursor::new(func, isa).at_first_insertion_point(entry_block); - let prologue_cfa_state = - insert_common_prologue(&mut pos, local_stack_size, reg_type, &csrs, None, isa); + insert_common_prologue(&mut pos, local_stack_size, reg_type, &csrs, None, isa); // Reset the cursor and insert the epilogue let mut pos = pos.at_position(CursorPosition::Nowhere); - insert_common_epilogues( - &mut pos, - local_stack_size, - reg_type, - &csrs, - None, - isa, - prologue_cfa_state, - ); + insert_common_epilogues(&mut pos, local_stack_size, reg_type, &csrs, None); Ok(()) } @@ -728,8 +684,7 @@ fn insert_common_prologue( csrs: &RegisterSet, fpr_slot: Option<&StackSlot>, isa: &dyn TargetIsa, -) -> Option { - let word_size = isa.pointer_bytes() as isize; +) { if stack_size > 0 { // Check if there is a special stack limit parameter. If so insert stack check. if let Some(stack_limit_arg) = pos.func.special_param(ArgumentPurpose::StackLimit) { @@ -739,7 +694,8 @@ fn insert_common_prologue( // also should be accounted for. // If any FPR are present, count them as well as necessary alignment space. // TODO: Check if the function body actually contains a `call` instruction. - let mut total_stack_size = (csrs.iter(GPR).len() + 1 + 1) as i64 * word_size as i64; + let mut total_stack_size = + (csrs.iter(GPR).len() + 1 + 1) as i64 * (isa.pointer_bytes() as isize) as i64; total_stack_size += csrs.iter(FPR).len() as i64 * types::F64X2.bytes() as i64; @@ -747,104 +703,29 @@ fn insert_common_prologue( } } - let mut cfa_state = if let Some(ref mut frame_layout) = pos.func.frame_layout { - let cfa_state = CFAState { - cf_ptr_reg: RU::rsp as RegUnit, - cf_ptr_offset: word_size, - current_depth: -word_size, - }; - - frame_layout.initial = vec![ - FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }, - FrameLayoutChange::ReturnAddressAt { - cfa_offset: cfa_state.current_depth, - }, - ] - .into_boxed_slice(); - - Some(cfa_state) - } else { - None - }; - // Append param to entry block let block = pos.current_block().expect("missing block under cursor"); let fp = pos.func.dfg.append_block_param(block, reg_type); pos.func.locations[fp] = ir::ValueLoc::Reg(RU::rbp as RegUnit); - let push_fp_inst = pos.ins().x86_push(fp); - - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - cfa_state.current_depth -= word_size; - cfa_state.cf_ptr_offset += word_size; - frame_layout.instructions.insert( - push_fp_inst, - vec![ - FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }, - FrameLayoutChange::RegAt { - reg: RU::rbp as RegUnit, - cfa_offset: cfa_state.current_depth, - }, - ] - .into_boxed_slice(), - ); - } + pos.ins().x86_push(fp); let mov_sp_inst = pos .ins() .copy_special(RU::rsp as RegUnit, RU::rbp as RegUnit); - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let mut cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - cfa_state.cf_ptr_reg = RU::rbp as RegUnit; - frame_layout.instructions.insert( - mov_sp_inst, - vec![FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }] - .into_boxed_slice(), - ); - } - + let mut last_csr_push = None; for reg in csrs.iter(GPR) { // Append param to entry block let csr_arg = pos.func.dfg.append_block_param(block, reg_type); // Assign it a location pos.func.locations[csr_arg] = ir::ValueLoc::Reg(reg); - - // Remember it so we can push it momentarily - let reg_push_inst = pos.ins().x86_push(csr_arg); - - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let mut cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - cfa_state.current_depth -= word_size; - frame_layout.instructions.insert( - reg_push_inst, - vec![FrameLayoutChange::RegAt { - reg, - cfa_offset: cfa_state.current_depth, - }] - .into_boxed_slice(), - ); - } + last_csr_push = Some(pos.ins().x86_push(csr_arg)); } // Allocate stack frame storage. + let mut adjust_sp_inst = None; if stack_size > 0 { if isa.flags().enable_probestack() && stack_size > (1 << isa.flags().probestack_size_log2()) { @@ -880,15 +761,16 @@ fn insert_common_prologue( if !isa.flags().probestack_func_adjusts_sp() { let result = pos.func.dfg.inst_results(call)[0]; pos.func.locations[result] = rax_val; - pos.func.prologue_end = Some(pos.ins().adjust_sp_down(result)); + adjust_sp_inst = Some(pos.ins().adjust_sp_down(result)); } } else { // Simply decrement the stack pointer. - pos.func.prologue_end = Some(pos.ins().adjust_sp_down_imm(Imm64::new(stack_size))); + adjust_sp_inst = Some(pos.ins().adjust_sp_down_imm(Imm64::new(stack_size))); } } // Now that RSP is prepared for the function, we can use stack slots: + let mut last_fpr_save = None; if let Some(fpr_slot) = fpr_slot { debug_assert!(csrs.iter(FPR).len() != 0); @@ -911,33 +793,22 @@ fn insert_common_prologue( // Since regalloc has already run, we must assign a location. pos.func.locations[csr_arg] = ir::ValueLoc::Reg(reg); - let reg_store_inst = - pos.ins() - .store(ir::MemFlags::trusted(), csr_arg, stack_addr, fpr_offset); + last_fpr_save = + Some( + pos.ins() + .store(ir::MemFlags::trusted(), csr_arg, stack_addr, fpr_offset), + ); - // If we preserve FPRs, they occur after SP is adjusted, so also fix up the end point - // to this new instruction. - pos.func.prologue_end = Some(reg_store_inst); fpr_offset += types::F64X2.bytes() as i32; - - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let mut cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - cfa_state.current_depth -= types::F64X2.bytes() as isize; - frame_layout.instructions.insert( - reg_store_inst, - vec![FrameLayoutChange::RegAt { - reg, - cfa_offset: cfa_state.current_depth, - }] - .into_boxed_slice(), - ); - } } } - cfa_state + pos.func.prologue_end = Some( + last_fpr_save + .or(adjust_sp_inst) + .or(last_csr_push) + .unwrap_or(mov_sp_inst), + ); } /// Insert a check that generates a trap if the stack pointer goes @@ -970,25 +841,12 @@ fn insert_common_epilogues( reg_type: ir::types::Type, csrs: &RegisterSet, fpr_slot: Option<&StackSlot>, - isa: &dyn TargetIsa, - cfa_state: Option, ) { while let Some(block) = pos.next_block() { pos.goto_last_inst(block); if let Some(inst) = pos.current_inst() { if pos.func.dfg[inst].opcode().is_return() { - let is_last = pos.func.layout.last_block() == Some(block); - insert_common_epilogue( - inst, - stack_size, - pos, - reg_type, - csrs, - fpr_slot, - isa, - is_last, - cfa_state.clone(), - ); + insert_common_epilogue(inst, stack_size, pos, reg_type, csrs, fpr_slot); } } } @@ -1003,17 +861,13 @@ fn insert_common_epilogue( reg_type: ir::types::Type, csrs: &RegisterSet, fpr_slot: Option<&StackSlot>, - isa: &dyn TargetIsa, - is_last: bool, - mut cfa_state: Option, ) { - let word_size = isa.pointer_bytes() as isize; - // Even though instructions to restore FPRs are inserted first, we have to append them after // restored GPRs to satisfy parameter order in the return. let mut restored_fpr_values = Vec::new(); // Restore FPRs before we move RSP and invalidate stack slots. + let mut first_fpr_load = None; if let Some(fpr_slot) = fpr_slot { debug_assert!(csrs.iter(FPR).len() != 0); @@ -1024,6 +878,8 @@ fn insert_common_epilogue( // See also: https://github.com/bytecodealliance/wasmtime/pull/1198 let stack_addr = pos.ins().stack_addr(types::I64, *fpr_slot, 0); + first_fpr_load.get_or_insert(pos.current_inst().expect("current inst")); + // Use r11 as fastcall allows it to be clobbered, and it won't have a meaningful value at // function exit. pos.func.locations[stack_addr] = ir::ValueLoc::Reg(RU::r11 as u16); @@ -1039,13 +895,6 @@ fn insert_common_epilogue( ); fpr_offset += types::F64X2.bytes() as i32; - if let Some(ref mut cfa_state) = cfa_state.as_mut() { - // Note: don't bother recording a frame layout change because the popped value is - // still correct in memory, and won't be overwritten until we've returned where the - // current frame's layout would no longer matter. Only adjust `current_depth` for a - // consistency check later. - cfa_state.current_depth += types::F64X2.bytes() as isize; - } // Unlike GPRs before, we don't need to step back after reach restoration because FPR // restoration is order-insensitive. Furthermore: we want GPR restoration to begin // after FPR restoration, so that stack adjustments occur after we're done relying on @@ -1056,114 +905,56 @@ fn insert_common_epilogue( } } + let mut sp_adjust_inst = None; if stack_size > 0 { - pos.ins().adjust_sp_up_imm(Imm64::new(stack_size)); - } - - // Pop all the callee-saved registers, stepping backward each time to - // preserve the correct order. - let fp_ret = pos.ins().x86_pop(reg_type); - let fp_pop_inst = pos.built_inst(); - - if let Some(ref mut cfa_state) = cfa_state.as_mut() { - // Account for CFA state in the reverse of `insert_common_prologue`. - cfa_state.current_depth += word_size; - cfa_state.cf_ptr_offset -= word_size; - // And now that we're going to overwrite `rbp`, `rsp` is the only way to get to the call frame. - // We don't apply a frame layout change *yet* because we check that at return the depth is - // exactly one `word_size`. - cfa_state.cf_ptr_reg = RU::rsp as RegUnit; + sp_adjust_inst = Some(pos.ins().adjust_sp_up_imm(Imm64::new(stack_size))); } - pos.prev_inst(); - - pos.func.locations[fp_ret] = ir::ValueLoc::Reg(RU::rbp as RegUnit); - pos.func.dfg.append_inst_arg(inst, fp_ret); + // Insert the pop of the frame pointer + let fp_pop = pos.ins().x86_pop(reg_type); + let fp_pop_inst = pos.prev_inst().unwrap(); + pos.func.locations[fp_pop] = ir::ValueLoc::Reg(RU::rbp as RegUnit); + pos.func.dfg.append_inst_arg(inst, fp_pop); + // Insert the CSR pops + let mut first_csr_pop_inst = None; for reg in csrs.iter(GPR) { - let csr_ret = pos.ins().x86_pop(reg_type); - if let Some(ref mut cfa_state) = cfa_state.as_mut() { - // Note: don't bother recording a frame layout change because the popped value is - // still correct in memory, and won't be overwritten until we've returned where the - // current frame's layout would no longer matter. Only adjust `current_depth` for a - // consistency check later. - cfa_state.current_depth += word_size; - } - pos.prev_inst(); - - pos.func.locations[csr_ret] = ir::ValueLoc::Reg(reg); - pos.func.dfg.append_inst_arg(inst, csr_ret); + let csr_pop = pos.ins().x86_pop(reg_type); + first_csr_pop_inst = Some(pos.prev_inst().unwrap()); + pos.func.locations[csr_pop] = ir::ValueLoc::Reg(reg); + pos.func.dfg.append_inst_arg(inst, csr_pop); } for value in restored_fpr_values.into_iter() { pos.func.dfg.append_inst_arg(inst, value); } - if let Some(ref mut frame_layout) = pos.func.frame_layout { - let cfa_state = cfa_state - .as_mut() - .expect("cfa state exists when recording frame layout"); - // Validity checks - if we accounted correctly, CFA state at a return will match CFA state - // at the entry of a function. - // - // Current_depth starts assuming a return address is pushed, and cf_ptr_offset is one - // pointer below current_depth. - assert_eq!(cfa_state.current_depth, -word_size); - assert_eq!(cfa_state.cf_ptr_offset, word_size); - - // Inserting preserve CFA state operation after FP pop instructions. - let new_cfa = FrameLayoutChange::CallFrameAddressAt { - reg: cfa_state.cf_ptr_reg, - offset: cfa_state.cf_ptr_offset, - }; - let new_cfa = if is_last { - vec![new_cfa] - } else { - vec![FrameLayoutChange::Preserve, new_cfa] - }; - - frame_layout - .instructions - .entry(fp_pop_inst) - .and_modify(|insts| { - *insts = insts - .iter() - .cloned() - .chain(new_cfa.clone().into_iter()) - .collect::>(); - }) - .or_insert_with(|| new_cfa.into_boxed_slice()); - - if !is_last { - // Inserting restore CFA state operation after each return. - frame_layout - .instructions - .insert(inst, vec![FrameLayoutChange::Restore].into_boxed_slice()); - } - } + pos.func.epilogues_start.push( + first_fpr_load + .or(sp_adjust_inst) + .or(first_csr_pop_inst) + .unwrap_or(fp_pop_inst), + ); } #[cfg(feature = "unwind")] -pub fn emit_unwind_info( +pub fn create_unwind_info( func: &ir::Function, isa: &dyn TargetIsa, - kind: FrameUnwindKind, - sink: &mut dyn FrameUnwindSink, -) -> CodegenResult<()> { - match kind { - FrameUnwindKind::Fastcall => { - // Assumption: RBP is being used as the frame pointer - // In the future, Windows fastcall codegen should usually omit the frame pointer - if let Some(info) = UnwindInfo::try_from_func(func, isa, Some(RU::rbp.into()))? { - info.emit(sink); - } +) -> CodegenResult> { + use crate::isa::unwind::UnwindInfo; + + // Assumption: RBP is being used as the frame pointer for both calling conventions + // In the future, we should be omitting frame pointer as an optimization, so this will change + Ok(match func.signature.call_conv { + CallConv::Fast | CallConv::Cold | CallConv::SystemV => { + super::unwind::systemv::create_unwind_info(func, isa, Some(RU::rbp.into()))? + .map(|u| UnwindInfo::SystemV(u)) } - FrameUnwindKind::Libunwind => { - if func.frame_layout.is_some() { - emit_fde(func, isa, sink)?; - } + CallConv::WindowsFastcall => { + super::unwind::winx64::create_unwind_info(func, isa, Some(RU::rbp.into()))? + .map(|u| UnwindInfo::WindowsX64(u)) } - } - - Ok(()) + _ => None, + }) } diff --git a/cranelift/codegen/src/isa/x86/fde.rs b/cranelift/codegen/src/isa/x86/fde.rs deleted file mode 100644 index 85ed5b5f2a75..000000000000 --- a/cranelift/codegen/src/isa/x86/fde.rs +++ /dev/null @@ -1,448 +0,0 @@ -//! Support for FDE data generation. - -use crate::binemit::{FrameUnwindOffset, FrameUnwindSink, Reloc}; -use crate::ir::{FrameLayoutChange, Function}; -use crate::isa::fde::RegisterMappingError; -use crate::isa::{CallConv, RegUnit, TargetIsa}; -use crate::result::CodegenResult; -use alloc::vec::Vec; -use core::convert::TryInto; -use gimli::write::{ - Address, CallFrameInstruction, CommonInformationEntry, EhFrame, EndianVec, - FrameDescriptionEntry, FrameTable, Result, Writer, -}; -use gimli::{Encoding, Format, LittleEndian, Register, X86_64}; - -pub type FDERelocEntry = (FrameUnwindOffset, Reloc); - -const FUNCTION_ENTRY_ADDRESS: Address = Address::Symbol { - symbol: 0, - addend: 0, -}; - -#[derive(Clone)] -struct FDEWriter { - vec: EndianVec, - relocs: Vec, -} - -impl FDEWriter { - fn new() -> Self { - Self { - vec: EndianVec::new(LittleEndian), - relocs: Vec::new(), - } - } - fn into_vec_and_relocs(self) -> (Vec, Vec) { - (self.vec.into_vec(), self.relocs) - } -} - -impl Writer for FDEWriter { - type Endian = LittleEndian; - fn endian(&self) -> Self::Endian { - LittleEndian - } - fn len(&self) -> usize { - self.vec.len() - } - fn write(&mut self, bytes: &[u8]) -> Result<()> { - self.vec.write(bytes) - } - fn write_at(&mut self, offset: usize, bytes: &[u8]) -> Result<()> { - self.vec.write_at(offset, bytes) - } - fn write_address(&mut self, address: Address, size: u8) -> Result<()> { - match address { - Address::Constant(_) => self.vec.write_address(address, size), - Address::Symbol { .. } => { - assert_eq!(address, FUNCTION_ENTRY_ADDRESS); - let rt = match size { - 4 => Reloc::Abs4, - 8 => Reloc::Abs8, - _ => { - panic!("Unexpected address size at FDEWriter::write_address"); - } - }; - self.relocs.push((self.vec.len().try_into().unwrap(), rt)); - self.vec.write_udata(0, size) - } - } - } -} - -fn return_address_reg(isa: &dyn TargetIsa) -> Register { - assert!(isa.name() == "x86" && isa.pointer_bits() == 64); - X86_64::RA -} - -/// Map Cranelift registers to their corresponding Gimli registers. -pub fn map_reg( - isa: &dyn TargetIsa, - reg: RegUnit, -) -> core::result::Result { - if isa.name() != "x86" || isa.pointer_bits() != 64 { - return Err(RegisterMappingError::UnsupportedArchitecture); - } - - // Mapping from https://github.com/bytecodealliance/cranelift/pull/902 by @iximeow - const X86_GP_REG_MAP: [gimli::Register; 16] = [ - X86_64::RAX, - X86_64::RCX, - X86_64::RDX, - X86_64::RBX, - X86_64::RSP, - X86_64::RBP, - X86_64::RSI, - X86_64::RDI, - X86_64::R8, - X86_64::R9, - X86_64::R10, - X86_64::R11, - X86_64::R12, - X86_64::R13, - X86_64::R14, - X86_64::R15, - ]; - const X86_XMM_REG_MAP: [gimli::Register; 16] = [ - X86_64::XMM0, - X86_64::XMM1, - X86_64::XMM2, - X86_64::XMM3, - X86_64::XMM4, - X86_64::XMM5, - X86_64::XMM6, - X86_64::XMM7, - X86_64::XMM8, - X86_64::XMM9, - X86_64::XMM10, - X86_64::XMM11, - X86_64::XMM12, - X86_64::XMM13, - X86_64::XMM14, - X86_64::XMM15, - ]; - - let reg_info = isa.register_info(); - let bank = reg_info - .bank_containing_regunit(reg) - .ok_or_else(|| RegisterMappingError::MissingBank)?; - match bank.name { - "IntRegs" => { - // x86 GP registers have a weird mapping to DWARF registers, so we use a - // lookup table. - Ok(X86_GP_REG_MAP[(reg - bank.first_unit) as usize]) - } - "FloatRegs" => Ok(X86_XMM_REG_MAP[(reg - bank.first_unit) as usize]), - _ => Err(RegisterMappingError::UnsupportedRegisterBank(bank.name)), - } -} - -fn to_cfi( - isa: &dyn TargetIsa, - change: &FrameLayoutChange, - cfa_def_reg: &mut Register, - cfa_def_offset: &mut i32, -) -> Option { - Some(match change { - FrameLayoutChange::CallFrameAddressAt { reg, offset } => { - let mapped = map_reg(isa, *reg).expect("a register mapping from cranelift to gimli"); - let offset = (*offset) as i32; - if mapped != *cfa_def_reg && offset != *cfa_def_offset { - *cfa_def_reg = mapped; - *cfa_def_offset = offset; - CallFrameInstruction::Cfa(mapped, offset) - } else if offset != *cfa_def_offset { - *cfa_def_offset = offset; - CallFrameInstruction::CfaOffset(offset) - } else if mapped != *cfa_def_reg { - *cfa_def_reg = mapped; - CallFrameInstruction::CfaRegister(mapped) - } else { - return None; - } - } - FrameLayoutChange::RegAt { reg, cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - let mapped = map_reg(isa, *reg).expect("a register mapping from cranelift to gimli"); - CallFrameInstruction::Offset(mapped, cfa_offset) - } - FrameLayoutChange::ReturnAddressAt { cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - CallFrameInstruction::Offset(X86_64::RA, cfa_offset) - } - FrameLayoutChange::Preserve => CallFrameInstruction::RememberState, - FrameLayoutChange::Restore => CallFrameInstruction::RestoreState, - }) -} - -/// Creates FDE structure from FrameLayout. -pub fn emit_fde( - func: &Function, - isa: &dyn TargetIsa, - sink: &mut dyn FrameUnwindSink, -) -> CodegenResult<()> { - assert!(isa.name() == "x86"); - - // Expecting function with System V prologue - assert!( - func.signature.call_conv == CallConv::Fast - || func.signature.call_conv == CallConv::Cold - || func.signature.call_conv == CallConv::SystemV - ); - - assert!(func.frame_layout.is_some(), "expected func.frame_layout"); - let frame_layout = func.frame_layout.as_ref().unwrap(); - - let mut blocks = func.layout.blocks().collect::>(); - blocks.sort_by_key(|block| func.offsets[*block]); // Ensure inst offsets always increase - - let encinfo = isa.encoding_info(); - let mut last_offset = 0; - let mut changes = Vec::new(); - for block in blocks { - for (offset, inst, size) in func.inst_offsets(block, &encinfo) { - let address_offset = (offset + size) as usize; - assert!(last_offset <= address_offset); - if let Some(cmds) = frame_layout.instructions.get(&inst) { - for cmd in cmds.iter() { - changes.push((address_offset, *cmd)); - } - } - last_offset = address_offset; - } - } - - let len = last_offset as u32; - - let word_size = isa.pointer_bytes() as i32; - - let encoding = Encoding { - format: Format::Dwarf32, - version: 1, - address_size: word_size as u8, - }; - let mut frames = FrameTable::default(); - - let mut cfa_def_reg = return_address_reg(isa); - let mut cfa_def_offset = 0i32; - - let mut cie = CommonInformationEntry::new( - encoding, - /* code_alignment_factor = */ 1, - /* data_alignment_factor = */ -word_size as i8, - return_address_reg(isa), - ); - for ch in frame_layout.initial.iter() { - if let Some(cfi) = to_cfi(isa, ch, &mut cfa_def_reg, &mut cfa_def_offset) { - cie.add_instruction(cfi); - } - } - - let cie_id = frames.add_cie(cie); - - let mut fde = FrameDescriptionEntry::new(FUNCTION_ENTRY_ADDRESS, len); - - for (addr, ch) in changes.iter() { - if let Some(cfi) = to_cfi(isa, ch, &mut cfa_def_reg, &mut cfa_def_offset) { - fde.add_instruction((*addr) as u32, cfi); - } - } - - frames.add_fde(cie_id, fde); - - let mut eh_frame = EhFrame::from(FDEWriter::new()); - frames.write_eh_frame(&mut eh_frame).unwrap(); - - let (bytes, relocs) = eh_frame.clone().into_vec_and_relocs(); - - let unwind_start = sink.len(); - sink.bytes(&bytes); - - for (off, r) in relocs { - sink.reloc(r, off + unwind_start); - } - - let cie_len = u32::from_le_bytes(bytes.as_slice()[..4].try_into().unwrap()); - let fde_offset = cie_len as usize + 4; - sink.set_entry_offset(unwind_start + fde_offset); - - // Need 0 marker for GCC unwind to end FDE "list". - sink.bytes(&[0, 0, 0, 0]); - - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::binemit::{FrameUnwindOffset, Reloc}; - use crate::cursor::{Cursor, FuncCursor}; - use crate::ir::{ - types, AbiParam, ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind, - TrapCode, - }; - use crate::isa::{lookup, CallConv}; - use crate::settings::{builder, Flags}; - use crate::Context; - use std::str::FromStr; - use target_lexicon::triple; - - struct SimpleUnwindSink(pub Vec, pub usize, pub Vec<(Reloc, usize)>); - impl FrameUnwindSink for SimpleUnwindSink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reloc(&mut self, r: Reloc, off: FrameUnwindOffset) { - self.2.push((r, off)); - } - fn set_entry_offset(&mut self, off: FrameUnwindOffset) { - self.1 = off; - } - } - - #[test] - fn test_simple_func() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::SystemV, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), - )); - context.func.collect_frame_layout_info(); - - context.compile(&*isa).expect("expected compilation"); - - let mut sink = SimpleUnwindSink(Vec::new(), 0, Vec::new()); - emit_fde(&context.func, &*isa, &mut sink).expect("can emit fde"); - - assert_eq!( - sink.0, - vec![ - 20, 0, 0, 0, // CIE len - 0, 0, 0, 0, // CIE marker - 1, // version - 0, // augmentation string - 1, // code aligment = 1 - 120, // data alignment = -8 - 16, // RA = r16 - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0x90, 0x01, // DW_CFA_offset r16, -8 * 1 - 0, 0, 0, 0, 0, 0, // padding - 36, 0, 0, 0, // FDE len - 28, 0, 0, 0, // CIE offset - 0, 0, 0, 0, 0, 0, 0, 0, // addr reloc - 16, 0, 0, 0, 0, 0, 0, 0, // function length - 0x42, // DW_CFA_advance_loc 2 - 0x0e, 0x10, // DW_CFA_def_cfa_offset 16 - 0x86, 0x02, // DW_CFA_offset r6, -8 * 2 - 0x43, // DW_CFA_advance_loc 3 - 0x0d, 0x06, // DW_CFA_def_cfa_register - 0x4a, // DW_CFA_advance_loc 10 - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0, 0, 0, 0, // padding - 0, 0, 0, 0, // End of FDEs - ] - ); - assert_eq!(sink.1, 24); - assert_eq!(sink.2.len(), 1); - } - - fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); - - let block0 = func.dfg.make_block(); - let mut pos = FuncCursor::new(&mut func); - pos.insert_block(block0); - pos.ins().return_(&[]); - - if let Some(stack_slot) = stack_slot { - func.stack_slots.push(stack_slot); - } - - func - } - - #[test] - fn test_multi_return_func() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); - context.func.collect_frame_layout_info(); - - context.compile(&*isa).expect("expected compilation"); - - let mut sink = SimpleUnwindSink(Vec::new(), 0, Vec::new()); - emit_fde(&context.func, &*isa, &mut sink).expect("can emit fde"); - - assert_eq!( - sink.0, - vec![ - 20, 0, 0, 0, // CIE len - 0, 0, 0, 0, // CIE marker - 1, // version - 0, // augmentation string - 1, // code aligment = 1 - 120, // data alignment = -8 - 16, // RA = r16 - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0x90, 0x01, // DW_CFA_offset r16, -8 * 1 - 0, 0, 0, 0, 0, 0, // padding - 36, 0, 0, 0, // FDE len - 28, 0, 0, 0, // CIE offset - 0, 0, 0, 0, 0, 0, 0, 0, // addr reloc - 15, 0, 0, 0, 0, 0, 0, 0, // function length - 0x42, // DW_CFA_advance_loc 2 - 0x0e, 0x10, // DW_CFA_def_cfa_offset 16 - 0x86, 0x02, // DW_CFA_offset r6, -8 * 2 - 0x43, // DW_CFA_advance_loc 3 - 0x0d, 0x06, // DW_CFA_def_cfa_register - 0x47, // DW_CFA_advance_loc 10 - 0x0a, // DW_CFA_preserve_state - 0x0c, 0x07, 0x08, // DW_CFA_def_cfa r7, 8 - 0x41, // DW_CFA_advance_loc 1 - 0x0b, // DW_CFA_restore_state - // NOTE: no additional CFA directives -- DW_CFA_restore_state - // is done before trap and it is last instruction in the function. - 0, // padding - 0, 0, 0, 0, // End of FDEs - ] - ); - assert_eq!(sink.1, 24); - assert_eq!(sink.2.len(), 1); - } - - fn create_multi_return_function(call_conv: CallConv) -> Function { - let mut sig = Signature::new(call_conv); - sig.params.push(AbiParam::new(types::I32)); - let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); - - let block0 = func.dfg.make_block(); - let v0 = func.dfg.append_block_param(block0, types::I32); - let block1 = func.dfg.make_block(); - let block2 = func.dfg.make_block(); - - let mut pos = FuncCursor::new(&mut func); - pos.insert_block(block0); - pos.ins().brnz(v0, block2, &[]); - pos.ins().jump(block1, &[]); - - pos.insert_block(block1); - pos.ins().return_(&[]); - - pos.insert_block(block2); - pos.ins().trap(TrapCode::User(0)); - - func - } -} diff --git a/cranelift/codegen/src/isa/x86/mod.rs b/cranelift/codegen/src/isa/x86/mod.rs index 881f12bdb1b4..03e167e07d02 100644 --- a/cranelift/codegen/src/isa/x86/mod.rs +++ b/cranelift/codegen/src/isa/x86/mod.rs @@ -3,27 +3,20 @@ mod abi; mod binemit; mod enc_tables; -#[cfg(feature = "unwind")] -mod fde; mod registers; pub mod settings; #[cfg(feature = "unwind")] -mod unwind; - -#[cfg(feature = "unwind")] -pub use fde::map_reg; +pub mod unwind; use super::super::settings as shared_settings; #[cfg(feature = "testing_hooks")] use crate::binemit::CodeSink; use crate::binemit::{emit_function, MemoryCodeSink}; -#[cfg(feature = "unwind")] -use crate::binemit::{FrameUnwindKind, FrameUnwindSink}; use crate::ir; use crate::isa::enc_tables::{self as shared_enc_tables, lookup_enclist, Encodings}; use crate::isa::Builder as IsaBuilder; #[cfg(feature = "unwind")] -use crate::isa::{fde::RegisterMappingError, RegUnit}; +use crate::isa::{unwind::systemv::RegisterMappingError, RegUnit}; use crate::isa::{EncInfo, RegClass, RegInfo, TargetIsa}; use crate::regalloc; use crate::result::CodegenResult; @@ -95,7 +88,7 @@ impl TargetIsa for Isa { #[cfg(feature = "unwind")] fn map_dwarf_register(&self, reg: RegUnit) -> Result { - map_reg(self, reg).map(|r| r.0) + unwind::systemv::map_reg(self, reg).map(|r| r.0) } fn encoding_info(&self) -> EncInfo { @@ -168,17 +161,17 @@ impl TargetIsa for Isa { ir::condcodes::IntCC::UnsignedLessThan } - /// Emit unwind information for the given function. - /// - /// Only some calling conventions (e.g. Windows fastcall) will have unwind information. #[cfg(feature = "unwind")] - fn emit_unwind_info( + fn create_unwind_info( &self, func: &ir::Function, - kind: FrameUnwindKind, - sink: &mut dyn FrameUnwindSink, - ) -> CodegenResult<()> { - abi::emit_unwind_info(func, self, kind, sink) + ) -> CodegenResult> { + abi::create_unwind_info(func, self) + } + + #[cfg(feature = "unwind")] + fn create_systemv_cie(&self) -> Option { + Some(unwind::systemv::create_cie()) } } diff --git a/cranelift/codegen/src/isa/x86/unwind.rs b/cranelift/codegen/src/isa/x86/unwind.rs index 707b9e6d2c2b..0f8a8ef92794 100644 --- a/cranelift/codegen/src/isa/x86/unwind.rs +++ b/cranelift/codegen/src/isa/x86/unwind.rs @@ -1,697 +1,4 @@ -//! Unwind information for x64 Windows. +//! Module for x86 unwind generation for supported ABIs. -use super::registers::{FPR, GPR, RU}; -use crate::binemit::FrameUnwindSink; -use crate::ir::{Function, InstructionData, Opcode, ValueLoc}; -use crate::isa::{CallConv, RegUnit, TargetIsa}; -use crate::result::{CodegenError, CodegenResult}; -use alloc::vec::Vec; -use byteorder::{ByteOrder, LittleEndian}; -use log::warn; - -/// Maximum (inclusive) size of a "small" stack allocation -const SMALL_ALLOC_MAX_SIZE: u32 = 128; -/// Maximum (inclusive) size of a "large" stack allocation that can represented in 16-bits -const LARGE_ALLOC_16BIT_MAX_SIZE: u32 = 524280; - -fn write_u8(sink: &mut dyn FrameUnwindSink, v: u8) { - sink.bytes(&[v]); -} - -fn write_u16(sink: &mut dyn FrameUnwindSink, v: u16) { - let mut buf = [0; 2]; - T::write_u16(&mut buf, v); - sink.bytes(&buf); -} - -fn write_u32(sink: &mut dyn FrameUnwindSink, v: u32) { - let mut buf = [0; 4]; - T::write_u32(&mut buf, v); - sink.bytes(&buf); -} - -/// The supported unwind codes for the x64 Windows ABI. -/// -/// See: https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 -/// Only what is needed to describe the prologues generated by the Cranelift x86 ISA are represented here. -/// Note: the Cranelift x86 ISA RU enum matches the Windows unwind GPR encoding values. -#[derive(Debug, PartialEq, Eq)] -enum UnwindCode { - PushRegister { - offset: u8, - reg: RegUnit, - }, - SaveXmm { - offset: u8, - reg: RegUnit, - stack_offset: u32, - }, - StackAlloc { - offset: u8, - size: u32, - }, - SetFramePointer { - offset: u8, - sp_offset: u8, - }, -} - -impl UnwindCode { - fn emit(&self, sink: &mut dyn FrameUnwindSink) { - enum UnwindOperation { - PushNonvolatileRegister = 0, - LargeStackAlloc = 1, - SmallStackAlloc = 2, - SetFramePointer = 3, - SaveXmm128 = 8, - SaveXmm128Far = 9, - } - - match self { - Self::PushRegister { offset, reg } => { - write_u8(sink, *offset); - write_u8( - sink, - ((GPR.index_of(*reg) as u8) << 4) - | (UnwindOperation::PushNonvolatileRegister as u8), - ); - } - Self::SaveXmm { - offset, - reg, - stack_offset, - } => { - write_u8(sink, *offset); - let stack_offset = stack_offset / 16; - if stack_offset <= core::u16::MAX as u32 { - write_u8( - sink, - (FPR.index_of(*reg) << 4) as u8 | (UnwindOperation::SaveXmm128 as u8), - ); - write_u16::(sink, stack_offset as u16); - } else { - write_u8( - sink, - (FPR.index_of(*reg) << 4) as u8 | (UnwindOperation::SaveXmm128Far as u8), - ); - write_u16::(sink, stack_offset as u16); - write_u16::(sink, (stack_offset >> 16) as u16); - } - } - Self::StackAlloc { offset, size } => { - // Stack allocations on Windows must be a multiple of 8 and be at least 1 slot - assert!(*size >= 8); - assert!((*size % 8) == 0); - - write_u8(sink, *offset); - if *size <= SMALL_ALLOC_MAX_SIZE { - write_u8( - sink, - ((((*size - 8) / 8) as u8) << 4) | UnwindOperation::SmallStackAlloc as u8, - ); - } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { - write_u8(sink, UnwindOperation::LargeStackAlloc as u8); - write_u16::(sink, (*size / 8) as u16); - } else { - write_u8(sink, (1 << 4) | (UnwindOperation::LargeStackAlloc as u8)); - write_u32::(sink, *size); - } - } - Self::SetFramePointer { offset, sp_offset } => { - write_u8(sink, *offset); - write_u8( - sink, - (*sp_offset << 4) | (UnwindOperation::SetFramePointer as u8), - ); - } - }; - } - - fn node_count(&self) -> usize { - match self { - Self::StackAlloc { size, .. } => { - if *size <= SMALL_ALLOC_MAX_SIZE { - 1 - } else if *size <= LARGE_ALLOC_16BIT_MAX_SIZE { - 2 - } else { - 3 - } - } - Self::SaveXmm { stack_offset, .. } => { - if *stack_offset <= core::u16::MAX as u32 { - 2 - } else { - 3 - } - } - _ => 1, - } - } -} - -/// Represents Windows x64 unwind information. -/// -/// For information about Windows x64 unwind info, see: -/// https://docs.microsoft.com/en-us/cpp/build/exception-handling-x64 -#[derive(Debug, PartialEq, Eq)] -pub struct UnwindInfo { - flags: u8, - prologue_size: u8, - frame_register: Option, - frame_register_offset: u8, - unwind_codes: Vec, -} - -impl UnwindInfo { - pub fn try_from_func( - func: &Function, - isa: &dyn TargetIsa, - frame_register: Option, - ) -> CodegenResult> { - // Only Windows fastcall is supported for unwind information - if func.signature.call_conv != CallConv::WindowsFastcall || func.prologue_end.is_none() { - return Ok(None); - } - - let prologue_end = func.prologue_end.unwrap(); - let entry_block = func.layout.blocks().nth(0).expect("missing entry block"); - - // Stores the stack size when SP is not adjusted via an immediate value - let mut stack_size = None; - let mut prologue_size = 0; - let mut unwind_codes = Vec::new(); - let mut found_end = false; - - // Have we saved at least one FPR? if so, we might have to check additional constraints. - let mut saved_fpr = false; - - // In addition to the min offset for a callee-save, we need to know the offset from the - // frame base to the stack pointer, so that we can record an unwind offset that spans only - // to the end of callee-save space. - let mut static_frame_allocation_size = 0u32; - - // For the time being, FPR preservation is split into a stack_addr and later store/load. - // Store the register used for stack store and ensure it is the same register with no - // intervening changes to the frame size. - let mut callee_save_region_reg = None; - // Also record the callee-save region's offset from RSP, because it must be added to FPR - // save offsets to compute an offset from the frame base. - let mut callee_save_offset = None; - - for (offset, inst, size) in func.inst_offsets(entry_block, &isa.encoding_info()) { - // x64 ABI prologues cannot exceed 255 bytes in length - if (offset + size) > 255 { - warn!("function prologues cannot exceed 255 bytes in size for Windows x64"); - return Err(CodegenError::CodeTooLarge); - } - - prologue_size += size; - - let unwind_offset = (offset + size) as u8; - - match func.dfg[inst] { - InstructionData::Unary { opcode, arg } => { - match opcode { - Opcode::X86Push => { - static_frame_allocation_size += 8; - - unwind_codes.push(UnwindCode::PushRegister { - offset: unwind_offset, - reg: func.locations[arg].unwrap_reg(), - }); - } - Opcode::AdjustSpDown => { - let stack_size = - stack_size.expect("expected a previous stack size instruction"); - static_frame_allocation_size += stack_size; - - // This is used when calling a stack check function - // We need to track the assignment to RAX which has the size of the stack - unwind_codes.push(UnwindCode::StackAlloc { - offset: unwind_offset, - size: stack_size, - }); - } - _ => {} - } - } - InstructionData::CopySpecial { src, dst, .. } => { - if let Some(frame_register) = frame_register { - if src == (RU::rsp as RegUnit) && dst == frame_register { - // Constructing an rbp-based stack frame, so the static frame - // allocation restarts at 0 from here. - static_frame_allocation_size = 0; - - unwind_codes.push(UnwindCode::SetFramePointer { - offset: unwind_offset, - sp_offset: 0, - }); - } - } - } - InstructionData::UnaryImm { opcode, imm } => { - match opcode { - Opcode::Iconst => { - let imm: i64 = imm.into(); - assert!(imm <= core::u32::MAX as i64); - assert!(stack_size.is_none()); - - // This instruction should only appear in a prologue to pass an - // argument of the stack size to a stack check function. - // Record the stack size so we know what it is when we encounter the adjustment - // instruction (which will adjust via the register assigned to this instruction). - stack_size = Some(imm as u32); - } - Opcode::AdjustSpDownImm => { - let imm: i64 = imm.into(); - assert!(imm <= core::u32::MAX as i64); - - static_frame_allocation_size += imm as u32; - - unwind_codes.push(UnwindCode::StackAlloc { - offset: unwind_offset, - size: imm as u32, - }); - } - _ => {} - } - } - InstructionData::StackLoad { - opcode: Opcode::StackAddr, - stack_slot, - offset: _, - } => { - let result = func.dfg.inst_results(inst).get(0).unwrap(); - if let ValueLoc::Reg(frame_reg) = func.locations[*result] { - callee_save_region_reg = Some(frame_reg); - - // Figure out the offset in the call frame that `frame_reg` will have. - let frame_size = func - .stack_slots - .layout_info - .expect("func's stack slots have layout info if stack operations exist") - .frame_size; - // Because we're well after the prologue has been constructed, stack slots - // must have been laid out... - let slot_offset = func.stack_slots[stack_slot] - .offset - .expect("callee-save slot has an offset computed"); - let frame_offset = frame_size as i32 + slot_offset; - - callee_save_offset = Some(frame_offset as u32); - } - } - InstructionData::Store { - opcode: Opcode::Store, - args: [arg1, arg2], - flags: _flags, - offset, - } => { - if let (ValueLoc::Reg(ru), ValueLoc::Reg(base_ru)) = - (func.locations[arg1], func.locations[arg2]) - { - if Some(base_ru) == callee_save_region_reg { - let offset_int: i32 = offset.into(); - assert!(offset_int >= 0, "negative fpr offset would store outside the stack frame, and is almost certainly an error"); - let offset_int: u32 = offset_int as u32 + callee_save_offset.expect("FPR presevation requires an FPR save region, which has some stack offset"); - if FPR.contains(ru) { - saved_fpr = true; - unwind_codes.push(UnwindCode::SaveXmm { - offset: unwind_offset, - reg: ru, - stack_offset: offset_int, - }); - } - } - } - } - _ => {} - }; - - if inst == prologue_end { - found_end = true; - break; - } - } - - if !found_end { - return Ok(None); - } - - if saved_fpr { - if static_frame_allocation_size > 240 && saved_fpr { - warn!("stack frame is too large ({} bytes) to use with Windows x64 SEH when preserving FPRs. \ - This is a Cranelift implementation limit, see \ - https://github.com/bytecodealliance/wasmtime/issues/1475", - static_frame_allocation_size); - return Err(CodegenError::ImplLimitExceeded); - } - // Only test static frame size is 16-byte aligned when an FPR is saved to avoid - // panicking when alignment is elided because no FPRs are saved and no child calls are - // made. - assert!( - static_frame_allocation_size % 16 == 0, - "static frame allocation must be a multiple of 16" - ); - } - - // Hack to avoid panicking unnecessarily. Because Cranelift generates prologues with RBP at - // one end of the call frame, and RSP at the other, required offsets are arbitrarily large. - // Windows x64 SEH only allows this offset be up to 240 bytes, however, meaning large - // frames are inexpressible, and we cannot actually compile the function. In case there are - // no preserved FPRs, we can lie without error and claim the offset to RBP is 0 - nothing - // will actually check it. This, then, avoids panics when compiling functions with large - // call frames. - let reported_frame_offset = if saved_fpr { - (static_frame_allocation_size / 16) as u8 - } else { - 0 - }; - - Ok(Some(Self { - flags: 0, // this assumes cranelift functions have no SEH handlers - prologue_size: prologue_size as u8, - frame_register, - frame_register_offset: reported_frame_offset, - unwind_codes, - })) - } - - pub fn size(&self) -> usize { - let node_count = self.node_count(); - - // Calculation of the size requires no SEH handler or chained info - assert!(self.flags == 0); - - // Size of fixed part of UNWIND_INFO is 4 bytes - // Then comes the UNWIND_CODE nodes (2 bytes each) - // Then comes 2 bytes of padding for the unwind codes if necessary - // Next would come the SEH data, but we assert above that the function doesn't have SEH data - - 4 + (node_count * 2) + if (node_count & 1) == 1 { 2 } else { 0 } - } - - pub fn node_count(&self) -> usize { - self.unwind_codes - .iter() - .fold(0, |nodes, c| nodes + c.node_count()) - } - - pub fn emit(&self, sink: &mut dyn FrameUnwindSink) { - const UNWIND_INFO_VERSION: u8 = 1; - - let size = self.size(); - let offset = sink.len(); - - // Ensure the memory is 32-bit aligned - assert_eq!(offset % 4, 0); - - sink.reserve(offset + size); - - let node_count = self.node_count(); - assert!(node_count <= 256); - - write_u8(sink, (self.flags << 3) | UNWIND_INFO_VERSION); - write_u8(sink, self.prologue_size); - write_u8(sink, node_count as u8); - - if let Some(reg) = self.frame_register { - write_u8( - sink, - (self.frame_register_offset << 4) | GPR.index_of(reg) as u8, - ); - } else { - write_u8(sink, 0); - } - - // Unwind codes are written in reverse order (prologue offset descending) - for code in self.unwind_codes.iter().rev() { - code.emit(sink); - } - - // To keep a 32-bit alignment, emit 2 bytes of padding if there's an odd number of 16-bit nodes - if (node_count & 1) == 1 { - write_u16::(sink, 0); - } - - // Ensure the correct number of bytes was emitted - assert_eq!(sink.len() - offset, size); - } -} - -#[cfg(test)] -mod tests { - use super::*; - use crate::binemit::{FrameUnwindOffset, Reloc}; - use crate::cursor::{Cursor, FuncCursor}; - use crate::ir::{ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind}; - use crate::isa::{lookup, CallConv}; - use crate::settings::{builder, Flags}; - use crate::Context; - use std::str::FromStr; - use target_lexicon::triple; - - struct SimpleUnwindSink(pub Vec); - impl FrameUnwindSink for SimpleUnwindSink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reloc(&mut self, _: Reloc, _: FrameUnwindOffset) {} - fn set_entry_offset(&mut self, _: FrameUnwindOffset) {} - } - - #[test] - fn test_wrong_calling_convention() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function(CallConv::SystemV, None)); - - context.compile(&*isa).expect("expected compilation"); - - assert_eq!( - UnwindInfo::try_from_func(&context.func, &*isa, None).expect("can emit unwind info"), - None - ); - } - - #[test] - fn test_small_alloc() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::WindowsFastcall, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), - )); - - context.compile(&*isa).expect("expected compilation"); - - let unwind = UnwindInfo::try_from_func(&context.func, &*isa, Some(RU::rbp.into())) - .expect("can emit unwind info") - .expect("expected unwind info"); - - assert_eq!( - unwind, - UnwindInfo { - flags: 0, - prologue_size: 9, - frame_register: Some(RU::rbp.into()), - frame_register_offset: 0, - unwind_codes: vec![ - UnwindCode::PushRegister { - offset: 2, - reg: RU::rbp.into() - }, - UnwindCode::SetFramePointer { - offset: 5, - sp_offset: 0 - }, - UnwindCode::StackAlloc { - offset: 9, - size: 64 + 32 - } - ] - } - ); - - assert_eq!(unwind.size(), 12); - - let mut sink = SimpleUnwindSink(Vec::new()); - unwind.emit(&mut sink); - - assert_eq!( - sink.0, - [ - 0x01, // Version and flags (version 1, no flags) - 0x09, // Prologue size - 0x03, // Unwind code count (1 for stack alloc, 1 for save frame reg, 1 for push reg) - 0x05, // Frame register + offset (RBP with 0 offset) - 0x09, // Prolog offset - 0xB2, // Operation 2 (small stack alloc), size = 0xB slots (e.g. (0xB * 8) + 8 = 96 (64 + 32) bytes) - 0x05, // Prolog offset - 0x03, // Operation 3 (save frame register), stack pointer offset = 0 - 0x02, // Prolog offset - 0x50, // Operation 0 (save nonvolatile register), reg = 5 (RBP) - 0x00, // Padding byte - 0x00, // Padding byte - ] - ); - } - - #[test] - fn test_medium_alloc() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::WindowsFastcall, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 10000)), - )); - - context.compile(&*isa).expect("expected compilation"); - - let unwind = UnwindInfo::try_from_func(&context.func, &*isa, Some(RU::rbp.into())) - .expect("can emit unwind info") - .expect("expected unwind info"); - - assert_eq!( - unwind, - UnwindInfo { - flags: 0, - prologue_size: 27, - frame_register: Some(RU::rbp.into()), - frame_register_offset: 0, - unwind_codes: vec![ - UnwindCode::PushRegister { - offset: 2, - reg: RU::rbp.into() - }, - UnwindCode::SetFramePointer { - offset: 5, - sp_offset: 0 - }, - UnwindCode::StackAlloc { - offset: 27, - size: 10000 + 32 - } - ] - } - ); - - assert_eq!(unwind.size(), 12); - - let mut sink = SimpleUnwindSink(Vec::new()); - unwind.emit(&mut sink); - - assert_eq!( - sink.0, - [ - 0x01, // Version and flags (version 1, no flags) - 0x1B, // Prologue size - 0x04, // Unwind code count (2 for stack alloc, 1 for save frame reg, 1 for push reg) - 0x05, // Frame register + offset (RBP with 0 offset) - 0x1B, // Prolog offset - 0x01, // Operation 1 (large stack alloc), size is scaled 16-bits (info = 0) - 0xE6, // Low size byte - 0x04, // High size byte (e.g. 0x04E6 * 8 = 100032 (10000 + 32) bytes) - 0x05, // Prolog offset - 0x03, // Operation 3 (save frame register), stack pointer offset = 0 - 0x02, // Prolog offset - 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) - ] - ); - } - - #[test] - fn test_large_alloc() { - let isa = lookup(triple!("x86_64")) - .expect("expect x86 ISA") - .finish(Flags::new(builder())); - - let mut context = Context::for_function(create_function( - CallConv::WindowsFastcall, - Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 1000000)), - )); - - context.compile(&*isa).expect("expected compilation"); - - let unwind = UnwindInfo::try_from_func(&context.func, &*isa, Some(RU::rbp.into())) - .expect("can emit unwind info") - .expect("expected unwind info"); - - assert_eq!( - unwind, - UnwindInfo { - flags: 0, - prologue_size: 27, - frame_register: Some(RU::rbp.into()), - frame_register_offset: 0, - unwind_codes: vec![ - UnwindCode::PushRegister { - offset: 2, - reg: RU::rbp.into() - }, - UnwindCode::SetFramePointer { - offset: 5, - sp_offset: 0 - }, - UnwindCode::StackAlloc { - offset: 27, - size: 1000000 + 32 - } - ] - } - ); - - assert_eq!(unwind.size(), 16); - - let mut sink = SimpleUnwindSink(Vec::new()); - unwind.emit(&mut sink); - - assert_eq!( - sink.0, - [ - 0x01, // Version and flags (version 1, no flags) - 0x1B, // Prologue size - 0x05, // Unwind code count (3 for stack alloc, 1 for save frame reg, 1 for push reg) - 0x05, // Frame register + offset (RBP with 0 offset) - 0x1B, // Prolog offset - 0x11, // Operation 1 (large stack alloc), size is unscaled 32-bits (info = 1) - 0x60, // Byte 1 of size - 0x42, // Byte 2 of size - 0x0F, // Byte 3 of size - 0x00, // Byte 4 of size (size is 0xF4260 = 1000032 (1000000 + 32) bytes) - 0x05, // Prolog offset - 0x03, // Operation 3 (save frame register), stack pointer offset = 0 - 0x02, // Prolog offset - 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) - 0x00, // Padding byte - 0x00, // Padding byte - ] - ); - } - - fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { - let mut func = - Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); - - let block0 = func.dfg.make_block(); - let mut pos = FuncCursor::new(&mut func); - pos.insert_block(block0); - pos.ins().return_(&[]); - - if let Some(stack_slot) = stack_slot { - func.stack_slots.push(stack_slot); - } - - func - } -} +pub mod systemv; +pub mod winx64; diff --git a/cranelift/codegen/src/isa/x86/unwind/systemv.rs b/cranelift/codegen/src/isa/x86/unwind/systemv.rs new file mode 100644 index 000000000000..ecbcbb1dd185 --- /dev/null +++ b/cranelift/codegen/src/isa/x86/unwind/systemv.rs @@ -0,0 +1,487 @@ +//! Unwind information for System V ABI (x86-64). + +use crate::ir::{Function, Inst, InstructionData, Opcode, Value}; +use crate::isa::{ + unwind::systemv::{CallFrameInstruction, RegisterMappingError, UnwindInfo}, + x86::registers::RU, + CallConv, RegUnit, TargetIsa, +}; +use crate::result::{CodegenError, CodegenResult}; +use alloc::vec::Vec; +use gimli::{write::CommonInformationEntry, Encoding, Format, Register, X86_64}; + +/// Creates a new x86-64 common information entry (CIE). +pub fn create_cie() -> CommonInformationEntry { + use gimli::write::CallFrameInstruction; + + let mut entry = CommonInformationEntry::new( + Encoding { + address_size: 8, + format: Format::Dwarf32, + version: 1, + }, + 1, // Code alignment factor + -8, // Data alignment factor + X86_64::RA, + ); + + // Every frame will start with the call frame address (CFA) at RSP+8 + // It is +8 to account for the push of the return address by the call instruction + entry.add_instruction(CallFrameInstruction::Cfa(X86_64::RSP, 8)); + + // Every frame will start with the return address at RSP (CFA-8 = RSP+8-8 = RSP) + entry.add_instruction(CallFrameInstruction::Offset(X86_64::RA, -8)); + + entry +} + +/// Map Cranelift registers to their corresponding Gimli registers. +pub fn map_reg(isa: &dyn TargetIsa, reg: RegUnit) -> Result { + if isa.name() != "x86" || isa.pointer_bits() != 64 { + return Err(RegisterMappingError::UnsupportedArchitecture); + } + + // Mapping from https://github.com/bytecodealliance/cranelift/pull/902 by @iximeow + const X86_GP_REG_MAP: [gimli::Register; 16] = [ + X86_64::RAX, + X86_64::RCX, + X86_64::RDX, + X86_64::RBX, + X86_64::RSP, + X86_64::RBP, + X86_64::RSI, + X86_64::RDI, + X86_64::R8, + X86_64::R9, + X86_64::R10, + X86_64::R11, + X86_64::R12, + X86_64::R13, + X86_64::R14, + X86_64::R15, + ]; + const X86_XMM_REG_MAP: [gimli::Register; 16] = [ + X86_64::XMM0, + X86_64::XMM1, + X86_64::XMM2, + X86_64::XMM3, + X86_64::XMM4, + X86_64::XMM5, + X86_64::XMM6, + X86_64::XMM7, + X86_64::XMM8, + X86_64::XMM9, + X86_64::XMM10, + X86_64::XMM11, + X86_64::XMM12, + X86_64::XMM13, + X86_64::XMM14, + X86_64::XMM15, + ]; + + let reg_info = isa.register_info(); + let bank = reg_info + .bank_containing_regunit(reg) + .ok_or_else(|| RegisterMappingError::MissingBank)?; + match bank.name { + "IntRegs" => { + // x86 GP registers have a weird mapping to DWARF registers, so we use a + // lookup table. + Ok(X86_GP_REG_MAP[(reg - bank.first_unit) as usize]) + } + "FloatRegs" => Ok(X86_XMM_REG_MAP[(reg - bank.first_unit) as usize]), + _ => Err(RegisterMappingError::UnsupportedRegisterBank(bank.name)), + } +} + +struct InstructionBuilder<'a> { + func: &'a Function, + isa: &'a dyn TargetIsa, + cfa_offset: i32, + frame_register: Option, + instructions: Vec<(u32, CallFrameInstruction)>, + stack_size: Option, + epilogue_pop_offsets: Vec, +} + +impl<'a> InstructionBuilder<'a> { + fn new(func: &'a Function, isa: &'a dyn TargetIsa, frame_register: Option) -> Self { + Self { + func, + isa, + cfa_offset: 8, // CFA offset starts at 8 to account to return address on stack + frame_register, + instructions: Vec::new(), + stack_size: None, + epilogue_pop_offsets: Vec::new(), + } + } + + fn push_reg(&mut self, offset: u32, arg: Value) -> Result<(), RegisterMappingError> { + self.cfa_offset += 8; + + let reg = self.func.locations[arg].unwrap_reg(); + + // Update the CFA if this is the save of the frame pointer register or if a frame pointer isn't being used + // When using a frame pointer, we only need to update the CFA to account for the push of the frame pointer itself + if match self.frame_register { + Some(fp) => reg == fp, + None => true, + } { + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + // Pushes in the prologue are register saves, so record an offset of the save + self.instructions.push(( + offset, + CallFrameInstruction::Offset(map_reg(self.isa, reg)?.0, -self.cfa_offset), + )); + + Ok(()) + } + + fn adjust_sp_down(&mut self, offset: u32) { + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset += self + .stack_size + .expect("expected a previous stack size instruction"); + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn adjust_sp_down_imm(&mut self, offset: u32, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset += imm as i32; + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn adjust_sp_up_imm(&mut self, offset: u32, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + + // Don't adjust the CFA if we're using a frame pointer + if self.frame_register.is_some() { + return; + } + + self.cfa_offset -= imm as i32; + self.instructions + .push((offset, CallFrameInstruction::CfaOffset(self.cfa_offset))); + } + + fn move_reg( + &mut self, + offset: u32, + src: RegUnit, + dst: RegUnit, + ) -> Result<(), RegisterMappingError> { + if let Some(fp) = self.frame_register { + // Check for change in CFA register (RSP is always the starting CFA) + if src == (RU::rsp as RegUnit) && dst == fp { + self.instructions.push(( + offset, + CallFrameInstruction::CfaRegister(map_reg(self.isa, dst)?.0), + )); + } + } + + Ok(()) + } + + fn prologue_imm_const(&mut self, imm: i64) { + assert!(imm <= core::u32::MAX as i64); + assert!(self.stack_size.is_none()); + + // This instruction should only appear in a prologue to pass an + // argument of the stack size to a stack check function. + // Record the stack size so we know what it is when we encounter the adjustment + // instruction (which will adjust via the register assigned to this instruction). + self.stack_size = Some(imm as i32); + } + + fn ret(&mut self, inst: Inst) -> Result<(), RegisterMappingError> { + let args = self.func.dfg.inst_args(inst); + + for (i, arg) in args.iter().rev().enumerate() { + // Only walk back the args for the pop instructions encountered + if i >= self.epilogue_pop_offsets.len() { + break; + } + + self.cfa_offset -= 8; + let reg = self.func.locations[*arg].unwrap_reg(); + + // Update the CFA if this is the restore of the frame pointer register or if a frame pointer isn't being used + match self.frame_register { + Some(fp) => { + if reg == fp { + self.instructions.push(( + self.epilogue_pop_offsets[i], + CallFrameInstruction::Cfa( + map_reg(self.isa, RU::rsp as RegUnit)?.0, + self.cfa_offset, + ), + )); + } + } + None => { + self.instructions.push(( + self.epilogue_pop_offsets[i], + CallFrameInstruction::CfaOffset(self.cfa_offset), + )); + + // Pops in the epilogue are register restores, so record a "same value" for the register + // This isn't necessary when using a frame pointer as the CFA doesn't change for CSR restores + self.instructions.push(( + self.epilogue_pop_offsets[i], + CallFrameInstruction::SameValue(map_reg(self.isa, reg)?.0), + )); + } + }; + } + + self.epilogue_pop_offsets.clear(); + + Ok(()) + } + + fn insert_pop_offset(&mut self, offset: u32) { + self.epilogue_pop_offsets.push(offset); + } + + fn remember_state(&mut self, offset: u32) { + self.instructions + .push((offset, CallFrameInstruction::RememberState)); + } + + fn restore_state(&mut self, offset: u32) { + self.instructions + .push((offset, CallFrameInstruction::RestoreState)); + } + + fn is_prologue_end(&self, inst: Inst) -> bool { + self.func.prologue_end == Some(inst) + } + + fn is_epilogue_start(&self, inst: Inst) -> bool { + self.func.epilogues_start.contains(&inst) + } +} + +pub(crate) fn create_unwind_info( + func: &Function, + isa: &dyn TargetIsa, + frame_register: Option, +) -> CodegenResult> { + // Only System V-like calling conventions are supported + match func.signature.call_conv { + CallConv::Fast | CallConv::Cold | CallConv::SystemV => {} + _ => return Ok(None), + } + + if func.prologue_end.is_none() || isa.name() != "x86" || isa.pointer_bits() != 64 { + return Ok(None); + } + + let mut builder = InstructionBuilder::new(func, isa, frame_register); + let mut in_prologue = true; + let mut in_epilogue = false; + let mut len = 0; + + let mut blocks = func.layout.blocks().collect::>(); + blocks.sort_by_key(|b| func.offsets[*b]); + + for (i, block) in blocks.iter().enumerate() { + for (offset, inst, size) in func.inst_offsets(*block, &isa.encoding_info()) { + let offset = offset + size; + assert!(len <= offset); + len = offset; + + let is_last_block = i == blocks.len() - 1; + + if in_prologue { + // Check for prologue end (inclusive) + in_prologue = !builder.is_prologue_end(inst); + } else if !in_epilogue && builder.is_epilogue_start(inst) { + // Now in an epilogue, emit a remember state instruction if not last block + in_epilogue = true; + + if !is_last_block { + builder.remember_state(offset); + } + } else if !in_epilogue { + // Ignore normal instructions + continue; + } + + match builder.func.dfg[inst] { + InstructionData::Unary { opcode, arg } => match opcode { + Opcode::X86Push => { + builder + .push_reg(offset, arg) + .map_err(CodegenError::RegisterMappingError)?; + } + Opcode::AdjustSpDown => { + builder.adjust_sp_down(offset); + } + _ => {} + }, + InstructionData::CopySpecial { src, dst, .. } => { + builder + .move_reg(offset, src, dst) + .map_err(CodegenError::RegisterMappingError)?; + } + InstructionData::NullAry { opcode } => match opcode { + Opcode::X86Pop => { + builder.insert_pop_offset(offset); + } + _ => {} + }, + InstructionData::UnaryImm { opcode, imm } => match opcode { + Opcode::Iconst => { + builder.prologue_imm_const(imm.into()); + } + Opcode::AdjustSpDownImm => { + builder.adjust_sp_down_imm(offset, imm.into()); + } + Opcode::AdjustSpUpImm => { + builder.adjust_sp_up_imm(offset, imm.into()); + } + _ => {} + }, + InstructionData::MultiAry { opcode, .. } => match opcode { + Opcode::Return => { + builder + .ret(inst) + .map_err(CodegenError::RegisterMappingError)?; + + if !is_last_block { + builder.restore_state(offset); + } + + in_epilogue = false; + } + _ => {} + }, + _ => {} + }; + } + } + + Ok(Some(UnwindInfo::new(builder.instructions, len))) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cursor::{Cursor, FuncCursor}; + use crate::ir::{ + types, AbiParam, ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind, + }; + use crate::isa::{lookup, CallConv}; + use crate::settings::{builder, Flags}; + use crate::Context; + use gimli::write::Address; + use std::str::FromStr; + use target_lexicon::triple; + + #[test] + fn test_simple_func() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::SystemV, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let fde = match isa + .create_unwind_info(&context.func) + .expect("can create unwind info") + { + Some(crate::isa::unwind::UnwindInfo::SystemV(info)) => { + info.to_fde(Address::Constant(1234)) + } + _ => panic!("expected unwind information"), + }; + + assert_eq!(format!("{:?}", fde), "FrameDescriptionEntry { address: Constant(1234), length: 16, lsda: None, instructions: [(2, CfaOffset(16)), (2, Offset(Register(6), -16)), (5, CfaRegister(Register(6))), (15, Cfa(Register(7), 8))] }"); + } + + fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { + let mut func = + Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + + let block0 = func.dfg.make_block(); + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().return_(&[]); + + if let Some(stack_slot) = stack_slot { + func.stack_slots.push(stack_slot); + } + + func + } + + #[test] + fn test_multi_return_func() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_multi_return_function(CallConv::SystemV)); + + context.compile(&*isa).expect("expected compilation"); + + let fde = match isa + .create_unwind_info(&context.func) + .expect("can create unwind info") + { + Some(crate::isa::unwind::UnwindInfo::SystemV(info)) => { + info.to_fde(Address::Constant(4321)) + } + _ => panic!("expected unwind information"), + }; + + assert_eq!(format!("{:?}", fde), "FrameDescriptionEntry { address: Constant(4321), length: 16, lsda: None, instructions: [(2, CfaOffset(16)), (2, Offset(Register(6), -16)), (5, CfaRegister(Register(6))), (12, RememberState), (12, Cfa(Register(7), 8)), (13, RestoreState), (15, Cfa(Register(7), 0))] }"); + } + + fn create_multi_return_function(call_conv: CallConv) -> Function { + let mut sig = Signature::new(call_conv); + sig.params.push(AbiParam::new(types::I32)); + let mut func = Function::with_name_signature(ExternalName::user(0, 0), sig); + + let block0 = func.dfg.make_block(); + let v0 = func.dfg.append_block_param(block0, types::I32); + let block1 = func.dfg.make_block(); + let block2 = func.dfg.make_block(); + + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().brnz(v0, block2, &[]); + pos.ins().jump(block1, &[]); + + pos.insert_block(block1); + pos.ins().return_(&[]); + + pos.insert_block(block2); + pos.ins().return_(&[]); + + func + } +} diff --git a/cranelift/codegen/src/isa/x86/unwind/winx64.rs b/cranelift/codegen/src/isa/x86/unwind/winx64.rs new file mode 100644 index 000000000000..60aff23f197c --- /dev/null +++ b/cranelift/codegen/src/isa/x86/unwind/winx64.rs @@ -0,0 +1,466 @@ +//! Unwind information for Windows x64 ABI. + +use crate::ir::{Function, InstructionData, Opcode, ValueLoc}; +use crate::isa::x86::registers::{FPR, GPR, RU}; +use crate::isa::{ + unwind::winx64::{UnwindCode, UnwindInfo}, + CallConv, RegUnit, TargetIsa, +}; +use crate::result::{CodegenError, CodegenResult}; +use alloc::vec::Vec; +use log::warn; + +pub(crate) fn create_unwind_info( + func: &Function, + isa: &dyn TargetIsa, + frame_register: Option, +) -> CodegenResult> { + // Only Windows fastcall is supported for unwind information + if func.signature.call_conv != CallConv::WindowsFastcall || func.prologue_end.is_none() { + return Ok(None); + } + + let prologue_end = func.prologue_end.unwrap(); + let entry_block = func.layout.entry_block().expect("missing entry block"); + + // Stores the stack size when SP is not adjusted via an immediate value + let mut stack_size = None; + let mut prologue_size = 0; + let mut unwind_codes = Vec::new(); + let mut found_end = false; + + // Have we saved at least one FPR? if so, we might have to check additional constraints. + let mut saved_fpr = false; + + // In addition to the min offset for a callee-save, we need to know the offset from the + // frame base to the stack pointer, so that we can record an unwind offset that spans only + // to the end of callee-save space. + let mut static_frame_allocation_size = 0u32; + + // For the time being, FPR preservation is split into a stack_addr and later store/load. + // Store the register used for stack store and ensure it is the same register with no + // intervening changes to the frame size. + let mut callee_save_region_reg = None; + // Also record the callee-save region's offset from RSP, because it must be added to FPR + // save offsets to compute an offset from the frame base. + let mut callee_save_offset = None; + + for (offset, inst, size) in func.inst_offsets(entry_block, &isa.encoding_info()) { + // x64 ABI prologues cannot exceed 255 bytes in length + if (offset + size) > 255 { + warn!("function prologues cannot exceed 255 bytes in size for Windows x64"); + return Err(CodegenError::CodeTooLarge); + } + + prologue_size += size; + + let unwind_offset = (offset + size) as u8; + + match func.dfg[inst] { + InstructionData::Unary { opcode, arg } => { + match opcode { + Opcode::X86Push => { + static_frame_allocation_size += 8; + + unwind_codes.push(UnwindCode::PushRegister { + offset: unwind_offset, + reg: GPR.index_of(func.locations[arg].unwrap_reg()) as u8, + }); + } + Opcode::AdjustSpDown => { + let stack_size = + stack_size.expect("expected a previous stack size instruction"); + static_frame_allocation_size += stack_size; + + // This is used when calling a stack check function + // We need to track the assignment to RAX which has the size of the stack + unwind_codes.push(UnwindCode::StackAlloc { + offset: unwind_offset, + size: stack_size, + }); + } + _ => {} + } + } + InstructionData::CopySpecial { src, dst, .. } => { + if let Some(frame_register) = frame_register { + if src == (RU::rsp as RegUnit) && dst == frame_register { + // Constructing an rbp-based stack frame, so the static frame + // allocation restarts at 0 from here. + static_frame_allocation_size = 0; + + unwind_codes.push(UnwindCode::SetFramePointer { + offset: unwind_offset, + sp_offset: 0, + }); + } + } + } + InstructionData::UnaryImm { opcode, imm } => { + match opcode { + Opcode::Iconst => { + let imm: i64 = imm.into(); + assert!(imm <= core::u32::MAX as i64); + assert!(stack_size.is_none()); + + // This instruction should only appear in a prologue to pass an + // argument of the stack size to a stack check function. + // Record the stack size so we know what it is when we encounter the adjustment + // instruction (which will adjust via the register assigned to this instruction). + stack_size = Some(imm as u32); + } + Opcode::AdjustSpDownImm => { + let imm: i64 = imm.into(); + assert!(imm <= core::u32::MAX as i64); + + static_frame_allocation_size += imm as u32; + + unwind_codes.push(UnwindCode::StackAlloc { + offset: unwind_offset, + size: imm as u32, + }); + } + _ => {} + } + } + InstructionData::StackLoad { + opcode: Opcode::StackAddr, + stack_slot, + offset: _, + } => { + let result = func.dfg.inst_results(inst).get(0).unwrap(); + if let ValueLoc::Reg(frame_reg) = func.locations[*result] { + callee_save_region_reg = Some(frame_reg); + + // Figure out the offset in the call frame that `frame_reg` will have. + let frame_size = func + .stack_slots + .layout_info + .expect("func's stack slots have layout info if stack operations exist") + .frame_size; + // Because we're well after the prologue has been constructed, stack slots + // must have been laid out... + let slot_offset = func.stack_slots[stack_slot] + .offset + .expect("callee-save slot has an offset computed"); + let frame_offset = frame_size as i32 + slot_offset; + + callee_save_offset = Some(frame_offset as u32); + } + } + InstructionData::Store { + opcode: Opcode::Store, + args: [arg1, arg2], + flags: _flags, + offset, + } => { + if let (ValueLoc::Reg(ru), ValueLoc::Reg(base_ru)) = + (func.locations[arg1], func.locations[arg2]) + { + if Some(base_ru) == callee_save_region_reg { + let offset_int: i32 = offset.into(); + assert!(offset_int >= 0, "negative fpr offset would store outside the stack frame, and is almost certainly an error"); + let offset_int: u32 = offset_int as u32 + callee_save_offset.expect("FPR presevation requires an FPR save region, which has some stack offset"); + if FPR.contains(ru) { + saved_fpr = true; + unwind_codes.push(UnwindCode::SaveXmm { + offset: unwind_offset, + reg: ru as u8, + stack_offset: offset_int, + }); + } + } + } + } + _ => {} + }; + + if inst == prologue_end { + found_end = true; + break; + } + } + + assert!(found_end); + + if saved_fpr { + if static_frame_allocation_size > 240 && saved_fpr { + warn!("stack frame is too large ({} bytes) to use with Windows x64 SEH when preserving FPRs. \ + This is a Cranelift implementation limit, see \ + https://github.com/bytecodealliance/wasmtime/issues/1475", + static_frame_allocation_size); + return Err(CodegenError::ImplLimitExceeded); + } + // Only test static frame size is 16-byte aligned when an FPR is saved to avoid + // panicking when alignment is elided because no FPRs are saved and no child calls are + // made. + assert!( + static_frame_allocation_size % 16 == 0, + "static frame allocation must be a multiple of 16" + ); + } + + // Hack to avoid panicking unnecessarily. Because Cranelift generates prologues with RBP at + // one end of the call frame, and RSP at the other, required offsets are arbitrarily large. + // Windows x64 SEH only allows this offset be up to 240 bytes, however, meaning large + // frames are inexpressible, and we cannot actually compile the function. In case there are + // no preserved FPRs, we can lie without error and claim the offset to RBP is 0 - nothing + // will actually check it. This, then, avoids panics when compiling functions with large + // call frames. + let reported_frame_offset = if saved_fpr { + (static_frame_allocation_size / 16) as u8 + } else { + 0 + }; + + Ok(Some(UnwindInfo { + flags: 0, // this assumes cranelift functions have no SEH handlers + prologue_size: prologue_size as u8, + frame_register: frame_register.map(|r| GPR.index_of(r) as u8), + frame_register_offset: reported_frame_offset, + unwind_codes, + })) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cursor::{Cursor, FuncCursor}; + use crate::ir::{ExternalName, InstBuilder, Signature, StackSlotData, StackSlotKind}; + use crate::isa::{lookup, CallConv}; + use crate::settings::{builder, Flags}; + use crate::Context; + use std::str::FromStr; + use target_lexicon::triple; + + #[test] + fn test_wrong_calling_convention() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function(CallConv::SystemV, None)); + + context.compile(&*isa).expect("expected compilation"); + + assert_eq!( + create_unwind_info(&context.func, &*isa, None).expect("can create unwind info"), + None + ); + } + + #[test] + fn test_small_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 64)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, Some(RU::rbp.into())) + .expect("can create unwind info") + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + flags: 0, + prologue_size: 9, + frame_register: Some(GPR.index_of(RU::rbp.into()) as u8), + frame_register_offset: 0, + unwind_codes: vec![ + UnwindCode::PushRegister { + offset: 2, + reg: GPR.index_of(RU::rbp.into()) as u8 + }, + UnwindCode::SetFramePointer { + offset: 5, + sp_offset: 0 + }, + UnwindCode::StackAlloc { + offset: 9, + size: 64 + 32 + } + ] + } + ); + + assert_eq!(unwind.emit_size(), 12); + + let mut buf = [0u8; 12]; + unwind.emit(&mut buf); + + assert_eq!( + buf, + [ + 0x01, // Version and flags (version 1, no flags) + 0x09, // Prologue size + 0x03, // Unwind code count (1 for stack alloc, 1 for save frame reg, 1 for push reg) + 0x05, // Frame register + offset (RBP with 0 offset) + 0x09, // Prolog offset + 0xB2, // Operation 2 (small stack alloc), size = 0xB slots (e.g. (0xB * 8) + 8 = 96 (64 + 32) bytes) + 0x05, // Prolog offset + 0x03, // Operation 3 (save frame register), stack pointer offset = 0 + 0x02, // Prolog offset + 0x50, // Operation 0 (save nonvolatile register), reg = 5 (RBP) + 0x00, // Padding byte + 0x00, // Padding byte + ] + ); + } + + #[test] + fn test_medium_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 10000)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, Some(RU::rbp.into())) + .expect("can create unwind info") + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + flags: 0, + prologue_size: 27, + frame_register: Some(GPR.index_of(RU::rbp.into()) as u8), + frame_register_offset: 0, + unwind_codes: vec![ + UnwindCode::PushRegister { + offset: 2, + reg: GPR.index_of(RU::rbp.into()) as u8 + }, + UnwindCode::SetFramePointer { + offset: 5, + sp_offset: 0 + }, + UnwindCode::StackAlloc { + offset: 27, + size: 10000 + 32 + } + ] + } + ); + + assert_eq!(unwind.emit_size(), 12); + + let mut buf = [0u8; 12]; + unwind.emit(&mut buf); + + assert_eq!( + buf, + [ + 0x01, // Version and flags (version 1, no flags) + 0x1B, // Prologue size + 0x04, // Unwind code count (2 for stack alloc, 1 for save frame reg, 1 for push reg) + 0x05, // Frame register + offset (RBP with 0 offset) + 0x1B, // Prolog offset + 0x01, // Operation 1 (large stack alloc), size is scaled 16-bits (info = 0) + 0xE6, // Low size byte + 0x04, // High size byte (e.g. 0x04E6 * 8 = 100032 (10000 + 32) bytes) + 0x05, // Prolog offset + 0x03, // Operation 3 (save frame register), stack pointer offset = 0 + 0x02, // Prolog offset + 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) + ] + ); + } + + #[test] + fn test_large_alloc() { + let isa = lookup(triple!("x86_64")) + .expect("expect x86 ISA") + .finish(Flags::new(builder())); + + let mut context = Context::for_function(create_function( + CallConv::WindowsFastcall, + Some(StackSlotData::new(StackSlotKind::ExplicitSlot, 1000000)), + )); + + context.compile(&*isa).expect("expected compilation"); + + let unwind = create_unwind_info(&context.func, &*isa, Some(RU::rbp.into())) + .expect("can create unwind info") + .expect("expected unwind info"); + + assert_eq!( + unwind, + UnwindInfo { + flags: 0, + prologue_size: 27, + frame_register: Some(GPR.index_of(RU::rbp.into()) as u8), + frame_register_offset: 0, + unwind_codes: vec![ + UnwindCode::PushRegister { + offset: 2, + reg: GPR.index_of(RU::rbp.into()) as u8 + }, + UnwindCode::SetFramePointer { + offset: 5, + sp_offset: 0 + }, + UnwindCode::StackAlloc { + offset: 27, + size: 1000000 + 32 + } + ] + } + ); + + assert_eq!(unwind.emit_size(), 16); + + let mut buf = [0u8; 16]; + unwind.emit(&mut buf); + + assert_eq!( + buf, + [ + 0x01, // Version and flags (version 1, no flags) + 0x1B, // Prologue size + 0x05, // Unwind code count (3 for stack alloc, 1 for save frame reg, 1 for push reg) + 0x05, // Frame register + offset (RBP with 0 offset) + 0x1B, // Prolog offset + 0x11, // Operation 1 (large stack alloc), size is unscaled 32-bits (info = 1) + 0x60, // Byte 1 of size + 0x42, // Byte 2 of size + 0x0F, // Byte 3 of size + 0x00, // Byte 4 of size (size is 0xF4260 = 1000032 (1000000 + 32) bytes) + 0x05, // Prolog offset + 0x03, // Operation 3 (save frame register), stack pointer offset = 0 + 0x02, // Prolog offset + 0x50, // Operation 0 (push nonvolatile register), reg = 5 (RBP) + 0x00, // Padding byte + 0x00, // Padding byte + ] + ); + } + + fn create_function(call_conv: CallConv, stack_slot: Option) -> Function { + let mut func = + Function::with_name_signature(ExternalName::user(0, 0), Signature::new(call_conv)); + + let block0 = func.dfg.make_block(); + let mut pos = FuncCursor::new(&mut func); + pos.insert_block(block0); + pos.ins().return_(&[]); + + if let Some(stack_slot) = stack_slot { + func.stack_slots.push(stack_slot); + } + + func + } +} diff --git a/cranelift/codegen/src/result.rs b/cranelift/codegen/src/result.rs index 65169cba1972..4ed2c8f70e7a 100644 --- a/cranelift/codegen/src/result.rs +++ b/cranelift/codegen/src/result.rs @@ -30,6 +30,11 @@ pub enum CodegenError { /// is exceeded, compilation fails. #[error("Code for function is too large")] CodeTooLarge, + + /// A failure to map Cranelift register representation to a DWARF register representation. + #[cfg(feature = "unwind")] + #[error("Register mapping error")] + RegisterMappingError(crate::isa::unwind::systemv::RegisterMappingError), } /// A convenient alias for a `Result` that uses `CodegenError` as the error type. diff --git a/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif b/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif new file mode 100644 index 000000000000..7ff0ea861578 --- /dev/null +++ b/cranelift/filetests/filetests/isa/x86/systemv_x64_unwind.clif @@ -0,0 +1,197 @@ +test unwind +set opt_level=speed_and_size +set is_pic +target x86_64 haswell + +; check the unwind information with a function with no args +function %no_args() system_v { +block0: + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000024 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x0000000000000006 (end_addr = 0x0000000000000006) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop + +; check a function with medium-sized stack alloc +function %medium_stack() system_v { + ss0 = explicit_slot 100000 +block0: + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000024 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x000000000000001a (end_addr = 0x000000000000001a) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (21) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop + +; check a function with large-sized stack alloc +function %large_stack() system_v { + ss0 = explicit_slot 524288 +block0: + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000024 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x000000000000001a (end_addr = 0x000000000000001a) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (21) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: + +; check a function that has CSRs +function %lots_of_registers(i64, i64) system_v { +block0(v0: i64, v1: i64): + v2 = load.i32 v0+0 + v3 = load.i32 v0+8 + v4 = load.i32 v0+16 + v5 = load.i32 v0+24 + v6 = load.i32 v0+32 + v7 = load.i32 v0+40 + v8 = load.i32 v0+48 + v9 = load.i32 v0+56 + v10 = load.i32 v0+64 + v11 = load.i32 v0+72 + v12 = load.i32 v0+80 + v13 = load.i32 v0+88 + v14 = load.i32 v0+96 + store.i32 v2, v1+0 + store.i32 v3, v1+8 + store.i32 v4, v1+16 + store.i32 v5, v1+24 + store.i32 v6, v1+32 + store.i32 v7, v1+40 + store.i32 v8, v1+48 + store.i32 v9, v1+56 + store.i32 v10, v1+64 + store.i32 v11, v1+72 + store.i32 v12, v1+80 + store.i32 v13, v1+88 + store.i32 v14, v1+96 + return +} +; sameln: 0x00000000: CIE +; nextln: length: 0x00000014 +; nextln: version: 0x01 +; nextln: code_align: 1 +; nextln: data_align: -8 +; nextln: ra_register: 0x10 +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_offset (r16, 1) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: Instructions: Init State: +; nextln: +; nextln: +; nextln: 0x00000018: FDE +; nextln: length: 0x00000034 +; nextln: CIE_pointer: 0x00000000 +; nextln: start_addr: 0x0000000000000000 +; nextln: range_size: 0x0000000000000074 (end_addr = 0x0000000000000074) +; nextln: Instructions: +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_def_cfa_offset (16) +; nextln: DW_CFA_offset (r6, 2) +; nextln: DW_CFA_advance_loc (3) +; nextln: DW_CFA_def_cfa_register (r6) +; nextln: DW_CFA_advance_loc (1) +; nextln: DW_CFA_offset (r3, 3) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r12, 4) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r13, 5) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r14, 6) +; nextln: DW_CFA_advance_loc (2) +; nextln: DW_CFA_offset (r15, 7) +; nextln: DW_CFA_advance_loc (102) +; nextln: DW_CFA_def_cfa (r7, 8) +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop +; nextln: DW_CFA_nop diff --git a/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif b/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif index eeb89aa4ae4a..6997238bfbab 100644 --- a/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif +++ b/cranelift/filetests/filetests/isa/x86/windows_fastcall_x64_unwind.clif @@ -3,46 +3,29 @@ set opt_level=speed_and_size set is_pic target x86_64 haswell -; check that there is no unwind information for a system_v function -function %not_fastcall() system_v { -block0: - return -} -; sameln: No unwind information. - ; check the unwind information with a function with no args function %no_args() windows_fastcall { block0: return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 8, -; nextln: unwind_code_count_raw: 3, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 0, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 8, -; nextln: op: SmallStackAlloc, -; nextln: info: 3, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 8 +; nextln: frame register: 5 +; nextln: frame register offset: 0 +; nextln: unwind codes: 3 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 8 +; nextln: op: SmallStackAlloc +; nextln: info: 3 ; check a function with medium-sized stack alloc function %medium_stack() windows_fastcall { @@ -50,36 +33,25 @@ function %medium_stack() windows_fastcall { block0: return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 17, -; nextln: unwind_code_count_raw: 4, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 0, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 17, -; nextln: op: LargeStackAlloc, -; nextln: info: 0, -; nextln: value: U16( -; nextln: 12504, -; nextln: ), -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 17 +; nextln: frame register: 5 +; nextln: frame register offset: 0 +; nextln: unwind codes: 3 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 17 +; nextln: op: LargeStackAlloc +; nextln: info: 0 +; nextln: value: 12504 (u16) ; check a function with large-sized stack alloc function %large_stack() windows_fastcall { @@ -87,36 +59,25 @@ function %large_stack() windows_fastcall { block0: return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 17, -; nextln: unwind_code_count_raw: 5, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 0, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 17, -; nextln: op: LargeStackAlloc, -; nextln: info: 1, -; nextln: value: U32( -; nextln: 524320, -; nextln: ), -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 17 +; nextln: frame register: 5 +; nextln: frame register offset: 0 +; nextln: unwind codes: 3 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 17 +; nextln: op: LargeStackAlloc +; nextln: info: 1 +; nextln: value: 524320 (u32) function %fpr_with_function_call(i64, i64) windows_fastcall { fn0 = %foo(f64, f64, i64, i64, i64) windows_fastcall; @@ -150,20 +111,34 @@ block0(v0: i64, v1: i64): ; area that does not overlap either the callee's shadow space or stack argument ; space. ; -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 25, -; nextln: unwind_code_count_raw: 7, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 12, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 25, -; nextln: op: SaveXmm128, -; nextln: info: 15, -; nextln: value: U16( -; nextln: 3, +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 25 +; nextln: frame register: 5 +; nextln: frame register offset: 12 +; nextln: unwind codes: 5 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 6 +; nextln: op: PushNonvolatileRegister +; nextln: info: 15 +; nextln: +; nextln: offset: 13 +; nextln: op: LargeStackAlloc +; nextln: info: 0 +; nextln: value: 23 (u16) +; nextln: +; nextln: offset: 25 +; nextln: op: SaveXmm128 +; nextln: info: 15 +; nextln: value: 3 (u16) ; check a function that has CSRs function %lots_of_registers(i64, i64) windows_fastcall { @@ -214,97 +189,64 @@ block0(v0: i64, v1: i64): store.f64 v23, v1+168 return } -; sameln: UnwindInfo { -; nextln: version: 1, -; nextln: flags: 0, -; nextln: prologue_size: 41, -; nextln: unwind_code_count_raw: 16, -; nextln: frame_register: 5, -; nextln: frame_register_offset: 10, -; nextln: unwind_codes: [ -; nextln: UnwindCode { -; nextln: offset: 41, -; nextln: op: SaveXmm128, -; nextln: info: 8, -; nextln: value: U16( -; nextln: 2, -; nextln: ), -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 36, -; nextln: op: SaveXmm128, -; nextln: info: 7, -; nextln: value: U16( -; nextln: 1, -; nextln: ), -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 31, -; nextln: op: SaveXmm128, -; nextln: info: 6, -; nextln: value: U16( -; nextln: 0, -; nextln: ), -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 19, -; nextln: op: SmallStackAlloc, -; nextln: info: 12, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 15, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 15, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 13, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 14, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 11, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 13, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 9, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 12, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 7, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 7, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 6, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 6, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 5, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 3, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 4, -; nextln: op: SetFramePointer, -; nextln: info: 0, -; nextln: value: None, -; nextln: }, -; nextln: UnwindCode { -; nextln: offset: 1, -; nextln: op: PushNonvolatileRegister, -; nextln: info: 5, -; nextln: value: None, -; nextln: }, -; nextln: ], -; nextln: } +; sameln: version: 1 +; nextln: flags: 0 +; nextln: prologue size: 41 +; nextln: frame register: 5 +; nextln: frame register offset: 10 +; nextln: unwind codes: 13 +; nextln: +; nextln: offset: 1 +; nextln: op: PushNonvolatileRegister +; nextln: info: 5 +; nextln: +; nextln: offset: 4 +; nextln: op: SetFramePointer +; nextln: info: 0 +; nextln: +; nextln: offset: 5 +; nextln: op: PushNonvolatileRegister +; nextln: info: 3 +; nextln: +; nextln: offset: 6 +; nextln: op: PushNonvolatileRegister +; nextln: info: 6 +; nextln: +; nextln: offset: 7 +; nextln: op: PushNonvolatileRegister +; nextln: info: 7 +; nextln: +; nextln: offset: 9 +; nextln: op: PushNonvolatileRegister +; nextln: info: 12 +; nextln: +; nextln: offset: 11 +; nextln: op: PushNonvolatileRegister +; nextln: info: 13 +; nextln: +; nextln: offset: 13 +; nextln: op: PushNonvolatileRegister +; nextln: info: 14 +; nextln: +; nextln: offset: 15 +; nextln: op: PushNonvolatileRegister +; nextln: info: 15 +; nextln: +; nextln: offset: 19 +; nextln: op: SmallStackAlloc +; nextln: info: 12 +; nextln: +; nextln: offset: 31 +; nextln: op: SaveXmm128 +; nextln: info: 6 +; nextln: value: 0 (u16) +; nextln: +; nextln: offset: 36 +; nextln: op: SaveXmm128 +; nextln: info: 7 +; nextln: value: 1 (u16) +; nextln: +; nextln: offset: 41 +; nextln: op: SaveXmm128 +; nextln: info: 8 +; nextln: value: 2 (u16) diff --git a/cranelift/filetests/filetests/isa/x86/windows_systemv_x64_fde.clif b/cranelift/filetests/filetests/isa/x86/windows_systemv_x64_fde.clif deleted file mode 100644 index 31b75b6c16d9..000000000000 --- a/cranelift/filetests/filetests/isa/x86/windows_systemv_x64_fde.clif +++ /dev/null @@ -1,54 +0,0 @@ -test fde -set opt_level=speed_and_size -set is_pic -target x86_64 haswell - -; check that there is no libunwind information for a windows_fastcall function -function %not_fastcall() windows_fastcall { -block0: - return -} -; sameln: No unwind information. - -; check the libunwind information with a function with no args -function %no_args() system_v { -block0: - return -} -; sameln: 0x00000000: CIE -; nextln: length: 0x00000014 -; nextln: version: 0x01 -; nextln: code_align: 1 -; nextln: data_align: -8 -; nextln: ra_register: 0x10 -; nextln: DW_CFA_def_cfa (r7, 8) -; nextln: DW_CFA_offset (r16, 1) -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: Instructions: Init State: -; nextln: -; nextln: -; nextln: 0x00000018: FDE -; nextln: length: 0x00000024 -; nextln: CIE_pointer: 0x00000000 -; nextln: start_addr: 0x0000000000000000 -; nextln: range_size: 0x0000000000000006 (end_addr = 0x0000000000000006) -; nextln: Instructions: -; nextln: DW_CFA_advance_loc (1) -; nextln: DW_CFA_def_cfa_offset (16) -; nextln: DW_CFA_offset (r6, 2) -; nextln: DW_CFA_advance_loc (3) -; nextln: DW_CFA_def_cfa_register (r6) -; nextln: DW_CFA_advance_loc (1) -; nextln: DW_CFA_def_cfa (r7, 8) -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: DW_CFA_nop -; nextln: -; nextln: Entry: 24 -; nextln: Relocs: [(Abs8, 32)] diff --git a/cranelift/filetests/src/lib.rs b/cranelift/filetests/src/lib.rs index 0fee249f12c4..aa3d443ee2b9 100644 --- a/cranelift/filetests/src/lib.rs +++ b/cranelift/filetests/src/lib.rs @@ -42,7 +42,6 @@ mod test_cat; mod test_compile; mod test_dce; mod test_domtree; -mod test_fde; mod test_legalizer; mod test_licm; mod test_postopt; @@ -140,7 +139,6 @@ fn new_subtest(parsed: &TestCommand) -> subtest::SubtestResult test_preopt::subtest(parsed), "safepoint" => test_safepoint::subtest(parsed), "unwind" => test_unwind::subtest(parsed), - "fde" => test_fde::subtest(parsed), _ => Err(format!("unknown test command '{}'", parsed.command)), } } diff --git a/cranelift/filetests/src/test_fde.rs b/cranelift/filetests/src/test_fde.rs deleted file mode 100644 index 5a9305479bba..000000000000 --- a/cranelift/filetests/src/test_fde.rs +++ /dev/null @@ -1,417 +0,0 @@ -//! Test command for verifying the unwind emitted for each function. -//! -//! The `unwind` test command runs each function through the full code generator pipeline. -#![cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] - -use crate::subtest::{run_filecheck, Context, SubTest, SubtestResult}; -use cranelift_codegen; -use cranelift_codegen::binemit::{FrameUnwindKind, FrameUnwindOffset, FrameUnwindSink, Reloc}; -use cranelift_codegen::ir; -use cranelift_reader::TestCommand; -use std::borrow::Cow; -use std::fmt::Write; - -struct TestUnwind; - -pub fn subtest(parsed: &TestCommand) -> SubtestResult> { - assert_eq!(parsed.command, "fde"); - if !parsed.options.is_empty() { - Err(format!("No options allowed on {}", parsed)) - } else { - Ok(Box::new(TestUnwind)) - } -} - -impl SubTest for TestUnwind { - fn name(&self) -> &'static str { - "fde" - } - - fn is_mutating(&self) -> bool { - false - } - - fn needs_isa(&self) -> bool { - true - } - - fn run(&self, func: Cow, context: &Context) -> SubtestResult<()> { - let isa = context.isa.expect("unwind needs an ISA"); - - if func.signature.call_conv != cranelift_codegen::isa::CallConv::SystemV { - return run_filecheck(&"No unwind information.", context); - } - - let mut comp_ctx = cranelift_codegen::Context::for_function(func.into_owned()); - comp_ctx.func.collect_frame_layout_info(); - - comp_ctx.compile(isa).expect("failed to compile function"); - - struct SimpleUnwindSink(pub Vec, pub usize, pub Vec<(Reloc, usize)>); - impl FrameUnwindSink for SimpleUnwindSink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reloc(&mut self, r: Reloc, off: FrameUnwindOffset) { - self.2.push((r, off)); - } - fn set_entry_offset(&mut self, off: FrameUnwindOffset) { - self.1 = off; - } - } - - let mut sink = SimpleUnwindSink(Vec::new(), 0, Vec::new()); - comp_ctx - .emit_unwind_info(isa, FrameUnwindKind::Libunwind, &mut sink) - .expect("can emit unwind info"); - - let mut text = String::new(); - if sink.0.is_empty() { - writeln!(text, "No unwind information.").unwrap(); - } else { - print_unwind_info(&mut text, &sink.0, isa.pointer_bytes()); - writeln!(text, "Entry: {}", sink.1).unwrap(); - writeln!(text, "Relocs: {:?}", sink.2).unwrap(); - } - - run_filecheck(&text, context) - } -} - -fn register_name<'a>(register: gimli::Register) -> std::borrow::Cow<'a, str> { - Cow::Owned(format!("r{}", register.0)) -} - -fn print_unwind_info(text: &mut String, mem: &[u8], address_size: u8) { - let mut eh_frame = gimli::EhFrame::new(mem, gimli::LittleEndian); - eh_frame.set_address_size(address_size); - let bases = gimli::BaseAddresses::default(); - dwarfdump::dump_eh_frame(text, &eh_frame, &bases, ®ister_name).unwrap(); -} - -mod dwarfdump { - // Copied from https://github.com/gimli-rs/gimli/blob/1e49ffc9af4ec64a1b7316924d73c933dd7157c5/examples/dwarfdump.rs - use gimli::UnwindSection; - use std::borrow::Cow; - use std::collections::HashMap; - use std::fmt::{self, Debug, Write}; - use std::result; - - #[derive(Debug, Clone, Copy, PartialEq, Eq)] - pub(super) enum Error { - GimliError(gimli::Error), - IoError, - } - - impl fmt::Display for Error { - #[inline] - fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> { - Debug::fmt(self, f) - } - } - - impl From for Error { - fn from(err: gimli::Error) -> Self { - Self::GimliError(err) - } - } - - impl From for Error { - fn from(_: fmt::Error) -> Self { - Self::IoError - } - } - - pub(super) type Result = result::Result; - - pub(super) trait Reader: gimli::Reader + Send + Sync {} - - impl<'input, Endian> Reader for gimli::EndianSlice<'input, Endian> where - Endian: gimli::Endianity + Send + Sync - { - } - - pub(super) fn dump_eh_frame( - w: &mut W, - eh_frame: &gimli::EhFrame, - bases: &gimli::BaseAddresses, - register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, - ) -> Result<()> { - let mut cies = HashMap::new(); - - let mut entries = eh_frame.entries(bases); - loop { - match entries.next()? { - None => return Ok(()), - Some(gimli::CieOrFde::Cie(cie)) => { - writeln!(w, "{:#010x}: CIE", cie.offset())?; - writeln!(w, " length: {:#010x}", cie.entry_len())?; - // TODO: CIE_id - writeln!(w, " version: {:#04x}", cie.version())?; - // TODO: augmentation - writeln!(w, " code_align: {}", cie.code_alignment_factor())?; - writeln!(w, " data_align: {}", cie.data_alignment_factor())?; - writeln!(w, " ra_register: {:#x}", cie.return_address_register().0)?; - if let Some(encoding) = cie.lsda_encoding() { - writeln!(w, " lsda_encoding: {:#02x}", encoding.0)?; - } - if let Some((encoding, personality)) = cie.personality_with_encoding() { - write!(w, " personality: {:#02x} ", encoding.0)?; - dump_pointer(w, personality)?; - writeln!(w)?; - } - if let Some(encoding) = cie.fde_address_encoding() { - writeln!(w, " fde_encoding: {:#02x}", encoding.0)?; - } - dump_cfi_instructions( - w, - cie.instructions(eh_frame, bases), - true, - register_name, - )?; - writeln!(w)?; - } - Some(gimli::CieOrFde::Fde(partial)) => { - let mut offset = None; - let fde = partial.parse(|_, bases, o| { - offset = Some(o); - cies.entry(o) - .or_insert_with(|| eh_frame.cie_from_offset(bases, o)) - .clone() - })?; - - writeln!(w)?; - writeln!(w, "{:#010x}: FDE", fde.offset())?; - writeln!(w, " length: {:#010x}", fde.entry_len())?; - writeln!(w, " CIE_pointer: {:#010x}", offset.unwrap().0)?; - // TODO: symbolicate the start address like the canonical dwarfdump does. - writeln!(w, " start_addr: {:#018x}", fde.initial_address())?; - writeln!( - w, - " range_size: {:#018x} (end_addr = {:#018x})", - fde.len(), - fde.initial_address() + fde.len() - )?; - if let Some(lsda) = fde.lsda() { - write!(w, " lsda: ")?; - dump_pointer(w, lsda)?; - writeln!(w)?; - } - dump_cfi_instructions( - w, - fde.instructions(eh_frame, bases), - false, - register_name, - )?; - writeln!(w)?; - } - } - } - } - - fn dump_pointer(w: &mut W, p: gimli::Pointer) -> Result<()> { - match p { - gimli::Pointer::Direct(p) => { - write!(w, "{:#018x}", p)?; - } - gimli::Pointer::Indirect(p) => { - write!(w, "({:#018x})", p)?; - } - } - Ok(()) - } - - #[allow(clippy::unneeded_field_pattern)] - fn dump_cfi_instructions( - w: &mut W, - mut insns: gimli::CallFrameInstructionIter, - is_initial: bool, - register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, - ) -> Result<()> { - use gimli::CallFrameInstruction::*; - - // TODO: we need to actually evaluate these instructions as we iterate them - // so we can print the initialized state for CIEs, and each unwind row's - // registers for FDEs. - // - // TODO: We should print DWARF expressions for the CFI instructions that - // embed DWARF expressions within themselves. - - if !is_initial { - writeln!(w, " Instructions:")?; - } - - loop { - match insns.next() { - Err(e) => { - writeln!(w, "Failed to decode CFI instruction: {}", e)?; - return Ok(()); - } - Ok(None) => { - if is_initial { - writeln!(w, " Instructions: Init State:")?; - } - return Ok(()); - } - Ok(Some(op)) => match op { - SetLoc { address } => { - writeln!(w, " DW_CFA_set_loc ({:#x})", address)?; - } - AdvanceLoc { delta } => { - writeln!(w, " DW_CFA_advance_loc ({})", delta)?; - } - DefCfa { register, offset } => { - writeln!( - w, - " DW_CFA_def_cfa ({}, {})", - register_name(register), - offset - )?; - } - DefCfaSf { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_def_cfa_sf ({}, {})", - register_name(register), - factored_offset - )?; - } - DefCfaRegister { register } => { - writeln!( - w, - " DW_CFA_def_cfa_register ({})", - register_name(register) - )?; - } - DefCfaOffset { offset } => { - writeln!(w, " DW_CFA_def_cfa_offset ({})", offset)?; - } - DefCfaOffsetSf { factored_offset } => { - writeln!( - w, - " DW_CFA_def_cfa_offset_sf ({})", - factored_offset - )?; - } - DefCfaExpression { expression: _ } => { - writeln!(w, " DW_CFA_def_cfa_expression (...)")?; - } - Undefined { register } => { - writeln!( - w, - " DW_CFA_undefined ({})", - register_name(register) - )?; - } - SameValue { register } => { - writeln!( - w, - " DW_CFA_same_value ({})", - register_name(register) - )?; - } - Offset { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_offset ({}, {})", - register_name(register), - factored_offset - )?; - } - OffsetExtendedSf { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_offset_extended_sf ({}, {})", - register_name(register), - factored_offset - )?; - } - ValOffset { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_val_offset ({}, {})", - register_name(register), - factored_offset - )?; - } - ValOffsetSf { - register, - factored_offset, - } => { - writeln!( - w, - " DW_CFA_val_offset_sf ({}, {})", - register_name(register), - factored_offset - )?; - } - Register { - dest_register, - src_register, - } => { - writeln!( - w, - " DW_CFA_register ({}, {})", - register_name(dest_register), - register_name(src_register) - )?; - } - Expression { - register, - expression: _, - } => { - writeln!( - w, - " DW_CFA_expression ({}, ...)", - register_name(register) - )?; - } - ValExpression { - register, - expression: _, - } => { - writeln!( - w, - " DW_CFA_val_expression ({}, ...)", - register_name(register) - )?; - } - Restore { register } => { - writeln!( - w, - " DW_CFA_restore ({})", - register_name(register) - )?; - } - RememberState => { - writeln!(w, " DW_CFA_remember_state")?; - } - RestoreState => { - writeln!(w, " DW_CFA_restore_state")?; - } - ArgsSize { size } => { - writeln!(w, " DW_CFA_GNU_args_size ({})", size)?; - } - Nop => { - writeln!(w, " DW_CFA_nop")?; - } - }, - } - } - } -} diff --git a/cranelift/filetests/src/test_unwind.rs b/cranelift/filetests/src/test_unwind.rs index bf51cb73db0f..faa4ec6cee05 100644 --- a/cranelift/filetests/src/test_unwind.rs +++ b/cranelift/filetests/src/test_unwind.rs @@ -4,13 +4,13 @@ #![cfg_attr(feature = "cargo-clippy", allow(clippy::cast_ptr_alignment))] use crate::subtest::{run_filecheck, Context, SubTest, SubtestResult}; -use byteorder::{ByteOrder, LittleEndian}; -use cranelift_codegen; -use cranelift_codegen::binemit::{FrameUnwindKind, FrameUnwindOffset, FrameUnwindSink, Reloc}; -use cranelift_codegen::ir; +use cranelift_codegen::{self, ir, isa::unwind::UnwindInfo}; use cranelift_reader::TestCommand; +use gimli::{ + write::{Address, EhFrame, EndianVec, FrameTable}, + LittleEndian, +}; use std::borrow::Cow; -use std::fmt::Write; struct TestUnwind; @@ -42,175 +42,526 @@ impl SubTest for TestUnwind { comp_ctx.compile(isa).expect("failed to compile function"); - struct Sink(Vec); - impl FrameUnwindSink for Sink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() + let mut text = String::new(); + match comp_ctx.create_unwind_info(isa).expect("unwind info") { + Some(UnwindInfo::WindowsX64(info)) => { + let mut mem = vec![0; info.emit_size()]; + info.emit(&mut mem); + windowsx64::dump(&mut text, &mem); } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); + Some(UnwindInfo::SystemV(info)) => { + let mut table = FrameTable::default(); + let cie = isa + .create_systemv_cie() + .expect("the ISA should support a System V CIE"); + + let cie_id = table.add_cie(cie); + table.add_fde(cie_id, info.to_fde(Address::Constant(0))); + + let mut eh_frame = EhFrame(EndianVec::new(LittleEndian)); + table.write_eh_frame(&mut eh_frame).unwrap(); + systemv::dump(&mut text, &eh_frame.0.into_vec(), isa.pointer_bytes()) } - fn reloc(&mut self, _: Reloc, _: FrameUnwindOffset) { - unimplemented!(); + None => {} + } + + run_filecheck(&text, context) + } +} + +mod windowsx64 { + use byteorder::{ByteOrder, LittleEndian}; + use std::fmt::Write; + + pub fn dump(text: &mut W, mem: &[u8]) { + let info = UnwindInfo::from_slice(mem); + + writeln!(text, " version: {}", info.version).unwrap(); + writeln!(text, " flags: {}", info.flags).unwrap(); + writeln!(text, " prologue size: {}", info.prologue_size).unwrap(); + writeln!(text, " frame register: {}", info.frame_register).unwrap(); + writeln!( + text, + "frame register offset: {}", + info.frame_register_offset + ) + .unwrap(); + writeln!(text, " unwind codes: {}", info.unwind_codes.len()).unwrap(); + + for code in info.unwind_codes.iter().rev() { + writeln!(text).unwrap(); + writeln!(text, " offset: {}", code.offset).unwrap(); + writeln!(text, " op: {:?}", code.op).unwrap(); + writeln!(text, " info: {}", code.info).unwrap(); + match code.value { + UnwindValue::None => {} + UnwindValue::U16(v) => { + writeln!(text, " value: {} (u16)", v).unwrap() + } + UnwindValue::U32(v) => { + writeln!(text, " value: {} (u32)", v).unwrap() + } + }; + } + } + + #[derive(Debug)] + struct UnwindInfo { + version: u8, + flags: u8, + prologue_size: u8, + unwind_code_count_raw: u8, + frame_register: u8, + frame_register_offset: u8, + unwind_codes: Vec, + } + + impl UnwindInfo { + fn from_slice(mem: &[u8]) -> Self { + let version_and_flags = mem[0]; + let prologue_size = mem[1]; + let unwind_code_count_raw = mem[2]; + let frame_register_and_offset = mem[3]; + let mut unwind_codes = Vec::new(); + + let mut i = 0; + while i < unwind_code_count_raw { + let code = UnwindCode::from_slice(&mem[(4 + (i * 2) as usize)..]); + + i += match &code.value { + UnwindValue::None => 1, + UnwindValue::U16(_) => 2, + UnwindValue::U32(_) => 3, + }; + + unwind_codes.push(code); } - fn set_entry_offset(&mut self, _: FrameUnwindOffset) { - unimplemented!(); + + Self { + version: version_and_flags & 0x3, + flags: (version_and_flags & 0xF8) >> 3, + prologue_size, + unwind_code_count_raw, + frame_register: frame_register_and_offset & 0xF, + frame_register_offset: (frame_register_and_offset & 0xF0) >> 4, + unwind_codes, } } + } - let mut sink = Sink(Vec::new()); - comp_ctx - .emit_unwind_info(isa, FrameUnwindKind::Fastcall, &mut sink) - .expect("can emit unwind info"); + #[derive(Debug)] + struct UnwindCode { + offset: u8, + op: UnwindOperation, + info: u8, + value: UnwindValue, + } - let mut text = String::new(); - if sink.0.is_empty() { - writeln!(text, "No unwind information.").unwrap(); - } else { - print_unwind_info(&mut text, &sink.0); + impl UnwindCode { + fn from_slice(mem: &[u8]) -> Self { + let offset = mem[0]; + let op_and_info = mem[1]; + let op = UnwindOperation::from(op_and_info & 0xF); + let info = (op_and_info & 0xF0) >> 4; + + let value = match op { + UnwindOperation::LargeStackAlloc => match info { + 0 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), + 1 => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), + _ => panic!("unexpected stack alloc info value"), + }, + UnwindOperation::SaveNonvolatileRegister => { + UnwindValue::U16(LittleEndian::read_u16(&mem[2..])) + } + UnwindOperation::SaveNonvolatileRegisterFar => { + UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + } + UnwindOperation::SaveXmm128 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), + UnwindOperation::SaveXmm128Far => { + UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + } + _ => UnwindValue::None, + }; + + Self { + offset, + op, + info, + value, + } } + } - run_filecheck(&text, context) + #[derive(Debug)] + enum UnwindOperation { + PushNonvolatileRegister = 0, + LargeStackAlloc = 1, + SmallStackAlloc = 2, + SetFramePointer = 3, + SaveNonvolatileRegister = 4, + SaveNonvolatileRegisterFar = 5, + SaveXmm128 = 8, + SaveXmm128Far = 9, + PushMachineFrame = 10, } -} -fn print_unwind_info(text: &mut String, mem: &[u8]) { - let info = UnwindInfo::from_slice(mem); - - // Assert correct alignment and padding of the unwind information - assert!(mem.len() % 4 == 0); - assert_eq!( - mem.len(), - 4 + ((info.unwind_code_count_raw as usize) * 2) - + if (info.unwind_code_count_raw & 1) == 1 { - 2 - } else { - 0 + impl From for UnwindOperation { + fn from(value: u8) -> Self { + // The numerical value is specified as part of the Windows x64 ABI + match value { + 0 => Self::PushNonvolatileRegister, + 1 => Self::LargeStackAlloc, + 2 => Self::SmallStackAlloc, + 3 => Self::SetFramePointer, + 4 => Self::SaveNonvolatileRegister, + 5 => Self::SaveNonvolatileRegisterFar, + 8 => Self::SaveXmm128, + 9 => Self::SaveXmm128Far, + 10 => Self::PushMachineFrame, + _ => panic!("unsupported unwind operation"), } - ); + } + } - writeln!(text, "{:#?}", info).unwrap(); + #[derive(Debug)] + enum UnwindValue { + None, + U16(u16), + U32(u32), + } } -#[derive(Debug)] -struct UnwindInfo { - pub version: u8, - pub flags: u8, - pub prologue_size: u8, - pub unwind_code_count_raw: u8, - pub frame_register: u8, - pub frame_register_offset: u8, - pub unwind_codes: Vec, -} +mod systemv { + fn register_name<'a>(register: gimli::Register) -> std::borrow::Cow<'a, str> { + Cow::Owned(format!("r{}", register.0)) + } -impl UnwindInfo { - fn from_slice(mem: &[u8]) -> Self { - let version_and_flags = mem[0]; - let prologue_size = mem[1]; - let unwind_code_count_raw = mem[2]; - let frame_register_and_offset = mem[3]; - let mut unwind_codes = Vec::new(); - - let mut i = 0; - while i < unwind_code_count_raw { - let code = UnwindCode::from_slice(&mem[(4 + (i * 2) as usize)..]); - - i += match &code.value { - UnwindValue::None => 1, - UnwindValue::U16(_) => 2, - UnwindValue::U32(_) => 3, - }; + pub fn dump(text: &mut W, bytes: &[u8], address_size: u8) { + let mut eh_frame = gimli::EhFrame::new(bytes, gimli::LittleEndian); + eh_frame.set_address_size(address_size); + let bases = gimli::BaseAddresses::default(); + dump_eh_frame(text, &eh_frame, &bases, ®ister_name).unwrap(); + } + + // Remainder copied from https://github.com/gimli-rs/gimli/blob/1e49ffc9af4ec64a1b7316924d73c933dd7157c5/examples/dwarfdump.rs + use gimli::UnwindSection; + use std::borrow::Cow; + use std::collections::HashMap; + use std::fmt::{self, Debug, Write}; + use std::result; + + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + pub(super) enum Error { + GimliError(gimli::Error), + IoError, + } - unwind_codes.push(code); + impl fmt::Display for Error { + #[inline] + fn fmt(&self, f: &mut fmt::Formatter) -> ::std::result::Result<(), fmt::Error> { + Debug::fmt(self, f) } + } - Self { - version: version_and_flags & 0x3, - flags: (version_and_flags & 0xF8) >> 3, - prologue_size, - unwind_code_count_raw, - frame_register: frame_register_and_offset & 0xF, - frame_register_offset: (frame_register_and_offset & 0xF0) >> 4, - unwind_codes, + impl From for Error { + fn from(err: gimli::Error) -> Self { + Self::GimliError(err) } } -} -#[derive(Debug)] -struct UnwindCode { - pub offset: u8, - pub op: UnwindOperation, - pub info: u8, - pub value: UnwindValue, -} + impl From for Error { + fn from(_: fmt::Error) -> Self { + Self::IoError + } + } + + pub(super) type Result = result::Result; + + pub(super) trait Reader: gimli::Reader + Send + Sync {} -impl UnwindCode { - fn from_slice(mem: &[u8]) -> Self { - let offset = mem[0]; - let op_and_info = mem[1]; - let op = UnwindOperation::from(op_and_info & 0xF); - let info = (op_and_info & 0xF0) >> 4; - - let value = match op { - UnwindOperation::LargeStackAlloc => match info { - 0 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - 1 => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), - _ => panic!("unexpected stack alloc info value"), - }, - UnwindOperation::SaveNonvolatileRegister => { - UnwindValue::U16(LittleEndian::read_u16(&mem[2..])) + impl<'input, Endian> Reader for gimli::EndianSlice<'input, Endian> where + Endian: gimli::Endianity + Send + Sync + { + } + + pub(super) fn dump_eh_frame( + w: &mut W, + eh_frame: &gimli::EhFrame, + bases: &gimli::BaseAddresses, + register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, + ) -> Result<()> { + let mut cies = HashMap::new(); + + let mut entries = eh_frame.entries(bases); + loop { + match entries.next()? { + None => return Ok(()), + Some(gimli::CieOrFde::Cie(cie)) => { + writeln!(w, "{:#010x}: CIE", cie.offset())?; + writeln!(w, " length: {:#010x}", cie.entry_len())?; + // TODO: CIE_id + writeln!(w, " version: {:#04x}", cie.version())?; + // TODO: augmentation + writeln!(w, " code_align: {}", cie.code_alignment_factor())?; + writeln!(w, " data_align: {}", cie.data_alignment_factor())?; + writeln!(w, " ra_register: {:#x}", cie.return_address_register().0)?; + if let Some(encoding) = cie.lsda_encoding() { + writeln!(w, " lsda_encoding: {:#02x}", encoding.0)?; + } + if let Some((encoding, personality)) = cie.personality_with_encoding() { + write!(w, " personality: {:#02x} ", encoding.0)?; + dump_pointer(w, personality)?; + writeln!(w)?; + } + if let Some(encoding) = cie.fde_address_encoding() { + writeln!(w, " fde_encoding: {:#02x}", encoding.0)?; + } + dump_cfi_instructions( + w, + cie.instructions(eh_frame, bases), + true, + register_name, + )?; + writeln!(w)?; + } + Some(gimli::CieOrFde::Fde(partial)) => { + let mut offset = None; + let fde = partial.parse(|_, bases, o| { + offset = Some(o); + cies.entry(o) + .or_insert_with(|| eh_frame.cie_from_offset(bases, o)) + .clone() + })?; + + writeln!(w)?; + writeln!(w, "{:#010x}: FDE", fde.offset())?; + writeln!(w, " length: {:#010x}", fde.entry_len())?; + writeln!(w, " CIE_pointer: {:#010x}", offset.unwrap().0)?; + // TODO: symbolicate the start address like the canonical dwarfdump does. + writeln!(w, " start_addr: {:#018x}", fde.initial_address())?; + writeln!( + w, + " range_size: {:#018x} (end_addr = {:#018x})", + fde.len(), + fde.initial_address() + fde.len() + )?; + if let Some(lsda) = fde.lsda() { + write!(w, " lsda: ")?; + dump_pointer(w, lsda)?; + writeln!(w)?; + } + dump_cfi_instructions( + w, + fde.instructions(eh_frame, bases), + false, + register_name, + )?; + writeln!(w)?; + } + } + } + } + + fn dump_pointer(w: &mut W, p: gimli::Pointer) -> Result<()> { + match p { + gimli::Pointer::Direct(p) => { + write!(w, "{:#018x}", p)?; } - UnwindOperation::SaveNonvolatileRegisterFar => { - UnwindValue::U32(LittleEndian::read_u32(&mem[2..])) + gimli::Pointer::Indirect(p) => { + write!(w, "({:#018x})", p)?; } - UnwindOperation::SaveXmm128 => UnwindValue::U16(LittleEndian::read_u16(&mem[2..])), - UnwindOperation::SaveXmm128Far => UnwindValue::U32(LittleEndian::read_u32(&mem[2..])), - _ => UnwindValue::None, - }; - - Self { - offset, - op, - info, - value, } + Ok(()) } -} -#[derive(Debug)] -enum UnwindOperation { - PushNonvolatileRegister = 0, - LargeStackAlloc = 1, - SmallStackAlloc = 2, - SetFramePointer = 3, - SaveNonvolatileRegister = 4, - SaveNonvolatileRegisterFar = 5, - SaveXmm128 = 8, - SaveXmm128Far = 9, - PushMachineFrame = 10, -} + #[allow(clippy::unneeded_field_pattern)] + fn dump_cfi_instructions( + w: &mut W, + mut insns: gimli::CallFrameInstructionIter, + is_initial: bool, + register_name: &dyn Fn(gimli::Register) -> Cow<'static, str>, + ) -> Result<()> { + use gimli::CallFrameInstruction::*; -impl From for UnwindOperation { - fn from(value: u8) -> Self { - // The numerical value is specified as part of the Windows x64 ABI - match value { - 0 => Self::PushNonvolatileRegister, - 1 => Self::LargeStackAlloc, - 2 => Self::SmallStackAlloc, - 3 => Self::SetFramePointer, - 4 => Self::SaveNonvolatileRegister, - 5 => Self::SaveNonvolatileRegisterFar, - 8 => Self::SaveXmm128, - 9 => Self::SaveXmm128Far, - 10 => Self::PushMachineFrame, - _ => panic!("unsupported unwind operation"), + // TODO: we need to actually evaluate these instructions as we iterate them + // so we can print the initialized state for CIEs, and each unwind row's + // registers for FDEs. + // + // TODO: We should print DWARF expressions for the CFI instructions that + // embed DWARF expressions within themselves. + + if !is_initial { + writeln!(w, " Instructions:")?; } - } -} -#[derive(Debug)] -enum UnwindValue { - None, - U16(u16), - U32(u32), + loop { + match insns.next() { + Err(e) => { + writeln!(w, "Failed to decode CFI instruction: {}", e)?; + return Ok(()); + } + Ok(None) => { + if is_initial { + writeln!(w, " Instructions: Init State:")?; + } + return Ok(()); + } + Ok(Some(op)) => match op { + SetLoc { address } => { + writeln!(w, " DW_CFA_set_loc ({:#x})", address)?; + } + AdvanceLoc { delta } => { + writeln!(w, " DW_CFA_advance_loc ({})", delta)?; + } + DefCfa { register, offset } => { + writeln!( + w, + " DW_CFA_def_cfa ({}, {})", + register_name(register), + offset + )?; + } + DefCfaSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_def_cfa_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + DefCfaRegister { register } => { + writeln!( + w, + " DW_CFA_def_cfa_register ({})", + register_name(register) + )?; + } + DefCfaOffset { offset } => { + writeln!(w, " DW_CFA_def_cfa_offset ({})", offset)?; + } + DefCfaOffsetSf { factored_offset } => { + writeln!( + w, + " DW_CFA_def_cfa_offset_sf ({})", + factored_offset + )?; + } + DefCfaExpression { expression: _ } => { + writeln!(w, " DW_CFA_def_cfa_expression (...)")?; + } + Undefined { register } => { + writeln!( + w, + " DW_CFA_undefined ({})", + register_name(register) + )?; + } + SameValue { register } => { + writeln!( + w, + " DW_CFA_same_value ({})", + register_name(register) + )?; + } + Offset { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_offset ({}, {})", + register_name(register), + factored_offset + )?; + } + OffsetExtendedSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_offset_extended_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + ValOffset { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_val_offset ({}, {})", + register_name(register), + factored_offset + )?; + } + ValOffsetSf { + register, + factored_offset, + } => { + writeln!( + w, + " DW_CFA_val_offset_sf ({}, {})", + register_name(register), + factored_offset + )?; + } + Register { + dest_register, + src_register, + } => { + writeln!( + w, + " DW_CFA_register ({}, {})", + register_name(dest_register), + register_name(src_register) + )?; + } + Expression { + register, + expression: _, + } => { + writeln!( + w, + " DW_CFA_expression ({}, ...)", + register_name(register) + )?; + } + ValExpression { + register, + expression: _, + } => { + writeln!( + w, + " DW_CFA_val_expression ({}, ...)", + register_name(register) + )?; + } + Restore { register } => { + writeln!( + w, + " DW_CFA_restore ({})", + register_name(register) + )?; + } + RememberState => { + writeln!(w, " DW_CFA_remember_state")?; + } + RestoreState => { + writeln!(w, " DW_CFA_restore_state")?; + } + ArgsSize { size } => { + writeln!(w, " DW_CFA_GNU_args_size ({})", size)?; + } + Nop => { + writeln!(w, " DW_CFA_nop")?; + } + }, + } + } + } } diff --git a/crates/api/src/trampoline/func.rs b/crates/api/src/trampoline/func.rs index fbfa0d3e811a..072d99dd2f3b 100644 --- a/crates/api/src/trampoline/func.rs +++ b/crates/api/src/trampoline/func.rs @@ -10,9 +10,7 @@ use std::mem; use std::panic::{self, AssertUnwindSafe}; use wasmtime_environ::entity::PrimaryMap; use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::{ - ir, settings, CompiledFunction, CompiledFunctionUnwindInfo, Export, Module, -}; +use wasmtime_environ::{ir, settings, CompiledFunction, Export, Module}; use wasmtime_jit::trampoline::ir::{ ExternalName, Function, InstBuilder, MemFlags, StackSlotData, StackSlotKind, }; @@ -112,7 +110,6 @@ fn make_trampoline( let mut context = Context::new(); context.func = Function::with_name_signature(ExternalName::user(0, 0), signature.clone()); - context.func.collect_frame_layout_info(); let ss = context.func.create_stack_slot(StackSlotData::new( StackSlotKind::ExplicitSlot, @@ -188,9 +185,10 @@ fn make_trampoline( .map_err(|error| pretty_error(&context.func, Some(isa), error)) .expect("compile_and_emit"); - let unwind_info = CompiledFunctionUnwindInfo::new(isa, &context) + let unwind_info = context + .create_unwind_info(isa) .map_err(|error| pretty_error(&context.func, Some(isa), error)) - .expect("emit unwind info"); + .expect("create unwind information"); code_memory .allocate_for_function(&CompiledFunction { @@ -251,7 +249,7 @@ pub fn create_handle_with_function( // Next up we wrap everything up into an `InstanceHandle` by publishing our // code memory (makes it executable) and ensuring all our various bits of // state make it into the instance constructors. - code_memory.publish(); + code_memory.publish(isa.as_ref()); let trampoline_state = TrampolineState { func, code_memory }; create_handle( module, diff --git a/crates/debug/src/frame.rs b/crates/debug/src/frame.rs deleted file mode 100644 index ab5b60de8455..000000000000 --- a/crates/debug/src/frame.rs +++ /dev/null @@ -1,139 +0,0 @@ -use std::collections::HashMap; -use wasmtime_environ::entity::EntityRef; -use wasmtime_environ::isa::{CallConv, TargetIsa}; -use wasmtime_environ::wasm::DefinedFuncIndex; -use wasmtime_environ::{FrameLayoutChange, FrameLayouts}; - -use gimli::write::{ - Address, CallFrameInstruction, CommonInformationEntry as CIEEntry, Error, - FrameDescriptionEntry as FDEEntry, FrameTable, -}; -use gimli::{Encoding, Format, Register, X86_64}; - -fn to_cfi( - isa: &dyn TargetIsa, - change: &FrameLayoutChange, - cfa_def_reg: &mut Register, - cfa_def_offset: &mut i32, -) -> Option { - Some(match change { - FrameLayoutChange::CallFrameAddressAt { reg, offset } => { - let mapped = match isa.map_dwarf_register(*reg) { - Ok(r) => Register(r), - Err(_) => return None, - }; - let offset = (*offset) as i32; - if mapped != *cfa_def_reg && offset != *cfa_def_offset { - *cfa_def_reg = mapped; - *cfa_def_offset = offset; - CallFrameInstruction::Cfa(mapped, offset) - } else if offset != *cfa_def_offset { - *cfa_def_offset = offset; - CallFrameInstruction::CfaOffset(offset) - } else if mapped != *cfa_def_reg { - *cfa_def_reg = mapped; - CallFrameInstruction::CfaRegister(mapped) - } else { - return None; - } - } - FrameLayoutChange::RegAt { reg, cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - let mapped = match isa.map_dwarf_register(*reg) { - Ok(r) => Register(r), - Err(_) => return None, - }; - CallFrameInstruction::Offset(mapped, cfa_offset) - } - FrameLayoutChange::ReturnAddressAt { cfa_offset } => { - assert!(cfa_offset % -8 == 0); - let cfa_offset = *cfa_offset as i32; - CallFrameInstruction::Offset(X86_64::RA, cfa_offset) - } - FrameLayoutChange::Preserve => CallFrameInstruction::RememberState, - FrameLayoutChange::Restore => CallFrameInstruction::RestoreState, - }) -} - -pub fn get_debug_frame_bytes( - funcs: &[(*const u8, usize)], - isa: &dyn TargetIsa, - layouts: &FrameLayouts, -) -> Result, Error> { - // FIXME Only x86-64 at this moment. - if isa.name() != "x86" || isa.pointer_bits() != 64 { - return Ok(None); - } - - let address_size = isa.pointer_bytes(); - let encoding = Encoding { - format: Format::Dwarf64, - version: 4, - address_size, - }; - - let mut frames = FrameTable::default(); - - let mut cached_cies = HashMap::new(); - - for (i, f) in funcs.into_iter().enumerate() { - let layout = &layouts[DefinedFuncIndex::new(i)]; - - // FIXME Can only process functions with SystemV-like prologue. - if layout.call_conv != CallConv::Fast - && layout.call_conv != CallConv::Cold - && layout.call_conv != CallConv::SystemV - { - continue; - } - - // Caching CIE with similar initial_commands. - let (cie_id, mut cfa_def_reg, mut cfa_def_offset) = { - use std::collections::hash_map::Entry; - match cached_cies.entry(&layout.initial_commands) { - Entry::Occupied(o) => *o.get(), - Entry::Vacant(v) => { - // cfa_def_reg and cfa_def_offset initialized with some random values. - let mut cfa_def_reg = X86_64::RA; - let mut cfa_def_offset = 0i32; - - // TODO adjust code_alignment_factor and data_alignment_factor based on ISA. - let mut cie = CIEEntry::new( - encoding, - /* code_alignment_factor = */ 1, - /* data_alignment_factor = */ -8, - /* return_address_register = */ X86_64::RA, - ); - for cmd in layout.initial_commands.iter() { - if let Some(instr) = to_cfi(isa, cmd, &mut cfa_def_reg, &mut cfa_def_offset) - { - cie.add_instruction(instr); - } - } - let cie_id = frames.add_cie(cie); - *v.insert((cie_id, cfa_def_reg, cfa_def_offset)) - } - } - }; - - let f_len = f.1 as u32; - let mut fde = FDEEntry::new( - Address::Symbol { - symbol: i, - addend: 0, - }, - f_len, - ); - - for (offset, cmd) in layout.commands.into_iter() { - if let Some(instr) = to_cfi(isa, cmd, &mut cfa_def_reg, &mut cfa_def_offset) { - fde.add_instruction(*offset as u32, instr); - } - } - - frames.add_fde(cie_id, fde); - } - - Ok(Some(frames)) -} diff --git a/crates/debug/src/lib.rs b/crates/debug/src/lib.rs index 2ca0e747c12d..e831570cf0cc 100644 --- a/crates/debug/src/lib.rs +++ b/crates/debug/src/lib.rs @@ -2,19 +2,18 @@ #![allow(clippy::cast_ptr_alignment)] -use crate::frame::get_debug_frame_bytes; use anyhow::Error; use faerie::{Artifact, Decl}; +use gimli::write::{Address, FrameTable}; use more_asserts::assert_gt; use target_lexicon::BinaryFormat; -use wasmtime_environ::isa::TargetIsa; -use wasmtime_environ::{FrameLayouts, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges}; +use wasmtime_environ::isa::{unwind::UnwindInfo, TargetIsa}; +use wasmtime_environ::{Compilation, ModuleAddressMap, ModuleVmctxInfo, ValueLabelsRanges}; pub use crate::read_debuginfo::{read_debuginfo, DebugInfoData, WasmFileInfo}; pub use crate::transform::transform_dwarf; pub use crate::write_debuginfo::{emit_dwarf, ResolvedSymbol, SymbolResolver}; -mod frame; mod gc; mod read_debuginfo; mod transform; @@ -28,6 +27,29 @@ impl SymbolResolver for FunctionRelocResolver { } } +fn create_frame_table<'a>( + isa: &dyn TargetIsa, + infos: impl Iterator>, +) -> Option { + let mut table = FrameTable::default(); + + let cie_id = table.add_cie(isa.create_systemv_cie()?); + + for (i, info) in infos.enumerate() { + if let Some(UnwindInfo::SystemV(info)) = info { + table.add_fde( + cie_id, + info.to_fde(Address::Symbol { + symbol: i, + addend: 0, + }), + ); + } + } + + Some(table) +} + pub fn emit_debugsections( obj: &mut Artifact, vmctx_info: &ModuleVmctxInfo, @@ -35,21 +57,13 @@ pub fn emit_debugsections( debuginfo_data: &DebugInfoData, at: &ModuleAddressMap, ranges: &ValueLabelsRanges, - frame_layouts: &FrameLayouts, + compilation: &Compilation, ) -> Result<(), Error> { let resolver = FunctionRelocResolver {}; let dwarf = transform_dwarf(isa, debuginfo_data, at, vmctx_info, ranges)?; + let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info)); - let max = at.values().map(|v| v.body_len).fold(0, usize::max); - let mut funcs_bodies = Vec::with_capacity(max as usize); - funcs_bodies.resize(max as usize, 0); - let funcs = at - .values() - .map(|v| (::std::ptr::null(), v.body_len)) - .collect::>(); - let frames = get_debug_frame_bytes(&funcs, isa, frame_layouts)?; - - emit_dwarf(obj, dwarf, &resolver, frames)?; + emit_dwarf(obj, dwarf, &resolver, frame_table)?; Ok(()) } @@ -70,8 +84,8 @@ pub fn emit_debugsections_image( vmctx_info: &ModuleVmctxInfo, at: &ModuleAddressMap, ranges: &ValueLabelsRanges, - frame_layouts: &FrameLayouts, funcs: &[(*const u8, usize)], + compilation: &Compilation, ) -> Result, Error> { let func_offsets = &funcs .iter() @@ -93,8 +107,8 @@ pub fn emit_debugsections_image( let body = unsafe { std::slice::from_raw_parts(segment_body.0, segment_body.1) }; obj.declare_with("all", Decl::function(), body.to_vec())?; - let frames = get_debug_frame_bytes(funcs, isa, frame_layouts)?; - emit_dwarf(&mut obj, dwarf, &resolver, frames)?; + let frame_table = create_frame_table(isa, compilation.into_iter().map(|f| &f.unwind_info)); + emit_dwarf(&mut obj, dwarf, &resolver, frame_table)?; // LLDB is too "magical" about mach-o, generating elf let mut bytes = obj.emit_as(BinaryFormat::Elf)?; diff --git a/crates/environ/src/cache.rs b/crates/environ/src/cache.rs index fc7e7c643d64..290a54033f88 100644 --- a/crates/environ/src/cache.rs +++ b/crates/environ/src/cache.rs @@ -1,6 +1,5 @@ use crate::address_map::{ModuleAddressMap, ValueLabelsRanges}; use crate::compilation::{Compilation, Relocations, Traps}; -use crate::frame_layout::FrameLayouts; use cranelift_codegen::ir; use cranelift_entity::PrimaryMap; use cranelift_wasm::DefinedFuncIndex; @@ -36,7 +35,6 @@ pub struct ModuleCacheData { value_ranges: ValueLabelsRanges, stack_slots: PrimaryMap, traps: Traps, - frame_layouts: FrameLayouts, } /// A type alias over the module cache data as a tuple. @@ -47,7 +45,6 @@ pub type ModuleCacheDataTupleType = ( ValueLabelsRanges, PrimaryMap, Traps, - FrameLayouts, ); struct Sha256Hasher(Sha256); @@ -207,7 +204,6 @@ impl ModuleCacheData { value_ranges: data.3, stack_slots: data.4, traps: data.5, - frame_layouts: data.6, } } @@ -219,7 +215,6 @@ impl ModuleCacheData { self.value_ranges, self.stack_slots, self.traps, - self.frame_layouts, ) } } diff --git a/crates/environ/src/cache/tests.rs b/crates/environ/src/cache/tests.rs index 35dc6c613e36..d454371178e0 100644 --- a/crates/environ/src/cache/tests.rs +++ b/crates/environ/src/cache/tests.rs @@ -100,6 +100,5 @@ fn new_module_cache_data() -> Result { PrimaryMap::new(), PrimaryMap::new(), PrimaryMap::new(), - PrimaryMap::new(), )) } diff --git a/crates/environ/src/compilation.rs b/crates/environ/src/compilation.rs index b56ac7b097b8..3382d89ae421 100644 --- a/crates/environ/src/compilation.rs +++ b/crates/environ/src/compilation.rs @@ -4,138 +4,13 @@ use crate::cache::ModuleCacheDataTupleType; use crate::CacheConfig; use crate::ModuleTranslation; -use cranelift_codegen::{binemit, ir, isa, CodegenResult, Context}; +use cranelift_codegen::{binemit, ir, isa, isa::unwind::UnwindInfo}; use cranelift_entity::PrimaryMap; use cranelift_wasm::{DefinedFuncIndex, FuncIndex, WasmError}; use serde::{Deserialize, Serialize}; use std::ops::Range; use thiserror::Error; -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct FDERelocEntry(pub i64, pub usize, pub u8); - -/// Relocation entry for unwind info. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub struct CompiledFunctionUnwindInfoReloc { - /// Entry offest in the code block. - pub offset: u32, - /// Entry addend relative to the code block. - pub addend: u32, -} - -/// Compiled function unwind information. -#[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] -pub enum CompiledFunctionUnwindInfo { - /// No info. - None, - /// Windows UNWIND_INFO. - Windows(Vec), - /// Frame layout info. - FrameLayout(Vec, usize, Vec), -} - -impl CompiledFunctionUnwindInfo { - /// Constructs unwind info object. - pub fn new(isa: &dyn isa::TargetIsa, context: &Context) -> CodegenResult { - use cranelift_codegen::binemit::{ - FrameUnwindKind, FrameUnwindOffset, FrameUnwindSink, Reloc, - }; - use cranelift_codegen::isa::CallConv; - - struct Sink(Vec, usize, Vec); - impl FrameUnwindSink for Sink { - fn len(&self) -> FrameUnwindOffset { - self.0.len() - } - fn bytes(&mut self, b: &[u8]) { - self.0.extend_from_slice(b); - } - fn reserve(&mut self, len: usize) { - self.0.reserve(len) - } - fn reloc(&mut self, r: Reloc, off: FrameUnwindOffset) { - self.2.push(FDERelocEntry( - 0, - off, - match r { - Reloc::Abs4 => 4, - Reloc::Abs8 => 8, - _ => { - panic!("unexpected reloc type"); - } - }, - )) - } - fn set_entry_offset(&mut self, off: FrameUnwindOffset) { - self.1 = off; - } - } - - let kind = match context.func.signature.call_conv { - CallConv::SystemV | CallConv::Fast | CallConv::Cold => FrameUnwindKind::Libunwind, - CallConv::WindowsFastcall => FrameUnwindKind::Fastcall, - _ => { - return Ok(CompiledFunctionUnwindInfo::None); - } - }; - - let mut sink = Sink(Vec::new(), 0, Vec::new()); - context.emit_unwind_info(isa, kind, &mut sink)?; - - let Sink(data, offset, relocs) = sink; - if data.is_empty() { - return Ok(CompiledFunctionUnwindInfo::None); - } - - let info = match kind { - FrameUnwindKind::Fastcall => CompiledFunctionUnwindInfo::Windows(data), - FrameUnwindKind::Libunwind => { - CompiledFunctionUnwindInfo::FrameLayout(data, offset, relocs) - } - }; - - Ok(info) - } - - /// Retuns true is no unwind info data. - pub fn is_empty(&self) -> bool { - match self { - CompiledFunctionUnwindInfo::None => true, - CompiledFunctionUnwindInfo::Windows(d) => d.is_empty(), - CompiledFunctionUnwindInfo::FrameLayout(c, _, _) => c.is_empty(), - } - } - - /// Returns size of serilized unwind info. - pub fn len(&self) -> usize { - match self { - CompiledFunctionUnwindInfo::None => 0, - CompiledFunctionUnwindInfo::Windows(d) => d.len(), - CompiledFunctionUnwindInfo::FrameLayout(c, _, _) => c.len(), - } - } - - /// Serializes data into byte array. - pub fn serialize(&self, dest: &mut [u8], relocs: &mut Vec) { - match self { - CompiledFunctionUnwindInfo::None => (), - CompiledFunctionUnwindInfo::Windows(d) => { - dest.copy_from_slice(d); - } - CompiledFunctionUnwindInfo::FrameLayout(code, _fde_offset, r) => { - dest.copy_from_slice(code); - r.iter().for_each(move |r| { - assert_eq!(r.2, 8); - relocs.push(CompiledFunctionUnwindInfoReloc { - offset: r.1 as u32, - addend: r.0 as u32, - }) - }); - } - } - } -} - /// Compiled function: machine code body, jump table offsets, and unwind information. #[derive(Serialize, Deserialize, Debug, Clone, PartialEq, Eq)] pub struct CompiledFunction { @@ -146,7 +21,7 @@ pub struct CompiledFunction { pub jt_offsets: ir::JumpTableOffsets, /// The unwind information. - pub unwind_info: CompiledFunctionUnwindInfo, + pub unwind_info: Option, } type Functions = PrimaryMap; @@ -167,15 +42,15 @@ impl Compilation { /// Allocates the compilation result with the given function bodies. pub fn from_buffer( buffer: Vec, - functions: impl IntoIterator, ir::JumpTableOffsets, Range)>, + functions: impl IntoIterator, ir::JumpTableOffsets)>, ) -> Self { Self::new( functions .into_iter() - .map(|(body_range, jt_offsets, unwind_range)| CompiledFunction { + .map(|(body_range, jt_offsets)| CompiledFunction { body: buffer[body_range].to_vec(), jt_offsets, - unwind_info: CompiledFunctionUnwindInfo::Windows(buffer[unwind_range].to_vec()), + unwind_info: None, // not implemented for lightbeam currently }) .collect(), ) diff --git a/crates/environ/src/cranelift.rs b/crates/environ/src/cranelift.rs index 1b5a83072a82..ee5ec884abac 100644 --- a/crates/environ/src/cranelift.rs +++ b/crates/environ/src/cranelift.rs @@ -3,10 +3,8 @@ use crate::address_map::{FunctionAddressMap, InstructionAddressMap}; use crate::cache::{ModuleCacheDataTupleType, ModuleCacheEntry}; use crate::compilation::{ - Compilation, CompileError, CompiledFunction, CompiledFunctionUnwindInfo, Relocation, - RelocationTarget, TrapInformation, + Compilation, CompileError, CompiledFunction, Relocation, RelocationTarget, TrapInformation, }; -use crate::frame_layout::FrameLayout; use crate::func_environ::{get_func_name, FuncEnvironment}; use crate::{CacheConfig, FunctionBodyData, ModuleLocal, ModuleTranslation, Tunables}; use cranelift_codegen::ir::{self, ExternalName}; @@ -154,38 +152,6 @@ fn get_function_address_map<'data>( } } -fn get_frame_layout( - context: &Context, - isa: &dyn isa::TargetIsa, -) -> ( - Box<[ir::FrameLayoutChange]>, - Box<[(usize, ir::FrameLayoutChange)]>, -) { - let func = &context.func; - assert!(func.frame_layout.is_some(), "expected func.frame_layout"); - - let mut blocks = func.layout.blocks().collect::>(); - blocks.sort_by_key(|b| func.offsets[*b]); // Ensure inst offsets always increase - - let encinfo = isa.encoding_info(); - let mut last_offset = 0; - let mut commands = Vec::new(); - for b in blocks { - for (offset, inst, size) in func.inst_offsets(b, &encinfo) { - if let Some(cmds) = func.frame_layout.as_ref().unwrap().instructions.get(&inst) { - let address_offset = (offset + size) as usize; - assert!(last_offset < address_offset); - for cmd in cmds.iter() { - commands.push((address_offset, cmd.clone())); - } - last_offset = address_offset; - } - } - } - let initial = func.frame_layout.as_ref().unwrap().initial.clone(); - (initial, commands.into_boxed_slice()) -} - /// A compiler that compiles a WebAssembly module with Cranelift, translating the Wasm to Cranelift IR, /// optimizing it and then translating to assembly. pub struct Cranelift; @@ -224,7 +190,6 @@ fn compile(env: CompileEnv<'_>) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result) -> Result, - /// Frame commands at specific offset. - pub commands: Box<[(usize, FrameLayoutChange)]>, -} - -/// Functions frame layouts. -pub type FrameLayouts = PrimaryMap; diff --git a/crates/environ/src/lib.rs b/crates/environ/src/lib.rs index a4bad0274d56..f259fa45d186 100644 --- a/crates/environ/src/lib.rs +++ b/crates/environ/src/lib.rs @@ -27,7 +27,6 @@ mod address_map; mod compilation; mod data_structures; -mod frame_layout; mod func_environ; mod module; mod module_environ; @@ -47,13 +46,11 @@ pub use crate::address_map::{ pub use crate::cache::create_new_config as cache_create_new_config; pub use crate::cache::CacheConfig; pub use crate::compilation::{ - Compilation, CompileError, CompiledFunction, CompiledFunctionUnwindInfo, - CompiledFunctionUnwindInfoReloc, Compiler, Relocation, RelocationTarget, Relocations, - TrapInformation, Traps, + Compilation, CompileError, CompiledFunction, Compiler, Relocation, RelocationTarget, + Relocations, TrapInformation, Traps, }; pub use crate::cranelift::Cranelift; pub use crate::data_structures::*; -pub use crate::frame_layout::{FrameLayout, FrameLayoutChange, FrameLayouts}; pub use crate::func_environ::BuiltinFunctionIndex; #[cfg(feature = "lightbeam")] pub use crate::lightbeam::Lightbeam; diff --git a/crates/environ/src/lightbeam.rs b/crates/environ/src/lightbeam.rs index d182129047c5..5e9c8f57c344 100644 --- a/crates/environ/src/lightbeam.rs +++ b/crates/environ/src/lightbeam.rs @@ -55,7 +55,7 @@ impl crate::compilation::Compiler for Lightbeam { let code_section_ranges_and_jt = code_section .funcs() .into_iter() - .map(|r| (r, SecondaryMap::new(), 0..0)); + .map(|r| (r, SecondaryMap::new())); Ok(( Compilation::from_buffer(code_section.buffer().to_vec(), code_section_ranges_and_jt), @@ -64,7 +64,6 @@ impl crate::compilation::Compiler for Lightbeam { ValueLabelsRanges::new(), PrimaryMap::new(), Traps::new(), - PrimaryMap::new(), )) } } diff --git a/crates/jit/Cargo.toml b/crates/jit/Cargo.toml index 16e804e7d93d..7b6c9b9f9d7c 100644 --- a/crates/jit/Cargo.toml +++ b/crates/jit/Cargo.toml @@ -29,6 +29,7 @@ more-asserts = "0.2.1" anyhow = "1.0" cfg-if = "0.1.9" log = "0.4" +gimli = { version = "0.20.0", default-features = false, features = ["write"] } [target.'cfg(target_os = "windows")'.dependencies] winapi = { version = "0.3.8", features = ["winnt", "impl-default"] } diff --git a/crates/jit/src/code_memory.rs b/crates/jit/src/code_memory.rs index a31695527b16..6d90c85710f4 100644 --- a/crates/jit/src/code_memory.rs +++ b/crates/jit/src/code_memory.rs @@ -1,37 +1,33 @@ //! Memory management for executable code. -use crate::function_table::FunctionTable; +use crate::unwind::UnwindRegistry; use region; use std::mem::ManuallyDrop; use std::{cmp, mem}; -use wasmtime_environ::{Compilation, CompiledFunction}; +use wasmtime_environ::{ + isa::{unwind::UnwindInfo, TargetIsa}, + Compilation, CompiledFunction, +}; use wasmtime_runtime::{Mmap, VMFunctionBody}; struct CodeMemoryEntry { mmap: ManuallyDrop, - table: ManuallyDrop, + registry: ManuallyDrop, } impl CodeMemoryEntry { - fn new() -> Self { - Self { - mmap: ManuallyDrop::new(Mmap::new()), - table: ManuallyDrop::new(FunctionTable::new()), - } - } fn with_capacity(cap: usize) -> Result { - Ok(Self { - mmap: ManuallyDrop::new(Mmap::with_at_least(cap)?), - table: ManuallyDrop::new(FunctionTable::new()), - }) + let mmap = ManuallyDrop::new(Mmap::with_at_least(cap)?); + let registry = ManuallyDrop::new(UnwindRegistry::new(mmap.as_ptr() as usize)); + Ok(Self { mmap, registry }) } } impl Drop for CodeMemoryEntry { fn drop(&mut self) { unsafe { - // Table needs to be freed before mmap. - ManuallyDrop::drop(&mut self.table); + // The registry needs to be dropped before the mmap + ManuallyDrop::drop(&mut self.registry); ManuallyDrop::drop(&mut self.mmap); } } @@ -39,7 +35,7 @@ impl Drop for CodeMemoryEntry { /// Memory manager for executable code. pub struct CodeMemory { - current: CodeMemoryEntry, + current: Option, entries: Vec, position: usize, published: usize, @@ -54,7 +50,7 @@ impl CodeMemory { /// Create a new `CodeMemory` instance. pub fn new() -> Self { Self { - current: CodeMemoryEntry::new(), + current: None, entries: Vec::new(), position: 0, published: 0, @@ -70,16 +66,14 @@ impl CodeMemory { ) -> Result<&mut [VMFunctionBody], String> { let size = Self::function_allocation_size(func); - let (buf, table, start) = self.allocate(size)?; + let (buf, registry, start) = self.allocate(size)?; - let (_, _, _, vmfunc) = Self::copy_function(func, start as u32, buf, table); + let (_, _, vmfunc) = Self::copy_function(func, start as u32, buf, registry); Ok(vmfunc) } /// Allocate a continuous memory block for a compilation. - /// - /// Allocates memory for both the function bodies as well as function unwind data. pub fn allocate_for_compilation( &mut self, compilation: &Compilation, @@ -88,33 +82,35 @@ impl CodeMemory { .into_iter() .fold(0, |acc, func| acc + Self::function_allocation_size(func)); - let (mut buf, mut table, start) = self.allocate(total_len)?; + let (mut buf, registry, start) = self.allocate(total_len)?; let mut result = Vec::with_capacity(compilation.len()); let mut start = start as u32; for func in compilation.into_iter() { - let (next_start, next_buf, next_table, vmfunc) = - Self::copy_function(func, start, buf, table); + let (next_start, next_buf, vmfunc) = Self::copy_function(func, start, buf, registry); result.push(vmfunc); start = next_start; buf = next_buf; - table = next_table; } Ok(result.into_boxed_slice()) } /// Make all allocated memory executable. - pub fn publish(&mut self) { + pub fn publish(&mut self, isa: &dyn TargetIsa) { self.push_current(0) .expect("failed to push current memory map"); - for CodeMemoryEntry { mmap: m, table: t } in &mut self.entries[self.published..] { + for CodeMemoryEntry { + mmap: m, + registry: r, + } in &mut self.entries[self.published..] + { // Remove write access to the pages due to the relocation fixups. - t.publish(m.as_ptr() as u64) - .expect("failed to publish function table"); + r.publish(isa) + .expect("failed to publish function unwind registry"); if !m.is_empty() { unsafe { @@ -139,73 +135,79 @@ impl CodeMemory { /// * The offset within the current mmap that the slice starts at /// /// TODO: Add an alignment flag. - fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut FunctionTable, usize), String> { - if self.current.mmap.len() - self.position < size { + fn allocate(&mut self, size: usize) -> Result<(&mut [u8], &mut UnwindRegistry, usize), String> { + assert!(size > 0); + + if match &self.current { + Some(e) => e.mmap.len() - self.position < size, + None => true, + } { self.push_current(cmp::max(0x10000, size))?; } let old_position = self.position; self.position += size; + let e = self.current.as_mut().unwrap(); + Ok(( - &mut self.current.mmap.as_mut_slice()[old_position..self.position], - &mut self.current.table, + &mut e.mmap.as_mut_slice()[old_position..self.position], + &mut e.registry, old_position, )) } /// Calculates the allocation size of the given compiled function. fn function_allocation_size(func: &CompiledFunction) -> usize { - if func.unwind_info.is_empty() { - func.body.len() - } else { - // Account for necessary unwind information alignment padding (32-bit) - ((func.body.len() + 3) & !3) + func.unwind_info.len() + match &func.unwind_info { + Some(UnwindInfo::WindowsX64(info)) => { + // Windows unwind information is required to be emitted into code memory + // This is because it must be a positive relative offset from the start of the memory + // Account for necessary unwind information alignment padding (32-bit alignment) + ((func.body.len() + 3) & !3) + info.emit_size() + } + _ => func.body.len(), } } /// Copies the data of the compiled function to the given buffer. /// - /// This will also add the function to the current function table. + /// This will also add the function to the current unwind registry. fn copy_function<'a>( func: &CompiledFunction, func_start: u32, buf: &'a mut [u8], - table: &'a mut FunctionTable, - ) -> ( - u32, - &'a mut [u8], - &'a mut FunctionTable, - &'a mut [VMFunctionBody], - ) { - let func_end = func_start + (func.body.len() as u32); - - let (body, remainder) = buf.split_at_mut(func.body.len()); + registry: &mut UnwindRegistry, + ) -> (u32, &'a mut [u8], &'a mut [VMFunctionBody]) { + let func_len = func.body.len(); + let mut func_end = func_start + (func_len as u32); + + let (body, mut remainder) = buf.split_at_mut(func_len); body.copy_from_slice(&func.body); let vmfunc = Self::view_as_mut_vmfunc_slice(body); - if func.unwind_info.is_empty() { - return (func_end, remainder, table, vmfunc); - } + if let Some(UnwindInfo::WindowsX64(info)) = &func.unwind_info { + // Windows unwind information is written following the function body + // Keep unwind information 32-bit aligned (round up to the nearest 4 byte boundary) + let unwind_start = (func_end + 3) & !3; + let unwind_size = info.emit_size(); + let padding = (unwind_start - func_end) as usize; - // Keep unwind information 32-bit aligned (round up to the nearest 4 byte boundary) - let padding = ((func.body.len() + 3) & !3) - func.body.len(); - let (unwind, remainder) = remainder.split_at_mut(padding + func.unwind_info.len()); - let mut relocs = Vec::new(); - func.unwind_info - .serialize(&mut unwind[padding..], &mut relocs); + let (slice, r) = remainder.split_at_mut(padding + unwind_size); - let unwind_start = func_end + (padding as u32); - let unwind_end = unwind_start + (func.unwind_info.len() as u32); + info.emit(&mut slice[padding..]); - relocs.iter_mut().for_each(move |r| { - r.offset += unwind_start; - r.addend += func_start; - }); + func_end = unwind_start + (unwind_size as u32); + remainder = r; + } - table.add_function(func_start, func_end, unwind_start, &relocs); + if let Some(info) = &func.unwind_info { + registry + .register(func_start, func_len as u32, info) + .expect("failed to register unwind information"); + } - (unwind_end, remainder, table, vmfunc) + (func_end, remainder, vmfunc) } /// Convert mut a slice from u8 to VMFunctionBody. @@ -215,21 +217,19 @@ impl CodeMemory { unsafe { &mut *body_ptr } } - /// Pushes the current Mmap (and function table) and allocates a new Mmap of the given size. + /// Pushes the current entry and allocates a new one with the given size. fn push_current(&mut self, new_size: usize) -> Result<(), String> { let previous = mem::replace( &mut self.current, if new_size == 0 { - CodeMemoryEntry::new() + None } else { - CodeMemoryEntry::with_capacity(cmp::max(0x10000, new_size))? + Some(CodeMemoryEntry::with_capacity(cmp::max(0x10000, new_size))?) }, ); - if !previous.mmap.is_empty() { - self.entries.push(previous); - } else { - assert_eq!(previous.table.len(), 0); + if let Some(e) = previous { + self.entries.push(e); } self.position = 0; diff --git a/crates/jit/src/compiler.rs b/crates/jit/src/compiler.rs index e5c45b0a7c81..9e1e309eb381 100644 --- a/crates/jit/src/compiler.rs +++ b/crates/jit/src/compiler.rs @@ -14,9 +14,9 @@ use wasmtime_environ::entity::{EntityRef, PrimaryMap}; use wasmtime_environ::isa::{TargetFrontendConfig, TargetIsa}; use wasmtime_environ::wasm::{DefinedFuncIndex, DefinedMemoryIndex, MemoryIndex}; use wasmtime_environ::{ - CacheConfig, CompileError, CompiledFunction, CompiledFunctionUnwindInfo, Compiler as _C, - ModuleAddressMap, ModuleMemoryOffset, ModuleTranslation, ModuleVmctxInfo, Relocation, - RelocationTarget, Relocations, Traps, Tunables, VMOffsets, + CacheConfig, CompileError, CompiledFunction, Compiler as _C, ModuleAddressMap, + ModuleMemoryOffset, ModuleTranslation, ModuleVmctxInfo, Relocation, RelocationTarget, + Relocations, Traps, Tunables, VMOffsets, }; use wasmtime_runtime::{ InstantiationError, SignatureRegistry, VMFunctionBody, VMSharedSignatureIndex, VMTrampoline, @@ -46,7 +46,6 @@ pub enum CompilationStrategy { /// TODO: Consider using cranelift-module. pub struct Compiler { isa: Box, - code_memory: CodeMemory, signatures: SignatureRegistry, strategy: CompilationStrategy, @@ -102,34 +101,27 @@ impl Compiler { translation: &ModuleTranslation, debug_data: Option, ) -> Result { - let ( - compilation, - relocations, - address_transform, - value_ranges, - stack_slots, - traps, - frame_layouts, - ) = match self.strategy { - // For now, interpret `Auto` as `Cranelift` since that's the most stable - // implementation. - CompilationStrategy::Auto | CompilationStrategy::Cranelift => { - wasmtime_environ::cranelift::Cranelift::compile_module( - translation, - &*self.isa, - &self.cache_config, - ) - } - #[cfg(feature = "lightbeam")] - CompilationStrategy::Lightbeam => { - wasmtime_environ::lightbeam::Lightbeam::compile_module( - translation, - &*self.isa, - &self.cache_config, - ) + let (compilation, relocations, address_transform, value_ranges, stack_slots, traps) = + match self.strategy { + // For now, interpret `Auto` as `Cranelift` since that's the most stable + // implementation. + CompilationStrategy::Auto | CompilationStrategy::Cranelift => { + wasmtime_environ::cranelift::Cranelift::compile_module( + translation, + &*self.isa, + &self.cache_config, + ) + } + #[cfg(feature = "lightbeam")] + CompilationStrategy::Lightbeam => { + wasmtime_environ::lightbeam::Lightbeam::compile_module( + translation, + &*self.isa, + &self.cache_config, + ) + } } - } - .map_err(SetupError::Compile)?; + .map_err(SetupError::Compile)?; // Allocate all of the compiled functions into executable memory, // copying over their contents. @@ -202,8 +194,8 @@ impl Compiler { &module_vmctx_info, &address_transform, &value_ranges, - &frame_layouts, &funcs, + &compilation, ) .map_err(SetupError::DebugInfo)?; Some(bytes) @@ -227,7 +219,7 @@ impl Compiler { /// Make memory containing compiled code executable. pub(crate) fn publish_compiled_code(&mut self) { - self.code_memory.publish(); + self.code_memory.publish(self.isa.as_ref()); } /// Shared signature registry. @@ -264,7 +256,6 @@ pub fn make_trampoline( let mut context = Context::new(); context.func = ir::Function::with_name_signature(ir::ExternalName::user(0, 0), wrapper_sig); - context.func.collect_frame_layout_info(); { let mut builder = FunctionBuilder::new(&mut context.func, fn_builder_ctx); @@ -343,7 +334,7 @@ pub fn make_trampoline( ))) })?; - let unwind_info = CompiledFunctionUnwindInfo::new(isa, &context).map_err(|error| { + let unwind_info = context.create_unwind_info(isa).map_err(|error| { SetupError::Compile(CompileError::Codegen(pretty_error( &context.func, Some(isa), @@ -369,6 +360,10 @@ fn allocate_functions( code_memory: &mut CodeMemory, compilation: &wasmtime_environ::Compilation, ) -> Result, String> { + if compilation.is_empty() { + return Ok(PrimaryMap::new()); + } + let fat_ptrs = code_memory.allocate_for_compilation(compilation)?; // Second, create a PrimaryMap from result vector of pointers. @@ -377,6 +372,7 @@ fn allocate_functions( let fat_ptr: *mut [VMFunctionBody] = fat_ptrs[i]; result.push(fat_ptr); } + Ok(result) } diff --git a/crates/jit/src/function_table.rs b/crates/jit/src/function_table.rs deleted file mode 100644 index ed982cf7a2d4..000000000000 --- a/crates/jit/src/function_table.rs +++ /dev/null @@ -1,224 +0,0 @@ -//! Runtime function table. -//! -//! This module is primarily used to track JIT functions on Windows for stack walking and unwind. - -type FunctionTableReloc = wasmtime_environ::CompiledFunctionUnwindInfoReloc; - -/// Represents a runtime function table. -/// -/// This is used to register JIT code with the operating system to enable stack walking and unwinding. -#[cfg(all(target_os = "windows", target_arch = "x86_64"))] -pub(crate) struct FunctionTable { - functions: Vec, - published: bool, -} - -#[cfg(all(target_os = "windows", target_arch = "x86_64"))] -impl FunctionTable { - /// Creates a new function table. - pub fn new() -> Self { - Self { - functions: Vec::new(), - published: false, - } - } - - /// Returns the number of functions in the table, also referred to as its 'length'. - pub fn len(&self) -> usize { - self.functions.len() - } - - /// Adds a function to the table based off of the start offset, end offset, and unwind offset. - /// - /// The offsets are from the "module base", which is provided when the table is published. - pub fn add_function( - &mut self, - start: u32, - end: u32, - unwind: u32, - _relocs: &[FunctionTableReloc], - ) { - assert_eq!(_relocs.len(), 0); - use winapi::um::winnt; - - assert!(!self.published, "table has already been published"); - - let mut entry = winnt::RUNTIME_FUNCTION::default(); - - entry.BeginAddress = start; - entry.EndAddress = end; - - unsafe { - *entry.u.UnwindInfoAddress_mut() = unwind; - } - - self.functions.push(entry); - } - - /// Publishes the function table using the given base address. - /// - /// A published function table will automatically be deleted when it is dropped. - pub fn publish(&mut self, base_address: u64) -> Result<(), String> { - use winapi::um::winnt; - - if self.published { - return Err("function table was already published".into()); - } - - self.published = true; - - if self.functions.is_empty() { - return Ok(()); - } - - unsafe { - // Windows heap allocations are 32-bit aligned, but assert just in case - assert_eq!( - (self.functions.as_mut_ptr() as u64) % 4, - 0, - "function table allocation was not aligned" - ); - - if winnt::RtlAddFunctionTable( - self.functions.as_mut_ptr(), - self.functions.len() as u32, - base_address, - ) == 0 - { - return Err("failed to add function table".into()); - } - } - - Ok(()) - } -} - -#[cfg(target_os = "windows")] -impl Drop for FunctionTable { - fn drop(&mut self) { - use winapi::um::winnt; - - if self.published { - unsafe { - winnt::RtlDeleteFunctionTable(self.functions.as_mut_ptr()); - } - } - } -} - -/// Represents a runtime function table. -/// -/// This is used to register JIT code with the operating system to enable stack walking and unwinding. -#[cfg(unix)] -pub(crate) struct FunctionTable { - functions: Vec, - relocs: Vec, - published: Option>, -} - -#[cfg(unix)] -impl FunctionTable { - /// Creates a new function table. - pub fn new() -> Self { - Self { - functions: Vec::new(), - relocs: Vec::new(), - published: None, - } - } - - /// Returns the number of functions in the table, also referred to as its 'length'. - pub fn len(&self) -> usize { - self.functions.len() - } - - /// Adds a function to the table based off of the start offset, end offset, and unwind offset. - /// - /// The offsets are from the "module base", which is provided when the table is published. - pub fn add_function( - &mut self, - _start: u32, - _end: u32, - unwind: u32, - relocs: &[FunctionTableReloc], - ) { - assert!(self.published.is_none(), "table has already been published"); - self.functions.push(unwind); - self.relocs.extend_from_slice(relocs); - } - - /// Publishes the function table using the given base address. - /// - /// A published function table will automatically be deleted when it is dropped. - pub fn publish(&mut self, base_address: u64) -> Result<(), String> { - if self.published.is_some() { - return Err("function table was already published".into()); - } - - if self.functions.is_empty() { - assert_eq!(self.relocs.len(), 0); - self.published = Some(vec![]); - return Ok(()); - } - - extern "C" { - // libunwind import - fn __register_frame(fde: *const u8); - } - - for reloc in self.relocs.iter() { - let addr = base_address + (reloc.offset as u64); - let target = base_address + (reloc.addend as u64); - unsafe { - std::ptr::write(addr as *mut u64, target); - } - } - - let mut fdes = Vec::with_capacity(self.functions.len()); - for unwind_offset in self.functions.iter() { - let addr = base_address + (*unwind_offset as u64); - let off = unsafe { std::ptr::read::(addr as *const u32) } as usize + 4; - - let fde = (addr + off as u64) as usize; - unsafe { - __register_frame(fde as *const _); - } - fdes.push(fde); - } - - self.published = Some(fdes); - Ok(()) - } -} - -#[cfg(unix)] -impl Drop for FunctionTable { - fn drop(&mut self) { - extern "C" { - // libunwind import - fn __deregister_frame(fde: *const u8); - } - - if let Some(published) = &self.published { - unsafe { - // I'm not really sure why, but it appears to be way faster to - // unregister frames in reverse order rather than in-order. This - // way we're deregistering in LIFO order, and maybe there's some - // vec shifting or something like that in libgcc? - // - // Locally on Ubuntu 18.04 a wasm module with 40k empty - // functions takes 0.1s to compile and drop with reverse - // iteration. With forward iteration it takes 3s to compile and - // drop! - // - // Poking around libgcc sources seems to indicate that some sort - // of linked list is being traversed... We may need to figure - // out something else for backtraces in the future since this - // API may not be long-lived to keep calling. - for fde in published.iter().rev() { - __deregister_frame(*fde as *const _); - } - } - } - } -} diff --git a/crates/jit/src/lib.rs b/crates/jit/src/lib.rs index f6148e21cb39..bf6d3dd46150 100644 --- a/crates/jit/src/lib.rs +++ b/crates/jit/src/lib.rs @@ -23,11 +23,11 @@ mod code_memory; mod compiler; -mod function_table; mod imports; mod instantiate; mod link; mod resolver; +mod unwind; pub mod native; pub mod trampoline; diff --git a/crates/jit/src/unwind.rs b/crates/jit/src/unwind.rs new file mode 100644 index 000000000000..8bb655137d74 --- /dev/null +++ b/crates/jit/src/unwind.rs @@ -0,0 +1,11 @@ +cfg_if::cfg_if! { + if #[cfg(all(windows, target_arch = "x86_64"))] { + mod winx64; + pub use self::winx64::*; + } else if #[cfg(unix)] { + mod systemv; + pub use self::systemv::*; + } else { + compile_error!("unsupported target platform for unwind"); + } +} diff --git a/crates/jit/src/unwind/systemv.rs b/crates/jit/src/unwind/systemv.rs new file mode 100644 index 000000000000..7c505f71d028 --- /dev/null +++ b/crates/jit/src/unwind/systemv.rs @@ -0,0 +1,150 @@ +//! Module for System V ABI unwind registry. + +use anyhow::{bail, Result}; +use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; +use gimli::{ + write::{Address, EhFrame, EndianVec, FrameTable, Writer}, + RunTimeEndian, +}; + +/// Represents a registry of function unwind information for System V ABI. +pub struct UnwindRegistry { + base_address: usize, + functions: Vec, + frame_table: Vec, + registrations: Vec, + published: bool, +} + +extern "C" { + // libunwind import + fn __register_frame(fde: *const u8); + fn __deregister_frame(fde: *const u8); +} + +impl UnwindRegistry { + /// Creates a new unwind registry with the given base address. + pub fn new(base_address: usize) -> Self { + Self { + base_address, + functions: Vec::new(), + frame_table: Vec::new(), + registrations: Vec::new(), + published: false, + } + } + + /// Registers a function given the start offset, length, and unwind information. + pub fn register(&mut self, func_start: u32, _func_len: u32, info: &UnwindInfo) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + match info { + UnwindInfo::SystemV(info) => { + self.functions.push(info.to_fde(Address::Constant( + self.base_address as u64 + func_start as u64, + ))); + } + _ => bail!("unsupported unwind information"), + } + + Ok(()) + } + + /// Publishes all registered functions. + pub fn publish(&mut self, isa: &dyn TargetIsa) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + if self.functions.is_empty() { + self.published = true; + return Ok(()); + } + + self.set_frame_table(isa)?; + + unsafe { + self.register_frames(); + } + + self.published = true; + + Ok(()) + } + + fn set_frame_table(&mut self, isa: &dyn TargetIsa) -> Result<()> { + let mut table = FrameTable::default(); + let cie_id = table.add_cie(match isa.create_systemv_cie() { + Some(cie) => cie, + None => bail!("ISA does not support System V unwind information"), + }); + + let functions = std::mem::replace(&mut self.functions, Vec::new()); + + for func in functions { + table.add_fde(cie_id, func); + } + + let mut eh_frame = EhFrame(EndianVec::new(RunTimeEndian::default())); + table.write_eh_frame(&mut eh_frame).unwrap(); + + // GCC expects a terminating "empty" length, so write a 0 length at the end of the table. + eh_frame.0.write_u32(0).unwrap(); + + self.frame_table = eh_frame.0.into_vec(); + + Ok(()) + } + + unsafe fn register_frames(&mut self) { + cfg_if::cfg_if! { + if #[cfg(target_os = "macos")] { + // On macOS, `__register_frame` takes a pointer to a single FDE + let start = self.frame_table.as_ptr(); + let end = start.add(self.frame_table.len()); + let mut current = start; + + // Walk all of the entries in the frame table and register them + while current < end { + let len = std::ptr::read::(current as *const u32) as usize; + + // Skip over the CIE + if current != start { + __register_frame(current); + self.registrations.push(current as usize); + } + + // Move to the next table entry (+4 because the length itself is not inclusive) + current = current.add(len + 4); + } + } else { + // On other platforms, `__register_frame` will walk the FDEs until an entry of length 0 + let ptr = self.frame_table.as_ptr(); + __register_frame(ptr); + self.registrations.push(ptr as usize); + } + } + } +} + +impl Drop for UnwindRegistry { + fn drop(&mut self) { + if self.published { + unsafe { + // libgcc stores the frame entries as a linked list in decreasing sort order + // based on the PC value of the registered entry. + // + // As we store the registrations in increasing order, it would be O(N^2) to + // deregister in that order. + // + // To ensure that we just pop off the first element in the list upon every + // deregistration, walk our list of registrations backwards. + for fde in self.registrations.iter().rev() { + __deregister_frame(*fde as *const _); + } + } + } + } +} diff --git a/crates/jit/src/unwind/winx64.rs b/crates/jit/src/unwind/winx64.rs new file mode 100644 index 000000000000..c2f37067ab02 --- /dev/null +++ b/crates/jit/src/unwind/winx64.rs @@ -0,0 +1,91 @@ +//! Module for Windows x64 ABI unwind registry. + +use anyhow::{bail, Result}; +use cranelift_codegen::isa::{unwind::UnwindInfo, TargetIsa}; +use winapi::um::winnt; + +/// Represents a registry of function unwind information for Windows x64 ABI. +pub struct UnwindRegistry { + base_address: usize, + functions: Vec, + published: bool, +} + +impl UnwindRegistry { + /// Creates a new unwind registry with the given base address. + pub fn new(base_address: usize) -> Self { + Self { + base_address, + functions: Vec::new(), + published: false, + } + } + + /// Registers a function given the start offset, length, and unwind information. + pub fn register(&mut self, func_start: u32, func_len: u32, info: &UnwindInfo) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + match info { + UnwindInfo::WindowsX64(_) => { + let mut entry = winnt::RUNTIME_FUNCTION::default(); + + entry.BeginAddress = func_start; + entry.EndAddress = func_start + func_len; + + // The unwind information should be immediately following the function + // with padding for 4 byte alignment + unsafe { + *entry.u.UnwindInfoAddress_mut() = (entry.EndAddress + 3) & !3; + } + + self.functions.push(entry); + + Ok(()) + } + _ => bail!("unsupported unwind information"), + } + } + + /// Publishes all registered functions. + pub fn publish(&mut self, _isa: &dyn TargetIsa) -> Result<()> { + if self.published { + bail!("unwind registry has already been published"); + } + + self.published = true; + + if !self.functions.is_empty() { + // Windows heap allocations are 32-bit aligned, but assert just in case + assert_eq!( + (self.functions.as_mut_ptr() as u64) % 4, + 0, + "function table allocation was not aligned" + ); + + unsafe { + if winnt::RtlAddFunctionTable( + self.functions.as_mut_ptr(), + self.functions.len() as u32, + self.base_address as u64, + ) == 0 + { + bail!("failed to register function table"); + } + } + } + + Ok(()) + } +} + +impl Drop for UnwindRegistry { + fn drop(&mut self) { + if self.published { + unsafe { + winnt::RtlDeleteFunctionTable(self.functions.as_mut_ptr()); + } + } + } +} diff --git a/src/obj.rs b/src/obj.rs index f51022843bd3..765a185d5301 100644 --- a/src/obj.rs +++ b/src/obj.rs @@ -64,25 +64,18 @@ pub fn compile_to_obj( .context("failed to translate module")?; // TODO: use the traps information - let ( - compilation, - relocations, - address_transform, - value_ranges, - stack_slots, - _traps, - frame_layouts, - ) = match strategy { - Strategy::Auto | Strategy::Cranelift => { - Cranelift::compile_module(&translation, &*isa, cache_config) + let (compilation, relocations, address_transform, value_ranges, stack_slots, _traps) = + match strategy { + Strategy::Auto | Strategy::Cranelift => { + Cranelift::compile_module(&translation, &*isa, cache_config) + } + #[cfg(feature = "lightbeam")] + Strategy::Lightbeam => Lightbeam::compile_module(&translation, &*isa, cache_config), + #[cfg(not(feature = "lightbeam"))] + Strategy::Lightbeam => bail!("lightbeam support not enabled"), + other => bail!("unsupported compilation strategy {:?}", other), } - #[cfg(feature = "lightbeam")] - Strategy::Lightbeam => Lightbeam::compile_module(&translation, &*isa, cache_config), - #[cfg(not(feature = "lightbeam"))] - Strategy::Lightbeam => bail!("lightbeam support not enabled"), - other => bail!("unsupported compilation strategy {:?}", other), - } - .context("failed to compile module")?; + .context("failed to compile module")?; if compilation.is_empty() { bail!("no functions were found/compiled"); @@ -127,7 +120,7 @@ pub fn compile_to_obj( &debug_data, &address_transform, &value_ranges, - &frame_layouts, + &compilation, ) .context("failed to emit debug sections")?; }