|
| 1 | +// SPDX-License-Identifier: Apache-2.0 |
| 2 | +// Copyright (C) 2022 Akira Moroo |
| 3 | +// Copyright (c) 2021-2022 Andre Richter <andre.o.richter@gmail.com> |
| 4 | + |
| 5 | +use core::ops::RangeInclusive; |
| 6 | + |
| 7 | +use cortex_a::{asm::barrier, registers::*}; |
| 8 | +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; |
| 9 | + |
| 10 | +use self::interface::Mmu; |
| 11 | + |
| 12 | +use super::{layout::KernelAddrSpace, translation::TranslationTable}; |
| 13 | + |
| 14 | +/// MMU enable errors variants. |
| 15 | +#[derive(Debug)] |
| 16 | +pub enum MmuEnableError { |
| 17 | + AlreadyEnabled, |
| 18 | + Other(&'static str), |
| 19 | +} |
| 20 | + |
| 21 | +/// Memory Management interfaces. |
| 22 | +pub mod interface { |
| 23 | + use super::*; |
| 24 | + |
| 25 | + /// MMU functions. |
| 26 | + pub trait Mmu { |
| 27 | + unsafe fn enable_mmu_and_caching(&self) -> Result<(), MmuEnableError>; |
| 28 | + |
| 29 | + fn is_enabled(&self) -> bool; |
| 30 | + } |
| 31 | +} |
| 32 | + |
| 33 | +/// Describes the characteristics of a translation granule. |
| 34 | +pub struct TranslationGranule<const GRANULE_SIZE: usize>; |
| 35 | + |
| 36 | +/// Describes properties of an address space. |
| 37 | +pub struct AddressSpace<const AS_SIZE: usize>; |
| 38 | + |
| 39 | +/// Architecture agnostic translation types. |
| 40 | +#[allow(dead_code)] |
| 41 | +#[derive(Copy, Clone)] |
| 42 | +pub enum Translation { |
| 43 | + Identity, |
| 44 | + Offset(usize), |
| 45 | +} |
| 46 | + |
| 47 | +/// Architecture agnostic memory attributes. |
| 48 | +#[derive(Copy, Clone)] |
| 49 | +pub enum MemAttributes { |
| 50 | + CacheableDRAM, |
| 51 | + Device, |
| 52 | +} |
| 53 | + |
| 54 | +/// Architecture agnostic access permissions. |
| 55 | +#[derive(Copy, Clone)] |
| 56 | +pub enum AccessPermissions { |
| 57 | + ReadOnly, |
| 58 | + ReadWrite, |
| 59 | +} |
| 60 | + |
| 61 | +/// Collection of memory attributes. |
| 62 | +#[derive(Copy, Clone)] |
| 63 | +pub struct AttributeFields { |
| 64 | + pub mem_attributes: MemAttributes, |
| 65 | + pub acc_perms: AccessPermissions, |
| 66 | + pub execute_never: bool, |
| 67 | +} |
| 68 | + |
| 69 | +impl Default for AttributeFields { |
| 70 | + fn default() -> AttributeFields { |
| 71 | + AttributeFields { |
| 72 | + mem_attributes: MemAttributes::CacheableDRAM, |
| 73 | + acc_perms: AccessPermissions::ReadWrite, |
| 74 | + execute_never: true, |
| 75 | + } |
| 76 | + } |
| 77 | +} |
| 78 | + |
| 79 | +/// Architecture agnostic descriptor for a memory range. |
| 80 | +pub struct TranslationDescriptor { |
| 81 | + pub name: &'static str, |
| 82 | + pub virtual_range: RangeInclusive<usize>, |
| 83 | + pub physical_range_translation: Translation, |
| 84 | + pub attribute_fields: AttributeFields, |
| 85 | +} |
| 86 | + |
| 87 | +/// Type for expressing the kernel's virtual memory layout. |
| 88 | +pub struct KernelVirtualLayout<const NUM_SPECIAL_RANGES: usize> { |
| 89 | + /// The last (inclusive) address of the address space. |
| 90 | + max_virt_addr_inclusive: usize, |
| 91 | + |
| 92 | + /// Array of descriptors for non-standard (normal cacheable DRAM) memory regions. |
| 93 | + inner: [TranslationDescriptor; NUM_SPECIAL_RANGES], |
| 94 | +} |
| 95 | + |
| 96 | +impl<const GRANULE_SIZE: usize> TranslationGranule<GRANULE_SIZE> { |
| 97 | + /// The granule's size. |
| 98 | + pub const SIZE: usize = Self::size_checked(); |
| 99 | + |
| 100 | + /// The granule's shift, aka log2(size). |
| 101 | + pub const SHIFT: usize = Self::SIZE.trailing_zeros() as usize; |
| 102 | + |
| 103 | + const fn size_checked() -> usize { |
| 104 | + assert!(GRANULE_SIZE.is_power_of_two()); |
| 105 | + |
| 106 | + GRANULE_SIZE |
| 107 | + } |
| 108 | +} |
| 109 | + |
| 110 | +impl<const AS_SIZE: usize> AddressSpace<AS_SIZE> { |
| 111 | + /// The address space size. |
| 112 | + pub const SIZE: usize = Self::size_checked(); |
| 113 | + |
| 114 | + /// The address space shift, aka log2(size). |
| 115 | + pub const SIZE_SHIFT: usize = Self::SIZE.trailing_zeros() as usize; |
| 116 | + |
| 117 | + const fn size_checked() -> usize { |
| 118 | + assert!(AS_SIZE.is_power_of_two()); |
| 119 | + |
| 120 | + // Check for architectural restrictions as well. |
| 121 | + Self::arch_address_space_size_sanity_checks(); |
| 122 | + |
| 123 | + AS_SIZE |
| 124 | + } |
| 125 | +} |
| 126 | + |
| 127 | +impl<const NUM_SPECIAL_RANGES: usize> KernelVirtualLayout<{ NUM_SPECIAL_RANGES }> { |
| 128 | + /// Create a new instance. |
| 129 | + pub const fn new(max: usize, layout: [TranslationDescriptor; NUM_SPECIAL_RANGES]) -> Self { |
| 130 | + Self { |
| 131 | + max_virt_addr_inclusive: max, |
| 132 | + inner: layout, |
| 133 | + } |
| 134 | + } |
| 135 | + |
| 136 | + /// For a virtual address, find and return the physical output address and corresponding |
| 137 | + /// attributes. |
| 138 | + /// |
| 139 | + /// If the address is not found in `inner`, return an identity mapped default with normal |
| 140 | + /// cacheable DRAM attributes. |
| 141 | + pub fn virt_addr_properties( |
| 142 | + &self, |
| 143 | + virt_addr: usize, |
| 144 | + ) -> Result<(usize, AttributeFields), &'static str> { |
| 145 | + if virt_addr > self.max_virt_addr_inclusive { |
| 146 | + return Err("Address out of range"); |
| 147 | + } |
| 148 | + |
| 149 | + for i in self.inner.iter() { |
| 150 | + if i.virtual_range.contains(&virt_addr) { |
| 151 | + let output_addr = match i.physical_range_translation { |
| 152 | + Translation::Identity => virt_addr, |
| 153 | + Translation::Offset(a) => a + (virt_addr - (i.virtual_range).start()), |
| 154 | + }; |
| 155 | + |
| 156 | + return Ok((output_addr, i.attribute_fields)); |
| 157 | + } |
| 158 | + } |
| 159 | + |
| 160 | + Ok((virt_addr, AttributeFields::default())) |
| 161 | + } |
| 162 | +} |
| 163 | + |
| 164 | +/// Memory Management Unit type. |
| 165 | +struct MemoryManagementUnit; |
| 166 | + |
| 167 | +pub type Granule512MiB = TranslationGranule<{ 512 * 1024 * 1024 }>; |
| 168 | +pub type Granule64KiB = TranslationGranule<{ 64 * 1024 }>; |
| 169 | + |
| 170 | +/// Constants for indexing the MAIR_EL1. |
| 171 | +pub mod mair { |
| 172 | + pub const DEVICE: u64 = 0; |
| 173 | + pub const NORMAL: u64 = 1; |
| 174 | +} |
| 175 | + |
| 176 | +/// The kernel translation tables. |
| 177 | +/// |
| 178 | +/// # Safety |
| 179 | +/// |
| 180 | +/// - Supposed to land in `.bss`. Therefore, ensure that all initial member values boil down to "0". |
| 181 | +static mut KERNEL_TABLES: TranslationTable = TranslationTable::new(); |
| 182 | + |
| 183 | +static MMU: MemoryManagementUnit = MemoryManagementUnit; |
| 184 | + |
| 185 | +impl<const AS_SIZE: usize> AddressSpace<AS_SIZE> { |
| 186 | + /// Checks for architectural restrictions. |
| 187 | + pub const fn arch_address_space_size_sanity_checks() { |
| 188 | + // Size must be at least one full 512 MiB table. |
| 189 | + assert!((AS_SIZE % Granule512MiB::SIZE) == 0); |
| 190 | + |
| 191 | + // Check for 48 bit virtual address size as maximum, which is supported by any ARMv8 |
| 192 | + // version. |
| 193 | + assert!(AS_SIZE <= (1 << 48)); |
| 194 | + } |
| 195 | +} |
| 196 | + |
| 197 | +impl MemoryManagementUnit { |
| 198 | + /// Setup function for the MAIR_EL1 register. |
| 199 | + fn setup_mair(&self) { |
| 200 | + // Define the memory types being mapped. |
| 201 | + MAIR_EL1.write( |
| 202 | + // Attribute 1 - Cacheable normal DRAM. |
| 203 | + MAIR_EL1::Attr1_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc |
| 204 | + + MAIR_EL1::Attr1_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc |
| 205 | + // Attribute 0 - Device. |
| 206 | + + MAIR_EL1::Attr0_Device::nonGathering_nonReordering_EarlyWriteAck, |
| 207 | + ); |
| 208 | + } |
| 209 | + |
| 210 | + /// Configure various settings of stage 1 of the EL1 translation regime. |
| 211 | + fn configure_translation_control(&self) { |
| 212 | + let t0sz = (64 - KernelAddrSpace::SIZE_SHIFT) as u64; |
| 213 | + |
| 214 | + TCR_EL1.write( |
| 215 | + TCR_EL1::TBI0::Used |
| 216 | + + TCR_EL1::IPS::Bits_40 |
| 217 | + + TCR_EL1::TG0::KiB_64 |
| 218 | + + TCR_EL1::SH0::Inner |
| 219 | + + TCR_EL1::ORGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable |
| 220 | + + TCR_EL1::IRGN0::WriteBack_ReadAlloc_WriteAlloc_Cacheable |
| 221 | + + TCR_EL1::EPD0::EnableTTBR0Walks |
| 222 | + + TCR_EL1::A1::TTBR0 |
| 223 | + + TCR_EL1::T0SZ.val(t0sz) |
| 224 | + + TCR_EL1::EPD1::DisableTTBR1Walks, |
| 225 | + ); |
| 226 | + } |
| 227 | +} |
| 228 | + |
| 229 | +/// Return a reference to the MMU instance. |
| 230 | +fn mmu() -> &'static impl interface::Mmu { |
| 231 | + &MMU |
| 232 | +} |
| 233 | + |
| 234 | +impl interface::Mmu for MemoryManagementUnit { |
| 235 | + unsafe fn enable_mmu_and_caching(&self) -> Result<(), MmuEnableError> { |
| 236 | + if self.is_enabled() { |
| 237 | + return Err(MmuEnableError::AlreadyEnabled); |
| 238 | + } |
| 239 | + |
| 240 | + // Fail early if translation granule is not supported. |
| 241 | + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran64::Supported) { |
| 242 | + return Err(MmuEnableError::Other( |
| 243 | + "Translation granule not supported in HW", |
| 244 | + )); |
| 245 | + } |
| 246 | + |
| 247 | + // Prepare the memory attribute indirection register. |
| 248 | + self.setup_mair(); |
| 249 | + |
| 250 | + // Populate translation tables. |
| 251 | + KERNEL_TABLES |
| 252 | + .populate_tt_entries() |
| 253 | + .map_err(MmuEnableError::Other)?; |
| 254 | + |
| 255 | + // Set the "Translation Table Base Register". |
| 256 | + TTBR0_EL1.set_baddr(KERNEL_TABLES.phys_base_address()); |
| 257 | + |
| 258 | + self.configure_translation_control(); |
| 259 | + |
| 260 | + // Switch the MMU on. |
| 261 | + // |
| 262 | + // First, force all previous changes to be seen before the MMU is enabled. |
| 263 | + barrier::isb(barrier::SY); |
| 264 | + |
| 265 | + // Enable the MMU and turn on data and instruction caching. |
| 266 | + SCTLR_EL1.modify(SCTLR_EL1::M::Enable + SCTLR_EL1::C::Cacheable + SCTLR_EL1::I::Cacheable); |
| 267 | + |
| 268 | + // Force MMU init to complete before next instruction. |
| 269 | + barrier::isb(barrier::SY); |
| 270 | + |
| 271 | + Ok(()) |
| 272 | + } |
| 273 | + |
| 274 | + #[inline(always)] |
| 275 | + fn is_enabled(&self) -> bool { |
| 276 | + SCTLR_EL1.matches_all(SCTLR_EL1::M::Enable) |
| 277 | + } |
| 278 | +} |
| 279 | + |
| 280 | +pub fn setup() { |
| 281 | + unsafe { |
| 282 | + if let Err(e) = mmu().enable_mmu_and_caching() { |
| 283 | + panic!("Failed to setup paging: {:?}", e); |
| 284 | + } |
| 285 | + } |
| 286 | +} |
0 commit comments