diff --git a/kernel/src/arch/aarch64/intrin.rs b/kernel/src/arch/aarch64/intrin.rs index 4ba7a58..ff2c09d 100644 --- a/kernel/src/arch/aarch64/intrin.rs +++ b/kernel/src/arch/aarch64/intrin.rs @@ -17,8 +17,8 @@ pub unsafe fn irq_disable() { /// /// Unsafe: requires EL0 #[inline(always)] -pub unsafe fn flush_tlb_virt(_addr: usize) { - todo!() +pub unsafe fn flush_tlb_virt(addr: usize) { + asm!("tlbi vaae1, {}", in(reg) addr); } /// Discards all entries related to `asid` from TLB cache diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index e0ea0dc..4a8f3ed 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -10,6 +10,7 @@ pub mod intrin; pub mod irq; pub mod reg; pub mod timer; +pub mod virt; cfg_if! { if #[cfg(feature = "mach_qemu")] { diff --git a/kernel/src/mem/virt/fixed.rs b/kernel/src/arch/aarch64/virt/fixed.rs similarity index 61% rename from kernel/src/mem/virt/fixed.rs rename to kernel/src/arch/aarch64/virt/fixed.rs index 5275576..6dfed98 100644 --- a/kernel/src/mem/virt/fixed.rs +++ b/kernel/src/arch/aarch64/virt/fixed.rs @@ -1,21 +1,16 @@ -//! Fixed-size table group for device MMIO mappings - -use crate::mem::{ - self, - virt::{Entry, MapAttributes, Table}, -}; +use super::{EntryImpl, TableImpl}; +use crate::mem; +use crate::mem::virt::table::{Entry, MapAttributes}; use cortex_a::asm::barrier::{self, dsb, isb}; use libsys::error::Errno; -const DEVICE_MAP_OFFSET: usize = mem::KERNEL_OFFSET + (256usize << 30); - /// Fixed-layout group of tables describing device MMIO and kernel identity /// mappings #[repr(C, align(0x1000))] pub struct FixedTableGroup { - l0: Table, - l1: Table, - l2: Table, + l0: TableImpl, + l1: TableImpl, + l2: TableImpl, pages_4k: usize, pages_2m: usize, @@ -27,9 +22,9 @@ impl FixedTableGroup { /// entries pub const fn empty() -> Self { Self { - l0: Table::empty(), - l1: Table::empty(), - l2: Table::empty(), + l0: TableImpl::empty(), + l1: TableImpl::empty(), + l2: TableImpl::empty(), pages_4k: 0, pages_2m: 1, @@ -44,7 +39,7 @@ impl FixedTableGroup { pub fn map_region(&mut self, phys: usize, count: usize) -> Result { // TODO generalize region allocation let phys_page = phys & !0xFFF; - let attrs = MapAttributes::SH_OUTER | MapAttributes::DEVICE | MapAttributes::ACCESS; + let attrs = MapAttributes::SHARE_OUTER | MapAttributes::DEVICE_MEMORY; match count { 262144 => { @@ -54,7 +49,7 @@ impl FixedTableGroup { } self.pages_1g += 1; - self.l0[count + 256] = Entry::block(phys_page, attrs); + self.l0[count + 256] = EntryImpl::block(phys_page, attrs | MapAttributes::ACCESS); unsafe { dsb(barrier::SY); isb(barrier::SY); @@ -69,7 +64,7 @@ impl FixedTableGroup { } self.pages_2m += 1; - self.l1[count] = Entry::block(phys_page, attrs); + self.l1[count] = EntryImpl::block(phys_page, attrs | MapAttributes::ACCESS); unsafe { dsb(barrier::SY); isb(barrier::SY); @@ -84,7 +79,7 @@ impl FixedTableGroup { } self.pages_4k += 1; - self.l2[count] = Entry::table(phys_page, attrs); + self.l2[count] = EntryImpl::normal(phys_page, attrs | MapAttributes::ACCESS); unsafe { dsb(barrier::SY); isb(barrier::SY); @@ -95,13 +90,28 @@ impl FixedTableGroup { _ => unimplemented!(), } } - - /// Sets up initial mappings for 4K, 2M and 1G device memory page translation - pub fn init_device_map(&mut self) { - let l1_phys = (&self.l1 as *const _) as usize - mem::KERNEL_OFFSET; - let l2_phys = (&self.l2 as *const _) as usize - mem::KERNEL_OFFSET; - - self.l0[256] = Entry::table(l1_phys, MapAttributes::empty()); - self.l1[0] = Entry::table(l2_phys, MapAttributes::empty()); - } +} + +const DEVICE_MAP_OFFSET: usize = mem::KERNEL_OFFSET + (256usize << 30); + +#[no_mangle] +static mut KERNEL_TTBR1: FixedTableGroup = FixedTableGroup::empty(); + +/// Allocates a range of virtual memory of requested size and maps +/// it to specified device memory +pub fn map_device_memory(phys: usize, count: usize) -> Result { + unsafe { KERNEL_TTBR1.map_region(phys, count) } +} + +/// Sets up initial mappings for device-memory virtual tables. +/// +/// # Safety +/// +/// Only safe to be called once during virtual memory init. +pub unsafe fn init_device_map() { + let l1_phys = (&KERNEL_TTBR1.l1 as *const _) as usize - mem::KERNEL_OFFSET; + let l2_phys = (&KERNEL_TTBR1.l2 as *const _) as usize - mem::KERNEL_OFFSET; + + KERNEL_TTBR1.l0[256] = Entry::normal(l1_phys, MapAttributes::empty()); + KERNEL_TTBR1.l1[0] = Entry::normal(l2_phys, MapAttributes::empty()); } diff --git a/kernel/src/arch/aarch64/virt/mod.rs b/kernel/src/arch/aarch64/virt/mod.rs new file mode 100644 index 0000000..0360c74 --- /dev/null +++ b/kernel/src/arch/aarch64/virt/mod.rs @@ -0,0 +1,81 @@ +//! AArch64 virtual memory management implementation + +use crate::mem::virt::table::MapAttributes; + +mod fixed; +mod table; + +pub use fixed::{init_device_map, map_device_memory}; +pub use table::{alloc_empty_space, fork_space, release_space, EntryImpl, TableImpl}; + +bitflags! { + /// Raw attributes for AArch64 [Entry] implementation + pub struct RawAttributesImpl: u64 { + // TODO use 2 lower bits to determine mapping size? + /// nG bit -- determines whether a TLB entry associated with this mapping + /// applies only to current ASID or all ASIDs. + const NOT_GLOBAL = 1 << 11; + /// AF bit -- must be set by software, otherwise Access Error exception is + /// generated when the page is accessed + const ACCESS = 1 << 10; + /// The memory region is outer-shareable + const SH_OUTER = 2 << 8; + /// This page is used for device-MMIO mapping and uses MAIR attribute #1 + const DEVICE = 1 << 2; + + /// Pages marked with this bit are Copy-on-Write + const EX_COW = 1 << 55; + + /// UXN bit -- if set, page may not be used for instruction fetching from EL0 + const UXN = 1 << 54; + /// PXN bit -- if set, page may not be used for instruction fetching from EL1 + const PXN = 1 << 53; + + // AP field + // Default behavior is: read-write for EL1, no access for EL0 + /// If set, the page referred to by this entry is read-only for both EL0/EL1 + const AP_BOTH_READONLY = 3 << 6; + /// If set, the page referred to by this entry is read-write for both EL0/EL1 + const AP_BOTH_READWRITE = 1 << 6; + } +} + +impl From for RawAttributesImpl { + fn from(src: MapAttributes) -> Self { + let mut res = RawAttributesImpl::empty(); + + if src.contains(MapAttributes::SHARE_OUTER) { + res |= RawAttributesImpl::SH_OUTER; + } + + if !src.contains(MapAttributes::GLOBAL) { + res |= RawAttributesImpl::NOT_GLOBAL; + } + + if !src.contains(MapAttributes::USER_EXEC) { + res |= RawAttributesImpl::UXN; + } + + if !src.contains(MapAttributes::KERNEL_EXEC) { + res |= RawAttributesImpl::PXN; + } + + if src.contains(MapAttributes::USER_READ) { + if src.contains(MapAttributes::USER_WRITE) { + res |= RawAttributesImpl::AP_BOTH_READWRITE; + } else { + res |= RawAttributesImpl::AP_BOTH_READONLY; + } + } + + if src.contains(MapAttributes::DEVICE_MEMORY) { + res |= RawAttributesImpl::DEVICE; + } + + if src.contains(MapAttributes::ACCESS) { + res |= RawAttributesImpl::ACCESS; + } + + res + } +} diff --git a/kernel/src/arch/aarch64/virt/table.rs b/kernel/src/arch/aarch64/virt/table.rs new file mode 100644 index 0000000..ad660a7 --- /dev/null +++ b/kernel/src/arch/aarch64/virt/table.rs @@ -0,0 +1,305 @@ +use crate::arch::aarch64::intrin::flush_tlb_virt; +use crate::mem::{ + self, + phys::{self, PageUsage}, + virt::table::{Entry, MapAttributes, Space, Table}, +}; +use core::ops::{Index, IndexMut}; +use libsys::{error::Errno, mem::memset}; + +use super::RawAttributesImpl; + +/// Transparent wrapper structure representing a single +/// translation table entry +#[derive(Clone, Copy)] +#[repr(transparent)] +pub struct EntryImpl(u64); + +/// Structure describing a single level of translation mappings +#[repr(C, align(0x1000))] +pub struct TableImpl { + entries: [EntryImpl; 512], +} + +impl EntryImpl { + const PRESENT: u64 = 1 << 0; + const TABLE: u64 = 1 << 1; + const PHYS_MASK: u64 = 0x0000FFFFFFFFF000; +} + +impl Entry for EntryImpl { + type RawAttributes = RawAttributesImpl; + const EMPTY: Self = Self(0); + + #[inline] + fn normal(addr: usize, attrs: MapAttributes) -> Self { + Self((addr as u64) | RawAttributesImpl::from(attrs).bits() | (1 << 1) | (1 << 0)) + } + + #[inline] + fn block(addr: usize, attrs: MapAttributes) -> Self { + Self((addr as u64) | RawAttributesImpl::from(attrs).bits() | (1 << 0)) + } + + #[inline] + fn address(self) -> usize { + (self.0 & Self::PHYS_MASK) as usize + } + + #[inline] + fn set_address(&mut self, virt: usize) { + self.0 = (self.0 & !Self::PHYS_MASK) | ((virt as u64) & Self::PHYS_MASK); + } + + #[inline] + fn is_present(self) -> bool { + self.0 & Self::PRESENT != 0 + } + + #[inline] + fn is_normal(self) -> bool { + self.0 & Self::TABLE != 0 + } + + #[inline] + fn fork_with_cow(&mut self) -> Self { + self.0 |= (RawAttributesImpl::AP_BOTH_READONLY | RawAttributesImpl::EX_COW).bits(); + *self + } + + #[inline] + fn copy_from_cow(self, new_addr: usize) -> Self { + let attrs = self.0 + & !(Self::PHYS_MASK + | RawAttributesImpl::AP_BOTH_READONLY.bits() + | RawAttributesImpl::EX_COW.bits()); + Self( + ((new_addr as u64) & Self::PHYS_MASK) + | (attrs | RawAttributesImpl::AP_BOTH_READWRITE.bits()), + ) + } + + #[inline] + fn is_cow(self) -> bool { + self.0 & RawAttributesImpl::EX_COW.bits() != 0 + } + + #[inline] + fn is_user_writable(self) -> bool { + self.0 & RawAttributesImpl::AP_BOTH_READONLY.bits() + == RawAttributesImpl::AP_BOTH_READWRITE.bits() + } +} + +impl Table for TableImpl { + type Entry = EntryImpl; + + unsafe fn write_last_level( + &mut self, + virt: usize, + entry: Self::Entry, + _create_intermediate: bool, // TODO handle this properly + overwrite: bool, + ) -> Result<(), Errno> { + let l0i = virt >> 30; + let l1i = (virt >> 21) & 0x1FF; + let l2i = (virt >> 12) & 0x1FF; + + let l1_table = self.next_level_table_or_alloc(l0i)?; + let l2_table = l1_table.next_level_table_or_alloc(l1i)?; + + if l2_table[l2i].is_present() && !overwrite { + return Err(Errno::AlreadyExists); + }; + + l2_table[l2i] = entry; + #[cfg(feature = "verbose")] + debugln!( + "{:#p} Map {:#x} -> {:#x}, {:#x}", + self, + virt, + entry.address(), + entry.0 & !EntryImpl::PHYS_MASK + ); + flush_tlb_virt(virt); + Ok(()) + } + + fn read_last_level(&self, virt: usize) -> Result { + let l0i = virt >> 30; + let l1i = (virt >> 21) & 0x1FF; + let l2i = (virt >> 12) & 0x1FF; + + let l1_table = self.next_level_table(l0i).ok_or(Errno::DoesNotExist)?; + let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?; + + let entry = l2_table[l2i]; + if entry.is_present() { + Ok(entry) + } else { + Err(Errno::DoesNotExist) + } + } +} + +impl TableImpl { + /// Constructs a table with no valid mappings + pub const fn empty() -> Self { + Self { + entries: [EntryImpl::EMPTY; 512], + } + } + + /// Returns next-level translation table reference for `index`, if one is present. + /// If `index` represents a `Block`-type mapping, will return an error. + /// If `index` does not map to any translation table, will try to allocate, init and + /// map a new one, returning it after doing so. + pub fn next_level_table_or_alloc(&mut self, index: usize) -> Result<&'static mut Self, Errno> { + let entry = self[index]; + if entry.is_present() { + if !entry.is_normal() { + return Err(Errno::InvalidArgument); + } + + Ok(unsafe { &mut *(mem::virtualize(entry.address()) as *mut _) }) + } else { + let phys = phys::alloc_page(PageUsage::Paging)?; + let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) }; + self[index] = EntryImpl::normal(phys, MapAttributes::empty()); + res.entries.fill(EntryImpl::EMPTY); + Ok(res) + } + } + + /// Returns next-level translation table reference for `index`, if one is present. + /// Same as [next_level_table_or_alloc], but returns `None` if no table is mapped. + pub fn next_level_table(&self, index: usize) -> Option<&'static Self> { + let entry = self[index]; + if entry.is_present() { + if !entry.is_normal() { + panic!("Entry is not a table: idx={}", index); + } + + Some(unsafe { &*(mem::virtualize(entry.address()) as *const _) }) + } else { + None + } + } + + /// Returns mutable next-level translation table reference for `index`, + /// if one is present. Same as [next_level_table_or_alloc], but returns + /// `None` if no table is mapped. + pub fn next_level_table_mut(&mut self, index: usize) -> Option<&'static mut Self> { + let entry = self[index]; + if entry.is_present() { + if !entry.is_normal() { + panic!("Entry is not a table: idx={}", index); + } + + Some(unsafe { &mut *(mem::virtualize(entry.address()) as *mut _) }) + } else { + None + } + } +} + +impl Index for TableImpl { + type Output = EntryImpl; + + fn index(&self, index: usize) -> &Self::Output { + &self.entries[index] + } +} + +impl IndexMut for TableImpl { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.entries[index] + } +} + +/// Allocates a page and constructs an empty space from it +pub fn alloc_empty_space() -> Result<&'static mut Space, Errno> { + let phys = phys::alloc_page(PageUsage::Paging)?; + let res = unsafe { &mut *(mem::virtualize(phys) as *mut Space) }; + res.0.entries.fill(EntryImpl::EMPTY); + Ok(res) +} + +/// Forks a memory space +pub fn fork_space(src: &mut Space) -> Result<&'static mut Space, Errno> { + let res = alloc_empty_space()?; + for l0i in 0..512 { + if let Some(l1_table) = src.0.next_level_table_mut(l0i) { + for l1i in 0..512 { + if let Some(l2_table) = l1_table.next_level_table_mut(l1i) { + for l2i in 0..512 { + let entry = &mut l2_table[l2i]; + if !entry.is_present() { + continue; + } + + assert!(entry.is_normal()); + let src_phys = entry.address(); + let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12); + let dst_phys = unsafe { phys::fork_page(src_phys)? }; + + let new_entry = if dst_phys != src_phys { + todo!() + } else if entry.is_user_writable() { + entry.fork_with_cow() + } else { + *entry + }; + + unsafe { + flush_tlb_virt(virt_addr); + res.0.write_last_level(virt_addr, new_entry, true, false)?; + } + } + } + } + } + } + Ok(res) +} + +/// Releases the intermediate paging structures and data pages +/// used by the memory space. +/// +/// # Safety +/// +/// Only safe to call on spaces not currently in use, otherwise will +/// trigger undefined behavior and/or page fault. +pub unsafe fn release_space(space: &mut Space) { + for l0i in 0..512 { + let l0_entry = space.0[l0i]; + if !l0_entry.is_present() { + continue; + } + + assert!(l0_entry.is_normal()); + let l1_table = &mut *(mem::virtualize(l0_entry.address()) as *mut TableImpl); + + for l1i in 0..512 { + let l1_entry = l1_table[l1i]; + if !l1_entry.is_present() { + continue; + } + assert!(l1_entry.is_normal()); + let l2_table = &mut *(mem::virtualize(l1_entry.address()) as *mut TableImpl); + + for l2i in 0..512 { + let entry = l2_table[l2i]; + if !entry.is_present() { + continue; + } + + assert!(entry.is_normal()); + phys::free_page(entry.address()).unwrap(); + } + phys::free_page(l1_entry.address()).unwrap(); + } + phys::free_page(l0_entry.address()).unwrap(); + } + memset(space as *mut Space as *mut u8, 0, 4096); +} diff --git a/kernel/src/mem/virt/mod.rs b/kernel/src/mem/virt/mod.rs index 5e879e5..be56206 100644 --- a/kernel/src/mem/virt/mod.rs +++ b/kernel/src/mem/virt/mod.rs @@ -8,12 +8,7 @@ use libsys::error::Errno; use tock_registers::interfaces::Writeable; pub mod table; -pub use table::{Entry, MapAttributes, Space, Table}; -pub mod fixed; -pub use fixed::FixedTableGroup; - -#[no_mangle] -static mut KERNEL_TTBR1: FixedTableGroup = FixedTableGroup::empty(); +use crate::arch::platform::virt as virt_impl; /// Structure representing a region of memory used for MMIO/device access // TODO: this shouldn't be trivially-cloneable and should instead incorporate @@ -45,7 +40,7 @@ impl DeviceMemory { /// /// See [FixedTableGroup::map_region] pub fn map(name: &'static str, phys: usize, count: usize) -> Result { - let base = unsafe { KERNEL_TTBR1.map_region(phys, count) }?; + let base = virt_impl::map_device_memory(phys, count)?; debugln!( "Mapping {:#x}..{:#x} -> {:#x} for {:?}", base, @@ -91,7 +86,7 @@ impl Deref for DeviceMemoryIo { /// identity-mapped translation pub fn enable() -> Result<(), Errno> { unsafe { - KERNEL_TTBR1.init_device_map(); + virt_impl::init_device_map(); dsb(barrier::ISH); isb(barrier::SY); diff --git a/kernel/src/mem/virt/table.rs b/kernel/src/mem/virt/table.rs index 5877b21..c8e080c 100644 --- a/kernel/src/mem/virt/table.rs +++ b/kernel/src/mem/virt/table.rs @@ -1,255 +1,153 @@ //! Translation table manipulation facilities +use crate::arch::platform::virt as virt_impl; use crate::mem::{ self, phys::{self, PageUsage}, }; -use core::ops::{Index, IndexMut}; -use libsys::{error::Errno, mem::memset}; -use core::arch::asm; +use libsys::error::Errno; +pub use virt_impl::{EntryImpl, TableImpl}; -/// Transparent wrapper structure representing a single -/// translation table entry -#[derive(Clone, Copy)] -#[repr(transparent)] -pub struct Entry(u64); +bitflags! { + /// Virtual space entry attributes + pub struct MapAttributes: u64 { + /// Entry is readable by user threads + const USER_READ = 1 << 0; + /// Entry is writable by user threads + const USER_WRITE = 1 << 1; + /// Data from entry can be executed by user threads + const USER_EXEC = 1 << 2; -/// Structure describing a single level of translation mappings -#[repr(C, align(0x1000))] -pub struct Table { - entries: [Entry; 512], + /// Entry is writable by kernel + const KERNEL_WRITE = 1 << 3; + /// Data from entry can be executed by kernel + const KERNEL_EXEC = 1 << 4; + + /// TODO TBD + const SHARE_OUTER = 1 << 5; + /// Memory is used for device interaction + const DEVICE_MEMORY = 1 << 6; + + /// Entry is marked as Copy-on-Write + const COPY_ON_WRITE = 1 << 7; + + /// Access flag for entry + const ACCESS = 1 << 8; + /// Entry is global across virtual address spaces + const GLOBAL = 1 << 9; + } +} + +/// Interface for a single element of paging mapping +pub trait Entry: Clone + Copy { + /// Platform-specific entry attribute representation + type RawAttributes: From + Copy + Clone; + /// Invalid entry with no association + const EMPTY: Self; + + /// Constructs an entry pointing to next-level table or page + fn normal(addr: usize, attrs: MapAttributes) -> Self; + /// Constructs an entry pointing to a contiguous block + fn block(addr: usize, attrs: MapAttributes) -> Self; + + /// Returns physical address the entry points to + fn address(self) -> usize; + /// Changes the entry physical address + fn set_address(&mut self, value: usize); + + /// Marks page as CoW and removes user write ability + fn fork_with_cow(&mut self) -> Self; + /// Clones a CoW entry + fn copy_from_cow(self, new_addr: usize) -> Self; + + /// Returns `true` if entry maps a paging element + fn is_present(self) -> bool; + /// Returns `true` if page is a 4KiB one + fn is_normal(self) -> bool; + /// Returns `true` if page is marked as Copy-on-Write + fn is_cow(self) -> bool; + /// Returns `true` if page is write-accessible for user threads + fn is_user_writable(self) -> bool; +} + +// TODO maybe make this `Space` instead of `Table`? +/// Interface for a single level of virtual memory mapping hierarchy +pub trait Table { + /// Table element type + type Entry: Entry; + + /// Writes an [Entry] to last-level table specifying `virt` address. + /// + /// # Safety + /// + /// Unsafe: allows virtual address space manipulations. Must not affect + /// pages currently in use. + unsafe fn write_last_level( + &mut self, + virt: usize, + entry: Self::Entry, + create_intermediate: bool, + overwrite: bool, + ) -> Result<(), Errno>; + + /// Reads an [Entry] from last-level table specifying `virt` address + fn read_last_level(&self, virt: usize) -> Result; } /// Wrapper for top-most level of address translation tables #[repr(transparent)] -pub struct Space(Table); +pub struct Space(pub TableImpl); -bitflags! { - /// Attributes attached to each translation [Entry] - pub struct MapAttributes: u64 { - // TODO use 2 lower bits to determine mapping size? - /// nG bit -- determines whether a TLB entry associated with this mapping - /// applies only to current ASID or all ASIDs. - const NOT_GLOBAL = 1 << 11; - /// AF bit -- must be set by software, otherwise Access Error exception is - /// generated when the page is accessed - const ACCESS = 1 << 10; - /// The memory region is outer-shareable - const SH_OUTER = 2 << 8; - /// This page is used for device-MMIO mapping and uses MAIR attribute #1 - const DEVICE = 1 << 2; - - /// Pages marked with this bit are Copy-on-Write - const EX_COW = 1 << 55; - - /// UXN bit -- if set, page may not be used for instruction fetching from EL0 - const UXN = 1 << 54; - /// PXN bit -- if set, page may not be used for instruction fetching from EL1 - const PXN = 1 << 53; - - // AP field - // Default behavior is: read-write for EL1, no access for EL0 - /// If set, the page referred to by this entry is read-only for both EL0/EL1 - const AP_BOTH_READONLY = 3 << 6; - /// If set, the page referred to by this entry is read-write for both EL0/EL1 - const AP_BOTH_READWRITE = 1 << 6; - } -} - -impl Table { - /// Returns next-level translation table reference for `index`, if one is present. - /// If `index` represents a `Block`-type mapping, will return an error. - /// If `index` does not map to any translation table, will try to allocate, init and - /// map a new one, returning it after doing so. - pub fn next_level_table_or_alloc(&mut self, index: usize) -> Result<&'static mut Table, Errno> { - let entry = self[index]; - if entry.is_present() { - if !entry.is_table() { - return Err(Errno::InvalidArgument); - } - - Ok(unsafe { &mut *(mem::virtualize(entry.address_unchecked()) as *mut _) }) - } else { - let phys = phys::alloc_page(PageUsage::Paging)?; - let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) }; - self[index] = Entry::table(phys, MapAttributes::empty()); - res.entries.fill(Entry::invalid()); - Ok(res) - } +impl Space { + /// Creates a new address space with only kernel entries mapped + pub fn alloc_empty() -> Result<&'static mut Self, Errno> { + virt_impl::alloc_empty_space() } - /// Returns next-level translation table reference for `index`, if one is present. - /// Same as [next_level_table_or_alloc], but returns `None` if no table is mapped. - pub fn next_level_table(&mut self, index: usize) -> Option<&'static mut Table> { - let entry = self[index]; - if entry.is_present() { - if !entry.is_table() { - panic!("Entry is not a table: idx={}", index); - } - - Some(unsafe { &mut *(mem::virtualize(entry.address_unchecked()) as *mut _) }) - } else { - None - } - } - - /// Constructs and fills a [Table] with non-present mappings - pub const fn empty() -> Table { - Table { - entries: [Entry::invalid(); 512], - } - } -} - -impl Index for Table { - type Output = Entry; - - fn index(&self, index: usize) -> &Self::Output { - &self.entries[index] - } -} - -impl IndexMut for Table { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.entries[index] - } -} - -impl Entry { - const PRESENT: u64 = 1 << 0; - const TABLE: u64 = 1 << 1; - const PHYS_MASK: u64 = 0x0000FFFFFFFFF000; - - /// Constructs a single non-present mapping - pub const fn invalid() -> Self { - Self(0) - } - - /// Constructs a `Block`-type memory mapping - pub const fn block(phys: usize, attrs: MapAttributes) -> Self { - Self((phys as u64 & Self::PHYS_MASK) | attrs.bits() | Self::PRESENT) - } - - /// Constructs a `Table` or `Page`-type mapping depending on translation level - /// this entry is used at - pub const fn table(phys: usize, attrs: MapAttributes) -> Self { - Self((phys as u64 & Self::PHYS_MASK) | attrs.bits() | Self::PRESENT | Self::TABLE) - } - - /// Returns `true` if this entry is not invalid - pub const fn is_present(self) -> bool { - self.0 & Self::PRESENT != 0 - } - - /// Returns `true` if this entry is a `Table` or `Page`-type mapping - pub const fn is_table(self) -> bool { - self.0 & Self::TABLE != 0 - } - - /// Returns the target address of this translation entry. + /// Releases the intermediate paging structures and data pages + /// used by the memory space. /// /// # Safety /// - /// Does not check if the entry is actually valid. - pub const unsafe fn address_unchecked(self) -> usize { - (self.0 & Self::PHYS_MASK) as usize + /// Only safe to call on spaces not currently in use, otherwise will + /// trigger undefined behavior and/or page fault. + pub unsafe fn release(&mut self) { + virt_impl::release_space(self) } - unsafe fn set_address(&mut self, address: usize) { - self.0 &= !Self::PHYS_MASK; - self.0 |= (address as u64) & Self::PHYS_MASK; + /// Performs process address space forking + pub fn fork(&mut self) -> Result<&'static mut Self, Errno> { + virt_impl::fork_space(self) } - unsafe fn fork_flags(self) -> MapAttributes { - MapAttributes::from_bits_unchecked(self.0 & !Self::PHYS_MASK) + /// Returns physical address of this table + pub fn address_phys(&mut self) -> usize { + self as *mut _ as usize - mem::KERNEL_OFFSET } - fn set_cow(&mut self) { - self.0 |= (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits(); - } - - fn clear_cow(&mut self) { - self.0 &= !(MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits(); - self.0 |= MapAttributes::AP_BOTH_READWRITE.bits(); - } - - #[inline] - fn is_cow(self) -> bool { - let attrs = (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits(); - self.0 & attrs == attrs - } -} - -impl Space { - /// Creates a new virtual address space and fills it with [Entry::invalid()] - /// mappings. Does physical memory page allocation. - pub fn alloc_empty() -> Result<&'static mut Self, Errno> { - let phys = phys::alloc_page(PageUsage::Paging)?; - let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) }; - res.0.entries.fill(Entry::invalid()); - Ok(res) - } - - /// Inserts a single `virt` -> `phys` translation entry to this address space. - /// - /// TODO: only works with 4K-sized pages at this moment. - pub fn map(&mut self, virt: usize, phys: usize, flags: MapAttributes) -> Result<(), Errno> { - let l0i = virt >> 30; - let l1i = (virt >> 21) & 0x1FF; - let l2i = (virt >> 12) & 0x1FF; - - let l1_table = self.0.next_level_table_or_alloc(l0i)?; - let l2_table = l1_table.next_level_table_or_alloc(l1i)?; - - if l2_table[l2i].is_present() { - Err(Errno::AlreadyExists) - } else { - l2_table[l2i] = Entry::table(phys, flags | MapAttributes::ACCESS); - #[cfg(feature = "verbose")] - debugln!("{:#p} Map {:#x} -> {:#x}, {:?}", self, virt, phys, flags); - Ok(()) + /// Creates a new virtual -> physical memory mapping. Will fail if one is + /// already associated with given virtual address. + pub fn map(&mut self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Errno> { + unsafe { + self.0.write_last_level( + virt, + Entry::normal(phys, attrs | MapAttributes::ACCESS), + true, + false, + ) } } - /// Translates a virtual address into a corresponding physical one. - /// - /// Only works for 4K pages atm. - // TODO extract attributes + /// Returns a virtual address physical mapping destination pub fn translate(&mut self, virt: usize) -> Result { - let l0i = virt >> 30; - let l1i = (virt >> 21) & 0x1FF; - let l2i = (virt >> 12) & 0x1FF; - - let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?; - let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?; - - let entry = l2_table[l2i]; - if entry.is_present() { - Ok(unsafe { entry.address_unchecked() }) - } else { - Err(Errno::DoesNotExist) - } + self.0.read_last_level(virt).map(Entry::address) } - /// Attempts to resolve a page fault at `virt` address by copying the - /// underlying Copy-on-Write mapping (if any is present) + /// Performs Copy-on-Write cloning on page fault pub fn try_cow_copy(&mut self, virt: usize) -> Result<(), Errno> { - let virt = virt & !0xFFF; - let l0i = virt >> 30; - let l1i = (virt >> 21) & 0x1FF; - let l2i = (virt >> 12) & 0x1FF; + let entry = self.0.read_last_level(virt)?; + let src_phys = entry.address(); - let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?; - let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?; - - let entry = l2_table[l2i]; - - if !entry.is_present() { - warnln!("Entry is not present: {:#x}", virt); - return Err(Errno::DoesNotExist); - } - - let src_phys = unsafe { entry.address_unchecked() }; if !entry.is_cow() { warnln!( "Entry is not marked as CoW: {:#x}, points to {:#x}", @@ -260,11 +158,11 @@ impl Space { } let dst_phys = unsafe { phys::copy_cow_page(src_phys)? }; - unsafe { - l2_table[l2i].set_address(dst_phys); - } - l2_table[l2i].clear_cow(); + unsafe { + self.0 + .write_last_level(virt, entry.copy_from_cow(dst_phys), false, true)?; + } Ok(()) } @@ -294,135 +192,14 @@ impl Space { Err(Errno::OutOfMemory) } - /// Removes a single 4K page mapping from the table and - /// releases the underlying physical memory - pub fn unmap_single(&mut self, page: usize) -> Result<(), Errno> { - let l0i = page >> 30; - let l1i = (page >> 21) & 0x1FF; - let l2i = (page >> 12) & 0x1FF; - - let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?; - let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?; - - let entry = l2_table[l2i]; - - if !entry.is_present() { - return Err(Errno::DoesNotExist); - } - - let phys = unsafe { entry.address_unchecked() }; - unsafe { - phys::free_page(phys)?; - } - l2_table[l2i] = Entry::invalid(); - - unsafe { - asm!("tlbi vaae1, {}", in(reg) page); - } - - // TODO release paging structure memory - - Ok(()) - } - - /// Releases a range of virtual pages and their corresponding physical pages + /// Releases memory from virtual address range `start`..`start + len * 0x1000` pub fn free(&mut self, start: usize, len: usize) -> Result<(), Errno> { for i in 0..len { - self.unmap_single(start + i * 0x1000)?; + unsafe { + self.0 + .write_last_level(start + i * 0x1000, EntryImpl::EMPTY, false, true)?; + } } Ok(()) } - - /// Performs a copy of the address space, cloning data owned by it - pub fn fork(&mut self) -> Result<&'static mut Self, Errno> { - let res = Self::alloc_empty()?; - for l0i in 0..512 { - if let Some(l1_table) = self.0.next_level_table(l0i) { - for l1i in 0..512 { - if let Some(l2_table) = l1_table.next_level_table(l1i) { - for l2i in 0..512 { - let entry = l2_table[l2i]; - - if !entry.is_present() { - continue; - } - - assert!(entry.is_table()); - let src_phys = unsafe { entry.address_unchecked() }; - let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12); - let dst_phys = unsafe { phys::fork_page(src_phys)? }; - - let mut flags = unsafe { entry.fork_flags() }; - if dst_phys != src_phys { - todo!(); - // res.map(virt_addr, dst_phys, flags)?; - } else { - let writable = flags & MapAttributes::AP_BOTH_READONLY - == MapAttributes::AP_BOTH_READWRITE; - - if writable { - flags |= - MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW; - l2_table[l2i].set_cow(); - - unsafe { - asm!("tlbi vaae1, {}", in(reg) virt_addr); - } - } - - res.map(virt_addr, dst_phys, flags)?; - } - } - } - } - } - } - Ok(res) - } - - /// Releases all the mappings from the address space. Frees all - /// memory pages referenced by this space as well as those used for - /// its paging tables. - /// - /// # Safety - /// - /// Unsafe: may invalidate currently active address space - pub unsafe fn release(space: &mut Self) { - for l0i in 0..512 { - let l0_entry = space.0[l0i]; - if !l0_entry.is_present() { - continue; - } - - assert!(l0_entry.is_table()); - let l1_table = &mut *(mem::virtualize(l0_entry.address_unchecked()) as *mut Table); - - for l1i in 0..512 { - let l1_entry = l1_table[l1i]; - if !l1_entry.is_present() { - continue; - } - assert!(l1_entry.is_table()); - let l2_table = &mut *(mem::virtualize(l1_entry.address_unchecked()) as *mut Table); - - for l2i in 0..512 { - let entry = l2_table[l2i]; - if !entry.is_present() { - continue; - } - - assert!(entry.is_table()); - phys::free_page(entry.address_unchecked()).unwrap(); - } - phys::free_page(l1_entry.address_unchecked()).unwrap(); - } - phys::free_page(l0_entry.address_unchecked()).unwrap(); - } - memset(space as *mut Space as *mut u8, 0, 4096); - } - - /// Returns the physical address of this structure - pub fn address_phys(&mut self) -> usize { - (self as *mut _ as usize) - mem::KERNEL_OFFSET - } } diff --git a/kernel/src/proc/elf.rs b/kernel/src/proc/elf.rs index cfe3125..045fb7e 100644 --- a/kernel/src/proc/elf.rs +++ b/kernel/src/proc/elf.rs @@ -2,7 +2,7 @@ use crate::mem::{ self, phys::{self, PageUsage}, - virt::{MapAttributes, Space}, + virt::table::{Space, MapAttributes}, }; use core::mem::{size_of, MaybeUninit}; use libsys::{ @@ -66,10 +66,10 @@ struct Phdr { } fn map_flags(elf_flags: usize) -> MapAttributes { - let mut dst_flags = MapAttributes::NOT_GLOBAL | MapAttributes::SH_OUTER; + let mut dst_flags = MapAttributes::SHARE_OUTER; - if elf_flags & (1 << 0) /* PF_X */ == 0 { - dst_flags |= MapAttributes::UXN | MapAttributes::PXN; + if elf_flags & (1 << 0) /* PF_X */ != 0 { + dst_flags |= MapAttributes::USER_EXEC; } match (elf_flags & (3 << 1)) >> 1 { @@ -78,9 +78,9 @@ fn map_flags(elf_flags: usize) -> MapAttributes { // Write-only: not sure if such mapping should exist at all 1 => todo!(), // Read-only - 2 => dst_flags |= MapAttributes::AP_BOTH_READONLY, + 2 => dst_flags |= MapAttributes::USER_READ, // Read+Write - 3 => dst_flags |= MapAttributes::AP_BOTH_READWRITE, + 3 => dst_flags |= MapAttributes::USER_WRITE | MapAttributes::USER_READ, _ => unreachable!(), }; diff --git a/kernel/src/proc/process.rs b/kernel/src/proc/process.rs index 3190632..7991385 100644 --- a/kernel/src/proc/process.rs +++ b/kernel/src/proc/process.rs @@ -3,7 +3,7 @@ use crate::arch::{aarch64::exception::ExceptionFrame, intrin}; use crate::mem::{ self, phys::{self, PageUsage}, - virt::{MapAttributes, Space}, + virt::table::{MapAttributes, Space}, }; use crate::proc::{ wait::Wait, Context, ProcessIo, Thread, ThreadRef, ThreadState, Tid, PROCESSES, SCHED, @@ -383,11 +383,7 @@ impl Process { phys } else { let page = phys::alloc_page(PageUsage::UserPrivate)?; - let flags = MapAttributes::SH_OUTER - | MapAttributes::NOT_GLOBAL - | MapAttributes::UXN - | MapAttributes::PXN - | MapAttributes::AP_BOTH_READONLY; + let flags = MapAttributes::SHARE_OUTER | MapAttributes::USER_READ; space.map(page_virt, page, flags)?; page }; @@ -407,11 +403,7 @@ impl Process { phys } else { let page = phys::alloc_page(PageUsage::UserPrivate)?; - let flags = MapAttributes::SH_OUTER - | MapAttributes::NOT_GLOBAL - | MapAttributes::UXN - | MapAttributes::PXN - | MapAttributes::AP_BOTH_READONLY; + let flags = MapAttributes::SHARE_OUTER | MapAttributes::USER_READ; space.map(page_virt, page, flags)?; page }; @@ -518,11 +510,10 @@ impl Process { let ustack_virt_bottom = Self::USTACK_VIRT_TOP - Self::USTACK_PAGES * mem::PAGE_SIZE; for i in 0..Self::USTACK_PAGES { let page = phys::alloc_page(PageUsage::UserPrivate).unwrap(); - let flags = MapAttributes::SH_OUTER - | MapAttributes::NOT_GLOBAL - | MapAttributes::UXN - | MapAttributes::PXN - | MapAttributes::AP_BOTH_READWRITE; + let flags = MapAttributes::SHARE_OUTER + | MapAttributes::USER_READ + | MapAttributes::USER_WRITE + | MapAttributes::KERNEL_WRITE; new_space .map(ustack_virt_bottom + i * mem::PAGE_SIZE, page, flags) .unwrap(); diff --git a/kernel/src/syscall/mod.rs b/kernel/src/syscall/mod.rs index 1590725..caa6358 100644 --- a/kernel/src/syscall/mod.rs +++ b/kernel/src/syscall/mod.rs @@ -4,7 +4,7 @@ use crate::arch::{machine, platform::exception::ExceptionFrame}; use crate::debug::Level; use crate::dev::timer::TimestampSource; use crate::fs::create_filesystem; -use crate::mem::{phys::PageUsage, virt::MapAttributes}; +use crate::mem::{phys::PageUsage, virt::table::MapAttributes}; use crate::proc::{self, elf, wait, Process, ProcessIo, Thread}; use core::mem::size_of; use core::ops::DerefMut; @@ -210,7 +210,7 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result { let _flags = MemoryAccess::from_bits(args[3] as u32).ok_or(Errno::InvalidArgument)?; let mut attrs = - MapAttributes::NOT_GLOBAL | MapAttributes::SH_OUTER | MapAttributes::PXN; + MapAttributes::SHARE_OUTER; if !acc.contains(MemoryAccess::READ) { return Err(Errno::NotImplemented); } @@ -218,12 +218,12 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result { if acc.contains(MemoryAccess::EXEC) { return Err(Errno::PermissionDenied); } - attrs |= MapAttributes::AP_BOTH_READWRITE; + attrs |= MapAttributes::USER_WRITE | MapAttributes::USER_READ; } else { - attrs |= MapAttributes::AP_BOTH_READONLY; + attrs |= MapAttributes::USER_READ; } - if !acc.contains(MemoryAccess::EXEC) { - attrs |= MapAttributes::UXN; + if acc.contains(MemoryAccess::EXEC) { + attrs |= MapAttributes::USER_EXEC; } // TODO don't ignore flags