alnyan/osdev5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

refactor: detach aarch64 virt implementation

Browse Source
This commit is contained in:
Mark Poliakov
2022-03-15 12:40:33 +02:00
parent 4b844a8774
commit 5ea133d2bd
10 changed files with 575 additions and 415 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/src/arch/aarch64/intrin.rs
+2 -2
View File
@@ -17,8 +17,8 @@ pub unsafe fn irq_disable() {
///
/// Unsafe: requires EL0
#[inline(always)]
pub unsafe fn flush_tlb_virt(_addr: usize) {
todo!()
pub unsafe fn flush_tlb_virt(addr: usize) {
asm!("tlbi vaae1, {}", in(reg) addr);
}
/// Discards all entries related to `asid` from TLB cache
kernel/src/arch/aarch64/mod.rs
+1
View File
@@ -10,6 +10,7 @@ pub mod intrin;
pub mod irq;
pub mod reg;
pub mod timer;
pub mod virt;
cfg_if! {
if #[cfg(feature = "mach_qemu")] {
kernel/src/mem/virt/fixed.rs → kernel/src/arch/aarch64/virt/fixed.rs
+37 -27
View File
@@ -1,21 +1,16 @@
//! Fixed-size table group for device MMIO mappings
use crate::mem::{
self,
virt::{Entry, MapAttributes, Table},
};
use super::{EntryImpl, TableImpl};
use crate::mem;
use crate::mem::virt::table::{Entry, MapAttributes};
use cortex_a::asm::barrier::{self, dsb, isb};
use libsys::error::Errno;
const DEVICE_MAP_OFFSET: usize = mem::KERNEL_OFFSET + (256usize << 30);
/// Fixed-layout group of tables describing device MMIO and kernel identity
/// mappings
#[repr(C, align(0x1000))]
pub struct FixedTableGroup {
l0: Table,
l1: Table,
l2: Table,
l0: TableImpl,
l1: TableImpl,
l2: TableImpl,
pages_4k: usize,
pages_2m: usize,
@@ -27,9 +22,9 @@ impl FixedTableGroup {
/// entries
pub const fn empty() -> Self {
Self {
l0: Table::empty(),
l1: Table::empty(),
l2: Table::empty(),
l0: TableImpl::empty(),
l1: TableImpl::empty(),
l2: TableImpl::empty(),
pages_4k: 0,
pages_2m: 1,
@@ -44,7 +39,7 @@ impl FixedTableGroup {
pub fn map_region(&mut self, phys: usize, count: usize) -> Result<usize, Errno> {
// TODO generalize region allocation
let phys_page = phys & !0xFFF;
let attrs = MapAttributes::SH_OUTER | MapAttributes::DEVICE | MapAttributes::ACCESS;
let attrs = MapAttributes::SHARE_OUTER | MapAttributes::DEVICE_MEMORY;
match count {
262144 => {
@@ -54,7 +49,7 @@ impl FixedTableGroup {
}
self.pages_1g += 1;
self.l0[count + 256] = Entry::block(phys_page, attrs);
self.l0[count + 256] = EntryImpl::block(phys_page, attrs | MapAttributes::ACCESS);
unsafe {
dsb(barrier::SY);
isb(barrier::SY);
@@ -69,7 +64,7 @@ impl FixedTableGroup {
}
self.pages_2m += 1;
self.l1[count] = Entry::block(phys_page, attrs);
self.l1[count] = EntryImpl::block(phys_page, attrs | MapAttributes::ACCESS);
unsafe {
dsb(barrier::SY);
isb(barrier::SY);
@@ -84,7 +79,7 @@ impl FixedTableGroup {
}
self.pages_4k += 1;
self.l2[count] = Entry::table(phys_page, attrs);
self.l2[count] = EntryImpl::normal(phys_page, attrs | MapAttributes::ACCESS);
unsafe {
dsb(barrier::SY);
isb(barrier::SY);
@@ -95,13 +90,28 @@ impl FixedTableGroup {
_ => unimplemented!(),
}
}
/// Sets up initial mappings for 4K, 2M and 1G device memory page translation
pub fn init_device_map(&mut self) {
let l1_phys = (&self.l1 as *const _) as usize - mem::KERNEL_OFFSET;
let l2_phys = (&self.l2 as *const _) as usize - mem::KERNEL_OFFSET;
self.l0[256] = Entry::table(l1_phys, MapAttributes::empty());
self.l1[0] = Entry::table(l2_phys, MapAttributes::empty());
}
}
const DEVICE_MAP_OFFSET: usize = mem::KERNEL_OFFSET + (256usize << 30);
#[no_mangle]
static mut KERNEL_TTBR1: FixedTableGroup = FixedTableGroup::empty();
/// Allocates a range of virtual memory of requested size and maps
/// it to specified device memory
pub fn map_device_memory(phys: usize, count: usize) -> Result<usize, Errno> {
unsafe { KERNEL_TTBR1.map_region(phys, count) }
}
/// Sets up initial mappings for device-memory virtual tables.
///
/// # Safety
///
/// Only safe to be called once during virtual memory init.
pub unsafe fn init_device_map() {
let l1_phys = (&KERNEL_TTBR1.l1 as *const _) as usize - mem::KERNEL_OFFSET;
let l2_phys = (&KERNEL_TTBR1.l2 as *const _) as usize - mem::KERNEL_OFFSET;
KERNEL_TTBR1.l0[256] = Entry::normal(l1_phys, MapAttributes::empty());
KERNEL_TTBR1.l1[0] = Entry::normal(l2_phys, MapAttributes::empty());
}
kernel/src/arch/aarch64/virt/mod.rs
+81
View File
@@ -0,0 +1,81 @@
//! AArch64 virtual memory management implementation
use crate::mem::virt::table::MapAttributes;
mod fixed;
mod table;
pub use fixed::{init_device_map, map_device_memory};
pub use table::{alloc_empty_space, fork_space, release_space, EntryImpl, TableImpl};
bitflags! {
/// Raw attributes for AArch64 [Entry] implementation
pub struct RawAttributesImpl: u64 {
// TODO use 2 lower bits to determine mapping size?
/// nG bit -- determines whether a TLB entry associated with this mapping
/// applies only to current ASID or all ASIDs.
const NOT_GLOBAL = 1 << 11;
/// AF bit -- must be set by software, otherwise Access Error exception is
/// generated when the page is accessed
const ACCESS = 1 << 10;
/// The memory region is outer-shareable
const SH_OUTER = 2 << 8;
/// This page is used for device-MMIO mapping and uses MAIR attribute #1
const DEVICE = 1 << 2;
/// Pages marked with this bit are Copy-on-Write
const EX_COW = 1 << 55;
/// UXN bit -- if set, page may not be used for instruction fetching from EL0
const UXN = 1 << 54;
/// PXN bit -- if set, page may not be used for instruction fetching from EL1
const PXN = 1 << 53;
// AP field
// Default behavior is: read-write for EL1, no access for EL0
/// If set, the page referred to by this entry is read-only for both EL0/EL1
const AP_BOTH_READONLY = 3 << 6;
/// If set, the page referred to by this entry is read-write for both EL0/EL1
const AP_BOTH_READWRITE = 1 << 6;
}
}
impl From<MapAttributes> for RawAttributesImpl {
fn from(src: MapAttributes) -> Self {
let mut res = RawAttributesImpl::empty();
if src.contains(MapAttributes::SHARE_OUTER) {
res |= RawAttributesImpl::SH_OUTER;
}
if !src.contains(MapAttributes::GLOBAL) {
res |= RawAttributesImpl::NOT_GLOBAL;
}
if !src.contains(MapAttributes::USER_EXEC) {
res |= RawAttributesImpl::UXN;
}
if !src.contains(MapAttributes::KERNEL_EXEC) {
res |= RawAttributesImpl::PXN;
}
if src.contains(MapAttributes::USER_READ) {
if src.contains(MapAttributes::USER_WRITE) {
res |= RawAttributesImpl::AP_BOTH_READWRITE;
} else {
res |= RawAttributesImpl::AP_BOTH_READONLY;
}
}
if src.contains(MapAttributes::DEVICE_MEMORY) {
res |= RawAttributesImpl::DEVICE;
}
if src.contains(MapAttributes::ACCESS) {
res |= RawAttributesImpl::ACCESS;
}
res
}
}
kernel/src/arch/aarch64/virt/table.rs
+305
View File
@@ -0,0 +1,305 @@
use crate::arch::aarch64::intrin::flush_tlb_virt;
use crate::mem::{
self,
phys::{self, PageUsage},
virt::table::{Entry, MapAttributes, Space, Table},
};
use core::ops::{Index, IndexMut};
use libsys::{error::Errno, mem::memset};
use super::RawAttributesImpl;
/// Transparent wrapper structure representing a single
/// translation table entry
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct EntryImpl(u64);
/// Structure describing a single level of translation mappings
#[repr(C, align(0x1000))]
pub struct TableImpl {
entries: [EntryImpl; 512],
}
impl EntryImpl {
const PRESENT: u64 = 1 << 0;
const TABLE: u64 = 1 << 1;
const PHYS_MASK: u64 = 0x0000FFFFFFFFF000;
}
impl Entry for EntryImpl {
type RawAttributes = RawAttributesImpl;
const EMPTY: Self = Self(0);
#[inline]
fn normal(addr: usize, attrs: MapAttributes) -> Self {
Self((addr as u64) | RawAttributesImpl::from(attrs).bits() | (1 << 1) | (1 << 0))
}
#[inline]
fn block(addr: usize, attrs: MapAttributes) -> Self {
Self((addr as u64) | RawAttributesImpl::from(attrs).bits() | (1 << 0))
}
#[inline]
fn address(self) -> usize {
(self.0 & Self::PHYS_MASK) as usize
}
#[inline]
fn set_address(&mut self, virt: usize) {
self.0 = (self.0 & !Self::PHYS_MASK) | ((virt as u64) & Self::PHYS_MASK);
}
#[inline]
fn is_present(self) -> bool {
self.0 & Self::PRESENT != 0
}
#[inline]
fn is_normal(self) -> bool {
self.0 & Self::TABLE != 0
}
#[inline]
fn fork_with_cow(&mut self) -> Self {
self.0 |= (RawAttributesImpl::AP_BOTH_READONLY | RawAttributesImpl::EX_COW).bits();
*self
}
#[inline]
fn copy_from_cow(self, new_addr: usize) -> Self {
let attrs = self.0
& !(Self::PHYS_MASK
| RawAttributesImpl::AP_BOTH_READONLY.bits()
| RawAttributesImpl::EX_COW.bits());
Self(
((new_addr as u64) & Self::PHYS_MASK)
| (attrs | RawAttributesImpl::AP_BOTH_READWRITE.bits()),
)
}
#[inline]
fn is_cow(self) -> bool {
self.0 & RawAttributesImpl::EX_COW.bits() != 0
}
#[inline]
fn is_user_writable(self) -> bool {
self.0 & RawAttributesImpl::AP_BOTH_READONLY.bits()
== RawAttributesImpl::AP_BOTH_READWRITE.bits()
}
}
impl Table for TableImpl {
type Entry = EntryImpl;
unsafe fn write_last_level(
&mut self,
virt: usize,
entry: Self::Entry,
_create_intermediate: bool, // TODO handle this properly
overwrite: bool,
) -> Result<(), Errno> {
let l0i = virt >> 30;
let l1i = (virt >> 21) & 0x1FF;
let l2i = (virt >> 12) & 0x1FF;
let l1_table = self.next_level_table_or_alloc(l0i)?;
let l2_table = l1_table.next_level_table_or_alloc(l1i)?;
if l2_table[l2i].is_present() && !overwrite {
return Err(Errno::AlreadyExists);
};
l2_table[l2i] = entry;
#[cfg(feature = "verbose")]
debugln!(
"{:#p} Map {:#x} -> {:#x}, {:#x}",
self,
virt,
entry.address(),
entry.0 & !EntryImpl::PHYS_MASK
);
flush_tlb_virt(virt);
Ok(())
}
fn read_last_level(&self, virt: usize) -> Result<Self::Entry, Errno> {
let l0i = virt >> 30;
let l1i = (virt >> 21) & 0x1FF;
let l2i = (virt >> 12) & 0x1FF;
let l1_table = self.next_level_table(l0i).ok_or(Errno::DoesNotExist)?;
let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?;
let entry = l2_table[l2i];
if entry.is_present() {
Ok(entry)
} else {
Err(Errno::DoesNotExist)
}
}
}
impl TableImpl {
/// Constructs a table with no valid mappings
pub const fn empty() -> Self {
Self {
entries: [EntryImpl::EMPTY; 512],
}
}
/// Returns next-level translation table reference for `index`, if one is present.
/// If `index` represents a `Block`-type mapping, will return an error.
/// If `index` does not map to any translation table, will try to allocate, init and
/// map a new one, returning it after doing so.
pub fn next_level_table_or_alloc(&mut self, index: usize) -> Result<&'static mut Self, Errno> {
let entry = self[index];
if entry.is_present() {
if !entry.is_normal() {
return Err(Errno::InvalidArgument);
}
Ok(unsafe { &mut *(mem::virtualize(entry.address()) as *mut _) })
} else {
let phys = phys::alloc_page(PageUsage::Paging)?;
let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) };
self[index] = EntryImpl::normal(phys, MapAttributes::empty());
res.entries.fill(EntryImpl::EMPTY);
Ok(res)
}
}
/// Returns next-level translation table reference for `index`, if one is present.
/// Same as [next_level_table_or_alloc], but returns `None` if no table is mapped.
pub fn next_level_table(&self, index: usize) -> Option<&'static Self> {
let entry = self[index];
if entry.is_present() {
if !entry.is_normal() {
panic!("Entry is not a table: idx={}", index);
}
Some(unsafe { &*(mem::virtualize(entry.address()) as *const _) })
} else {
None
}
}
/// Returns mutable next-level translation table reference for `index`,
/// if one is present. Same as [next_level_table_or_alloc], but returns
/// `None` if no table is mapped.
pub fn next_level_table_mut(&mut self, index: usize) -> Option<&'static mut Self> {
let entry = self[index];
if entry.is_present() {
if !entry.is_normal() {
panic!("Entry is not a table: idx={}", index);
}
Some(unsafe { &mut *(mem::virtualize(entry.address()) as *mut _) })
} else {
None
}
}
}
impl Index<usize> for TableImpl {
type Output = EntryImpl;
fn index(&self, index: usize) -> &Self::Output {
&self.entries[index]
}
}
impl IndexMut<usize> for TableImpl {
fn index_mut(&mut self, index: usize) -> &mut Self::Output {
&mut self.entries[index]
}
}
/// Allocates a page and constructs an empty space from it
pub fn alloc_empty_space() -> Result<&'static mut Space, Errno> {
let phys = phys::alloc_page(PageUsage::Paging)?;
let res = unsafe { &mut *(mem::virtualize(phys) as *mut Space) };
res.0.entries.fill(EntryImpl::EMPTY);
Ok(res)
}
/// Forks a memory space
pub fn fork_space(src: &mut Space) -> Result<&'static mut Space, Errno> {
let res = alloc_empty_space()?;
for l0i in 0..512 {
if let Some(l1_table) = src.0.next_level_table_mut(l0i) {
for l1i in 0..512 {
if let Some(l2_table) = l1_table.next_level_table_mut(l1i) {
for l2i in 0..512 {
let entry = &mut l2_table[l2i];
if !entry.is_present() {
continue;
}
assert!(entry.is_normal());
let src_phys = entry.address();
let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12);
let dst_phys = unsafe { phys::fork_page(src_phys)? };
let new_entry = if dst_phys != src_phys {
todo!()
} else if entry.is_user_writable() {
entry.fork_with_cow()
} else {
*entry
};
unsafe {
flush_tlb_virt(virt_addr);
res.0.write_last_level(virt_addr, new_entry, true, false)?;
}
}
}
}
}
}
Ok(res)
}
/// Releases the intermediate paging structures and data pages
/// used by the memory space.
///
/// # Safety
///
/// Only safe to call on spaces not currently in use, otherwise will
/// trigger undefined behavior and/or page fault.
pub unsafe fn release_space(space: &mut Space) {
for l0i in 0..512 {
let l0_entry = space.0[l0i];
if !l0_entry.is_present() {
continue;
}
assert!(l0_entry.is_normal());
let l1_table = &mut *(mem::virtualize(l0_entry.address()) as *mut TableImpl);
for l1i in 0..512 {
let l1_entry = l1_table[l1i];
if !l1_entry.is_present() {
continue;
}
assert!(l1_entry.is_normal());
let l2_table = &mut *(mem::virtualize(l1_entry.address()) as *mut TableImpl);
for l2i in 0..512 {
let entry = l2_table[l2i];
if !entry.is_present() {
continue;
}
assert!(entry.is_normal());
phys::free_page(entry.address()).unwrap();
}
phys::free_page(l1_entry.address()).unwrap();
}
phys::free_page(l0_entry.address()).unwrap();
}
memset(space as *mut Space as *mut u8, 0, 4096);
}
kernel/src/mem/virt/mod.rs
+3 -8
View File
@@ -8,12 +8,7 @@ use libsys::error::Errno;
use tock_registers::interfaces::Writeable;
pub mod table;
pub use table::{Entry, MapAttributes, Space, Table};
pub mod fixed;
pub use fixed::FixedTableGroup;
#[no_mangle]
static mut KERNEL_TTBR1: FixedTableGroup = FixedTableGroup::empty();
use crate::arch::platform::virt as virt_impl;
/// Structure representing a region of memory used for MMIO/device access
// TODO: this shouldn't be trivially-cloneable and should instead incorporate
@@ -45,7 +40,7 @@ impl DeviceMemory {
///
/// See [FixedTableGroup::map_region]
pub fn map(name: &'static str, phys: usize, count: usize) -> Result<Self, Errno> {
let base = unsafe { KERNEL_TTBR1.map_region(phys, count) }?;
let base = virt_impl::map_device_memory(phys, count)?;
debugln!(
"Mapping {:#x}..{:#x} -> {:#x} for {:?}",
base,
@@ -91,7 +86,7 @@ impl<T> Deref for DeviceMemoryIo<T> {
/// identity-mapped translation
pub fn enable() -> Result<(), Errno> {
unsafe {
KERNEL_TTBR1.init_device_map();
virt_impl::init_device_map();
dsb(barrier::ISH);
isb(barrier::SY);
kernel/src/mem/virt/table.rs
+127 -350
View File
@@ -1,255 +1,153 @@
//! Translation table manipulation facilities
use crate::arch::platform::virt as virt_impl;
use crate::mem::{
self,
phys::{self, PageUsage},
};
use core::ops::{Index, IndexMut};
use libsys::{error::Errno, mem::memset};
use core::arch::asm;
use libsys::error::Errno;
pub use virt_impl::{EntryImpl, TableImpl};
/// Transparent wrapper structure representing a single
/// translation table entry
#[derive(Clone, Copy)]
#[repr(transparent)]
pub struct Entry(u64);
bitflags! {
/// Virtual space entry attributes
pub struct MapAttributes: u64 {
/// Entry is readable by user threads
const USER_READ = 1 << 0;
/// Entry is writable by user threads
const USER_WRITE = 1 << 1;
/// Data from entry can be executed by user threads
const USER_EXEC = 1 << 2;
/// Structure describing a single level of translation mappings
#[repr(C, align(0x1000))]
pub struct Table {
entries: [Entry; 512],
/// Entry is writable by kernel
const KERNEL_WRITE = 1 << 3;
/// Data from entry can be executed by kernel
const KERNEL_EXEC = 1 << 4;
/// TODO TBD
const SHARE_OUTER = 1 << 5;
/// Memory is used for device interaction
const DEVICE_MEMORY = 1 << 6;
/// Entry is marked as Copy-on-Write
const COPY_ON_WRITE = 1 << 7;
/// Access flag for entry
const ACCESS = 1 << 8;
/// Entry is global across virtual address spaces
const GLOBAL = 1 << 9;
}
}
/// Interface for a single element of paging mapping
pub trait Entry: Clone + Copy {
/// Platform-specific entry attribute representation
type RawAttributes: From<MapAttributes> + Copy + Clone;
/// Invalid entry with no association
const EMPTY: Self;
/// Constructs an entry pointing to next-level table or page
fn normal(addr: usize, attrs: MapAttributes) -> Self;
/// Constructs an entry pointing to a contiguous block
fn block(addr: usize, attrs: MapAttributes) -> Self;
/// Returns physical address the entry points to
fn address(self) -> usize;
/// Changes the entry physical address
fn set_address(&mut self, value: usize);
/// Marks page as CoW and removes user write ability
fn fork_with_cow(&mut self) -> Self;
/// Clones a CoW entry
fn copy_from_cow(self, new_addr: usize) -> Self;
/// Returns `true` if entry maps a paging element
fn is_present(self) -> bool;
/// Returns `true` if page is a 4KiB one
fn is_normal(self) -> bool;
/// Returns `true` if page is marked as Copy-on-Write
fn is_cow(self) -> bool;
/// Returns `true` if page is write-accessible for user threads
fn is_user_writable(self) -> bool;
}
// TODO maybe make this `Space` instead of `Table`?
/// Interface for a single level of virtual memory mapping hierarchy
pub trait Table {
/// Table element type
type Entry: Entry;
/// Writes an [Entry] to last-level table specifying `virt` address.
///
/// # Safety
///
/// Unsafe: allows virtual address space manipulations. Must not affect
/// pages currently in use.
unsafe fn write_last_level(
&mut self,
virt: usize,
entry: Self::Entry,
create_intermediate: bool,
overwrite: bool,
) -> Result<(), Errno>;
/// Reads an [Entry] from last-level table specifying `virt` address
fn read_last_level(&self, virt: usize) -> Result<Self::Entry, Errno>;
}
/// Wrapper for top-most level of address translation tables
#[repr(transparent)]
pub struct Space(Table);
pub struct Space(pub TableImpl);
bitflags! {
/// Attributes attached to each translation [Entry]
pub struct MapAttributes: u64 {
// TODO use 2 lower bits to determine mapping size?
/// nG bit -- determines whether a TLB entry associated with this mapping
/// applies only to current ASID or all ASIDs.
const NOT_GLOBAL = 1 << 11;
/// AF bit -- must be set by software, otherwise Access Error exception is
/// generated when the page is accessed
const ACCESS = 1 << 10;
/// The memory region is outer-shareable
const SH_OUTER = 2 << 8;
/// This page is used for device-MMIO mapping and uses MAIR attribute #1
const DEVICE = 1 << 2;
/// Pages marked with this bit are Copy-on-Write
const EX_COW = 1 << 55;
/// UXN bit -- if set, page may not be used for instruction fetching from EL0
const UXN = 1 << 54;
/// PXN bit -- if set, page may not be used for instruction fetching from EL1
const PXN = 1 << 53;
// AP field
// Default behavior is: read-write for EL1, no access for EL0
/// If set, the page referred to by this entry is read-only for both EL0/EL1
const AP_BOTH_READONLY = 3 << 6;
/// If set, the page referred to by this entry is read-write for both EL0/EL1
const AP_BOTH_READWRITE = 1 << 6;
}
}
impl Table {
/// Returns next-level translation table reference for `index`, if one is present.
/// If `index` represents a `Block`-type mapping, will return an error.
/// If `index` does not map to any translation table, will try to allocate, init and
/// map a new one, returning it after doing so.
pub fn next_level_table_or_alloc(&mut self, index: usize) -> Result<&'static mut Table, Errno> {
let entry = self[index];
if entry.is_present() {
if !entry.is_table() {
return Err(Errno::InvalidArgument);
}
Ok(unsafe { &mut *(mem::virtualize(entry.address_unchecked()) as *mut _) })
} else {
let phys = phys::alloc_page(PageUsage::Paging)?;
let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) };
self[index] = Entry::table(phys, MapAttributes::empty());
res.entries.fill(Entry::invalid());
Ok(res)
}
impl Space {
/// Creates a new address space with only kernel entries mapped
pub fn alloc_empty() -> Result<&'static mut Self, Errno> {
virt_impl::alloc_empty_space()
}
/// Returns next-level translation table reference for `index`, if one is present.
/// Same as [next_level_table_or_alloc], but returns `None` if no table is mapped.
pub fn next_level_table(&mut self, index: usize) -> Option<&'static mut Table> {
let entry = self[index];
if entry.is_present() {
if !entry.is_table() {
panic!("Entry is not a table: idx={}", index);
}
Some(unsafe { &mut *(mem::virtualize(entry.address_unchecked()) as *mut _) })
} else {
None
}
}
/// Constructs and fills a [Table] with non-present mappings
pub const fn empty() -> Table {
Table {
entries: [Entry::invalid(); 512],
}
}
}
impl Index<usize> for Table {
type Output = Entry;
fn index(&self, index: usize) -> &Self::Output {
&self.entries[index]
}
}
impl IndexMut<usize> for Table {
fn index_mut(&mut self, index: usize) -> &mut Self::Output {
&mut self.entries[index]
}
}
impl Entry {
const PRESENT: u64 = 1 << 0;
const TABLE: u64 = 1 << 1;
const PHYS_MASK: u64 = 0x0000FFFFFFFFF000;
/// Constructs a single non-present mapping
pub const fn invalid() -> Self {
Self(0)
}
/// Constructs a `Block`-type memory mapping
pub const fn block(phys: usize, attrs: MapAttributes) -> Self {
Self((phys as u64 & Self::PHYS_MASK) | attrs.bits() | Self::PRESENT)
}
/// Constructs a `Table` or `Page`-type mapping depending on translation level
/// this entry is used at
pub const fn table(phys: usize, attrs: MapAttributes) -> Self {
Self((phys as u64 & Self::PHYS_MASK) | attrs.bits() | Self::PRESENT | Self::TABLE)
}
/// Returns `true` if this entry is not invalid
pub const fn is_present(self) -> bool {
self.0 & Self::PRESENT != 0
}
/// Returns `true` if this entry is a `Table` or `Page`-type mapping
pub const fn is_table(self) -> bool {
self.0 & Self::TABLE != 0
}
/// Returns the target address of this translation entry.
/// Releases the intermediate paging structures and data pages
/// used by the memory space.
///
/// # Safety
///
/// Does not check if the entry is actually valid.
pub const unsafe fn address_unchecked(self) -> usize {
(self.0 & Self::PHYS_MASK) as usize
/// Only safe to call on spaces not currently in use, otherwise will
/// trigger undefined behavior and/or page fault.
pub unsafe fn release(&mut self) {
virt_impl::release_space(self)
}
unsafe fn set_address(&mut self, address: usize) {
self.0 &= !Self::PHYS_MASK;
self.0 |= (address as u64) & Self::PHYS_MASK;
/// Performs process address space forking
pub fn fork(&mut self) -> Result<&'static mut Self, Errno> {
virt_impl::fork_space(self)
}
unsafe fn fork_flags(self) -> MapAttributes {
MapAttributes::from_bits_unchecked(self.0 & !Self::PHYS_MASK)
/// Returns physical address of this table
pub fn address_phys(&mut self) -> usize {
self as *mut _ as usize - mem::KERNEL_OFFSET
}
fn set_cow(&mut self) {
self.0 |= (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits();
}
fn clear_cow(&mut self) {
self.0 &= !(MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits();
self.0 |= MapAttributes::AP_BOTH_READWRITE.bits();
}
#[inline]
fn is_cow(self) -> bool {
let attrs = (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits();
self.0 & attrs == attrs
}
}
impl Space {
/// Creates a new virtual address space and fills it with [Entry::invalid()]
/// mappings. Does physical memory page allocation.
pub fn alloc_empty() -> Result<&'static mut Self, Errno> {
let phys = phys::alloc_page(PageUsage::Paging)?;
let res = unsafe { &mut *(mem::virtualize(phys) as *mut Self) };
res.0.entries.fill(Entry::invalid());
Ok(res)
}
/// Inserts a single `virt` -> `phys` translation entry to this address space.
///
/// TODO: only works with 4K-sized pages at this moment.
pub fn map(&mut self, virt: usize, phys: usize, flags: MapAttributes) -> Result<(), Errno> {
let l0i = virt >> 30;
let l1i = (virt >> 21) & 0x1FF;
let l2i = (virt >> 12) & 0x1FF;
let l1_table = self.0.next_level_table_or_alloc(l0i)?;
let l2_table = l1_table.next_level_table_or_alloc(l1i)?;
if l2_table[l2i].is_present() {
Err(Errno::AlreadyExists)
} else {
l2_table[l2i] = Entry::table(phys, flags | MapAttributes::ACCESS);
#[cfg(feature = "verbose")]
debugln!("{:#p} Map {:#x} -> {:#x}, {:?}", self, virt, phys, flags);
Ok(())
/// Creates a new virtual -> physical memory mapping. Will fail if one is
/// already associated with given virtual address.
pub fn map(&mut self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Errno> {
unsafe {
self.0.write_last_level(
virt,
Entry::normal(phys, attrs | MapAttributes::ACCESS),
true,
false,
)
}
}
/// Translates a virtual address into a corresponding physical one.
///
/// Only works for 4K pages atm.
// TODO extract attributes
/// Returns a virtual address physical mapping destination
pub fn translate(&mut self, virt: usize) -> Result<usize, Errno> {
let l0i = virt >> 30;
let l1i = (virt >> 21) & 0x1FF;
let l2i = (virt >> 12) & 0x1FF;
let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?;
let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?;
let entry = l2_table[l2i];
if entry.is_present() {
Ok(unsafe { entry.address_unchecked() })
} else {
Err(Errno::DoesNotExist)
}
self.0.read_last_level(virt).map(Entry::address)
}
/// Attempts to resolve a page fault at `virt` address by copying the
/// underlying Copy-on-Write mapping (if any is present)
/// Performs Copy-on-Write cloning on page fault
pub fn try_cow_copy(&mut self, virt: usize) -> Result<(), Errno> {
let virt = virt & !0xFFF;
let l0i = virt >> 30;
let l1i = (virt >> 21) & 0x1FF;
let l2i = (virt >> 12) & 0x1FF;
let entry = self.0.read_last_level(virt)?;
let src_phys = entry.address();
let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?;
let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?;
let entry = l2_table[l2i];
if !entry.is_present() {
warnln!("Entry is not present: {:#x}", virt);
return Err(Errno::DoesNotExist);
}
let src_phys = unsafe { entry.address_unchecked() };
if !entry.is_cow() {
warnln!(
"Entry is not marked as CoW: {:#x}, points to {:#x}",
@@ -260,11 +158,11 @@ impl Space {
}
let dst_phys = unsafe { phys::copy_cow_page(src_phys)? };
unsafe {
l2_table[l2i].set_address(dst_phys);
}
l2_table[l2i].clear_cow();
unsafe {
self.0
.write_last_level(virt, entry.copy_from_cow(dst_phys), false, true)?;
}
Ok(())
}
@@ -294,135 +192,14 @@ impl Space {
Err(Errno::OutOfMemory)
}
/// Removes a single 4K page mapping from the table and
/// releases the underlying physical memory
pub fn unmap_single(&mut self, page: usize) -> Result<(), Errno> {
let l0i = page >> 30;
let l1i = (page >> 21) & 0x1FF;
let l2i = (page >> 12) & 0x1FF;
let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?;
let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?;
let entry = l2_table[l2i];
if !entry.is_present() {
return Err(Errno::DoesNotExist);
}
let phys = unsafe { entry.address_unchecked() };
unsafe {
phys::free_page(phys)?;
}
l2_table[l2i] = Entry::invalid();
unsafe {
asm!("tlbi vaae1, {}", in(reg) page);
}
// TODO release paging structure memory
Ok(())
}
/// Releases a range of virtual pages and their corresponding physical pages
/// Releases memory from virtual address range `start`..`start + len * 0x1000`
pub fn free(&mut self, start: usize, len: usize) -> Result<(), Errno> {
for i in 0..len {
self.unmap_single(start + i * 0x1000)?;
unsafe {
self.0
.write_last_level(start + i * 0x1000, EntryImpl::EMPTY, false, true)?;
}
}
Ok(())
}
/// Performs a copy of the address space, cloning data owned by it
pub fn fork(&mut self) -> Result<&'static mut Self, Errno> {
let res = Self::alloc_empty()?;
for l0i in 0..512 {
if let Some(l1_table) = self.0.next_level_table(l0i) {
for l1i in 0..512 {
if let Some(l2_table) = l1_table.next_level_table(l1i) {
for l2i in 0..512 {
let entry = l2_table[l2i];
if !entry.is_present() {
continue;
}
assert!(entry.is_table());
let src_phys = unsafe { entry.address_unchecked() };
let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12);
let dst_phys = unsafe { phys::fork_page(src_phys)? };
let mut flags = unsafe { entry.fork_flags() };
if dst_phys != src_phys {
todo!();
// res.map(virt_addr, dst_phys, flags)?;
} else {
let writable = flags & MapAttributes::AP_BOTH_READONLY
== MapAttributes::AP_BOTH_READWRITE;
if writable {
flags |=
MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW;
l2_table[l2i].set_cow();
unsafe {
asm!("tlbi vaae1, {}", in(reg) virt_addr);
}
}
res.map(virt_addr, dst_phys, flags)?;
}
}
}
}
}
}
Ok(res)
}
/// Releases all the mappings from the address space. Frees all
/// memory pages referenced by this space as well as those used for
/// its paging tables.
///
/// # Safety
///
/// Unsafe: may invalidate currently active address space
pub unsafe fn release(space: &mut Self) {
for l0i in 0..512 {
let l0_entry = space.0[l0i];
if !l0_entry.is_present() {
continue;
}
assert!(l0_entry.is_table());
let l1_table = &mut *(mem::virtualize(l0_entry.address_unchecked()) as *mut Table);
for l1i in 0..512 {
let l1_entry = l1_table[l1i];
if !l1_entry.is_present() {
continue;
}
assert!(l1_entry.is_table());
let l2_table = &mut *(mem::virtualize(l1_entry.address_unchecked()) as *mut Table);
for l2i in 0..512 {
let entry = l2_table[l2i];
if !entry.is_present() {
continue;
}
assert!(entry.is_table());
phys::free_page(entry.address_unchecked()).unwrap();
}
phys::free_page(l1_entry.address_unchecked()).unwrap();
}
phys::free_page(l0_entry.address_unchecked()).unwrap();
}
memset(space as *mut Space as *mut u8, 0, 4096);
}
/// Returns the physical address of this structure
pub fn address_phys(&mut self) -> usize {
(self as *mut _ as usize) - mem::KERNEL_OFFSET
}
}
kernel/src/proc/elf.rs
+6 -6
View File
@@ -2,7 +2,7 @@
use crate::mem::{
self,
phys::{self, PageUsage},
virt::{MapAttributes, Space},
virt::table::{Space, MapAttributes},
};
use core::mem::{size_of, MaybeUninit};
use libsys::{
@@ -66,10 +66,10 @@ struct Phdr<E: Elf> {
}
fn map_flags(elf_flags: usize) -> MapAttributes {
let mut dst_flags = MapAttributes::NOT_GLOBAL | MapAttributes::SH_OUTER;
let mut dst_flags = MapAttributes::SHARE_OUTER;
if elf_flags & (1 << 0) /* PF_X */ == 0 {
dst_flags |= MapAttributes::UXN | MapAttributes::PXN;
if elf_flags & (1 << 0) /* PF_X */ != 0 {
dst_flags |= MapAttributes::USER_EXEC;
}
match (elf_flags & (3 << 1)) >> 1 {
@@ -78,9 +78,9 @@ fn map_flags(elf_flags: usize) -> MapAttributes {
// Write-only: not sure if such mapping should exist at all
1 => todo!(),
// Read-only
2 => dst_flags |= MapAttributes::AP_BOTH_READONLY,
2 => dst_flags |= MapAttributes::USER_READ,
// Read+Write
3 => dst_flags |= MapAttributes::AP_BOTH_READWRITE,
3 => dst_flags |= MapAttributes::USER_WRITE | MapAttributes::USER_READ,
_ => unreachable!(),
};
kernel/src/proc/process.rs
+7 -16
View File
@@ -3,7 +3,7 @@ use crate::arch::{aarch64::exception::ExceptionFrame, intrin};
use crate::mem::{
self,
phys::{self, PageUsage},
virt::{MapAttributes, Space},
virt::table::{MapAttributes, Space},
};
use crate::proc::{
wait::Wait, Context, ProcessIo, Thread, ThreadRef, ThreadState, Tid, PROCESSES, SCHED,
@@ -383,11 +383,7 @@ impl Process {
phys
} else {
let page = phys::alloc_page(PageUsage::UserPrivate)?;
let flags = MapAttributes::SH_OUTER
| MapAttributes::NOT_GLOBAL
| MapAttributes::UXN
| MapAttributes::PXN
| MapAttributes::AP_BOTH_READONLY;
let flags = MapAttributes::SHARE_OUTER | MapAttributes::USER_READ;
space.map(page_virt, page, flags)?;
page
};
@@ -407,11 +403,7 @@ impl Process {
phys
} else {
let page = phys::alloc_page(PageUsage::UserPrivate)?;
let flags = MapAttributes::SH_OUTER
| MapAttributes::NOT_GLOBAL
| MapAttributes::UXN
| MapAttributes::PXN
| MapAttributes::AP_BOTH_READONLY;
let flags = MapAttributes::SHARE_OUTER | MapAttributes::USER_READ;
space.map(page_virt, page, flags)?;
page
};
@@ -518,11 +510,10 @@ impl Process {
let ustack_virt_bottom = Self::USTACK_VIRT_TOP - Self::USTACK_PAGES * mem::PAGE_SIZE;
for i in 0..Self::USTACK_PAGES {
let page = phys::alloc_page(PageUsage::UserPrivate).unwrap();
let flags = MapAttributes::SH_OUTER
| MapAttributes::NOT_GLOBAL
| MapAttributes::UXN
| MapAttributes::PXN
| MapAttributes::AP_BOTH_READWRITE;
let flags = MapAttributes::SHARE_OUTER
| MapAttributes::USER_READ
| MapAttributes::USER_WRITE
| MapAttributes::KERNEL_WRITE;
new_space
.map(ustack_virt_bottom + i * mem::PAGE_SIZE, page, flags)
.unwrap();
kernel/src/syscall/mod.rs
+6 -6
View File
@@ -4,7 +4,7 @@ use crate::arch::{machine, platform::exception::ExceptionFrame};
use crate::debug::Level;
use crate::dev::timer::TimestampSource;
use crate::fs::create_filesystem;
use crate::mem::{phys::PageUsage, virt::MapAttributes};
use crate::mem::{phys::PageUsage, virt::table::MapAttributes};
use crate::proc::{self, elf, wait, Process, ProcessIo, Thread};
use core::mem::size_of;
use core::ops::DerefMut;
@@ -210,7 +210,7 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
let _flags = MemoryAccess::from_bits(args[3] as u32).ok_or(Errno::InvalidArgument)?;
let mut attrs =
MapAttributes::NOT_GLOBAL | MapAttributes::SH_OUTER | MapAttributes::PXN;
MapAttributes::SHARE_OUTER;
if !acc.contains(MemoryAccess::READ) {
return Err(Errno::NotImplemented);
}
@@ -218,12 +218,12 @@ fn _syscall(num: SystemCall, args: &[usize]) -> Result<usize, Errno> {
if acc.contains(MemoryAccess::EXEC) {
return Err(Errno::PermissionDenied);
}
attrs |= MapAttributes::AP_BOTH_READWRITE;
attrs |= MapAttributes::USER_WRITE | MapAttributes::USER_READ;
} else {
attrs |= MapAttributes::AP_BOTH_READONLY;
attrs |= MapAttributes::USER_READ;
}
if !acc.contains(MemoryAccess::EXEC) {
attrs |= MapAttributes::UXN;
if acc.contains(MemoryAccess::EXEC) {
attrs |= MapAttributes::USER_EXEC;
}
// TODO don't ignore flags