alnyan/osdev5
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

feature: implement CoW for user pages:

Browse Source
This commit is contained in:
Mark Poliakov
2021-11-09 18:24:08 +02:00
parent e1c7ebc8e1
commit e34df46663
8 changed files with 212 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
kernel/src/arch/aarch64/boot/mod.rs
+5 -3
View File
@@ -33,8 +33,11 @@ fn init_device_tree(fdt_base_phys: usize) -> Result<(), Errno> {
return Ok(());
};
use crate::debug::Level;
fdt.dump(Level::Debug);
#[cfg(feature = "verbose")]
{
use crate::debug::Level;
fdt.dump(Level::Debug);
}
let mut cfg = CONFIG.lock();
@@ -102,7 +105,6 @@ extern "C" fn __aa64_bsp_main(fdt_base: usize) -> ! {
machine::init_board().unwrap();
debugln!("Config: {:#x?}", CONFIG.lock());
infoln!("Machine init finished");
unsafe {
kernel/src/arch/aarch64/exception.rs
+18 -1
View File
@@ -3,6 +3,8 @@
use crate::arch::machine;
use crate::debug::Level;
use crate::dev::irq::{IntController, IrqContext};
use crate::proc::Process;
use crate::mem;
use crate::syscall;
use ::syscall::abi;
use cortex_a::registers::{ESR_EL1, FAR_EL1};
@@ -84,10 +86,25 @@ extern "C" fn __aa64_exc_sync_handler(exc: &mut ExceptionFrame) {
#[allow(clippy::single_match)]
match err_code {
EC_DATA_ABORT_ELX | EC_DATA_ABORT_EL0 => {
EC_DATA_ABORT_ELX => {
let far = FAR_EL1.get();
dump_data_abort(Level::Error, esr, far);
}
EC_DATA_ABORT_EL0 => {
let far = FAR_EL1.get() as usize;
let proc = Process::current();
if far < mem::KERNEL_OFFSET {
if let Err(_) = proc.manipulate_space(|space| space.try_cow_copy(far)) {
// Kill program
todo!()
}
return;
} else {
// Kill program
todo!()
}
}
EC_SVC_AA64 => {
unsafe {
if exc.x[8] == abi::SYS_FORK {
kernel/src/mem/phys/manager.rs
+49 -14
View File
@@ -1,5 +1,5 @@
use super::{PageInfo, PageUsage};
use crate::mem::{memcpy, virtualize, PAGE_SIZE};
use crate::mem::{memcpy, memset, virtualize, PAGE_SIZE};
use crate::sync::IrqSafeSpinLock;
use core::mem;
use error::Errno;
@@ -8,7 +8,8 @@ pub unsafe trait Manager {
fn alloc_page(&mut self, pu: PageUsage) -> Result<usize, Errno>;
fn alloc_contiguous_pages(&mut self, pu: PageUsage, count: usize) -> Result<usize, Errno>;
fn free_page(&mut self, page: usize) -> Result<(), Errno>;
fn clone_page(&mut self, src: usize) -> Result<usize, Errno>;
fn copy_cow_page(&mut self, src: usize) -> Result<usize, Errno>;
fn fork_page(&mut self, src: usize) -> Result<usize, Errno>;
// TODO status()
}
pub struct SimpleManager {
@@ -60,7 +61,6 @@ unsafe impl Manager for SimpleManager {
fn alloc_page(&mut self, pu: PageUsage) -> Result<usize, Errno> {
self.alloc_single_index(pu)
.map(|r| (self.base_index + r) * PAGE_SIZE)
//return Ok((self.base_index + index) * PAGE_SIZE);
}
fn alloc_contiguous_pages(&mut self, pu: PageUsage, count: usize) -> Result<usize, Errno> {
'l0: for i in 0..self.pages.len() {
@@ -79,21 +79,56 @@ unsafe impl Manager for SimpleManager {
}
Err(Errno::OutOfMemory)
}
fn free_page(&mut self, _page: usize) -> Result<(), Errno> {
todo!()
fn free_page(&mut self, addr: usize) -> Result<(), Errno> {
let index = self.page_index(addr);
let page = &mut self.pages[index];
assert!(page.usage != PageUsage::Reserved && page.usage != PageUsage::Available);
if page.refcount > 1 {
page.refcount -= 1;
} else {
assert_eq!(page.refcount, 1);
page.usage = PageUsage::Available;
page.refcount = 0;
}
Ok(())
}
fn clone_page(&mut self, src: usize) -> Result<usize, Errno> {
fn copy_cow_page(&mut self, src: usize) -> Result<usize, Errno> {
let src_index = self.page_index(src);
let src_usage = self.pages[src_index].usage;
assert_eq!(self.pages[src_index].refcount, 1);
let dst_index = self.alloc_single_index(src_usage)?;
assert!(src_usage != PageUsage::Available && src_usage != PageUsage::Reserved);
let dst = (self.base_index + dst_index) * PAGE_SIZE;
unsafe {
memcpy(virtualize(dst) as *mut u8, virtualize(src) as *mut u8, 4096);
let page = &mut self.pages[src_index];
let usage = page.usage;
if usage != PageUsage::UserPrivate {
panic!("CoW not available for non-UserPrivate pages: {:?}", usage);
}
Ok(dst)
if page.refcount > 1 {
page.refcount -= 1;
drop(page);
let dst_index = self.alloc_single_index(usage)?;
let dst = (self.base_index + dst_index) * PAGE_SIZE;
unsafe {
memcpy(virtualize(dst) as *mut u8, virtualize(src) as *mut u8, 4096);
}
Ok(dst)
} else {
assert_eq!(page.refcount, 1);
// No additional operations needed
Ok(src)
}
}
fn fork_page(&mut self, src: usize) -> Result<usize, Errno> {
let src_index = self.page_index(src);
let page = &mut self.pages[src_index];
let usage = page.usage;
if usage != PageUsage::UserPrivate {
todo!("Handle page types other than UserPrivate")
} else {
page.refcount += 1;
}
Ok(src)
}
}
kernel/src/mem/phys/mod.rs
+6 -2
View File
@@ -92,8 +92,12 @@ pub unsafe fn free_page(page: usize) -> Result<(), Errno> {
/// # Safety
///
/// Unsafe: accepts arbitrary `page` arguments
pub unsafe fn clone_page(page: usize) -> Result<usize, Errno> {
MANAGER.lock().as_mut().unwrap().clone_page(page)
pub unsafe fn fork_page(page: usize) -> Result<usize, Errno> {
MANAGER.lock().as_mut().unwrap().fork_page(page)
}
pub unsafe fn copy_cow_page(page: usize) -> Result<usize, Errno> {
MANAGER.lock().as_mut().unwrap().copy_cow_page(page)
}
fn find_contiguous<T: Iterator<Item = MemoryRegion>>(iter: T, count: usize) -> Option<usize> {
kernel/src/mem/virt/table.rs
+117 -8
View File
@@ -38,6 +38,8 @@ bitflags! {
/// This page is used for device-MMIO mapping and uses MAIR attribute #1
const DEVICE = 1 << 2;
const EX_COW = 1 << 55;
/// UXN bit -- if set, page may not be used for instruction fetching from EL0
const UXN = 1 << 54;
/// PXN bit -- if set, page may not be used for instruction fetching from EL1
@@ -151,9 +153,29 @@ impl Entry {
(self.0 & Self::PHYS_MASK) as usize
}
unsafe fn set_address(&mut self, address: usize) {
self.0 &= !Self::PHYS_MASK;
self.0 |= (address as u64) & Self::PHYS_MASK;
}
unsafe fn fork_flags(self) -> MapAttributes {
MapAttributes::from_bits_unchecked(self.0 & !Self::PHYS_MASK)
}
fn set_cow(&mut self) {
self.0 |= (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits();
}
fn clear_cow(&mut self) {
self.0 &= !(MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits();
self.0 |= MapAttributes::AP_BOTH_READWRITE.bits();
}
#[inline]
fn is_cow(self) -> bool {
let attrs = (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits();
self.0 & attrs == attrs
}
}
impl Space {
@@ -181,11 +203,46 @@ impl Space {
Err(Errno::AlreadyExists)
} else {
l2_table[l2i] = Entry::table(phys, flags | MapAttributes::ACCESS);
#[cfg(feature = "verbose")]
debugln!("Map {:#x} -> {:#x}", virt, phys);
Ok(())
}
}
pub fn try_cow_copy(&mut self, virt: usize) -> Result<(), Errno> {
let l0i = virt >> 30;
let l1i = (virt >> 21) & 0x1FF;
let l2i = (virt >> 12) & 0x1FF;
let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?;
let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?;
let entry = l2_table[l2i];
if !entry.is_present() {
return Err(Errno::DoesNotExist);
}
let src_phys = unsafe { entry.address_unchecked() };
if !entry.is_cow() {
return Err(Errno::DoesNotExist);
}
let dst_phys = unsafe { phys::copy_cow_page(src_phys)? };
if src_phys != dst_phys {
unsafe {
asm!("tlbi vaae1, {}", in(reg) virt);
}
}
unsafe {
l2_table[l2i].set_address(dst_phys);
}
l2_table[l2i].clear_cow();
Ok(())
}
/// Performs a copy of the address space, cloning data owned by it
pub fn fork(&mut self) -> Result<&'static mut Self, Errno> {
let res = Self::alloc_empty()?;
@@ -196,15 +253,23 @@ impl Space {
for l2i in 0..512 {
let entry = l2_table[l2i];
// TODO copy-on-write
if entry.is_present() {
assert!(entry.is_table());
let src_phys = unsafe { entry.address_unchecked() };
let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12);
debugln!("Fork page {:#x}:{:#x}", virt_addr, src_phys);
let dst_phys = unsafe { phys::clone_page(src_phys)? };
if !entry.is_present() {
continue;
}
res.map(virt_addr, dst_phys, unsafe { entry.fork_flags() })?;
assert!(entry.is_table());
let src_phys = unsafe { entry.address_unchecked() };
let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12);
let dst_phys = unsafe { phys::fork_page(src_phys)? };
let mut flags = unsafe { entry.fork_flags() };
if dst_phys != src_phys {
res.map(virt_addr, dst_phys, flags)?;
} else {
// TODO only apply CoW to writable pages
flags |= MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW;
l2_table[l2i].set_cow();
res.map(virt_addr, dst_phys, flags);
}
}
}
@@ -213,4 +278,48 @@ impl Space {
}
Ok(res)
}
pub unsafe fn release(space: &mut Self) {
for l0i in 0..512 {
let l0_entry = space.0[l0i];
if !l0_entry.is_present() {
continue;
}
assert!(l0_entry.is_table());
let l1_table =
unsafe { &mut *(mem::virtualize(l0_entry.address_unchecked()) as *mut Table) };
for l1i in 0..512 {
let l1_entry = l1_table[l1i];
if !l1_entry.is_present() {
continue;
}
assert!(l1_entry.is_table());
let l2_table =
unsafe { &mut *(mem::virtualize(l1_entry.address_unchecked()) as *mut Table) };
for l2i in 0..512 {
let entry = l2_table[l2i];
if !entry.is_present() {
continue;
}
assert!(entry.is_table());
unsafe {
phys::free_page(unsafe { entry.address_unchecked() });
}
}
unsafe {
phys::free_page(unsafe { l1_entry.address_unchecked() });
}
}
unsafe {
phys::free_page(unsafe { l0_entry.address_unchecked() });
}
}
unsafe {
mem::memset(space as *mut Space as *mut u8, 0, 4096);
}
}
}
kernel/src/proc/elf.rs
+1 -1
View File
@@ -105,7 +105,7 @@ where
let page_off = (dst_page_off + off) % mem::PAGE_SIZE;
let count = core::cmp::min(rem, mem::PAGE_SIZE - page_off);
let page = phys::alloc_page(PageUsage::Kernel)?;
let page = phys::alloc_page(PageUsage::UserPrivate)?;
// TODO fetch existing mapping and test flag equality instead
// if flags differ, bail out
kernel/src/proc/process.rs
+15
View File
@@ -168,6 +168,14 @@ impl Process {
(&mut *ctx).enter()
}
#[inline]
pub fn manipulate_space<F: FnOnce(&mut Space) -> Result<(), Errno>>(
&self,
f: F,
) -> Result<(), Errno> {
f(self.inner.lock().space.as_mut().unwrap())
}
/// Schedules a next thread for execution
///
/// # Safety
@@ -285,6 +293,13 @@ impl Process {
lock.exit = Some(status);
lock.state = State::Finished;
if let Some(space) = lock.space.take() {
unsafe {
// TODO invalidate everything related to this ASID
Space::release(space);
}
}
self.io.lock().handle_exit();
SCHED.dequeue(lock.id);