diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index 08b3e5f..0675dec 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -26,6 +26,7 @@ cortex-a = { version = "6.x.x" } [features] pl011 = [] pl031 = [] +verbose = [] mach_qemu = ["pl011", "pl031"] mach_orangepi3 = [] diff --git a/kernel/src/arch/aarch64/boot/mod.rs b/kernel/src/arch/aarch64/boot/mod.rs index afe913f..7fcfae2 100644 --- a/kernel/src/arch/aarch64/boot/mod.rs +++ b/kernel/src/arch/aarch64/boot/mod.rs @@ -33,8 +33,11 @@ fn init_device_tree(fdt_base_phys: usize) -> Result<(), Errno> { return Ok(()); }; - use crate::debug::Level; - fdt.dump(Level::Debug); + #[cfg(feature = "verbose")] + { + use crate::debug::Level; + fdt.dump(Level::Debug); + } let mut cfg = CONFIG.lock(); @@ -102,7 +105,6 @@ extern "C" fn __aa64_bsp_main(fdt_base: usize) -> ! { machine::init_board().unwrap(); - debugln!("Config: {:#x?}", CONFIG.lock()); infoln!("Machine init finished"); unsafe { diff --git a/kernel/src/arch/aarch64/exception.rs b/kernel/src/arch/aarch64/exception.rs index 3aa9376..9cbe269 100644 --- a/kernel/src/arch/aarch64/exception.rs +++ b/kernel/src/arch/aarch64/exception.rs @@ -3,6 +3,8 @@ use crate::arch::machine; use crate::debug::Level; use crate::dev::irq::{IntController, IrqContext}; +use crate::proc::Process; +use crate::mem; use crate::syscall; use ::syscall::abi; use cortex_a::registers::{ESR_EL1, FAR_EL1}; @@ -84,10 +86,25 @@ extern "C" fn __aa64_exc_sync_handler(exc: &mut ExceptionFrame) { #[allow(clippy::single_match)] match err_code { - EC_DATA_ABORT_ELX | EC_DATA_ABORT_EL0 => { + EC_DATA_ABORT_ELX => { let far = FAR_EL1.get(); dump_data_abort(Level::Error, esr, far); } + EC_DATA_ABORT_EL0 => { + let far = FAR_EL1.get() as usize; + let proc = Process::current(); + + if far < mem::KERNEL_OFFSET { + if let Err(_) = proc.manipulate_space(|space| space.try_cow_copy(far)) { + // Kill program + todo!() + } + return; + } else { + // Kill program + todo!() + } + } EC_SVC_AA64 => { unsafe { if exc.x[8] == abi::SYS_FORK { diff --git a/kernel/src/mem/phys/manager.rs b/kernel/src/mem/phys/manager.rs index cf8f33f..0b3fdc4 100644 --- a/kernel/src/mem/phys/manager.rs +++ b/kernel/src/mem/phys/manager.rs @@ -1,5 +1,5 @@ use super::{PageInfo, PageUsage}; -use crate::mem::{memcpy, virtualize, PAGE_SIZE}; +use crate::mem::{memcpy, memset, virtualize, PAGE_SIZE}; use crate::sync::IrqSafeSpinLock; use core::mem; use error::Errno; @@ -8,7 +8,8 @@ pub unsafe trait Manager { fn alloc_page(&mut self, pu: PageUsage) -> Result; fn alloc_contiguous_pages(&mut self, pu: PageUsage, count: usize) -> Result; fn free_page(&mut self, page: usize) -> Result<(), Errno>; - fn clone_page(&mut self, src: usize) -> Result; + fn copy_cow_page(&mut self, src: usize) -> Result; + fn fork_page(&mut self, src: usize) -> Result; // TODO status() } pub struct SimpleManager { @@ -60,7 +61,6 @@ unsafe impl Manager for SimpleManager { fn alloc_page(&mut self, pu: PageUsage) -> Result { self.alloc_single_index(pu) .map(|r| (self.base_index + r) * PAGE_SIZE) - //return Ok((self.base_index + index) * PAGE_SIZE); } fn alloc_contiguous_pages(&mut self, pu: PageUsage, count: usize) -> Result { 'l0: for i in 0..self.pages.len() { @@ -79,21 +79,56 @@ unsafe impl Manager for SimpleManager { } Err(Errno::OutOfMemory) } - fn free_page(&mut self, _page: usize) -> Result<(), Errno> { - todo!() + fn free_page(&mut self, addr: usize) -> Result<(), Errno> { + let index = self.page_index(addr); + let page = &mut self.pages[index]; + + assert!(page.usage != PageUsage::Reserved && page.usage != PageUsage::Available); + + if page.refcount > 1 { + page.refcount -= 1; + } else { + assert_eq!(page.refcount, 1); + page.usage = PageUsage::Available; + page.refcount = 0; + } + Ok(()) } - fn clone_page(&mut self, src: usize) -> Result { + fn copy_cow_page(&mut self, src: usize) -> Result { let src_index = self.page_index(src); - let src_usage = self.pages[src_index].usage; - assert_eq!(self.pages[src_index].refcount, 1); - let dst_index = self.alloc_single_index(src_usage)?; - assert!(src_usage != PageUsage::Available && src_usage != PageUsage::Reserved); - let dst = (self.base_index + dst_index) * PAGE_SIZE; - unsafe { - memcpy(virtualize(dst) as *mut u8, virtualize(src) as *mut u8, 4096); + let page = &mut self.pages[src_index]; + let usage = page.usage; + if usage != PageUsage::UserPrivate { + panic!("CoW not available for non-UserPrivate pages: {:?}", usage); } - Ok(dst) + + if page.refcount > 1 { + page.refcount -= 1; + drop(page); + let dst_index = self.alloc_single_index(usage)?; + let dst = (self.base_index + dst_index) * PAGE_SIZE; + unsafe { + memcpy(virtualize(dst) as *mut u8, virtualize(src) as *mut u8, 4096); + } + Ok(dst) + } else { + assert_eq!(page.refcount, 1); + // No additional operations needed + Ok(src) + } + } + + fn fork_page(&mut self, src: usize) -> Result { + let src_index = self.page_index(src); + let page = &mut self.pages[src_index]; + let usage = page.usage; + if usage != PageUsage::UserPrivate { + todo!("Handle page types other than UserPrivate") + } else { + page.refcount += 1; + } + Ok(src) } } diff --git a/kernel/src/mem/phys/mod.rs b/kernel/src/mem/phys/mod.rs index 4489ba6..de00f34 100644 --- a/kernel/src/mem/phys/mod.rs +++ b/kernel/src/mem/phys/mod.rs @@ -92,8 +92,12 @@ pub unsafe fn free_page(page: usize) -> Result<(), Errno> { /// # Safety /// /// Unsafe: accepts arbitrary `page` arguments -pub unsafe fn clone_page(page: usize) -> Result { - MANAGER.lock().as_mut().unwrap().clone_page(page) +pub unsafe fn fork_page(page: usize) -> Result { + MANAGER.lock().as_mut().unwrap().fork_page(page) +} + +pub unsafe fn copy_cow_page(page: usize) -> Result { + MANAGER.lock().as_mut().unwrap().copy_cow_page(page) } fn find_contiguous>(iter: T, count: usize) -> Option { diff --git a/kernel/src/mem/virt/table.rs b/kernel/src/mem/virt/table.rs index 7617296..31a7e53 100644 --- a/kernel/src/mem/virt/table.rs +++ b/kernel/src/mem/virt/table.rs @@ -38,6 +38,8 @@ bitflags! { /// This page is used for device-MMIO mapping and uses MAIR attribute #1 const DEVICE = 1 << 2; + const EX_COW = 1 << 55; + /// UXN bit -- if set, page may not be used for instruction fetching from EL0 const UXN = 1 << 54; /// PXN bit -- if set, page may not be used for instruction fetching from EL1 @@ -151,9 +153,29 @@ impl Entry { (self.0 & Self::PHYS_MASK) as usize } + unsafe fn set_address(&mut self, address: usize) { + self.0 &= !Self::PHYS_MASK; + self.0 |= (address as u64) & Self::PHYS_MASK; + } + unsafe fn fork_flags(self) -> MapAttributes { MapAttributes::from_bits_unchecked(self.0 & !Self::PHYS_MASK) } + + fn set_cow(&mut self) { + self.0 |= (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits(); + } + + fn clear_cow(&mut self) { + self.0 &= !(MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits(); + self.0 |= MapAttributes::AP_BOTH_READWRITE.bits(); + } + + #[inline] + fn is_cow(self) -> bool { + let attrs = (MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW).bits(); + self.0 & attrs == attrs + } } impl Space { @@ -181,11 +203,46 @@ impl Space { Err(Errno::AlreadyExists) } else { l2_table[l2i] = Entry::table(phys, flags | MapAttributes::ACCESS); + #[cfg(feature = "verbose")] debugln!("Map {:#x} -> {:#x}", virt, phys); Ok(()) } } + pub fn try_cow_copy(&mut self, virt: usize) -> Result<(), Errno> { + let l0i = virt >> 30; + let l1i = (virt >> 21) & 0x1FF; + let l2i = (virt >> 12) & 0x1FF; + + let l1_table = self.0.next_level_table(l0i).ok_or(Errno::DoesNotExist)?; + let l2_table = l1_table.next_level_table(l1i).ok_or(Errno::DoesNotExist)?; + + let entry = l2_table[l2i]; + + if !entry.is_present() { + return Err(Errno::DoesNotExist); + } + + let src_phys = unsafe { entry.address_unchecked() }; + if !entry.is_cow() { + return Err(Errno::DoesNotExist); + } + + let dst_phys = unsafe { phys::copy_cow_page(src_phys)? }; + if src_phys != dst_phys { + unsafe { + asm!("tlbi vaae1, {}", in(reg) virt); + } + } + + unsafe { + l2_table[l2i].set_address(dst_phys); + } + l2_table[l2i].clear_cow(); + + Ok(()) + } + /// Performs a copy of the address space, cloning data owned by it pub fn fork(&mut self) -> Result<&'static mut Self, Errno> { let res = Self::alloc_empty()?; @@ -196,15 +253,23 @@ impl Space { for l2i in 0..512 { let entry = l2_table[l2i]; - // TODO copy-on-write - if entry.is_present() { - assert!(entry.is_table()); - let src_phys = unsafe { entry.address_unchecked() }; - let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12); - debugln!("Fork page {:#x}:{:#x}", virt_addr, src_phys); - let dst_phys = unsafe { phys::clone_page(src_phys)? }; + if !entry.is_present() { + continue; + } - res.map(virt_addr, dst_phys, unsafe { entry.fork_flags() })?; + assert!(entry.is_table()); + let src_phys = unsafe { entry.address_unchecked() }; + let virt_addr = (l0i << 30) | (l1i << 21) | (l2i << 12); + let dst_phys = unsafe { phys::fork_page(src_phys)? }; + + let mut flags = unsafe { entry.fork_flags() }; + if dst_phys != src_phys { + res.map(virt_addr, dst_phys, flags)?; + } else { + // TODO only apply CoW to writable pages + flags |= MapAttributes::AP_BOTH_READONLY | MapAttributes::EX_COW; + l2_table[l2i].set_cow(); + res.map(virt_addr, dst_phys, flags); } } } @@ -213,4 +278,48 @@ impl Space { } Ok(res) } + + pub unsafe fn release(space: &mut Self) { + for l0i in 0..512 { + let l0_entry = space.0[l0i]; + if !l0_entry.is_present() { + continue; + } + + assert!(l0_entry.is_table()); + let l1_table = + unsafe { &mut *(mem::virtualize(l0_entry.address_unchecked()) as *mut Table) }; + + for l1i in 0..512 { + let l1_entry = l1_table[l1i]; + if !l1_entry.is_present() { + continue; + } + assert!(l1_entry.is_table()); + let l2_table = + unsafe { &mut *(mem::virtualize(l1_entry.address_unchecked()) as *mut Table) }; + + for l2i in 0..512 { + let entry = l2_table[l2i]; + if !entry.is_present() { + continue; + } + + assert!(entry.is_table()); + unsafe { + phys::free_page(unsafe { entry.address_unchecked() }); + } + } + unsafe { + phys::free_page(unsafe { l1_entry.address_unchecked() }); + } + } + unsafe { + phys::free_page(unsafe { l0_entry.address_unchecked() }); + } + } + unsafe { + mem::memset(space as *mut Space as *mut u8, 0, 4096); + } + } } diff --git a/kernel/src/proc/elf.rs b/kernel/src/proc/elf.rs index 28df5b5..2f6c3d9 100644 --- a/kernel/src/proc/elf.rs +++ b/kernel/src/proc/elf.rs @@ -105,7 +105,7 @@ where let page_off = (dst_page_off + off) % mem::PAGE_SIZE; let count = core::cmp::min(rem, mem::PAGE_SIZE - page_off); - let page = phys::alloc_page(PageUsage::Kernel)?; + let page = phys::alloc_page(PageUsage::UserPrivate)?; // TODO fetch existing mapping and test flag equality instead // if flags differ, bail out diff --git a/kernel/src/proc/process.rs b/kernel/src/proc/process.rs index 04a0877..a63e786 100644 --- a/kernel/src/proc/process.rs +++ b/kernel/src/proc/process.rs @@ -168,6 +168,14 @@ impl Process { (&mut *ctx).enter() } + #[inline] + pub fn manipulate_space Result<(), Errno>>( + &self, + f: F, + ) -> Result<(), Errno> { + f(self.inner.lock().space.as_mut().unwrap()) + } + /// Schedules a next thread for execution /// /// # Safety @@ -285,6 +293,13 @@ impl Process { lock.exit = Some(status); lock.state = State::Finished; + if let Some(space) = lock.space.take() { + unsafe { + // TODO invalidate everything related to this ASID + Space::release(space); + } + } + self.io.lock().handle_exit(); SCHED.dequeue(lock.id);