#![feature( maybe_uninit_slice, slice_ptr_get, step_trait, const_trait_impl, maybe_uninit_as_bytes, negative_impls )] #![no_std] extern crate alloc; use core::{ alloc::Layout, fmt, marker::PhantomData, mem::{size_of, MaybeUninit}, ops::{Deref, DerefMut}, slice::SliceIndex, }; use address::Virtualize; use kernel_arch::{mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl}; use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, table::{MapAttributes, TableAllocator}, }; use phys::GlobalPhysicalAllocator; use yggdrasil_abi::error::Error; pub mod address; pub mod device; pub mod phys; pub mod pointer; pub mod process; #[cfg(any(target_os = "none", rust_analyzer))] pub mod heap; pub use libk_mm_interface::table; pub struct TableAllocatorImpl; impl TableAllocator for TableAllocatorImpl { fn allocate_page_table() -> Result { phys::alloc_page() } unsafe fn free_page_table(address: PhysicalAddress) { phys::free_page(address) } } // TODO find a way to integrate this nicely with Architecture? pub const L3_PAGE_SIZE: usize = 1 << 12; #[cfg(not(target_arch = "x86"))] pub const L2_PAGE_SIZE: usize = 1 << 21; #[cfg(target_arch = "x86")] pub const L2_PAGE_SIZE: usize = 1 << 22; pub trait PageProvider: Send + Sync { fn get_page(&self, offset: u64) -> Result; fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error>; fn clone_page( &self, offset: u64, src_phys: PhysicalAddress, src_attrs: MapAttributes, ) -> Result; } pub struct PageBox< T: ?Sized, A: PhysicalMemoryAllocator
= GlobalPhysicalAllocator, > { value: *mut T, page_count: usize, _pd: PhantomData, } pub struct PageSlice { data: [T], } impl PageBox { pub fn new(init: T) -> Result, Error> { PageBox::new_in(init) } pub fn new_slice(item: T, count: usize) -> Result, Error> where T: Copy, { PageBox::new_slice_in(item, count) } pub fn new_slice_with T>(f: F, count: usize) -> Result, Error> { PageBox::new_slice_in_with(f, count) } pub fn new_uninit() -> Result>, Error> { PageBox::new_uninit_in() } pub fn new_uninit_slice(count: usize) -> Result]>, Error> { PageBox::new_uninit_slice_in(count) } pub fn new_zeroed_slice(count: usize) -> Result]>, Error> { let (base, page_count) = PageBox::>::alloc_slice(count, true)?; let base_virt_ptr = base.virtualize() as *mut MaybeUninit; let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); let result = PageBox { value, page_count, _pd: PhantomData, }; result.trace_created(); Ok(result) } pub unsafe fn from_physical_raw(address: PhysicalAddress) -> PageBox { PageBox::from_physical_raw_in(address) } } impl> PageBox { #[inline] fn alloc_slice(count: usize, zeroed: bool) -> Result<(PhysicalAddress, usize), Error> { // TODO hardcoded page sizes let layout = Layout::array::(count).unwrap(); let page_count = layout.size().div_ceil(L3_PAGE_SIZE); let base = A::allocate_contiguous_pages(page_count)?; if zeroed { let ptr = base.virtualize() as *mut u8; let slice = unsafe { core::slice::from_raw_parts_mut(ptr, page_count * L3_PAGE_SIZE) }; slice.fill(0); } Ok((base, page_count)) } #[inline] fn alloc() -> Result<(PhysicalAddress, usize), Error> { let page_count = size_of::().div_ceil(L3_PAGE_SIZE); let phys = A::allocate_contiguous_pages(page_count)?; Ok((phys, page_count)) } pub fn new_in(init: T) -> Result, Error> { let (base, page_count) = PageBox::::alloc()?; let value = base.virtualize() as *mut T; unsafe { value.write(init); } let result = PageBox { value, page_count, _pd: PhantomData, }; result.trace_created(); Ok(result) } pub fn new_slice_in(item: T, count: usize) -> Result, Error> where T: Copy, { let (base, page_count) = PageBox::::alloc_slice(count, false)?; let base_virt_ptr = base.virtualize() as *mut T; let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); for i in 0..count { unsafe { value.get_unchecked_mut(i).write(item); } } let result = PageBox { value, page_count, _pd: PhantomData, }; result.trace_created(); Ok(result) } pub fn new_slice_in_with T>( f: F, count: usize, ) -> Result, Error> { let mut value = PageBox::::new_uninit_slice_in(count)?; for i in 0..count { value[i].write(f(i)); } Ok(unsafe { value.assume_init_slice() }) } pub fn new_uninit_in() -> Result, A>, Error> { let (base, page_count) = PageBox::, A>::alloc()?; let value = base.virtualize() as *mut MaybeUninit; let result = PageBox { value, page_count, _pd: PhantomData, }; result.trace_created(); Ok(result) } pub fn new_uninit_slice_in(count: usize) -> Result], A>, Error> { let (base, page_count) = PageBox::, A>::alloc_slice(count, false)?; let base_virt_ptr = base.virtualize() as *mut MaybeUninit; let value = core::ptr::slice_from_raw_parts_mut(base_virt_ptr, count); let result = PageBox { value, page_count, _pd: PhantomData, }; result.trace_created(); Ok(result) } pub unsafe fn from_physical_raw_in(address: PhysicalAddress) -> PageBox { let page_count = size_of::().div_ceil(L3_PAGE_SIZE); let value = address.virtualize() as *mut T; PageBox { value, page_count, _pd: PhantomData, } } } impl> PageBox { #[inline] pub fn as_ptr(&self) -> *const T { self.value as _ } pub fn into_physical_raw(self) -> PhysicalAddress { let address = unsafe { self.as_physical_address() }; core::mem::forget(self); address } #[inline] fn trace_created(&self) { log::trace!( "Alloc PageBox<{}> @ {:p}, {}", core::any::type_name::(), self.value, self.page_count ); } #[inline] fn trace_dropped(&self) { log::trace!( "Free PageBox<{}> @ {:p}, {}", core::any::type_name::(), self.value, self.page_count ); } } impl PageBox { pub fn from_iter_exact>(it: I) -> Result, Error> where I::IntoIter: ExactSizeIterator, { let it = it.into_iter(); let mut slice = PageBox::::new_uninit_slice(it.len())?; for (i, item) in it.enumerate() { slice[i].write(item); } let slice = unsafe { slice.assume_init_slice() }; Ok(slice) } } impl> PageBox<[T], A> { pub fn as_slice(&self) -> &PageSlice { unsafe { core::mem::transmute(&self[..]) } } pub fn as_slice_mut(&mut self) -> &mut PageSlice { unsafe { core::mem::transmute(&mut self[..]) } } } impl> PageBox, A> { /// Consumes the [PageBox], returning a new one with [MaybeUninit] removed. /// /// # Safety /// /// See [MaybeUninit::assume_init_mut]. pub unsafe fn assume_init(self) -> PageBox { // SAFETY: Memory-safe, as: // 1. MaybeUninit is transparent // 2. self.value still points to the same memory and is not deallocated let page_count = self.page_count; let value = MaybeUninit::assume_init_mut(&mut *self.value); // Prevent deallocation of the PageBox with MaybeUninit core::mem::forget(self); PageBox { value, page_count, _pd: PhantomData, } } pub fn as_bytes_mut(p: &mut Self) -> &mut PageSlice> { unsafe { core::mem::transmute(p.as_bytes_mut()) } } } impl> PageBox<[MaybeUninit], A> { /// Consumes the [PageBox], returning a new one with [MaybeUninit] removed. /// /// # Safety /// /// See [MaybeUninit::slice_assume_init_mut]. pub unsafe fn assume_init_slice(self) -> PageBox<[T], A> { // SAFETY: Memory-safe, as: // 1. MaybeUninit is transparent // 2. self.value still points to the same memory and is not deallocated let page_count = self.page_count; let value = MaybeUninit::slice_assume_init_mut(&mut *self.value); core::mem::forget(self); PageBox { value, page_count, _pd: PhantomData, } } /// Returns a reference to the slice data with [MaybeUninit] removed. /// /// # Safety /// /// See [MaybeUninit::slice_assume_init_ref] pub unsafe fn assume_init_slice_ref(&self) -> &[T] { MaybeUninit::slice_assume_init_ref(self.deref()) } /// Returns a mutable reference to the slice data with [MaybeUninit] removed. /// /// # Safety /// /// See [MaybeUninit::slice_assume_init_mut] pub unsafe fn assume_init_slice_mut(&mut self) -> &mut [T] { MaybeUninit::slice_assume_init_mut(self.deref_mut()) } /// Fills a slice of MaybeUninit with zeroes. /// /// # Safety /// /// Unsafe: will not drop possibly previously written data. Only meant for [Copy] and other /// trivial types. pub unsafe fn zero(p: &mut Self) { let ptr = p.as_mut_ptr() as *mut u8; let slice = core::slice::from_raw_parts_mut(ptr, p.page_count * L3_PAGE_SIZE); slice.fill(0); } } impl> AsPhysicalAddress for PageBox { #[inline] unsafe fn as_physical_address(&self) -> PhysicalAddress { PhysicalAddress::from_virtualized(self.value.addr()) } } impl> Deref for PageBox { type Target = T; #[inline(always)] fn deref(&self) -> &Self::Target { unsafe { &*self.value } } } impl> DerefMut for PageBox { #[inline(always)] fn deref_mut(&mut self) -> &mut Self::Target { unsafe { &mut *self.value } } } impl> Drop for PageBox { fn drop(&mut self) { self.trace_dropped(); unsafe { core::ptr::drop_in_place(self.value); } // SAFETY: Safe, pointer obtained through "virtualize" let base = PhysicalAddress::from_virtualized(self.value.addr()); for i in 0..self.page_count { // SAFETY: Safe, page allocated only by this PageBox unsafe { A::free_page(base.add(L3_PAGE_SIZE * i)); } } } } impl> fmt::Pointer for PageBox { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.value.fmt(f) } } impl> fmt::Debug for PageBox { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Debug::fmt(self.deref(), f) } } impl> fmt::Display for PageBox { #[inline] fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { fmt::Display::fmt(self.deref(), f) } } unsafe impl> Send for PageBox { } unsafe impl> Sync for PageBox { } impl PageSlice { pub fn subslice_mut>( &mut self, index: R, ) -> &mut PageSlice { unsafe { core::mem::transmute(&mut self.data[index]) } } pub fn subslice>(&self, index: R) -> &PageSlice { unsafe { core::mem::transmute(&self.data[index]) } } } impl AsPhysicalAddress for PageSlice { unsafe fn as_physical_address(&self) -> PhysicalAddress { PhysicalAddress::from_virtualized(self.data.as_ptr().addr()) } } impl Deref for PageSlice { type Target = [T]; fn deref(&self) -> &Self::Target { &self.data } } impl DerefMut for PageSlice { fn deref_mut(&mut self) -> &mut Self::Target { &mut self.data } } pub fn flush_cache_data(data: *const T) { ArchitectureImpl::flush_virtual_range(data.addr()..data.addr() + size_of::()); } pub fn flush_cache_data_slice(data: *const [T]) { ArchitectureImpl::flush_virtual_range(data.addr()..data.addr() + size_of::() * data.len()); }