diff --git a/driver/block/core/src/device.rs b/driver/block/core/src/device.rs index c0dd5579..7dab9f99 100644 --- a/driver/block/core/src/device.rs +++ b/driver/block/core/src/device.rs @@ -8,7 +8,10 @@ use core::{ use alloc::boxed::Box; use futures_util::{task::AtomicWaker, Future}; -use kernel_util::{mem::PageBox, runtime::QueueWaker}; +use kernel_util::{ + mem::{address::PhysicalAddress, PageBox, PageProvider}, + runtime::QueueWaker, +}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; use crate::{ @@ -237,6 +240,16 @@ impl<'a, D: NgBlockDevice + 'a> NgBlockDeviceWrapper<'a, D> { } } +impl<'a, D: NgBlockDevice + 'a> PageProvider for NgBlockDeviceWrapper<'a, D> { + fn get_page(&self, _offset: u64) -> Result { + todo!() + } + + fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> { + todo!() + } +} + impl<'a, D: NgBlockDevice + 'a> BlockDevice for NgBlockDeviceWrapper<'a, D> { fn poll_read( &self, diff --git a/driver/block/core/src/lib.rs b/driver/block/core/src/lib.rs index 1af33678..0ddaf8d6 100644 --- a/driver/block/core/src/lib.rs +++ b/driver/block/core/src/lib.rs @@ -4,7 +4,7 @@ extern crate alloc; use core::task::{Context, Poll}; -use kernel_util::mem::address::PhysicalAddress; +use kernel_util::mem::{address::PhysicalAddress, PageProvider}; use yggdrasil_abi::{error::Error, io::DeviceRequest}; pub mod device; @@ -57,7 +57,7 @@ pub fn probe_partitions< /// Block device interface #[allow(unused)] -pub trait BlockDevice: Sync { +pub trait BlockDevice: PageProvider + Sync { fn poll_read( &self, cx: &mut Context<'_>, @@ -99,10 +99,6 @@ pub trait BlockDevice: Sync { Err(Error::NotImplemented) } - fn get_page(&self, offset: u64) -> Result { - Err(Error::NotImplemented) - } - // fn read_exact(&'static self, pos: u64, buf: &mut [u8]) -> Result<(), Error> { // let count = self.read(pos, buf)?; // if count == buf.len() { diff --git a/lib/kernel-util/src/mem/mod.rs b/lib/kernel-util/src/mem/mod.rs index 7392a516..972d56bc 100644 --- a/lib/kernel-util/src/mem/mod.rs +++ b/lib/kernel-util/src/mem/mod.rs @@ -16,6 +16,11 @@ pub mod device; pub mod pointer; pub mod table; +pub trait PageProvider { + fn get_page(&self, offset: u64) -> Result; + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error>; +} + pub struct PageBox { value: *mut T, page_count: usize, diff --git a/lib/vfs/src/file/mod.rs b/lib/vfs/src/file/mod.rs index 4fac85d5..4f0ad75b 100644 --- a/lib/vfs/src/file/mod.rs +++ b/lib/vfs/src/file/mod.rs @@ -10,7 +10,10 @@ use alloc::{ collections::{btree_map::Entry, BTreeMap}, sync::Arc, }; -use kernel_util::{mem::address::PhysicalAddress, sync::IrqSafeSpinlock}; +use kernel_util::{ + mem::{address::PhysicalAddress, PageProvider}, + sync::IrqSafeSpinlock, +}; use yggdrasil_abi::{ error::Error, io::{DirectoryEntry, OpenOptions, RawFd, SeekFrom}, @@ -196,14 +199,6 @@ impl File { } } - /// Gets a page from a file to be mapped into virtual memory - pub fn get_page(&self, offset: u64) -> Result { - match self { - Self::Block(f) => f.device.0.get_page(offset), - _ => Err(Error::NotImplemented), - } - } - /// Interprets the file as a poll channel pub fn as_poll_channel(&self) -> Result<&FdPoll, Error> { if let Self::Poll(poll) = self { @@ -223,6 +218,22 @@ impl File { } } +impl PageProvider for File { + fn get_page(&self, offset: u64) -> Result { + match self { + Self::Block(f) => f.device.0.get_page(offset), + _ => Err(Error::InvalidOperation), + } + } + + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + match self { + Self::Block(f) => f.device.0.release_page(offset, phys), + _ => Err(Error::InvalidOperation), + } + } +} + impl Read for File { fn read(&self, buf: &mut [u8]) -> Result { match self { diff --git a/lib/vmalloc/Cargo.toml b/lib/vmalloc/Cargo.toml index 24ba6152..b4ec8bc5 100644 --- a/lib/vmalloc/Cargo.toml +++ b/lib/vmalloc/Cargo.toml @@ -9,6 +9,8 @@ authors = ["Mark Poliakov "] [dependencies] yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" } +discrete_range_map = { git = "https://git.alnyan.me/yggdrasil/discrete_range_map.git" } + [dev-dependencies] itertools = "0.11.0" proptest = "1.2.0" diff --git a/lib/vmalloc/src/lib.rs b/lib/vmalloc/src/lib.rs index b7b6b04f..70b918ab 100644 --- a/lib/vmalloc/src/lib.rs +++ b/lib/vmalloc/src/lib.rs @@ -1,64 +1,97 @@ +//! Virtual memory allocator for the Yggdrasil kernel. +//! +//! The allocator uses a [DiscreteRangeMap] to track the memory regions and allows attaching +//! metadata values to each region. "Touching" region coalescing is enabled through the [Eq] trait +//! implemented for [RangeData]. + +#![deny(missing_docs)] #![no_std] #![feature(linked_list_cursors, let_chains, btree_extract_if)] extern crate alloc; -use allocator::TreeAllocator; +use core::ops::Range; + +use discrete_range_map::{DiscreteRangeMap, InclusiveInterval, InclusiveRange}; use yggdrasil_abi::error::Error; -pub(crate) mod allocator; +#[cfg(target_pointer_width = "64")] +type PfnIndex = u64; -#[derive(Debug, PartialEq, Eq, Clone, Copy)] -pub struct VirtualMemoryRange { - start_pfn: usize, - end_pfn: usize, +/// Metadata associated with an allocated memory region. The [Eq] trait is used to coalesce "equal" +/// regions if they "touch". +pub trait RangeData: Eq {} + +fn ie(from: PfnIndex, to: PfnIndex) -> InclusiveInterval { + InclusiveInterval::from(from..to) } -pub struct VirtualMemoryAllocator { - inner: TreeAllocator, +/// Main virtual memory allocator +pub struct VirtualMemoryAllocator { + map: DiscreteRangeMap, D>, + outer_range: InclusiveInterval, } -impl VirtualMemoryRange { - pub fn start_pfn(&self) -> usize { - self.start_pfn - } - - pub fn end_pfn(&self) -> usize { - self.end_pfn - } - - pub fn pfn_count(&self) -> usize { - self.end_pfn - self.start_pfn - } - - pub fn contains(&self, other: &Self) -> bool { - other.start_pfn >= self.start_pfn && other.end_pfn <= self.end_pfn - } -} - -impl VirtualMemoryAllocator { - pub fn new(start_pfn: usize, end_pfn: usize) -> Self { +impl VirtualMemoryAllocator { + /// Creates a new virtual memory allocator, bounded by `lower_limit_pfn..upper_limit_pfn` + pub fn new(lower_limit_pfn: usize, upper_limit_pfn: usize) -> Self { Self { - inner: TreeAllocator::new(start_pfn, end_pfn), + map: DiscreteRangeMap::new(), + outer_range: ie(lower_limit_pfn as _, upper_limit_pfn as _), } } - pub fn allocate(&mut self, pfn_count: usize) -> Result { - self.inner.allocate(pfn_count).ok_or(Error::OutOfMemory) + /// Allocates a contiguous range of virtual address space and associates metadata with it + pub fn allocate(&mut self, page_count: usize, data: D) -> Result { + let start_pfn = self + .map + .gaps_trimmed(self.outer_range) + .find_map(|range| { + if range.size() >= page_count as _ { + Some(range.start() as usize) + } else { + None + } + }) + .ok_or(Error::OutOfMemory)?; + + // Should not fail + self.insert(start_pfn, page_count, data)?; + + Ok(start_pfn) } - pub fn free(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { - self.inner.free(start_pfn, pfn_count) + /// Tries to insert given PF# range with its associated metadata as allocated memory, + /// returning [Error] if requested range overlaps any existing allocated ranges + pub fn insert(&mut self, start_pfn: usize, page_count: usize, data: D) -> Result<(), Error> { + let end_pfn = (start_pfn + page_count) as PfnIndex; + let start_pfn = start_pfn as PfnIndex; + + self.map + .insert_merge_touching_if_values_equal(ie(start_pfn, end_pfn), data) + .map_err(|_| Error::AlreadyExists)?; + + Ok(()) } - pub fn insert(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> { - self.inner.insert(start_pfn, pfn_count) - } + /// Releases any pages overlapping the requested range, calling `release` on the ranges + pub fn free, D) -> Result<(), Error>>( + &mut self, + start_pfn: usize, + page_count: usize, + mut release: F, + ) -> Result<(), Error> + where + D: Clone, + { + let end_pfn = (start_pfn + page_count) as PfnIndex; + let start_pfn = start_pfn as PfnIndex; - pub fn ranges(&self) -> impl Iterator + '_ { - self.inner.ranges() + self.map + .cut_with_origin(ie(start_pfn, end_pfn)) + .try_for_each(|(origin, range, data)| { + let range = range.start() as usize..range.end() as usize + 1; + release(origin.start() as _, range, data) + }) } } - -#[cfg(test)] -mod tests {} diff --git a/src/device/display/linear_fb.rs b/src/device/display/linear_fb.rs index 6db637bc..8f13b060 100644 --- a/src/device/display/linear_fb.rs +++ b/src/device/display/linear_fb.rs @@ -8,7 +8,9 @@ use core::{ use abi::{error::Error, io::DeviceRequest}; use device_api::Device; use kernel_util::{ - mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel}, + mem::{ + address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel, PageProvider, + }, sync::IrqSafeSpinlock, }; use ygg_driver_block::BlockDevice; @@ -137,7 +139,9 @@ impl BlockDevice for LinearFramebuffer { _ => todo!(), } } +} +impl PageProvider for LinearFramebuffer { fn get_page(&self, offset: u64) -> Result { let offset = offset as usize; if offset + L3::SIZE > self.size { @@ -148,9 +152,14 @@ impl BlockDevice for LinearFramebuffer { ); Err(Error::InvalidMemoryOperation) } else { - Ok(self.phys_base.add(offset)) + let page = self.phys_base.add(offset); + Ok(page) } } + + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + Ok(()) + } } impl Device for LinearFramebuffer { diff --git a/src/mem/process.rs b/src/mem/process.rs index 6f1afffa..d7b02a2b 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -1,11 +1,17 @@ //! Process address space structures and management functions +use core::ops::Range; + use abi::error::Error; use cfg_if::cfg_if; -use kernel_util::sync::IrqSafeSpinlock; -use vmalloc::VirtualMemoryAllocator; +use kernel_util::{ + mem::{table::EntryLevelExt, PageProvider}, + sync::IrqSafeSpinlock, +}; +use vfs::FileRef; +use vmalloc::{RangeData, VirtualMemoryAllocator}; -use crate::mem::phys; +use crate::{arch::L3, mem::phys}; use super::{table::MapAttributes, PhysicalAddress}; @@ -57,8 +63,82 @@ pub trait ProcessAddressSpaceManager: Sized { fn as_address_with_asid(&self) -> u64; } +/// Describes how the physical memory is provided for the mapping +#[derive(Clone)] +pub enum VirtualRangeBacking { + /// Memory is taken from regular "anonymous" physical memory + Anonymous, + /// Mapping is backed by file blocks/device memory + File(FileBacking), +} + +/// Describes a file-backed memory range provider +#[derive(Clone)] +pub struct FileBacking { + offset: u64, + file: FileRef, +} + +impl VirtualRangeBacking { + /// Creates a file-backed memory range provider + pub fn file(offset: u64, file: FileRef) -> Result { + if !(offset as usize).is_page_aligned_for::() { + todo!(); + } + + Ok(Self::File(FileBacking { offset, file })) + } + + /// Creates a range of anonymous memory + pub fn anonymous() -> Self { + Self::Anonymous + } +} + +impl PageProvider for VirtualRangeBacking { + fn get_page(&self, offset: u64) -> Result { + match self { + Self::Anonymous => phys::alloc_page(), + Self::File(f) => f.file.get_page(f.offset + offset), + } + } + + fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { + match self { + Self::Anonymous => unsafe { + phys::free_page(phys); + Ok(()) + }, + Self::File(f) => f.file.release_page(f.offset + offset, phys), + } + } +} + +impl PartialEq for VirtualRangeBacking { + fn eq(&self, other: &Self) -> bool { + return matches!(self, Self::Anonymous) && matches!(other, Self::Anonymous); + } +} + +impl Eq for VirtualRangeBacking {} + +impl RangeData for VirtualRangeBacking {} + +impl core::fmt::Debug for VirtualRangeBacking { + fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result { + match self { + Self::Anonymous => f.debug_struct("VirtualRangeBacking::Anonymous").finish(), + Self::File(fb) => f + .debug_struct("VirtualRangeBacking::File") + .field("offset", &fb.offset) + .field("file", fb.file.as_ref()) + .finish(), + } + } +} + struct Inner { - allocator: VirtualMemoryAllocator, + allocator: VirtualMemoryAllocator, table: ProcessAddressSpaceImpl, } @@ -68,16 +148,17 @@ pub struct ProcessAddressSpace { } impl Inner { - fn try_map_pages Result>( + fn try_map_pages( &mut self, address: usize, page_count: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), (usize, Error)> { for i in 0..page_count { + let offset = (i * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; let virt = address + i * ProcessAddressSpaceImpl::PAGE_SIZE; - let phys = match get_page(i * ProcessAddressSpaceImpl::PAGE_SIZE) { + let phys = match backing.get_page(offset) { Ok(page) => page, Err(err) => { return Err((i, err)); @@ -92,18 +173,19 @@ impl Inner { Ok(()) } - fn map_range Result>( + fn map_range( &mut self, address: usize, page_count: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), Error> { // If inserting fails, the range cannot be mapped let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE; - self.allocator.insert(start_pfn, page_count)?; + self.allocator + .insert(start_pfn, page_count, backing.clone())?; - if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) { + if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) { // TODO rollback & remove the range todo!(); }; @@ -111,16 +193,41 @@ impl Inner { Ok(()) } - fn alloc_range Result>( + fn map_single( + &mut self, + address: usize, + backing: VirtualRangeBacking, + attributes: MapAttributes, + ) -> Result { + let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE; + self.allocator.insert(start_pfn, 1, backing.clone())?; + + let phys = match backing.get_page(0) { + Ok(page) => page, + Err(_err) => { + // TODO rollback + todo!(); + } + }; + + if let Err(_err) = unsafe { self.table.map_page(address, phys, attributes) } { + // TODO rollback + todo!(); + } + + Ok(phys) + } + + fn alloc_range( &mut self, page_count: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result { - let start_pfn = self.allocator.allocate(page_count)?; + let start_pfn = self.allocator.allocate(page_count, backing.clone())?; let address = start_pfn * ProcessAddressSpaceImpl::PAGE_SIZE; - if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) { + if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) { // TODO rollback todo!(); }; @@ -128,27 +235,22 @@ impl Inner { Ok(address) } - unsafe fn unmap_range( - &mut self, - start_address: usize, - page_count: usize, - free_page: F, - ) -> Result<(), Error> { + unsafe fn unmap_range(&mut self, start_address: usize, page_count: usize) -> Result<(), Error> { let start_pfn = start_address / ProcessAddressSpaceImpl::PAGE_SIZE; - // Deallocate the range first - self.allocator.free(start_pfn, page_count)?; + self.allocator + .free(start_pfn, page_count, |origin_pfn, pfn_range, backing| { + for pfn in pfn_range { + let offset = ((pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE) as u64; - // Then unmap it from the table - for i in 0..page_count { - let virt = start_address + i * ProcessAddressSpaceImpl::PAGE_SIZE; - // This should not fail under normal circumstances - // TODO handle failures here? - let phys = self.table.unmap_page(virt).unwrap(); + let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE; + let phys = self.table.unmap_page(virt)?; - // TODO this will fail spectacularly for memory-mapped files and devices - free_page(virt, phys); - } + backing.release_page(offset, phys)?; + } + + Ok(()) + })?; Ok(()) } @@ -169,11 +271,11 @@ impl ProcessAddressSpace { /// Allocates a region of virtual memory within the address space and maps the pages to the /// ones returned from `get_page` function - pub fn allocate Result>( + pub fn allocate( &self, _hint: Option, size: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result { assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); @@ -182,17 +284,17 @@ impl ProcessAddressSpace { lock.alloc_range( size / ProcessAddressSpaceImpl::PAGE_SIZE, - get_page, + backing, attributes, ) } /// Maps a region of memory in the address space - pub fn map Result>( + pub fn map( &self, address: usize, size: usize, - get_page: F, + backing: VirtualRangeBacking, attributes: MapAttributes, ) -> Result<(), Error> { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); @@ -203,7 +305,7 @@ impl ProcessAddressSpace { lock.map_range( address, size / ProcessAddressSpaceImpl::PAGE_SIZE, - get_page, + backing, attributes, ) } @@ -212,16 +314,14 @@ impl ProcessAddressSpace { pub fn map_single( &self, address: usize, - physical: PhysicalAddress, + backing: VirtualRangeBacking, attributes: MapAttributes, - ) -> Result<(), Error> { + ) -> Result { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); // XXX // assert!(physical.is_aligned_for::()); - self.inner - .lock() - .map_range(address, 1, |_| Ok(physical), attributes) + self.inner.lock().map_single(address, backing, attributes) } /// Returns the [PhysicalAddress] associated with given virtual `address`, @@ -245,11 +345,7 @@ impl ProcessAddressSpace { let mut lock = self.inner.lock(); - lock.unmap_range( - address, - size / ProcessAddressSpaceImpl::PAGE_SIZE, - |_, paddr| phys::free_page(paddr), - ) + lock.unmap_range(address, size / ProcessAddressSpaceImpl::PAGE_SIZE) } /// Returns the physical address of this table, with ASID applied diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 50193b1b..227b7e7d 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -12,7 +12,11 @@ use vfs::{FileRef, Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom}; use crate::{ - mem::{phys, process::ProcessAddressSpace, table::MapAttributes}, + mem::{ + phys, + process::{ProcessAddressSpace, VirtualRangeBacking}, + table::MapAttributes, + }, task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout}, }; @@ -68,7 +72,7 @@ pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result Result { // TODO growing buffer - let phys_page = phys::alloc_page()?; - space.map_single( + let phys_page = space.map_single( virt, - phys_page, + VirtualRangeBacking::anonymous(), MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; @@ -117,7 +121,7 @@ fn setup_binary>( space.map( virt_stack_base, USER_STACK_PAGES * 0x1000, - |_| phys::alloc_page(), + VirtualRangeBacking::anonymous(), MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; diff --git a/src/syscall/mod.rs b/src/syscall/mod.rs index b1fc9ba6..6a206946 100644 --- a/src/syscall/mod.rs +++ b/src/syscall/mod.rs @@ -20,7 +20,7 @@ use crate::{ arch::L3, debug::LogLevel, fs, - mem::{phys, table::MapAttributes}, + mem::{phys, process::VirtualRangeBacking, table::MapAttributes}, proc::{self, io::ProcessIo, random}, task::{ process::{Process, ProcessId}, @@ -94,34 +94,25 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result let space = thread.address_space(); len = len.page_align_up::(); - // if len & 0xFFF != 0 { - // todo!(); - // } - match source { - MappingSource::Anonymous => space.allocate( + run_with_io(process, |io| { + let backing = match source { + MappingSource::Anonymous => VirtualRangeBacking::anonymous(), + &MappingSource::File(fd, offset) => { + let file = io.files.file(fd)?; + VirtualRangeBacking::file(offset, file.clone())? + } + }; + + space.allocate( None, len, - |_| phys::alloc_page(), + backing, MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, - ), - &MappingSource::File(fd, offset) => run_with_io(process, |io| { - let file = io.files.file(fd)?; - log::info!("len = {:#x}", len); - - space.allocate( - None, - len, - |off| file.get_page(offset + off as u64), - // TODO make get_page return attributes for each page itself - MapAttributes::USER_WRITE - | MapAttributes::USER_READ - | MapAttributes::NON_GLOBAL, - ) - }), - } + ) + }) } SyscallFunction::UnmapMemory => { let addr = args[0] as usize;