mem: implement a better vmalloc

This commit is contained in:
Mark Poliakov 2023-12-31 12:50:16 +02:00
parent 293dcfea6a
commit ae7ba554d4
11 changed files with 303 additions and 139 deletions

View File

@ -8,7 +8,10 @@ use core::{
use alloc::boxed::Box;
use futures_util::{task::AtomicWaker, Future};
use kernel_util::{mem::PageBox, runtime::QueueWaker};
use kernel_util::{
mem::{address::PhysicalAddress, PageBox, PageProvider},
runtime::QueueWaker,
};
use yggdrasil_abi::{error::Error, io::DeviceRequest};
use crate::{
@ -237,6 +240,16 @@ impl<'a, D: NgBlockDevice + 'a> NgBlockDeviceWrapper<'a, D> {
}
}
impl<'a, D: NgBlockDevice + 'a> PageProvider for NgBlockDeviceWrapper<'a, D> {
fn get_page(&self, _offset: u64) -> Result<PhysicalAddress, Error> {
todo!()
}
fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> {
todo!()
}
}
impl<'a, D: NgBlockDevice + 'a> BlockDevice for NgBlockDeviceWrapper<'a, D> {
fn poll_read(
&self,

View File

@ -4,7 +4,7 @@ extern crate alloc;
use core::task::{Context, Poll};
use kernel_util::mem::address::PhysicalAddress;
use kernel_util::mem::{address::PhysicalAddress, PageProvider};
use yggdrasil_abi::{error::Error, io::DeviceRequest};
pub mod device;
@ -57,7 +57,7 @@ pub fn probe_partitions<
/// Block device interface
#[allow(unused)]
pub trait BlockDevice: Sync {
pub trait BlockDevice: PageProvider + Sync {
fn poll_read(
&self,
cx: &mut Context<'_>,
@ -99,10 +99,6 @@ pub trait BlockDevice: Sync {
Err(Error::NotImplemented)
}
fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
Err(Error::NotImplemented)
}
// fn read_exact(&'static self, pos: u64, buf: &mut [u8]) -> Result<(), Error> {
// let count = self.read(pos, buf)?;
// if count == buf.len() {

View File

@ -16,6 +16,11 @@ pub mod device;
pub mod pointer;
pub mod table;
pub trait PageProvider {
fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error>;
fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error>;
}
pub struct PageBox<T: ?Sized> {
value: *mut T,
page_count: usize,

View File

@ -10,7 +10,10 @@ use alloc::{
collections::{btree_map::Entry, BTreeMap},
sync::Arc,
};
use kernel_util::{mem::address::PhysicalAddress, sync::IrqSafeSpinlock};
use kernel_util::{
mem::{address::PhysicalAddress, PageProvider},
sync::IrqSafeSpinlock,
};
use yggdrasil_abi::{
error::Error,
io::{DirectoryEntry, OpenOptions, RawFd, SeekFrom},
@ -196,14 +199,6 @@ impl File {
}
}
/// Gets a page from a file to be mapped into virtual memory
pub fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
match self {
Self::Block(f) => f.device.0.get_page(offset),
_ => Err(Error::NotImplemented),
}
}
/// Interprets the file as a poll channel
pub fn as_poll_channel(&self) -> Result<&FdPoll, Error> {
if let Self::Poll(poll) = self {
@ -223,6 +218,22 @@ impl File {
}
}
impl PageProvider for File {
fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
match self {
Self::Block(f) => f.device.0.get_page(offset),
_ => Err(Error::InvalidOperation),
}
}
fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> {
match self {
Self::Block(f) => f.device.0.release_page(offset, phys),
_ => Err(Error::InvalidOperation),
}
}
}
impl Read for File {
fn read(&self, buf: &mut [u8]) -> Result<usize, Error> {
match self {

View File

@ -9,6 +9,8 @@ authors = ["Mark Poliakov <mark@alnyan.me>"]
[dependencies]
yggdrasil-abi = { git = "https://git.alnyan.me/yggdrasil/yggdrasil-abi.git" }
discrete_range_map = { git = "https://git.alnyan.me/yggdrasil/discrete_range_map.git" }
[dev-dependencies]
itertools = "0.11.0"
proptest = "1.2.0"

View File

@ -1,64 +1,97 @@
//! Virtual memory allocator for the Yggdrasil kernel.
//!
//! The allocator uses a [DiscreteRangeMap] to track the memory regions and allows attaching
//! metadata values to each region. "Touching" region coalescing is enabled through the [Eq] trait
//! implemented for [RangeData].
#![deny(missing_docs)]
#![no_std]
#![feature(linked_list_cursors, let_chains, btree_extract_if)]
extern crate alloc;
use allocator::TreeAllocator;
use core::ops::Range;
use discrete_range_map::{DiscreteRangeMap, InclusiveInterval, InclusiveRange};
use yggdrasil_abi::error::Error;
pub(crate) mod allocator;
#[cfg(target_pointer_width = "64")]
type PfnIndex = u64;
#[derive(Debug, PartialEq, Eq, Clone, Copy)]
pub struct VirtualMemoryRange {
start_pfn: usize,
end_pfn: usize,
/// Metadata associated with an allocated memory region. The [Eq] trait is used to coalesce "equal"
/// regions if they "touch".
pub trait RangeData: Eq {}
fn ie(from: PfnIndex, to: PfnIndex) -> InclusiveInterval<PfnIndex> {
InclusiveInterval::from(from..to)
}
pub struct VirtualMemoryAllocator {
inner: TreeAllocator,
/// Main virtual memory allocator
pub struct VirtualMemoryAllocator<D: RangeData> {
map: DiscreteRangeMap<PfnIndex, InclusiveInterval<PfnIndex>, D>,
outer_range: InclusiveInterval<PfnIndex>,
}
impl VirtualMemoryRange {
pub fn start_pfn(&self) -> usize {
self.start_pfn
}
pub fn end_pfn(&self) -> usize {
self.end_pfn
}
pub fn pfn_count(&self) -> usize {
self.end_pfn - self.start_pfn
}
pub fn contains(&self, other: &Self) -> bool {
other.start_pfn >= self.start_pfn && other.end_pfn <= self.end_pfn
}
}
impl VirtualMemoryAllocator {
pub fn new(start_pfn: usize, end_pfn: usize) -> Self {
impl<D: RangeData> VirtualMemoryAllocator<D> {
/// Creates a new virtual memory allocator, bounded by `lower_limit_pfn..upper_limit_pfn`
pub fn new(lower_limit_pfn: usize, upper_limit_pfn: usize) -> Self {
Self {
inner: TreeAllocator::new(start_pfn, end_pfn),
map: DiscreteRangeMap::new(),
outer_range: ie(lower_limit_pfn as _, upper_limit_pfn as _),
}
}
pub fn allocate(&mut self, pfn_count: usize) -> Result<usize, Error> {
self.inner.allocate(pfn_count).ok_or(Error::OutOfMemory)
/// Allocates a contiguous range of virtual address space and associates metadata with it
pub fn allocate(&mut self, page_count: usize, data: D) -> Result<usize, Error> {
let start_pfn = self
.map
.gaps_trimmed(self.outer_range)
.find_map(|range| {
if range.size() >= page_count as _ {
Some(range.start() as usize)
} else {
None
}
})
.ok_or(Error::OutOfMemory)?;
// Should not fail
self.insert(start_pfn, page_count, data)?;
Ok(start_pfn)
}
pub fn free(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> {
self.inner.free(start_pfn, pfn_count)
/// Tries to insert given PF# range with its associated metadata as allocated memory,
/// returning [Error] if requested range overlaps any existing allocated ranges
pub fn insert(&mut self, start_pfn: usize, page_count: usize, data: D) -> Result<(), Error> {
let end_pfn = (start_pfn + page_count) as PfnIndex;
let start_pfn = start_pfn as PfnIndex;
self.map
.insert_merge_touching_if_values_equal(ie(start_pfn, end_pfn), data)
.map_err(|_| Error::AlreadyExists)?;
Ok(())
}
pub fn insert(&mut self, start_pfn: usize, pfn_count: usize) -> Result<(), Error> {
self.inner.insert(start_pfn, pfn_count)
}
/// Releases any pages overlapping the requested range, calling `release` on the ranges
pub fn free<F: FnMut(usize, Range<usize>, D) -> Result<(), Error>>(
&mut self,
start_pfn: usize,
page_count: usize,
mut release: F,
) -> Result<(), Error>
where
D: Clone,
{
let end_pfn = (start_pfn + page_count) as PfnIndex;
let start_pfn = start_pfn as PfnIndex;
pub fn ranges(&self) -> impl Iterator<Item = (bool, VirtualMemoryRange)> + '_ {
self.inner.ranges()
self.map
.cut_with_origin(ie(start_pfn, end_pfn))
.try_for_each(|(origin, range, data)| {
let range = range.start() as usize..range.end() as usize + 1;
release(origin.start() as _, range, data)
})
}
}
#[cfg(test)]
mod tests {}

View File

@ -8,7 +8,9 @@ use core::{
use abi::{error::Error, io::DeviceRequest};
use device_api::Device;
use kernel_util::{
mem::{address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel},
mem::{
address::PhysicalAddress, device::RawDeviceMemoryMapping, table::EntryLevel, PageProvider,
},
sync::IrqSafeSpinlock,
};
use ygg_driver_block::BlockDevice;
@ -137,7 +139,9 @@ impl BlockDevice for LinearFramebuffer {
_ => todo!(),
}
}
}
impl PageProvider for LinearFramebuffer {
fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
let offset = offset as usize;
if offset + L3::SIZE > self.size {
@ -148,9 +152,14 @@ impl BlockDevice for LinearFramebuffer {
);
Err(Error::InvalidMemoryOperation)
} else {
Ok(self.phys_base.add(offset))
let page = self.phys_base.add(offset);
Ok(page)
}
}
fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> {
Ok(())
}
}
impl Device for LinearFramebuffer {

View File

@ -1,11 +1,17 @@
//! Process address space structures and management functions
use core::ops::Range;
use abi::error::Error;
use cfg_if::cfg_if;
use kernel_util::sync::IrqSafeSpinlock;
use vmalloc::VirtualMemoryAllocator;
use kernel_util::{
mem::{table::EntryLevelExt, PageProvider},
sync::IrqSafeSpinlock,
};
use vfs::FileRef;
use vmalloc::{RangeData, VirtualMemoryAllocator};
use crate::mem::phys;
use crate::{arch::L3, mem::phys};
use super::{table::MapAttributes, PhysicalAddress};
@ -57,8 +63,82 @@ pub trait ProcessAddressSpaceManager: Sized {
fn as_address_with_asid(&self) -> u64;
}
/// Describes how the physical memory is provided for the mapping
#[derive(Clone)]
pub enum VirtualRangeBacking {
/// Memory is taken from regular "anonymous" physical memory
Anonymous,
/// Mapping is backed by file blocks/device memory
File(FileBacking),
}
/// Describes a file-backed memory range provider
#[derive(Clone)]
pub struct FileBacking {
offset: u64,
file: FileRef,
}
impl VirtualRangeBacking {
/// Creates a file-backed memory range provider
pub fn file(offset: u64, file: FileRef) -> Result<Self, Error> {
if !(offset as usize).is_page_aligned_for::<L3>() {
todo!();
}
Ok(Self::File(FileBacking { offset, file }))
}
/// Creates a range of anonymous memory
pub fn anonymous() -> Self {
Self::Anonymous
}
}
impl PageProvider for VirtualRangeBacking {
fn get_page(&self, offset: u64) -> Result<PhysicalAddress, Error> {
match self {
Self::Anonymous => phys::alloc_page(),
Self::File(f) => f.file.get_page(f.offset + offset),
}
}
fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> {
match self {
Self::Anonymous => unsafe {
phys::free_page(phys);
Ok(())
},
Self::File(f) => f.file.release_page(f.offset + offset, phys),
}
}
}
impl PartialEq for VirtualRangeBacking {
fn eq(&self, other: &Self) -> bool {
return matches!(self, Self::Anonymous) && matches!(other, Self::Anonymous);
}
}
impl Eq for VirtualRangeBacking {}
impl RangeData for VirtualRangeBacking {}
impl core::fmt::Debug for VirtualRangeBacking {
fn fmt(&self, f: &mut core::fmt::Formatter<'_>) -> core::fmt::Result {
match self {
Self::Anonymous => f.debug_struct("VirtualRangeBacking::Anonymous").finish(),
Self::File(fb) => f
.debug_struct("VirtualRangeBacking::File")
.field("offset", &fb.offset)
.field("file", fb.file.as_ref())
.finish(),
}
}
}
struct Inner {
allocator: VirtualMemoryAllocator,
allocator: VirtualMemoryAllocator<VirtualRangeBacking>,
table: ProcessAddressSpaceImpl,
}
@ -68,16 +148,17 @@ pub struct ProcessAddressSpace {
}
impl Inner {
fn try_map_pages<F: Fn(usize) -> Result<PhysicalAddress, Error>>(
fn try_map_pages(
&mut self,
address: usize,
page_count: usize,
get_page: F,
backing: VirtualRangeBacking,
attributes: MapAttributes,
) -> Result<(), (usize, Error)> {
for i in 0..page_count {
let offset = (i * ProcessAddressSpaceImpl::PAGE_SIZE) as u64;
let virt = address + i * ProcessAddressSpaceImpl::PAGE_SIZE;
let phys = match get_page(i * ProcessAddressSpaceImpl::PAGE_SIZE) {
let phys = match backing.get_page(offset) {
Ok(page) => page,
Err(err) => {
return Err((i, err));
@ -92,18 +173,19 @@ impl Inner {
Ok(())
}
fn map_range<F: Fn(usize) -> Result<PhysicalAddress, Error>>(
fn map_range(
&mut self,
address: usize,
page_count: usize,
get_page: F,
backing: VirtualRangeBacking,
attributes: MapAttributes,
) -> Result<(), Error> {
// If inserting fails, the range cannot be mapped
let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE;
self.allocator.insert(start_pfn, page_count)?;
self.allocator
.insert(start_pfn, page_count, backing.clone())?;
if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) {
if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) {
// TODO rollback & remove the range
todo!();
};
@ -111,16 +193,41 @@ impl Inner {
Ok(())
}
fn alloc_range<F: Fn(usize) -> Result<PhysicalAddress, Error>>(
fn map_single(
&mut self,
address: usize,
backing: VirtualRangeBacking,
attributes: MapAttributes,
) -> Result<PhysicalAddress, Error> {
let start_pfn = address / ProcessAddressSpaceImpl::PAGE_SIZE;
self.allocator.insert(start_pfn, 1, backing.clone())?;
let phys = match backing.get_page(0) {
Ok(page) => page,
Err(_err) => {
// TODO rollback
todo!();
}
};
if let Err(_err) = unsafe { self.table.map_page(address, phys, attributes) } {
// TODO rollback
todo!();
}
Ok(phys)
}
fn alloc_range(
&mut self,
page_count: usize,
get_page: F,
backing: VirtualRangeBacking,
attributes: MapAttributes,
) -> Result<usize, Error> {
let start_pfn = self.allocator.allocate(page_count)?;
let start_pfn = self.allocator.allocate(page_count, backing.clone())?;
let address = start_pfn * ProcessAddressSpaceImpl::PAGE_SIZE;
if let Err(_e) = self.try_map_pages(address, page_count, get_page, attributes) {
if let Err(_e) = self.try_map_pages(address, page_count, backing, attributes) {
// TODO rollback
todo!();
};
@ -128,28 +235,23 @@ impl Inner {
Ok(address)
}
unsafe fn unmap_range<F: Fn(usize, PhysicalAddress)>(
&mut self,
start_address: usize,
page_count: usize,
free_page: F,
) -> Result<(), Error> {
unsafe fn unmap_range(&mut self, start_address: usize, page_count: usize) -> Result<(), Error> {
let start_pfn = start_address / ProcessAddressSpaceImpl::PAGE_SIZE;
// Deallocate the range first
self.allocator.free(start_pfn, page_count)?;
self.allocator
.free(start_pfn, page_count, |origin_pfn, pfn_range, backing| {
for pfn in pfn_range {
let offset = ((pfn - origin_pfn) * ProcessAddressSpaceImpl::PAGE_SIZE) as u64;
// Then unmap it from the table
for i in 0..page_count {
let virt = start_address + i * ProcessAddressSpaceImpl::PAGE_SIZE;
// This should not fail under normal circumstances
// TODO handle failures here?
let phys = self.table.unmap_page(virt).unwrap();
let virt = pfn * ProcessAddressSpaceImpl::PAGE_SIZE;
let phys = self.table.unmap_page(virt)?;
// TODO this will fail spectacularly for memory-mapped files and devices
free_page(virt, phys);
backing.release_page(offset, phys)?;
}
Ok(())
})?;
Ok(())
}
}
@ -169,11 +271,11 @@ impl ProcessAddressSpace {
/// Allocates a region of virtual memory within the address space and maps the pages to the
/// ones returned from `get_page` function
pub fn allocate<F: Fn(usize) -> Result<PhysicalAddress, Error>>(
pub fn allocate(
&self,
_hint: Option<usize>,
size: usize,
get_page: F,
backing: VirtualRangeBacking,
attributes: MapAttributes,
) -> Result<usize, Error> {
assert_eq!(size & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0);
@ -182,17 +284,17 @@ impl ProcessAddressSpace {
lock.alloc_range(
size / ProcessAddressSpaceImpl::PAGE_SIZE,
get_page,
backing,
attributes,
)
}
/// Maps a region of memory in the address space
pub fn map<F: Fn(usize) -> Result<PhysicalAddress, Error>>(
pub fn map(
&self,
address: usize,
size: usize,
get_page: F,
backing: VirtualRangeBacking,
attributes: MapAttributes,
) -> Result<(), Error> {
assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0);
@ -203,7 +305,7 @@ impl ProcessAddressSpace {
lock.map_range(
address,
size / ProcessAddressSpaceImpl::PAGE_SIZE,
get_page,
backing,
attributes,
)
}
@ -212,16 +314,14 @@ impl ProcessAddressSpace {
pub fn map_single(
&self,
address: usize,
physical: PhysicalAddress,
backing: VirtualRangeBacking,
attributes: MapAttributes,
) -> Result<(), Error> {
) -> Result<PhysicalAddress, Error> {
assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0);
// XXX
// assert!(physical.is_aligned_for::<L3>());
self.inner
.lock()
.map_range(address, 1, |_| Ok(physical), attributes)
self.inner.lock().map_single(address, backing, attributes)
}
/// Returns the [PhysicalAddress] associated with given virtual `address`,
@ -245,11 +345,7 @@ impl ProcessAddressSpace {
let mut lock = self.inner.lock();
lock.unmap_range(
address,
size / ProcessAddressSpaceImpl::PAGE_SIZE,
|_, paddr| phys::free_page(paddr),
)
lock.unmap_range(address, size / ProcessAddressSpaceImpl::PAGE_SIZE)
}
/// Returns the physical address of this table, with ASID applied

View File

@ -12,7 +12,11 @@ use vfs::{FileRef, Read, Seek};
use yggdrasil_abi::{error::Error, io::SeekFrom};
use crate::{
mem::{phys, process::ProcessAddressSpace, table::MapAttributes},
mem::{
phys,
process::{ProcessAddressSpace, VirtualRangeBacking},
table::MapAttributes,
},
task::process::{ProcessImage, ProcessTlsInfo, ProcessTlsLayout},
};
@ -68,7 +72,7 @@ pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result<us
let address = space.allocate(
None,
0x1000,
|_| phys::alloc_page(),
VirtualRangeBacking::anonymous(),
MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL,
)?;
@ -163,7 +167,7 @@ fn load_segment(
space.map(
aligned_start,
aligned_end - aligned_start,
|_| phys::alloc_page(),
VirtualRangeBacking::anonymous(),
attrs,
)?;
@ -219,7 +223,7 @@ fn tls_segment(
let base_address = space.allocate(
None,
aligned_size,
|_| phys::alloc_page(),
VirtualRangeBacking::anonymous(),
MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL,
)?;

View File

@ -13,7 +13,12 @@ use kernel_util::mem::pointer::PhysicalRefMut;
use vfs::{FileRef, IoContext, Read, Seek};
use crate::{
mem::{phys, process::ProcessAddressSpace, table::MapAttributes, ForeignPointer},
mem::{
phys,
process::{ProcessAddressSpace, VirtualRangeBacking},
table::MapAttributes,
ForeignPointer,
},
proc,
task::{
context::TaskContextImpl,
@ -86,10 +91,9 @@ fn setup_program_env(
env: &[&str],
) -> Result<usize, Error> {
// TODO growing buffer
let phys_page = phys::alloc_page()?;
space.map_single(
let phys_page = space.map_single(
virt,
phys_page,
VirtualRangeBacking::anonymous(),
MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL,
)?;
let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) };
@ -117,7 +121,7 @@ fn setup_binary<S: Into<String>>(
space.map(
virt_stack_base,
USER_STACK_PAGES * 0x1000,
|_| phys::alloc_page(),
VirtualRangeBacking::anonymous(),
MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL,
)?;

View File

@ -20,7 +20,7 @@ use crate::{
arch::L3,
debug::LogLevel,
fs,
mem::{phys, table::MapAttributes},
mem::{phys, process::VirtualRangeBacking, table::MapAttributes},
proc::{self, io::ProcessIo, random},
task::{
process::{Process, ProcessId},
@ -94,34 +94,25 @@ fn syscall_handler(func: SyscallFunction, args: &[u64]) -> Result<usize, Error>
let space = thread.address_space();
len = len.page_align_up::<L3>();
// if len & 0xFFF != 0 {
// todo!();
// }
match source {
MappingSource::Anonymous => space.allocate(
None,
len,
|_| phys::alloc_page(),
MapAttributes::USER_WRITE
| MapAttributes::USER_READ
| MapAttributes::NON_GLOBAL,
),
&MappingSource::File(fd, offset) => run_with_io(process, |io| {
run_with_io(process, |io| {
let backing = match source {
MappingSource::Anonymous => VirtualRangeBacking::anonymous(),
&MappingSource::File(fd, offset) => {
let file = io.files.file(fd)?;
log::info!("len = {:#x}", len);
VirtualRangeBacking::file(offset, file.clone())?
}
};
space.allocate(
None,
len,
|off| file.get_page(offset + off as u64),
// TODO make get_page return attributes for each page itself
backing,
MapAttributes::USER_WRITE
| MapAttributes::USER_READ
| MapAttributes::NON_GLOBAL,
)
}),
}
})
}
SyscallFunction::UnmapMemory => {
let addr = args[0] as usize;