From 72633eb339df2510fed1abe787c35f59db4a2e1a Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Wed, 26 Feb 2025 16:21:56 +0200 Subject: [PATCH] maint: sync up other architectures with mmap(file) --- kernel/arch/aarch64/src/mem/process.rs | 45 +++++++- kernel/arch/aarch64/src/mem/table.rs | 44 +++++++- kernel/arch/riscv64/src/mem/process.rs | 50 ++++++++- kernel/arch/riscv64/src/mem/table.rs | 24 +++- kernel/arch/x86_64/src/mem/process.rs | 36 +++++- kernel/arch/x86_64/src/mem/table.rs | 22 +++- kernel/driver/block/ahci/src/port.rs | 6 +- kernel/driver/block/nvme/src/drive.rs | 4 +- kernel/driver/block/scsi/src/lib.rs | 6 +- kernel/driver/virtio/blk/src/lib.rs | 6 +- kernel/driver/virtio/gpu/src/lib.rs | 66 +++++------ kernel/driver/virtio/net/src/lib.rs | 9 +- kernel/lib/memtables/src/riscv64.rs | 2 + kernel/libk/libk-mm/interface/src/lib.rs | 9 ++ kernel/libk/libk-mm/interface/src/process.rs | 16 +++ kernel/libk/libk-mm/interface/src/table.rs | 2 + kernel/libk/libk-mm/src/lib.rs | 11 +- kernel/libk/libk-mm/src/process.rs | 103 ++++++++++++------ kernel/libk/src/device/block/partition/mod.rs | 6 +- kernel/libk/src/device/display/mod.rs | 5 +- kernel/libk/src/task/binary/elf.rs | 2 +- kernel/libk/src/task/binary/mod.rs | 8 ++ kernel/libk/src/task/thread.rs | 6 +- kernel/libk/src/vfs/file/mod.rs | 6 +- kernel/libk/src/vfs/file/regular.rs | 23 ++-- kernel/libk/src/vfs/shared_memory.rs | 13 +-- kernel/src/arch/aarch64/boot/mod.rs | 1 + kernel/src/arch/aarch64/exception.rs | 102 ++++++++--------- kernel/src/arch/aarch64/gic/gicv2m.rs | 2 +- kernel/src/arch/aarch64/gic/mod.rs | 2 +- kernel/src/arch/aarch64/mod.rs | 9 +- kernel/src/arch/aarch64/timer.rs | 2 +- kernel/src/arch/x86_64/exception.rs | 11 +- kernel/src/device/clock/bcm2835_aux.rs | 8 +- kernel/src/device/display/linear_fb.rs | 6 +- kernel/src/device/power/arm_psci.rs | 2 +- kernel/src/device/serial/bcm2835_aux_uart.rs | 33 ++---- kernel/src/device/serial/pl011.rs | 2 +- test.c | 30 ++++- .../lib/libcolors/src/application/mod.rs | 5 +- .../lib/ygglibc/src/headers/strings/mod.rs | 2 +- userspace/lib/ygglibc/src/lib.rs | 7 -- xtask/src/build/c.rs | 30 ++--- 43 files changed, 522 insertions(+), 262 deletions(-) diff --git a/kernel/arch/aarch64/src/mem/process.rs b/kernel/arch/aarch64/src/mem/process.rs index 4626338c..730daa6a 100644 --- a/kernel/arch/aarch64/src/mem/process.rs +++ b/kernel/arch/aarch64/src/mem/process.rs @@ -7,7 +7,7 @@ use core::{ use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::PhysicalRefMut, - process::ProcessAddressSpaceManager, + process::{PageAttributeUpdate, ProcessAddressSpaceManager}, table::{ EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator, }, @@ -79,7 +79,15 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceI ) } - unsafe fn unmap_page(&mut self, address: usize) -> Result { + unsafe fn update_page_attributes( + &mut self, + address: usize, + update: &PageAttributeUpdate, + ) -> Result<(), Error> { + self.update_l3_entry(address, |entry| entry.update(update)) + } + + unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> { self.pop_l3_entry(address) } @@ -120,7 +128,32 @@ impl ProcessAddressSpaceImpl { Ok(()) } - fn pop_l3_entry(&mut self, virt: usize) -> Result { + fn update_l3_entry) -> Result<(), Error>>( + &mut self, + virt: usize, + mapper: F, + ) -> Result<(), Error> { + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); + + let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; + let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; + + let entry = &mut l3[l3i]; + if !entry.is_present() { + return Err(Error::DoesNotExist); + } + + mapper(entry)?; + ic_iallu(); + dc_cvac((&raw const l3[l3i]).addr()); + tlb_flush_vaae1(virt); + + Ok(()) + } + + fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> { let l1i = virt.page_index::(); let l2i = virt.page_index::(); let l3i = virt.page_index::(); @@ -129,14 +162,16 @@ impl ProcessAddressSpaceImpl { let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; - let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?; + let entry = l3[l3i]; + let page = entry.as_page().ok_or(Error::DoesNotExist)?; + let dirty = entry.is_dirty(); l3[l3i] = PageEntry::INVALID; ic_iallu(); dc_cvac((&raw const l3[l3i]).addr()); tlb_flush_vaae1(virt); - Ok(page) + Ok((page, dirty)) } fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { diff --git a/kernel/arch/aarch64/src/mem/table.rs b/kernel/arch/aarch64/src/mem/table.rs index 82a5eeed..880cbe03 100644 --- a/kernel/arch/aarch64/src/mem/table.rs +++ b/kernel/arch/aarch64/src/mem/table.rs @@ -9,6 +9,7 @@ use kernel_arch_interface::KERNEL_VIRT_OFFSET; use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, + process::PageAttributeUpdate, table::{ EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel, TableAllocator, @@ -46,6 +47,7 @@ bitflags! { const NON_GLOBAL = 1 << 11; + const DIRTY = 1 << 51; const PXN = 1 << 53; const UXN = 1 << 54; } @@ -136,8 +138,32 @@ impl PageTable { } impl PageEntry { + const ATTR_MASK: u64 = 0xFFF | (0xFFFF << 48); pub const INVALID: Self = Self(0, PhantomData); + pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> { + let mut attrs = PageAttributes::from_bits_retain(self.0); + if let Some(write) = update.user_write { + // Make writeable/non-writeable + if write { + attrs &= !PageAttributes::AP_ACCESS_MASK; + attrs |= PageAttributes::AP_BOTH_READWRITE; + } else { + todo!(); + } + } + if let Some(dirty) = update.dirty { + if dirty { + attrs |= PageAttributes::DIRTY; + } else { + attrs &= !PageAttributes::DIRTY; + } + } + self.0 &= !Self::ATTR_MASK; + self.0 |= attrs.bits() & Self::ATTR_MASK; + Ok(()) + } + pub const fn is_present(self) -> bool { self.0 & PageAttributes::PRESENT.bits() != 0 } @@ -271,7 +297,7 @@ impl PageEntry { if self.0 & PageAttributes::PRESENT.bits() != 0 && self.0 & PageAttributes::BLOCK.bits() == 0 { - Some(PhysicalAddress::from_u64(self.0 & !0xFFF)) + Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK)) } else { None } @@ -283,7 +309,7 @@ impl PageEntry { } else if let Some(table) = self.as_table() { EntryType::Table(table) } else { - EntryType::Page(PhysicalAddress::from_u64(self.0 & !0xFFF)) + EntryType::Page(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK)) } } } @@ -316,10 +342,14 @@ impl PageEntry { ) } + pub fn is_dirty(&self) -> bool { + self.0 & PageAttributes::DIRTY.bits() != 0 + } + pub fn as_page(&self) -> Option { let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); if self.0 & mask == mask { - Some(PhysicalAddress::from_u64(self.0 & !0xFFF)) + Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK)) } else { None } @@ -355,6 +385,10 @@ impl From for PageAttributes { out |= PageAttributes::AP_KERNEL_READONLY; } + if value.contains(MapAttributes::DIRTY) { + out |= PageAttributes::DIRTY; + } + if value.contains(MapAttributes::NON_GLOBAL) { out |= PageAttributes::NON_GLOBAL; } @@ -377,6 +411,10 @@ impl From for MapAttributes { _ => unreachable!(), }; + if value.contains(PageAttributes::DIRTY) { + out |= MapAttributes::DIRTY; + } + if value.contains(PageAttributes::NON_GLOBAL) { out |= MapAttributes::NON_GLOBAL; } diff --git a/kernel/arch/riscv64/src/mem/process.rs b/kernel/arch/riscv64/src/mem/process.rs index 0078684b..be7fcab2 100644 --- a/kernel/arch/riscv64/src/mem/process.rs +++ b/kernel/arch/riscv64/src/mem/process.rs @@ -6,7 +6,7 @@ use core::{ use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::PhysicalRefMut, - process::ProcessAddressSpaceManager, + process::{PageAttributeUpdate, ProcessAddressSpaceManager}, table::{ EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator, }, @@ -70,7 +70,15 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceI Ok(()) } - unsafe fn unmap_page(&mut self, address: usize) -> Result { + unsafe fn update_page_attributes( + &mut self, + address: usize, + update: &PageAttributeUpdate, + ) -> Result<(), Error> { + self.update_l3_entry(address, |entry| entry.update(update)) + } + + unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> { self.pop_l3_entry(address) } @@ -118,7 +126,11 @@ impl ProcessAddressSpaceImpl { Ok(()) } - fn pop_l3_entry(&mut self, virt: usize) -> Result { + fn update_l3_entry) -> Result<(), Error>>( + &mut self, + virt: usize, + mapper: F, + ) -> Result<(), Error> { let l1i = virt.page_index::(); let l2i = virt.page_index::(); let l3i = virt.page_index::(); @@ -127,12 +139,33 @@ impl ProcessAddressSpaceImpl { let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; - let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?; + let entry = &mut l3[l3i]; + if !entry.is_present() { + return Err(Error::DoesNotExist); + } + mapper(entry)?; + super::tlb_flush_va_asid(virt, self.asid as usize); + + Ok(()) + } + + fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> { + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); + + // TODO somehow drop tables if they're known to be empty? + let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; + let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; + + let entry = l3[l3i]; + let page = entry.as_page().ok_or(Error::DoesNotExist)?; + let dirty = entry.is_dirty(); l3[l3i] = PageEntry::INVALID; super::tlb_flush_va_asid(virt, self.asid as usize); - Ok(page) + Ok((page, dirty)) } fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { @@ -178,6 +211,9 @@ fn to_page_attributes(src: MapAttributes) -> PageAttributes { if src.intersects(MapAttributes::USER_READ | MapAttributes::USER_WRITE) { result |= PageAttributes::U; } + if src.contains(MapAttributes::DIRTY) { + result |= PageAttributes::SW_DIRTY; + } result } @@ -192,5 +228,9 @@ fn to_map_attributes(src: PageAttributes) -> MapAttributes { } } + if src.contains(PageAttributes::SW_DIRTY) { + result |= MapAttributes::DIRTY; + } + result } diff --git a/kernel/arch/riscv64/src/mem/table.rs b/kernel/arch/riscv64/src/mem/table.rs index a3481a0c..a1add287 100644 --- a/kernel/arch/riscv64/src/mem/table.rs +++ b/kernel/arch/riscv64/src/mem/table.rs @@ -6,6 +6,7 @@ use core::{ use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, + process::PageAttributeUpdate, table::{ page_index, EntryLevel, EntryLevelDrop, NextPageTable, NonTerminalEntryLevel, TableAllocator, @@ -88,6 +89,8 @@ impl PageTable { } impl PageEntry { + // Upper + lower 10 bits + const ATTR_MASK: u64 = 0xFFC00000000003FF; pub const INVALID: Self = Self(0, PhantomData); /// Constructs a [PageEntry] from its raw representation. @@ -103,6 +106,23 @@ impl PageEntry { self.0 & PageAttributes::V.bits() != 0 } + pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> { + let mut attrs = self.attributes(); + if let Some(write) = update.user_write { + attrs.set(PageAttributes::W, write); + } + if let Some(dirty) = update.dirty { + attrs.set(PageAttributes::SW_DIRTY, dirty); + } + self.0 &= !Self::ATTR_MASK; + self.0 |= attrs.bits() & Self::ATTR_MASK; + Ok(()) + } + + pub const fn is_dirty(&self) -> bool { + self.0 & PageAttributes::SW_DIRTY.bits() != 0 + } + pub fn attributes(self) -> PageAttributes { PageAttributes::from_bits_retain(self.0) } @@ -211,7 +231,7 @@ impl PageEntry { & (PageAttributes::R | PageAttributes::W | PageAttributes::X | PageAttributes::V) .bits() == PageAttributes::V.bits()) - .then_some((self.0 << 2) & !0xFFF) + .then_some((self.0 & !Self::ATTR_MASK) << 2) .map(PhysicalAddress::from_u64) } } @@ -232,7 +252,7 @@ impl PageEntry { pub fn as_page(&self) -> Option { (self.0 & PageAttributes::V.bits() != 0) - .then_some((self.0 << 2) & !0xFFF) + .then_some((self.0 & !Self::ATTR_MASK) << 2) .map(PhysicalAddress::from_u64) } } diff --git a/kernel/arch/x86_64/src/mem/process.rs b/kernel/arch/x86_64/src/mem/process.rs index 2738e36b..746107a3 100644 --- a/kernel/arch/x86_64/src/mem/process.rs +++ b/kernel/arch/x86_64/src/mem/process.rs @@ -4,7 +4,7 @@ use core::marker::PhantomData; use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::PhysicalRefMut, - process::ProcessAddressSpaceManager, + process::{PageAttributeUpdate, ProcessAddressSpaceManager}, table::{ EntryLevel, EntryLevelDrop, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator, }, @@ -61,6 +61,14 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceI self.write_l3_entry(address, PageEntry::page(physical, flags.into()), false) } + unsafe fn update_page_attributes( + &mut self, + address: usize, + update: &PageAttributeUpdate, + ) -> Result<(), Error> { + self.update_l3_entry(address, |entry| entry.update(update)) + } + unsafe fn unmap_page(&mut self, address: usize) -> Result<(PhysicalAddress, bool), Error> { self.pop_l3_entry(address) } @@ -111,6 +119,32 @@ impl ProcessAddressSpaceImpl { Ok(()) } + fn update_l3_entry) -> Result<(), Error>>( + &mut self, + virt: usize, + mapper: F, + ) -> Result<(), Error> { + let l0i = virt.page_index::(); + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); + + let mut l1 = self.l0.get_mut(l0i).ok_or(Error::DoesNotExist)?; + let mut l2 = l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; + let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; + + let entry = &mut l3[l3i]; + if !entry.is_present() { + return Err(Error::DoesNotExist); + } + mapper(entry)?; + unsafe { + flush_tlb_entry(virt); + } + + Ok(()) + } + fn pop_l3_entry(&mut self, virt: usize) -> Result<(PhysicalAddress, bool), Error> { let l0i = virt.page_index::(); let l1i = virt.page_index::(); diff --git a/kernel/arch/x86_64/src/mem/table.rs b/kernel/arch/x86_64/src/mem/table.rs index 58cb3f25..5d6b2d98 100644 --- a/kernel/arch/x86_64/src/mem/table.rs +++ b/kernel/arch/x86_64/src/mem/table.rs @@ -8,6 +8,7 @@ use bitflags::bitflags; use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, + process::PageAttributeUpdate, table::{ EntryLevel, EntryLevelDrop, MapAttributes, NextPageTable, NonTerminalEntryLevel, TableAllocator, @@ -100,7 +101,7 @@ impl PageEntry { /// not pub fn as_page(self) -> Option { if self.0 & PageAttributes::PRESENT.bits() != 0 { - Some(PhysicalAddress::from_u64(self.0 & !0xFFF)) + Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK)) } else { None } @@ -151,7 +152,7 @@ impl PageEntry { if self.0 & PageAttributes::PRESENT.bits() != 0 && self.0 & PageAttributes::BLOCK.bits() == 0 { - Some(PhysicalAddress::from_u64(self.0 & !0xFFF)) + Some(PhysicalAddress::from_u64(self.0 & !Self::ATTR_MASK)) } else { None } @@ -164,6 +165,8 @@ impl PageEntry { } impl PageEntry { + const ATTR_MASK: u64 = 0xFFF | (1 << 63); + /// An entry that is not mapped pub const INVALID: Self = Self(0, PhantomData); @@ -185,6 +188,21 @@ impl PageEntry { pub fn is_present(&self) -> bool { self.0 & PageAttributes::PRESENT.bits() != 0 } + + pub fn update(&mut self, update: &PageAttributeUpdate) -> Result<(), Error> { + let mut attrs = PageAttributes::from_bits_retain(self.0); + if let Some(write) = update.user_write { + if write { + attrs |= PageAttributes::WRITABLE; + } else { + attrs &= !PageAttributes::WRITABLE; + } + } + // Dirty is ignored, it's hardware-managed + self.0 &= !Self::ATTR_MASK; + self.0 |= attrs.bits() & Self::ATTR_MASK; + Ok(()) + } } impl PageTable { diff --git a/kernel/driver/block/ahci/src/port.rs b/kernel/driver/block/ahci/src/port.rs index a6ee3b85..84caba99 100644 --- a/kernel/driver/block/ahci/src/port.rs +++ b/kernel/driver/block/ahci/src/port.rs @@ -16,8 +16,8 @@ use libk::{ error::Error, }; use libk_mm::{ - address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, PageProvider, - VirtualPage, + address::PhysicalAddress, device::DeviceMemoryIo, table::MapAttributes, OnDemandPage, + PageProvider, VirtualPage, }; use libk_util::{sync::IrqSafeSpinlock, waker::QueueWaker, OneTimeInit}; use tock_registers::interfaces::{Readable, Writeable}; @@ -366,7 +366,7 @@ impl Device for AhciPort { } impl PageProvider for AhciPort { - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unimplemented!() } diff --git a/kernel/driver/block/nvme/src/drive.rs b/kernel/driver/block/nvme/src/drive.rs index 204011e3..56f38ce5 100644 --- a/kernel/driver/block/nvme/src/drive.rs +++ b/kernel/driver/block/nvme/src/drive.rs @@ -11,7 +11,7 @@ use libk::{ use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, table::MapAttributes, - PageProvider, PageSlice, VirtualPage, + OnDemandPage, PageProvider, PageSlice, VirtualPage, }; use crate::{command::IdentifyNamespaceRequest, register_nvme_namespace, IoDirection}; @@ -151,7 +151,7 @@ impl BlockDevice for NvmeNamespace { } impl PageProvider for NvmeNamespace { - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unimplemented!() } diff --git a/kernel/driver/block/scsi/src/lib.rs b/kernel/driver/block/scsi/src/lib.rs index 5ce63c28..6d0ad86c 100644 --- a/kernel/driver/block/scsi/src/lib.rs +++ b/kernel/driver/block/scsi/src/lib.rs @@ -22,7 +22,9 @@ use libk::{ fs::devfs, task::{runtime, sync::AsyncMutex}, }; -use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider, VirtualPage}; +use libk_mm::{ + address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage, +}; use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, OneTimeInit, @@ -278,7 +280,7 @@ impl BlockDevice for ScsiUnit { } impl PageProvider for ScsiUnit { - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unimplemented!() } diff --git a/kernel/driver/virtio/blk/src/lib.rs b/kernel/driver/virtio/blk/src/lib.rs index cb6cae66..7d9cb333 100644 --- a/kernel/driver/virtio/blk/src/lib.rs +++ b/kernel/driver/virtio/blk/src/lib.rs @@ -17,7 +17,9 @@ use libk::{ fs::devfs, task::runtime, }; -use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider, VirtualPage}; +use libk_mm::{ + address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage, +}; use libk_util::sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}; use ygg_driver_pci::{ device::{PciDeviceInfo, PreferredInterruptMode}, @@ -329,7 +331,7 @@ impl BlockDevice for VirtioBlk { } impl PageProvider for VirtioBlk { - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unimplemented!() } diff --git a/kernel/driver/virtio/gpu/src/lib.rs b/kernel/driver/virtio/gpu/src/lib.rs index 489b0a53..660994c8 100644 --- a/kernel/driver/virtio/gpu/src/lib.rs +++ b/kernel/driver/virtio/gpu/src/lib.rs @@ -5,8 +5,7 @@ extern crate alloc; use core::mem::MaybeUninit; -use alloc::{boxed::Box, sync::Arc, vec::Vec}; -use async_trait::async_trait; +use alloc::{sync::Arc, vec::Vec}; use command::{CommandExecution, ScanoutInfo}; use device_api::{ device::{Device, DeviceInitContext}, @@ -24,7 +23,7 @@ use libk::{ use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, table::MapAttributes, - PageProvider, VirtualPage, L3_PAGE_SIZE, + OnDemandPage, PageProvider, VirtualPage, L3_PAGE_SIZE, }; use libk_util::{ sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, @@ -294,52 +293,41 @@ impl Device for VirtioGpu { } impl PageProvider for VirtioGpu { - // fn get_page(&self, offset: u64) -> Result { - // // TODO check that the page is mapped by framebuffer owner - // let config = self.config.read(); - // let framebuffer = config.framebuffer.as_ref().ok_or(Error::DoesNotExist)?; - // if offset as usize + L3_PAGE_SIZE > framebuffer.dma_buffer.page_count() * L3_PAGE_SIZE { - // log::warn!( - // "virtio-gpu: offset {:#x} outside of framebuffer bounds {:#x}", - // offset, - // framebuffer.size - // ); - // return Err(Error::InvalidMemoryOperation); - // } - // let phys = unsafe { framebuffer.dma_buffer.as_physical_address() }.add(offset as usize); - // - // Ok(phys) - // } - // - // fn clone_page( - // &self, - // _offset: u64, - // _src_phys: PhysicalAddress, - // _src_attrs: MapAttributes, - // ) -> Result { - // todo!() - // } - // - // fn release_page(&self, _offset: u64, _phys: PhysicalAddress) -> Result<(), Error> { - // Ok(()) - // } - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unreachable!() } fn get_page(&self, offset: u64) -> Result { - unimplemented!() + // TODO check that the page is mapped by framebuffer owner + let config = self.config.read(); + let framebuffer = config.framebuffer.as_ref().ok_or(Error::DoesNotExist)?; + if offset as usize + L3_PAGE_SIZE > framebuffer.dma_buffer.page_count() * L3_PAGE_SIZE { + log::warn!( + "virtio-gpu: offset {:#x} outside of framebuffer bounds {:#x}", + offset, + framebuffer.size + ); + return Err(Error::InvalidMemoryOperation); + } + let phys = unsafe { framebuffer.dma_buffer.as_physical_address() }.add(offset as usize); + + Ok(VirtualPage::Immediate(phys)) } - fn release_page(&self, offset: u64, phys: PhysicalAddress, _dirty: bool) -> Result<(), Error> { - unimplemented!() + fn release_page( + &self, + _offset: u64, + _phys: PhysicalAddress, + _dirty: bool, + ) -> Result<(), Error> { + Ok(()) } fn clone_page( &self, - offset: u64, - src_phys: PhysicalAddress, - src_attrs: MapAttributes, + _offset: u64, + _src_phys: PhysicalAddress, + _src_attrs: MapAttributes, ) -> Result { unimplemented!() } diff --git a/kernel/driver/virtio/net/src/lib.rs b/kernel/driver/virtio/net/src/lib.rs index b513297d..76fc294c 100644 --- a/kernel/driver/virtio/net/src/lib.rs +++ b/kernel/driver/virtio/net/src/lib.rs @@ -140,7 +140,9 @@ impl VirtioNet { (Some(msis[0].vector as u16), Some(msis[1].vector as u16)) } else { // TODO support non-MSI-x/non-multivec setups - todo!(); + self.pci_device_info + .map_interrupt(InterruptAffinity::Any, self.clone())?; + (None, None) }; // Setup a callback to remove pending buffers and pass them to the network stack @@ -245,7 +247,10 @@ impl InterruptHandler for VirtioNet { true } // TODO non-multivec/legacy IRQ setup - IrqVector::Irq(_) => todo!(), + IrqVector::Irq(_) => { + self.softirq.signal((1 << Self::VQ_RX) | (1 << Self::VQ_TX)); + true + } } } } diff --git a/kernel/lib/memtables/src/riscv64.rs b/kernel/lib/memtables/src/riscv64.rs index 16e2e85d..8d30b574 100644 --- a/kernel/lib/memtables/src/riscv64.rs +++ b/kernel/lib/memtables/src/riscv64.rs @@ -11,6 +11,8 @@ bitflags! { #[derive(Clone, Copy, PartialEq, Eq, Debug)] pub struct PageAttributes: u64 { const N = 1 << 63; + /// Software-tracked dirty bit (RSW[0]) + const SW_DIRTY = 1 << 9; /// Dirty bit const D = 1 << 7; /// Access bit diff --git a/kernel/libk/libk-mm/interface/src/lib.rs b/kernel/libk/libk-mm/interface/src/lib.rs index eeb5814e..04cfb419 100644 --- a/kernel/libk/libk-mm/interface/src/lib.rs +++ b/kernel/libk/libk-mm/interface/src/lib.rs @@ -11,6 +11,15 @@ pub mod pointer; pub mod process; pub mod table; +/// Describes which kind of a page fault occured +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PageFaultKind { + TranslationFault { write: bool }, + WriteFault, + AccessFault, + Other, +} + /// Wrapper type to represent an object residing within the kernel #[repr(transparent)] pub struct KernelImageObject { diff --git a/kernel/libk/libk-mm/interface/src/process.rs b/kernel/libk/libk-mm/interface/src/process.rs index af340958..2ecfc8be 100644 --- a/kernel/libk/libk-mm/interface/src/process.rs +++ b/kernel/libk/libk-mm/interface/src/process.rs @@ -5,6 +5,11 @@ use crate::{ table::{MapAttributes, TableAllocator}, }; +pub struct PageAttributeUpdate { + pub user_write: Option, + pub dirty: Option, +} + /// Interface for virtual memory address space management pub trait ProcessAddressSpaceManager: Sized { /// PFN of a minimum address allowed for virtual region allocation @@ -27,6 +32,17 @@ pub trait ProcessAddressSpaceManager: Sized { flags: MapAttributes, ) -> Result<(), Error>; + /// Adds/removes attributes from a page entry in the address space. + /// + /// # Safety + /// + /// The caller must ensure the validity of this update. + unsafe fn update_page_attributes( + &mut self, + address: usize, + update: &PageAttributeUpdate, + ) -> Result<(), Error>; + /// Removes a single PAGE_SIZE mapping from the address space. /// /// # Safety diff --git a/kernel/libk/libk-mm/interface/src/table.rs b/kernel/libk/libk-mm/interface/src/table.rs index 2812a887..e2c29f20 100644 --- a/kernel/libk/libk-mm/interface/src/table.rs +++ b/kernel/libk/libk-mm/interface/src/table.rs @@ -37,6 +37,8 @@ bitflags! { const USER_WRITE = 1 << 1; /// The mapping is not global across the address spaces const NON_GLOBAL = 1 << 2; + /// The mapping is marked dirty by the OS + const DIRTY = 1 << 3; } } diff --git a/kernel/libk/libk-mm/src/lib.rs b/kernel/libk/libk-mm/src/lib.rs index d1f43381..2d5718b8 100644 --- a/kernel/libk/libk-mm/src/lib.rs +++ b/kernel/libk/libk-mm/src/lib.rs @@ -21,8 +21,6 @@ use core::{ }; use address::Virtualize; -use alloc::boxed::Box; -use async_trait::async_trait; use kernel_arch::{mem::PhysicalMemoryAllocator, Architecture, ArchitectureImpl}; use libk_mm_interface::{ address::{AsPhysicalAddress, PhysicalAddress}, @@ -40,7 +38,7 @@ pub mod process; #[cfg(any(target_os = "none", rust_analyzer))] pub mod heap; -pub use libk_mm_interface::table; +pub use libk_mm_interface::{table, PageFaultKind}; pub struct TableAllocatorImpl; @@ -67,8 +65,13 @@ pub enum VirtualPage { Immediate(PhysicalAddress), } +pub struct OnDemandPage { + pub physical: PhysicalAddress, + pub writeable: bool, +} + pub trait PageProvider: Send + Sync { - fn ondemand_fetch(&self, opaque: u64) -> Result; + fn ondemand_fetch(&self, offset: u64) -> Result; fn get_page(&self, offset: u64) -> Result; diff --git a/kernel/libk/libk-mm/src/process.rs b/kernel/libk/libk-mm/src/process.rs index 6b40b242..4629501b 100644 --- a/kernel/libk/libk-mm/src/process.rs +++ b/kernel/libk/libk-mm/src/process.rs @@ -1,15 +1,12 @@ -use core::{ - cmp, - ops::{Deref, Range}, -}; +use core::ops::{Deref, Range}; -use alloc::{boxed::Box, sync::Arc}; -use async_trait::async_trait; +use alloc::sync::Arc; use kernel_arch::ProcessAddressSpaceImpl; use libk_mm_interface::{ address::PhysicalAddress, - process::ProcessAddressSpaceManager, + process::{PageAttributeUpdate, ProcessAddressSpaceManager}, table::{MapAttributes, TableAllocator}, + PageFaultKind, }; use libk_util::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; use vmalloc::{RangeData, VirtualMemoryAllocator}; @@ -18,7 +15,7 @@ use yggdrasil_abi::error::Error; use crate::{ phys, pointer::{PhysicalRef, PhysicalRefMut}, - PageProvider, TableAllocatorImpl, VirtualPage, L3_PAGE_SIZE, + OnDemandPage, PageProvider, TableAllocatorImpl, VirtualPage, L3_PAGE_SIZE, }; /// Describes how the physical memory is provided for the mapping @@ -62,7 +59,7 @@ impl VirtualRangeBacking { } impl PageProvider for VirtualRangeBacking { - fn ondemand_fetch(&self, opaque: u64) -> Result { + fn ondemand_fetch(&self, opaque: u64) -> Result { match self { Self::Anonymous => unreachable!(), Self::File(f) => f.file.ondemand_fetch(f.offset + opaque), @@ -356,33 +353,75 @@ impl Inner { Ok(()) } - fn handle_fault(&mut self, address: usize) -> Result<(), Error> { - // TODO hardcoded "prefetch" hint - const PREFETCH_HINT: usize = 16; + fn handle_fault(&mut self, address: usize, kind: PageFaultKind) -> Result<(), Error> { + let (ondemand, write) = match kind { + PageFaultKind::WriteFault => (false, true), + PageFaultKind::TranslationFault { write: true } => (true, true), + PageFaultKind::TranslationFault { write: false } => (true, false), + _ => return Err(Error::DoesNotExist), + }; - let pfn = address / L3_PAGE_SIZE; - let (pfn_range, backing) = self.allocator.get(pfn).ok_or(Error::DoesNotExist)?; - let remaining = (pfn_range.end - pfn).min(PREFETCH_HINT); - // TODO rollback - for i in 0..remaining { - let address = (pfn + i) * L3_PAGE_SIZE; - let offset = (pfn - pfn_range.start + i) * L3_PAGE_SIZE; + if ondemand { + // TODO hardcoded "prefetch" hint + const PREFETCH_HINT: usize = 16; - if self.table.translate(address).is_ok() { - continue; - } + let pfn = address / L3_PAGE_SIZE; + let (pfn_range, backing) = self.allocator.get(pfn).ok_or(Error::DoesNotExist)?; + let remaining = (pfn_range.end - pfn).min(PREFETCH_HINT); - let phys = backing.ondemand_fetch(offset as u64)?; - // TODO don't guess attributes - match unsafe { self.table.map_page(address, phys, MapAttributes::USER_READ) } { - Ok(()) => (), - Err(error) => { - log::warn!("Failed to map on-demand resolved page: {error:?}"); - backing.release_page(offset as u64, phys, false).ok(); - return Err(error); + // TODO rollback + for i in 0..remaining { + let address = (pfn + i) * L3_PAGE_SIZE; + let offset = (pfn - pfn_range.start + i) * L3_PAGE_SIZE; + + if self.table.translate(address).is_ok() { + continue; + } + + // TODO query cpu flags to determine whether it supports hardware dirty bit + // tracking. If so, just map data read-write here + let page = backing.ondemand_fetch(offset as u64)?; + let mut attrs = MapAttributes::USER_READ; + if page.writeable { + // Used as an indicator the page can be made writeable, used on platforms + // without hardware dirty state tracking + attrs |= MapAttributes::DIRTY; + } + + match unsafe { self.table.map_page(address, page.physical, attrs) } { + Ok(()) => (), + Err(error) => { + log::warn!("Failed to map on-demand resolved page: {error:?}"); + backing + .release_page(offset as u64, page.physical, false) + .ok(); + return Err(error); + } } } } + + if write && !cfg!(target_arch = "x86_64") { + // TODO this will break permission architecture, need to check if the range is actually + // writeable first + let (_, attrs) = self.table.translate(address)?; + + if attrs.contains(MapAttributes::DIRTY) { + unsafe { + self.table.update_page_attributes( + address & !(L3_PAGE_SIZE - 1), + &PageAttributeUpdate { + user_write: Some(true), + dirty: Some(true), + }, + )?; + } + } else { + log::warn!("Write to non-demand/non-dirty page @ {address:#x}"); + return Err(Error::DoesNotExist); + } + } + Ok(()) } } @@ -523,8 +562,8 @@ impl ProcessAddressSpace { } /// Handles a translation fault, possibly fetching on-demand pages - pub fn handle_fault(&self, address: usize) -> Result<(), Error> { - self.inner.lock().handle_fault(address) + pub fn handle_fault(&self, address: usize, kind: PageFaultKind) -> Result<(), Error> { + self.inner.lock().handle_fault(address, kind) } } diff --git a/kernel/libk/src/device/block/partition/mod.rs b/kernel/libk/src/device/block/partition/mod.rs index 133ea948..7011bc52 100644 --- a/kernel/libk/src/device/block/partition/mod.rs +++ b/kernel/libk/src/device/block/partition/mod.rs @@ -3,7 +3,9 @@ use core::mem::MaybeUninit; use alloc::{boxed::Box, sync::Arc}; use async_trait::async_trait; use device_api::device::Device; -use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider, VirtualPage}; +use libk_mm::{ + address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage, +}; use yggdrasil_abi::error::Error; use crate::dma::{DmaBuffer, DmaSlice, DmaSliceMut}; @@ -32,7 +34,7 @@ impl Partition { } impl PageProvider for Partition { - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unimplemented!() } diff --git a/kernel/libk/src/device/display/mod.rs b/kernel/libk/src/device/display/mod.rs index 3676b22a..968e78de 100644 --- a/kernel/libk/src/device/display/mod.rs +++ b/kernel/libk/src/device/display/mod.rs @@ -4,7 +4,8 @@ use alloc::{boxed::Box, sync::Arc}; use async_trait::async_trait; use device_api::device::Device; use libk_mm::{ - address::PhysicalAddress, table::MapAttributes, PageProvider, VirtualPage, L3_PAGE_SIZE, + address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage, + L3_PAGE_SIZE, }; use yggdrasil_abi::{ bitflags, @@ -232,7 +233,7 @@ impl PageProvider for DisplayWrapper { // fn release_page(&self, offset: u64, phys: PhysicalAddress) -> Result<(), Error> { // self.device.release_page(offset, phys) // } - fn ondemand_fetch(&self, opaque: u64) -> Result { + fn ondemand_fetch(&self, opaque: u64) -> Result { self.device.ondemand_fetch(opaque) } diff --git a/kernel/libk/src/task/binary/elf.rs b/kernel/libk/src/task/binary/elf.rs index ea94f6c3..ac848473 100644 --- a/kernel/libk/src/task/binary/elf.rs +++ b/kernel/libk/src/task/binary/elf.rs @@ -189,7 +189,7 @@ pub fn load_elf_from_file( let (image_load_base, ip_offset) = elf_load_address(aslr, elf.ehdr.e_type, vaddr_min, image_load_size); - log::debug!( + log::info!( "Loading ELF virtual {:#x?} -> real {:#x?}", vaddr_min..vaddr_max, image_load_base..image_load_base + image_load_size diff --git a/kernel/libk/src/task/binary/mod.rs b/kernel/libk/src/task/binary/mod.rs index d1bc5437..54f8edf4 100644 --- a/kernel/libk/src/task/binary/mod.rs +++ b/kernel/libk/src/task/binary/mod.rs @@ -144,6 +144,8 @@ fn setup_program_env( _ => 0, }; + placer.align(size_of::())?; + // Put ProgramArgumentInner struct let arg_address = placer.position + virt; @@ -180,6 +182,12 @@ where MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; + log::info!( + "stack: {:#x}..{:#x}", + virt_stack_base, + virt_stack_base + USER_STACK_PAGES * 0x1000 + ); + let mut auxv = vec![]; if let Some(tls_image) = image.tls_image.as_ref() { if tls_image.master_copy_base != 0 { diff --git a/kernel/libk/src/task/thread.rs b/kernel/libk/src/task/thread.rs index 50149320..a2a4c379 100644 --- a/kernel/libk/src/task/thread.rs +++ b/kernel/libk/src/task/thread.rs @@ -11,7 +11,7 @@ use kernel_arch::{ task::{Scheduler, TaskContext, TaskFrame}, Architecture, ArchitectureImpl, CpuImpl, }; -use libk_mm::process::ProcessAddressSpace; +use libk_mm::{process::ProcessAddressSpace, PageFaultKind}; use libk_util::{ event::BoolEvent, sync::{spin_rwlock::IrqSafeRwLock, IrqGuard, IrqSafeSpinlock}, @@ -674,8 +674,8 @@ impl CurrentThread { } } - pub fn handle_page_fault(&self, address: usize) -> Result<(), Error> { - self.address_space().handle_fault(address) + pub fn handle_page_fault(&self, address: usize, kind: PageFaultKind) -> Result<(), Error> { + self.address_space().handle_fault(address, kind) } /// Sets up a return frame to handle a pending signal, if any is present in the task's queue. diff --git a/kernel/libk/src/vfs/file/mod.rs b/kernel/libk/src/vfs/file/mod.rs index 773c85c3..cce46181 100644 --- a/kernel/libk/src/vfs/file/mod.rs +++ b/kernel/libk/src/vfs/file/mod.rs @@ -12,7 +12,9 @@ use alloc::{ }; use async_trait::async_trait; use device::{BlockFile, CharFile}; -use libk_mm::{address::PhysicalAddress, table::MapAttributes, PageProvider, VirtualPage}; +use libk_mm::{ + address::PhysicalAddress, table::MapAttributes, OnDemandPage, PageProvider, VirtualPage, +}; use libk_util::{ io::{ReadAt, WriteAt}, sync::{spin_rwlock::IrqSafeRwLock, IrqSafeSpinlock}, @@ -421,7 +423,7 @@ impl File { } impl PageProvider for File { - fn ondemand_fetch(&self, opaque: u64) -> Result { + fn ondemand_fetch(&self, opaque: u64) -> Result { self.as_page_provider()?.ondemand_fetch(opaque) } diff --git a/kernel/libk/src/vfs/file/regular.rs b/kernel/libk/src/vfs/file/regular.rs index 25a17ef0..a63320a7 100644 --- a/kernel/libk/src/vfs/file/regular.rs +++ b/kernel/libk/src/vfs/file/regular.rs @@ -1,14 +1,12 @@ -use alloc::boxed::Box; -use async_trait::async_trait; use libk_mm::{ - address::PhysicalAddress, phys, pointer::PhysicalRef, table::MapAttributes, PageBox, - PageProvider, VirtualPage, L3_PAGE_SIZE, + address::PhysicalAddress, phys, pointer::PhysicalRef, table::MapAttributes, OnDemandPage, + PageBox, PageProvider, VirtualPage, L3_PAGE_SIZE, }; use libk_util::sync::IrqSafeSpinlock; use yggdrasil_abi::{error::Error, io::SeekFrom}; use super::InstanceData; -use crate::{debug, vfs::node::NodeRef}; +use crate::vfs::node::NodeRef; #[derive(Clone)] pub struct RegularFile { @@ -117,7 +115,7 @@ impl Drop for RegularFile { } impl PageProvider for RegularFile { - fn ondemand_fetch(&self, opaque: u64) -> Result { + fn ondemand_fetch(&self, opaque: u64) -> Result { assert_eq!(opaque as usize % L3_PAGE_SIZE, 0); let reg = self.node.as_regular()?; let mut page = PageBox::new_slice(0, L3_PAGE_SIZE)?; @@ -131,10 +129,13 @@ impl PageProvider for RegularFile { page[len..].fill(0); } let page = PageBox::into_physical_raw(page); - Ok(page) + Ok(OnDemandPage { + physical: page, + writeable: self.write, + }) } - fn get_page(&self, offset: u64) -> Result { + fn get_page(&self, _offset: u64) -> Result { Ok(VirtualPage::OnDemand) } @@ -161,9 +162,9 @@ impl PageProvider for RegularFile { fn clone_page( &self, - offset: u64, - src_phys: PhysicalAddress, - src_attrs: MapAttributes, + _offset: u64, + _src_phys: PhysicalAddress, + _src_attrs: MapAttributes, ) -> Result { unimplemented!() } diff --git a/kernel/libk/src/vfs/shared_memory.rs b/kernel/libk/src/vfs/shared_memory.rs index cc5ad03a..71c2fc45 100644 --- a/kernel/libk/src/vfs/shared_memory.rs +++ b/kernel/libk/src/vfs/shared_memory.rs @@ -1,11 +1,10 @@ use core::mem::MaybeUninit; -use alloc::{boxed::Box, vec::Vec}; -use async_trait::async_trait; +use alloc::vec::Vec; use libk_mm::{ address::{AsPhysicalAddress, PhysicalAddress}, table::MapAttributes, - PageBox, PageProvider, VirtualPage, L3_PAGE_SIZE, + OnDemandPage, PageBox, PageProvider, VirtualPage, L3_PAGE_SIZE, }; use yggdrasil_abi::error::Error; @@ -28,7 +27,7 @@ impl SharedMemory { } impl PageProvider for SharedMemory { - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unreachable!() } @@ -52,9 +51,9 @@ impl PageProvider for SharedMemory { fn clone_page( &self, - offset: u64, - src_phys: PhysicalAddress, - src_attrs: MapAttributes, + _offset: u64, + _src_phys: PhysicalAddress, + _src_attrs: MapAttributes, ) -> Result { todo!() } diff --git a/kernel/src/arch/aarch64/boot/mod.rs b/kernel/src/arch/aarch64/boot/mod.rs index 2e86625a..eaa37be5 100644 --- a/kernel/src/arch/aarch64/boot/mod.rs +++ b/kernel/src/arch/aarch64/boot/mod.rs @@ -42,6 +42,7 @@ unsafe fn pre_init_mmu() { TCR_EL1.write( TCR_EL1::AS::ASID8Bits + TCR_EL1::A1::TTBR0 + + TCR_EL1::HD::CLEAR + // General TCR_EL1::IPS::Bits_48 + // TTBR0 diff --git a/kernel/src/arch/aarch64/exception.rs b/kernel/src/arch/aarch64/exception.rs index f592c050..f18e0bc2 100644 --- a/kernel/src/arch/aarch64/exception.rs +++ b/kernel/src/arch/aarch64/exception.rs @@ -19,7 +19,7 @@ use kernel_arch_aarch64::{ mem::table::{EntryType, PageTable, L1, L2, L3}, }; use libk::{device::external_interrupt_controller, task::thread::Thread}; -use libk_mm::{address::PhysicalAddress, table::EntryLevelExt}; +use libk_mm::{address::PhysicalAddress, table::EntryLevelExt, PageFaultKind}; use tock_registers::interfaces::{Readable, Writeable}; use crate::{ @@ -82,54 +82,27 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { let far = FAR_EL1.get() as usize; let ttbr0 = TTBR0_EL1.get(); - log::error!(target: "raw", "SYNC exception:\n"); - log::error!(target: "raw", "FAR: {:#x}\n", far); - log::error!(target: "raw", "ELR: {:#x}\n", ELR_EL1.get()); - log::error!(target: "raw", "ESR: {:#x}\n", ESR_EL1.get()); - log::error!(target: "raw", "SP_EL0: {:#x}\n", SP_EL0.get()); - log::error!(target: "raw", "TTBR0_EL1: {:#x}\n", ttbr0); - log::error!(target: "raw", "TTBR1_EL1: {:#x}\n", TTBR1_EL1.get()); - log::error!(target: "raw", "Register dump:\n"); - log::error!(target: "raw", "{:?}\n", frame); + log::error!(target: ":raw", "SYNC exception:\n"); + log::error!(target: ":raw", "FAR: {:#x}\n", far); + log::error!(target: ":raw", "ELR: {:#x}\n", ELR_EL1.get()); + log::error!(target: ":raw", "ESR: {:#x}\n", ESR_EL1.get()); + log::error!(target: ":raw", "SP_EL0: {:#x}\n", SP_EL0.get()); + log::error!(target: ":raw", "TTBR0_EL1: {:#x}\n", ttbr0); + log::error!(target: ":raw", "TTBR1_EL1: {:#x}\n", TTBR1_EL1.get()); + log::error!(target: ":raw", "Register dump:\n"); + log::error!(target: ":raw", "{:?}\n", frame); - log::error!(target: "raw", "FAR_EL1 page table walk:\n"); + log::error!(target: ":raw", "FAR_EL1 page table walk:\n"); unsafe { perform_ptw(far, |level, result| { - log::error!(target: "raw", "[{level}] {result}\n"); + log::error!(target: ":raw", "[{level}] {result}\n"); }); } - // if let Some(cpu) = cpu { - // // let current = cpu.queue().current_process(); - // let current = cpu.current_thread_id().and_then(Thread::get); - - // if let Some(current) = current { - // log::error!(target: "raw", "In thread {}\n", current.id); - - // let space = current.address_space(); - // if far < KERNEL_VIRT_OFFSET && space.as_address_with_asid() == ttbr0 { - // match space.translate(far) { - // Ok(phys) => log::error!( - // target: "raw", - // "FAR translation (manual): {:#x} -> {:#x}\n", - // far, - // phys - // ), - // Err(_) => log::error!(target: "raw", "FAR does not translate\n"), - // } - - // let at_s1e0r = at_s1e0r(far); - // let at_s1e1r = at_s1e1r(far); - - // log::error!(target: "raw", "at s1e0r: {far:#x} -> {at_s1e0r:#x?}\n"); - // log::error!(target: "raw", "at s1e1r: {far:#x} -> {at_s1e1r:#x?}\n"); - // } - // } - // } match ec { // Data abort from lower level 0b100100 => { - log::error!(target: "raw", "Exception kind: Data Abort from EL0\n"); + log::error!(target: ":raw", "Exception kind: Data Abort from EL0\n"); let dfsc = iss & 0x3F; if iss & (1 << 24) != 0 { @@ -143,7 +116,7 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { let access_type_str = if iss & (1 << 6) != 0 { "write" } else { "read" }; log::error!( - target: "raw", + target: ":raw", "Invalid {} of a {} to/from {:#x}\n", access_type_str, access_size_str, @@ -151,26 +124,26 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { ); } - log::error!(target: "raw", "DFSC = {:#x}\n", dfsc); + log::error!(target: ":raw", "DFSC = {:#x}\n", dfsc); } // Instruction abort from lower level 0b100000 => { log::error!( - target: "raw", + target: ":raw", "Exception kind: Instruction Abort from EL0\n" ); let ifsc = iss & 0x3F; - log::error!(target: "raw", "IFSC = {:#x}\n", ifsc); + log::error!(target: ":raw", "IFSC = {:#x}\n", ifsc); } _ => (), } - log::error!(target: "raw", "System register dump:\n"); - log::error!(target: "raw", "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get()); - log::error!(target: "raw", "TCR_EL1 = {:#x}\n", TCR_EL1.get()); - log::error!(target: "raw", "TPIDR_EL1 = {:#x}\n", TPIDR_EL1.get()); - log::error!(target: "raw", "TPIDR_EL0 = {:#x}\n", TPIDR_EL0.get()); + log::error!(target: ":raw", "System register dump:\n"); + log::error!(target: ":raw", "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get()); + log::error!(target: ":raw", "TCR_EL1 = {:#x}\n", TCR_EL1.get()); + log::error!(target: ":raw", "TPIDR_EL1 = {:#x}\n", TPIDR_EL1.get()); + log::error!(target: ":raw", "TPIDR_EL0 = {:#x}\n", TPIDR_EL0.get()); } #[no_mangle] @@ -289,13 +262,28 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { *thread.name.read() ); thread.raise_signal(Signal::Aborted); - true + false } - _ => { + // Data abort from a lower EL + 0b100100 => { let thread = Thread::current(); - if ec == 0b100100 { - // Data abort from lower level - let thread = Thread::current(); + let dfsc = esr_el1 & 0x3F; + let wnr = esr_el1 & (1 << 6) != 0; + let fnv = esr_el1 & (1 << 10) != 0; + let far = FAR_EL1.get() as usize; + let kind = match dfsc { + // Translation fault, level 3 + 0b000111 => Some(PageFaultKind::TranslationFault { write: wnr }), + // Permission fault, level 3 + 0b001111 if wnr => Some(PageFaultKind::WriteFault), + _ => None, + }; + if let Some(kind) = kind + && !fnv + && thread.handle_page_fault(far, kind).is_ok() + { + false + } else { log::warn!( "Data abort in {} {:?} at {:#x} with address {:#x}", thread.id, @@ -303,8 +291,12 @@ fn el0_sync_inner(frame: &mut ExceptionFrame) { ELR_EL1.get(), FAR_EL1.get() ); + thread.raise_signal(Signal::MemoryAccessViolation); + true } - + } + _ => { + let thread = Thread::current(); thread.raise_signal(Signal::MemoryAccessViolation); true } diff --git a/kernel/src/arch/aarch64/gic/gicv2m.rs b/kernel/src/arch/aarch64/gic/gicv2m.rs index ec0571b7..b943c556 100644 --- a/kernel/src/arch/aarch64/gic/gicv2m.rs +++ b/kernel/src/arch/aarch64/gic/gicv2m.rs @@ -232,7 +232,7 @@ impl MessageInterruptController for Gicv2m { device_tree_driver! { compatible: ["arm,gic-v2m-frame"], driver: { - fn probe(&self, node: &Arc, context: &ProbeContext) -> Option> { + fn probe(&self, node: &Arc, context: &mut ProbeContext) -> Option> { let base = node.map_base(context, 0)?; let parent = node.parent()?.as_interrupt_controller()?; diff --git a/kernel/src/arch/aarch64/gic/mod.rs b/kernel/src/arch/aarch64/gic/mod.rs index 61075f76..f122d178 100644 --- a/kernel/src/arch/aarch64/gic/mod.rs +++ b/kernel/src/arch/aarch64/gic/mod.rs @@ -267,7 +267,7 @@ impl GicPerCpu { device_tree_driver! { compatible: ["arm,cortex-a15-gic", "arm,gic-400"], driver: { - fn probe(&self, node: &Arc, context: &ProbeContext) -> Option> { + fn probe(&self, node: &Arc, context: &mut ProbeContext) -> Option> { let gicd_range = node.map_range(context, 0)?; let gicc_range = node.map_range(context, 1)?; diff --git a/kernel/src/arch/aarch64/mod.rs b/kernel/src/arch/aarch64/mod.rs index ee8ba0da..0f2766c0 100644 --- a/kernel/src/arch/aarch64/mod.rs +++ b/kernel/src/arch/aarch64/mod.rs @@ -9,7 +9,10 @@ use device_api::{ interrupt::{Irq, LocalInterruptController}, ResetDevice, }; -use device_tree::{driver::unflatten_device_tree, DeviceTree, DeviceTreeNodeExt}; +use device_tree::{ + driver::{unflatten_device_tree, InitSequence}, + DeviceTree, DeviceTreeNodeExt, +}; use kernel_arch_aarch64::{ mem::{ self, @@ -214,7 +217,7 @@ impl AArch64 { let node = stdout_path.and_then(device_tree::driver::find_node); if let Some(node) = node { - node.force_init()?; + node.force_init(InitSequence::Early)?; } // No stdout @@ -285,6 +288,7 @@ impl AArch64 { |node, error| { log::error!("{}: {error:?}", node.name().unwrap_or("")); }, + InitSequence::Early, ); device_tree::driver::init_irqs( |_| (), @@ -294,6 +298,7 @@ impl AArch64 { node.name().unwrap_or("") ); }, + InitSequence::Early, ); PciBusManager::probe_bus_devices()?; diff --git a/kernel/src/arch/aarch64/timer.rs b/kernel/src/arch/aarch64/timer.rs index fbda12d5..5242ca86 100644 --- a/kernel/src/arch/aarch64/timer.rs +++ b/kernel/src/arch/aarch64/timer.rs @@ -91,7 +91,7 @@ impl ArmTimer { device_tree_driver! { compatible: ["arm,armv8-timer"], driver: { - fn probe(&self, node: &Arc, _context: &ProbeContext) -> Option> { + fn probe(&self, node: &Arc, _context: &mut ProbeContext) -> Option> { let irq = node.interrupt(1)?; Some(Arc::new(unsafe { ArmTimer::new(irq) })) } diff --git a/kernel/src/arch/x86_64/exception.rs b/kernel/src/arch/x86_64/exception.rs index 1c5dc99d..6d80cd8d 100644 --- a/kernel/src/arch/x86_64/exception.rs +++ b/kernel/src/arch/x86_64/exception.rs @@ -5,6 +5,7 @@ use abi::{bitflags, primitive_enum, process::Signal}; use kernel_arch_x86::registers::{CR2, CR3}; use kernel_arch_x86_64::context::ExceptionFrame; use libk::task::thread::Thread; +use libk_mm::PageFaultKind; use libk_util::sync::spin_rwlock::IrqSafeRwLock; use tock_registers::interfaces::Readable; @@ -146,8 +147,16 @@ fn user_exception_inner(kind: ExceptionKind, frame: &mut ExceptionFrame) { ExceptionKind::PageFault => { let cr2 = CR2.get(); let flags = PageFaultFlags::new(frame.exc_code as usize); + let kind = match ( + flags.contains(PageFaultFlags::P), + flags.contains(PageFaultFlags::W), + ) { + (true, true) => PageFaultKind::WriteFault, + (true, false) => PageFaultKind::Other, + (false, write) => PageFaultKind::TranslationFault { write }, + }; - if !flags.contains(PageFaultFlags::P) && thread.handle_page_fault(cr2).is_ok() { + if thread.handle_page_fault(cr2, kind).is_ok() { false } else { thread.raise_signal(Signal::MemoryAccessViolation); diff --git a/kernel/src/device/clock/bcm2835_aux.rs b/kernel/src/device/clock/bcm2835_aux.rs index 925cf7d9..92bf2086 100644 --- a/kernel/src/device/clock/bcm2835_aux.rs +++ b/kernel/src/device/clock/bcm2835_aux.rs @@ -38,10 +38,10 @@ struct Bcm2835Aux { } impl ClockController for Bcm2835Aux { - fn enable_clock(&self, clock: u32) -> Result<(), Error> { + fn enable_clock(&self, clock: Option) -> Result<(), Error> { let regs = self.regs.try_get().ok_or(Error::DoesNotExist)?.lock(); match clock { - 0 => { + Some(0) => { regs.AUX_ENABLES.modify(AUX_ENABLES::MU_ENABLE::SET); Ok(()) } @@ -49,7 +49,7 @@ impl ClockController for Bcm2835Aux { } } - fn disable_clock(&self, _clock: u32) -> Result<(), Error> { + fn disable_clock(&self, _clock: Option) -> Result<(), Error> { Err(Error::NotImplemented) } } @@ -74,7 +74,7 @@ impl Device for Bcm2835Aux { device_tree_driver! { compatible: ["brcm,bcm2835-aux"], driver: { - fn probe(&self, node: &Arc, context: &ProbeContext) -> Option> { + fn probe(&self, node: &Arc, context: &mut ProbeContext) -> Option> { let base = node.map_base(context, 0)?; Some(Arc::new(Bcm2835Aux { diff --git a/kernel/src/device/display/linear_fb.rs b/kernel/src/device/display/linear_fb.rs index 8847999d..0b761788 100644 --- a/kernel/src/device/display/linear_fb.rs +++ b/kernel/src/device/display/linear_fb.rs @@ -3,8 +3,6 @@ use core::mem::MaybeUninit; use abi::error::Error; -use alloc::boxed::Box; -use async_trait::async_trait; use device_api::device::Device; use libk::device::display::{ DisplayDevice, DisplayMode, DisplayOwner, DriverFlags, FramebufferInfo, PixelFormat, @@ -13,7 +11,7 @@ use libk_mm::{ address::PhysicalAddress, device::{DeviceMemoryAttributes, DeviceMemoryCaching, RawDeviceMemoryMapping}, table::{EntryLevel, MapAttributes}, - PageProvider, VirtualPage, + OnDemandPage, PageProvider, VirtualPage, }; use libk_util::sync::IrqSafeSpinlock; @@ -91,7 +89,7 @@ impl LinearFramebuffer { } impl PageProvider for LinearFramebuffer { - fn ondemand_fetch(&self, _opaque: u64) -> Result { + fn ondemand_fetch(&self, _opaque: u64) -> Result { unreachable!() } diff --git a/kernel/src/device/power/arm_psci.rs b/kernel/src/device/power/arm_psci.rs index 09d783cd..4fc85022 100644 --- a/kernel/src/device/power/arm_psci.rs +++ b/kernel/src/device/power/arm_psci.rs @@ -79,7 +79,7 @@ impl Psci { device_tree_driver! { compatible: ["arm,psci-1.0", "arm,psci"], driver: { - fn probe(&self, node: &Arc, _context: &ProbeContext) -> Option> { + fn probe(&self, node: &Arc, _context: &mut ProbeContext) -> Option> { let method = node.property("method")?; let method = match method.as_str()? { "hvc" => CallMethod::Hvc, diff --git a/kernel/src/device/serial/bcm2835_aux_uart.rs b/kernel/src/device/serial/bcm2835_aux_uart.rs index 1c35412d..d0329759 100644 --- a/kernel/src/device/serial/bcm2835_aux_uart.rs +++ b/kernel/src/device/serial/bcm2835_aux_uart.rs @@ -8,13 +8,11 @@ use abi::{ }; use alloc::sync::Arc; use device_api::{ + clock::ClockHandle, device::{Device, DeviceInitContext}, interrupt::{FullIrq, InterruptHandler, IrqVector}, }; -use device_tree::{ - driver::{device_tree_driver, lookup_phandle, Node, ProbeContext}, - DeviceTreePropertyRead, -}; +use device_tree::driver::{device_tree_driver, Node, ProbeContext}; use libk::{ debug::DebugSink, device::{external_interrupt_controller, manager::DEVICE_REGISTRY}, @@ -65,9 +63,9 @@ struct Inner { } pub struct Bcm2835AuxUart { - node: Arc, base: PhysicalAddress, irq: FullIrq, + clock: ClockHandle, inner: OneTimeInit>>, } @@ -151,8 +149,7 @@ impl Device for Bcm2835AuxUart { unsafe fn init(self: Arc, _cx: DeviceInitContext) -> Result<(), Error> { // TODO initialize pinctrl - // NOTE: might as well make it an error if clock cannot be initialized - self.enable_clock(); + self.clock.enable()?; let regs = unsafe { DeviceMemoryIo::map(self.base, Default::default()) }?; let config = TerminalOptions { @@ -195,35 +192,19 @@ impl Device for Bcm2835AuxUart { } } -impl Bcm2835AuxUart { - fn enable_clock(&self) -> Option<()> { - let clocks = self.node.property("clocks")?; - - let (phandle, index) = clocks.read_cells(0, (1, 1))?; - let phandle = phandle as u32; - let index = index as u32; - - let clock = lookup_phandle(phandle, true)?; - let clock = clock.as_clock_controller()?; - - clock.enable_clock(index).ok()?; - - Some(()) - } -} - // TODO handle pinctrl device_tree_driver! { compatible: ["brcm,bcm2835-aux-uart"], driver: { - fn probe(&self, node: &Arc, context: &ProbeContext) -> Option> { + fn probe(&self, node: &Arc, context: &mut ProbeContext) -> Option> { let base = node.map_base(context, 0)?; let irq = node.interrupt(0)?; + let clock = node.clock(0)?; Some(Arc::new(Bcm2835AuxUart { - node: node.clone(), base, irq, + clock, inner: OneTimeInit::new(), })) } diff --git a/kernel/src/device/serial/pl011.rs b/kernel/src/device/serial/pl011.rs index 4179338b..7d03d16b 100644 --- a/kernel/src/device/serial/pl011.rs +++ b/kernel/src/device/serial/pl011.rs @@ -181,7 +181,7 @@ impl Device for Pl011 { device_tree_driver! { compatible: ["arm,pl011"], driver: { - fn probe(&self, node: &Arc, context: &ProbeContext) -> Option> { + fn probe(&self, node: &Arc, context: &mut ProbeContext) -> Option> { let base = node.map_base(context, 0)?; let irq = node.interrupt(0)?; diff --git a/test.c b/test.c index 511564fd..63ef0614 100644 --- a/test.c +++ b/test.c @@ -1,12 +1,32 @@ +#include +#include #include #include #include +#include int main(int argc, char **argv) { - char buffer[9] = {1, 2, 3, 4, 5, 6, 7, 8, 9}; - strncpy(buffer, "abcdefghijkl", sizeof(buffer)); - for (int i = 0; i < sizeof(buffer); ++i) { - printf("[%d] 0x%hhx\n", i, buffer[i]); + int fd = open("test.dat", O_RDWR | O_CREAT, 0644); + if (fd < 0) { + perror("open()"); + return EXIT_FAILURE; } - return 0; + if (ftruncate(fd, 0x1000 * 4) != 0) { + perror("ftruncate()"); + return EXIT_FAILURE; + } + void *data; + if ((data = mmap(NULL, 0x1000 * 4, PROT_READ | PROT_WRITE, MAP_PRIVATE, fd, 0)) == MAP_FAILED) { + perror("mmap()"); + return EXIT_FAILURE; + } + + memset(data + 0x1000 * 0, 'a', 0x1000); + memset(data + 0x1000 * 1, 'b', 0x1000); + memset(data + 0x1000 * 2, 'c', 0x1000); + memset(data + 0x1000 * 3, 'd', 0x1000); + + close(fd); + + return EXIT_SUCCESS; } diff --git a/userspace/lib/libcolors/src/application/mod.rs b/userspace/lib/libcolors/src/application/mod.rs index c032c34c..a7d98a91 100644 --- a/userspace/lib/libcolors/src/application/mod.rs +++ b/userspace/lib/libcolors/src/application/mod.rs @@ -1,7 +1,10 @@ use std::{ collections::BTreeMap, process::ExitCode, - sync::{atomic::{AtomicBool, Ordering}, Arc, Mutex}, + sync::{ + atomic::{AtomicBool, Ordering}, + Arc, Mutex, + }, }; use cross::signal::set_sigint_handler; diff --git a/userspace/lib/ygglibc/src/headers/strings/mod.rs b/userspace/lib/ygglibc/src/headers/strings/mod.rs index 012c31a9..e42c2666 100644 --- a/userspace/lib/ygglibc/src/headers/strings/mod.rs +++ b/userspace/lib/ygglibc/src/headers/strings/mod.rs @@ -1,6 +1,6 @@ use core::{ cmp::Ordering, - ffi::{c_char, c_int, CStr}, + ffi::{c_char, c_int}, }; use super::locale::locale_t; diff --git a/userspace/lib/ygglibc/src/lib.rs b/userspace/lib/ygglibc/src/lib.rs index 0cbf522d..00092fc7 100644 --- a/userspace/lib/ygglibc/src/lib.rs +++ b/userspace/lib/ygglibc/src/lib.rs @@ -72,13 +72,6 @@ unsafe extern "C" fn __ygglibc_entry( signal::init(true); init::init(); - let sp = unsafe { - let sp: usize; - core::arch::asm!("mov %rsp, {0}", out(reg) sp, options(att_syntax)); - sp - }; - log::info!("sp = {sp:#x}"); - // Setup args let args = env::handle_kernel_argument(arg); let mut c_args = Vec::new(); diff --git a/xtask/src/build/c.rs b/xtask/src/build/c.rs index 63b7da4a..7348c883 100644 --- a/xtask/src/build/c.rs +++ b/xtask/src/build/c.rs @@ -1,4 +1,3 @@ -#![allow(unused)] use std::{ fs, path::{Path, PathBuf}, @@ -127,25 +126,19 @@ fn build_test_c_program( fn install_ygglibc(env: &BuildEnv, ygglibc: &Ygglibc) -> Result<(), Error> { log::info!("Installing ygglibc into LLVM sysroot"); - // let dst_lib_dir = env.llvm_sysroot.join("lib"); - // let dst_include_dir = env.llvm_sysroot.join("usr/include"); - let _ = env; - let _ = ygglibc.crt0_file; + let dst_lib_dir = env.llvm_sysroot.join("lib"); + let dst_include_dir = env.llvm_sysroot.join("usr/include"); - // fs::create_dir_all(&dst_lib_dir)?; - // fs::create_dir_all(&dst_include_dir)?; + fs::create_dir_all(&dst_lib_dir)?; + fs::create_dir_all(&dst_include_dir)?; - // fs::copy(&ygglibc.static_lib_file, dst_lib_dir.join("libygglibc.a"))?; - // fs::copy(&ygglibc.shared_lib_file, dst_lib_dir.join("libygglibc.so"))?; - // fs::copy(&ygglibc.crt0_file, dst_lib_dir.join("crt0.o"))?; + fs::copy(&ygglibc.static_lib_file, dst_lib_dir.join("libygglibc.a"))?; + fs::copy(&ygglibc.shared_lib_file, dst_lib_dir.join("libygglibc.so"))?; + fs::copy(&ygglibc.crt0_file, dst_lib_dir.join("crt0.o"))?; - let _ = ygglibc.static_lib_file; - let _ = ygglibc.shared_lib_file; - let _ = ygglibc.include_paths; - - // for path in ygglibc.include_paths.iter() { - // util::copy_dir_recursive(path, &dst_include_dir)?; - // } + for path in ygglibc.include_paths.iter() { + util::copy_dir_recursive(path, &dst_include_dir)?; + } Ok(()) } @@ -219,8 +212,7 @@ fn install_openlibm(env: &BuildEnv, libm: &Openlibm) -> Result<(), Error> { fs::copy(&libm.static_lib_file, env.llvm_sysroot.join("lib/libm.a"))?; fs::copy(&libm.shared_lib_file, env.llvm_sysroot.join("lib/libm.so"))?; - let _ = libm.include_path; - // util::copy_dir_recursive(&libm.include_path, env.llvm_sysroot.join("usr/include"))?; + util::copy_dir_recursive(&libm.include_path, env.llvm_sysroot.join("usr/include"))?; Ok(()) }