alnyan/yggdrasil
1
0
Fork 0
Code Issues 33 Pull Requests Packages Projects Releases Wiki Activity

mem: rework phys/virt mem management, get process working

Browse Source
This commit is contained in:
Mark Poliakov 2023-09-13 18:21:45 +03:00
parent 3330beedfd
commit 6949f8c44a
39 changed files with 2264 additions and 1335 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
Cargo.toml
View File

@ -14,12 +14,13 @@ vfs = { path = "lib/vfs" }
memfs = { path = "lib/memfs" }
device-api = { path = "lib/device-api", features = ["derive"] }
kernel-util = { path = "lib/kernel-util" }
memtables = { path = "lib/memtables" }
atomic_enum = "0.2.0"
bitflags = "2.3.3"
linked_list_allocator = "0.10.5"
spinning_top = "0.2.5"
# static_assertions = "1.1.0"
static_assertions = "1.1.0"
tock-registers = "0.8.1"
cfg-if = "1.0.0"
git-version = "0.3.5"
@ -48,7 +49,7 @@ acpi-system = { git = "https://github.com/alnyan/acpi-system.git", version = "0.
xhci_lib = { git = "https://github.com/rust-osdev/xhci.git", package = "xhci" }
[features]
default = []
default = ["fb_console"]
fb_console = []
aarch64_qemu = []
aarch64_orange_pi3 = []

9
lib/memtables/Cargo.toml Normal file
View File

@ -0,0 +1,9 @@
[package]
name = "memtables"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
bytemuck = { version = "1.14.0", features = ["derive"] }

39
lib/memtables/src/lib.rs Normal file
View File

@ -0,0 +1,39 @@
#![no_std]
use bytemuck::{Pod, Zeroable};
pub const KERNEL_L3_COUNT: usize = 16;
#[derive(Clone, Copy, Pod, Zeroable)]
#[repr(C, align(0x1000))]
pub struct RawTable {
pub data: [u64; 512],
}
#[derive(Clone, Copy, Pod, Zeroable)]
#[repr(C)]
pub struct FixedTables {
pub l0: RawTable,
pub kernel_l1: RawTable,
pub kernel_l2: RawTable,
pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
}
impl RawTable {
pub const fn zeroed() -> Self {
Self { data: [0; 512] }
}
}
impl FixedTables {
pub const fn zeroed() -> Self {
Self {
l0: RawTable::zeroed(),
kernel_l1: RawTable::zeroed(),
kernel_l2: RawTable::zeroed(),
kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
}
}
}

67
src/arch/mod.rs
View File

@ -27,6 +27,8 @@ use device_api::{
ResetDevice,
};
use crate::mem::{device::RawDeviceMemoryMapping, phys::PhysicalMemoryRegion, PhysicalAddress};
cfg_if! {
if #[cfg(target_arch = "aarch64")] {
pub mod aarch64;
@ -60,24 +62,31 @@ pub trait Architecture {
/// IRQ number type associated with the architecture
type IrqNumber;
/// Initializes the memory management unit and sets up virtual memory management.
/// `bsp` flag is provided to make sure mapping tables are only initialized once in a SMP
/// system.
///
/// # Safety
///
/// Unsafe to call if the MMU has already been initialized.
unsafe fn init_mmu(&self, bsp: bool);
/// Starts up the application processors that may be present in the system.
///
/// # Safety
///
/// Only safe to call once during system init.
unsafe fn start_application_processors(&self);
unsafe fn start_application_processors(&self) {}
/// Allocates a virtual mapping for the specified physical memory region
fn map_device_pages(&self, phys: usize, count: usize) -> Result<usize, Error>;
unsafe fn map_device_memory(
&self,
base: PhysicalAddress,
size: usize,
) -> Result<RawDeviceMemoryMapping, Error>;
unsafe fn unmap_device_memory(&self, map: RawDeviceMemoryMapping);
fn map_physical_memory<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
&self,
it: I,
memory_start: PhysicalAddress,
memory_end: PhysicalAddress,
) -> Result<(), Error>;
fn virtualize(address: PhysicalAddress) -> Result<usize, Error>;
fn physicalize(address: usize) -> Result<PhysicalAddress, Error>;
// Architecture intrinsics
@ -102,36 +111,50 @@ pub trait Architecture {
fn register_external_interrupt_controller(
&self,
intc: &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber>,
) -> Result<(), Error>;
) -> Result<(), Error> {
Err(Error::NotImplemented)
}
/// Adds a local interrupt controller to the system
fn register_local_interrupt_controller(
&self,
intc: &'static dyn LocalInterruptController<IpiMessage = CpuMessage>,
) -> Result<(), Error>;
) -> Result<(), Error> {
Err(Error::NotImplemented)
}
/// Adds a monotonic timer to the system
fn register_monotonic_timer(
&self,
timer: &'static dyn MonotonicTimestampProviderDevice,
) -> Result<(), Error>;
) -> Result<(), Error> {
Err(Error::NotImplemented)
}
/// Adds a reset device to the system
fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error>;
fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> {
Err(Error::NotImplemented)
}
// TODO only supports 1 extintc per system
/// Returns the primary external interrupt controller
fn external_interrupt_controller(
&'static self,
) -> &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber>;
) -> &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber> {
unimplemented!()
}
/// Returns the local interrupt controller
fn local_interrupt_controller(
&'static self,
) -> &'static dyn LocalInterruptController<IpiMessage = CpuMessage>;
) -> &'static dyn LocalInterruptController<IpiMessage = CpuMessage> {
unimplemented!()
}
/// Returns the monotonic timer
fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice;
fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
unimplemented!()
}
/// Sends a message to the requested set of CPUs through an interprocessor interrupt.
///
@ -143,7 +166,9 @@ pub trait Architecture {
/// # Safety
///
/// As the call may alter the flow of execution on CPUs, this function is unsafe.
unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error>;
unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> {
Ok(())
}
/// Performs a CPU reset.
///
@ -151,5 +176,7 @@ pub trait Architecture {
///
/// The caller must ensure it is actually safe to reset, i.e. no critical processes will be
/// aborted and no data will be lost.
unsafe fn reset(&self) -> !;
unsafe fn reset(&self) -> ! {
loop {}
}
}

91
src/arch/x86_64/acpi.rs
View File

@ -23,7 +23,7 @@ use crate::{
x86_64::{smp::CPU_COUNT, IrqNumber, SHUTDOWN_FENCE},
Architecture, CpuMessage, ARCHITECTURE,
},
mem::{heap::GLOBAL_HEAP, ConvertAddress},
mem::{address::FromRaw, heap::GLOBAL_HEAP, PhysicalAddress},
sync::IrqSafeSpinlock,
util,
};
@ -79,14 +79,19 @@ unsafe impl Allocator for AcpiAllocator {
impl acpi_system::Handler for AcpiHandlerImpl {
unsafe fn map_slice(address: u64, length: u64) -> &'static [u8] {
if address + length < 0x100000000 {
core::slice::from_raw_parts(
(address as usize).virtualize() as *const u8,
length as usize,
)
} else {
panic!("Unhandled address: {:#x}", address)
}
let slice = PhysicalAddress::from_raw(address).virtualize_slice::<u8>(length as usize);
todo!();
// PhysicalPointer::into_raw(slice)
// if address + length < 0x100000000 {
// core::slice::from_raw_parts(
// (address as usize).virtualize() as *const u8,
// length as usize,
// )
// } else {
// panic!("Unhandled address: {:#x}", address)
// }
}
fn io_read_u8(port: u16) -> u8 {
@ -123,71 +128,39 @@ impl acpi_system::Handler for AcpiHandlerImpl {
}
fn mem_read_u8(address: u64) -> u8 {
let value = unsafe { (address as *const u8).virtualize().read_volatile() };
log::trace!("mem_read_u8 {:#x} <- {:#x}", address, value);
value
todo!()
}
fn mem_read_u16(address: u64) -> u16 {
let value = if address & 0x1 == 0 {
unsafe { (address as *const u16).virtualize().read_volatile() }
} else {
unsafe { (address as *const u16).virtualize().read_unaligned() }
};
log::trace!("mem_read_u16 {:#x} <- {:#x}", address, value);
value
todo!()
}
fn mem_read_u32(address: u64) -> u32 {
let value = if address & 0x3 == 0 {
unsafe { (address as *const u32).virtualize().read_volatile() }
} else {
unsafe { (address as *const u32).virtualize().read_unaligned() }
};
log::trace!("mem_read_u32 {:#x} <- {:#x}", address, value);
value
todo!()
}
fn mem_read_u64(address: u64) -> u64 {
let value = if address & 0x7 == 0 {
unsafe { (address as *const u64).virtualize().read_volatile() }
} else {
unsafe { (address as *const u64).virtualize().read_unaligned() }
};
log::trace!("mem_read_u64 {:#x} <- {:#x}", address, value);
value
todo!()
}
fn mem_write_u8(address: u64, value: u8) {
log::trace!("mem_write_u8 {:#x}, {:#x}", address, value);
unsafe { (address as *mut u8).virtualize().write_volatile(value) };
todo!()
}
fn mem_write_u16(address: u64, value: u16) {
log::trace!("mem_write_u16 {:#x}, {:#x}", address, value);
if address & 0x1 == 0 {
unsafe { (address as *mut u16).virtualize().write_volatile(value) };
} else {
unsafe { (address as *mut u16).virtualize().write_unaligned(value) };
}
todo!()
}
fn mem_write_u32(address: u64, value: u32) {
log::trace!("mem_write_u32 {:#x}, {:#x}", address, value);
if address & 0x3 == 0 {
unsafe { (address as *mut u32).virtualize().write_volatile(value) };
} else {
unsafe { (address as *mut u32).virtualize().write_unaligned(value) };
}
todo!()
}
fn mem_write_u64(address: u64, value: u64) {
log::trace!("mem_write_u64 {:#x}, {:#x}", address, value);
if address & 0x7 == 0 {
unsafe { (address as *mut u64).virtualize().write_volatile(value) };
} else {
unsafe { (address as *mut u64).virtualize().write_unaligned(value) };
}
todo!()
}
fn install_interrupt_handler(irq: u32) -> Result<(), AcpiSystemError> {
@ -342,17 +315,15 @@ impl AcpiHandler for AcpiHandlerImpl {
physical_address: usize,
size: usize,
) -> PhysicalMapping<Self, T> {
if physical_address <= 0xFFFFFFFF {
PhysicalMapping::new(
physical_address,
NonNull::new_unchecked(physical_address.virtualize() as *mut T),
size,
size,
*self,
)
} else {
todo!()
}
PhysicalMapping::new(
physical_address,
NonNull::new_unchecked(
PhysicalAddress::from_raw(physical_address).virtualize_raw() as *mut T
),
size,
size,
*self,
)
}
// Unmap nothing, these addresses are "virtualized" to high address space

47
src/arch/x86_64/apic/ioapic.rs
View File

@ -8,10 +8,19 @@ use device_api::{
},
Device,
};
use tock_registers::{
interfaces::{Readable, Writeable},
register_structs,
registers::{ReadWrite, WriteOnly},
};
use crate::{
arch::x86_64::{acpi::AcpiAllocator, apic::local::BSP_APIC_ID, IrqNumber},
mem::ConvertAddress,
mem::{
address::FromRaw,
device::{DeviceMemoryIo, DeviceMemoryMapping},
PhysicalAddress,
},
sync::IrqSafeSpinlock,
};
@ -38,12 +47,18 @@ struct IsaRedirection {
trigger: IrqTrigger,
}
struct Regs {
base: usize,
register_structs! {
#[allow(non_snake_case)]
Regs {
(0x00 => Index: WriteOnly<u32>),
(0x04 => _0),
(0x10 => Data: ReadWrite<u32>),
(0x14 => @END),
}
}
struct Inner {
regs: Regs,
regs: DeviceMemoryIo<'static, Regs>,
max_gsi: u32,
}
@ -59,22 +74,14 @@ pub struct IoApic {
impl Regs {
#[inline]
fn read(&self, reg: u32) -> u32 {
let ptr = self.base as *mut u32;
unsafe {
ptr.write_volatile(reg & 0xFF);
ptr.add(4).read_volatile()
}
self.Index.set(reg);
self.Data.get()
}
#[inline]
fn write(&self, reg: u32, value: u32) {
let ptr = self.base as *mut u32;
unsafe {
ptr.write_volatile(reg & 0xFF);
ptr.add(4).write_volatile(value);
}
self.Index.set(reg);
self.Data.set(value);
}
}
@ -269,8 +276,12 @@ impl IoApic {
}
// TODO properly map this using DeviceMemory
let regs = Regs {
base: unsafe { (ioapic.address as usize).virtualize() },
// let regs = Regs {
// base: unsafe { PhysicalAddress::from_raw(ioapic.address as u64).virtualize_raw() },
// };
// let mapping = unsafe { DeviceMemoryMapping::map(base, size) };
let regs = unsafe {
DeviceMemoryIo::<'_, Regs>::map(PhysicalAddress::from_raw(ioapic.address as u64))?
};
let max_gsi = (regs.read(REG_IOAPIC_VERSION) >> 16) & 0xFF;

11
src/arch/x86_64/apic/local.rs
View File

@ -17,7 +17,7 @@ use crate::{
x86_64::{registers::MSR_IA32_APIC_BASE, smp::CPU_COUNT},
CpuMessage,
},
mem::ConvertAddress,
mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress},
task::Cpu,
};
@ -130,7 +130,7 @@ register_structs! {
/// Per-processor local APIC interface
pub struct LocalApic {
regs: &'static Regs,
regs: DeviceMemoryIo<'static, Regs>,
}
unsafe impl Send for LocalApic {}
@ -190,8 +190,7 @@ impl LocalApic {
///
/// Only meant to be called once per processor during their init.
pub unsafe fn new() -> Self {
let base = unsafe { Self::base().virtualize() };
let regs = unsafe { &*(base as *const Regs) };
let regs = DeviceMemoryIo::<Regs>::map(Self::base()).unwrap();
let id = regs.Id.read(Id::ApicId);
@ -294,8 +293,8 @@ impl LocalApic {
}
#[inline]
fn base() -> usize {
MSR_IA32_APIC_BASE.read_base() as usize
fn base() -> PhysicalAddress {
PhysicalAddress::from_raw(MSR_IA32_APIC_BASE.read_base())
}
#[inline]

144
src/arch/x86_64/boot/mod.rs
View File

@ -1,5 +1,5 @@
//! x86-64 boot and entry functions
use core::{arch::global_asm, sync::atomic::Ordering};
use core::arch::global_asm;
use tock_registers::interfaces::Writeable;
use yboot_proto::{
@ -8,26 +8,20 @@ use yboot_proto::{
};
use crate::{
arch::{
x86_64::{cpuid, exception, registers::MSR_IA32_KERNEL_GS_BASE, BootData},
Architecture, ArchitectureImpl, ARCHITECTURE,
},
arch::{x86_64::registers::MSR_IA32_KERNEL_GS_BASE, Architecture, ArchitectureImpl},
fs::devfs,
kernel_main, kernel_secondary_main,
mem::{
heap,
phys::{self, PageUsage},
ConvertAddress, KERNEL_VIRT_OFFSET,
},
kernel_main,
mem::KERNEL_VIRT_OFFSET,
task::runtime,
};
use super::smp::CPU_COUNT;
use super::{cpuid::init_cpuid, exception, ARCHITECTURE};
// use super::ARCHITECTURE;
pub enum BootData {
YBoot(&'static LoadProtocolV1),
}
const BOOT_STACK_SIZE: usize = 1024 * 1024;
const HEAP_PAGES: usize = 512;
#[repr(C, align(0x20))]
struct BootStack {
@ -41,7 +35,7 @@ static mut BSP_STACK: BootStack = BootStack {
#[used]
#[link_section = ".data.yboot"]
static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 {
static YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 {
header: LoadProtocolHeader {
kernel_magic: KERNEL_MAGIC,
version: PROTOCOL_VERSION_1,
@ -65,32 +59,80 @@ static mut YBOOT_DATA: LoadProtocolV1 = LoadProtocolV1 {
res_size: 0,
},
};
//
//
// unsafe extern "C" fn __x86_64_upper_entry() -> ! {
// }
//
// /// Application processor entry point
// pub extern "C" fn __x86_64_ap_entry() -> ! {
// let cpu_id = CPU_COUNT.load(Ordering::Acquire);
//
// MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64);
// unsafe {
// core::arch::asm!("swapgs");
// }
// MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64);
// unsafe {
// core::arch::asm!("swapgs");
// }
//
// // Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base
// cpuid::feature_gate();
//
// infoln!("cpu{} initializing", cpu_id);
// unsafe {
// ARCHITECTURE.init_mmu(false);
// core::arch::asm!("wbinvd");
//
// // Cpu::init_local(LocalApic::new(), cpu_id as u32);
// // syscall::init_syscall();
// exception::init_exceptions(cpu_id);
//
// ARCHITECTURE.init_platform(cpu_id);
// }
//
// CPU_COUNT.fetch_add(1, Ordering::Release);
//
// kernel_secondary_main()
// }
static UNINIT_CPU: usize = 0;
unsafe extern "C" fn __x86_64_upper_entry() -> ! {
ArchitectureImpl::set_interrupt_mask(true);
unsafe fn init_dummy_cpu() {
// TODO this is incorrect
static UNINIT_CPU_INNER: usize = 0;
static UNINIT_CPU_PTR: &'static usize = &UNINIT_CPU_INNER;
// Point %gs to a dummy structure so that Cpu::get_local() works properly even before the CPU
// data structure is initialized
MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64);
MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU_PTR as *const _ as u64);
core::arch::asm!("swapgs");
MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64);
MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU_PTR as *const _ as u64);
core::arch::asm!("swapgs");
}
ARCHITECTURE.init_mmu(true);
core::arch::asm!("wbinvd");
pub extern "C" fn __x86_64_ap_entry() -> ! {
loop {}
}
extern "C" fn __x86_64_upper_entry() -> ! {
// Safety: ok, CPU hasn't been initialized yet and it's the early kernel entry
unsafe {
init_dummy_cpu();
}
ARCHITECTURE.set_boot_data(BootData::YBoot(&YBOOT_DATA));
ARCHITECTURE
.init_physical_memory()
.expect("Failed to initialize the physical memory manager");
let heap_base = phys::alloc_pages_contiguous(HEAP_PAGES, PageUsage::Used)
.expect("Couldn't allocate memory for heap");
heap::init_heap(heap_base.virtualize(), HEAP_PAGES * 0x1000);
// Gather available CPU features
init_cpuid();
exception::init_exceptions(0);
// Setup memory management: kernel virtual memory tables, physical page manager and heap
unsafe {
ARCHITECTURE.init_memory_management();
}
unsafe {
exception::init_exceptions(0);
}
// Initialize async executor queue
runtime::init_task_queue();
@ -98,52 +140,21 @@ unsafe extern "C" fn __x86_64_upper_entry() -> ! {
devfs::init();
// Initializes: local CPU, platform devices (timers/serials/etc), debug output
ARCHITECTURE.init_platform(0);
cpuid::feature_gate();
unsafe {
ARCHITECTURE.init_platform(0);
}
kernel_main()
}
/// Application processor entry point
pub extern "C" fn __x86_64_ap_entry() -> ! {
let cpu_id = CPU_COUNT.load(Ordering::Acquire);
MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64);
unsafe {
core::arch::asm!("swapgs");
}
MSR_IA32_KERNEL_GS_BASE.set(&UNINIT_CPU as *const _ as u64);
unsafe {
core::arch::asm!("swapgs");
}
// Still not initialized: GDT, IDT, CPU features, syscall, kernel_gs_base
cpuid::feature_gate();
infoln!("cpu{} initializing", cpu_id);
unsafe {
ARCHITECTURE.init_mmu(false);
core::arch::asm!("wbinvd");
// Cpu::init_local(LocalApic::new(), cpu_id as u32);
// syscall::init_syscall();
exception::init_exceptions(cpu_id);
ARCHITECTURE.init_platform(cpu_id);
}
CPU_COUNT.fetch_add(1, Ordering::Release);
kernel_secondary_main()
}
global_asm!(
r#"
// {boot_data}
.global __x86_64_entry
.section .text.entry
__x86_64_entry:
cli
mov ${yboot_loader_magic}, %edi
cmp %edi, %eax
je 2f
@ -166,6 +177,7 @@ __x86_64_entry:
yboot_loader_magic = const LOADER_MAGIC,
stack_size = const BOOT_STACK_SIZE,
stack_bottom = sym BSP_STACK,
boot_data = sym YBOOT_DATA,
entry = sym __x86_64_upper_entry,
options(att_syntax)
);

29
src/arch/x86_64/context.rs
View File

@ -4,10 +4,10 @@ use core::{arch::global_asm, cell::UnsafeCell};
use abi::error::Error;
use crate::{
arch::x86_64::table::KERNEL_TABLES,
arch::x86_64::mem::KERNEL_TABLES,
mem::{
phys::{self, PageUsage},
ConvertAddress,
address::{AsPhysicalAddress, IntoRaw},
phys, PhysicalAddress,
},
task::context::TaskContextImpl,
};
@ -65,11 +65,11 @@ impl StackBuilder {
self.sp
}
fn init_common(&mut self, entry: usize, cr3: usize) {
fn init_common(&mut self, entry: usize, cr3: PhysicalAddress) {
self.push(entry); // address for ret
// End of common context
self.push(cr3); // %cr3
self.push(cr3.into_raw()); // %cr3
self.push(0); // %rbp
self.push(0); // %fs (TODO)
@ -89,9 +89,9 @@ impl TaskContextImpl for TaskContext {
fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
const KERNEL_TASK_PAGES: usize = 32;
let stack_base = unsafe {
phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize()
};
let stack_base =
unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw() };
let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000);
@ -100,7 +100,7 @@ impl TaskContextImpl for TaskContext {
stack.push(arg);
stack.init_common(__x86_64_task_enter_kernel as _, unsafe {
KERNEL_TABLES.physical_address()
KERNEL_TABLES.as_physical_address()
});
let sp = stack.build();
@ -115,10 +115,15 @@ impl TaskContextImpl for TaskContext {
})
}
fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result<Self, Error> {
fn user(
entry: usize,
arg: usize,
cr3: PhysicalAddress,
user_stack_sp: usize,
) -> Result<Self, Error> {
const USER_TASK_PAGES: usize = 8;
let stack_base =
unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() };
let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw() };
let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000);

67
src/arch/x86_64/cpuid.rs
View File

@ -1,9 +1,13 @@
//! x86-64 CPUID interface
use tock_registers::interfaces::ReadWriteable;
use crate::arch::x86_64::registers::CR4;
use bitflags::bitflags;
use kernel_util::util::OneTimeInit;
use super::registers::XCR0;
bitflags! {
pub struct ProcessorFeatures: u64 {
const PDPE1GB = 1 << 0;
}
}
unsafe fn cpuid(eax: u32, result: &mut [u32]) {
core::arch::asm!(
@ -21,60 +25,19 @@ unsafe fn cpuid(eax: u32, result: &mut [u32]) {
);
}
type RequiredBit = (u32, &'static str);
pub static PROCESSOR_FEATURES: OneTimeInit<ProcessorFeatures> = OneTimeInit::new();
const EAX1_ECX_REQUIRED_FEATURES: &[RequiredBit] = &[
(1 << 0, "SSE3"),
(1 << 19, "SSE4.1"),
(1 << 20, "SSE4.2"),
// (1 << 24, "TSC"),
(1 << 26, "XSAVE"),
(1 << 28, "AVX"),
];
const EAX1_EDX_REQUIRED_FEATURES: &[RequiredBit] = &[
(1 << 0, "FPU"),
(1 << 3, "PSE"),
(1 << 4, "TSC (%edx)"),
(1 << 5, "MSR"),
(1 << 6, "PAE"),
(1 << 9, "APIC"),
(1 << 13, "PGE"),
(1 << 23, "MMX"),
(1 << 24, "FXSR"),
(1 << 25, "SSE"),
(1 << 26, "SSE2"),
];
fn enable_cr4_features() {
// TODO maybe also include FSGSBASE here?
CR4.modify(CR4::OSXSAVE::SET + CR4::OSFXSR::SET + CR4::PGE::SET);
}
fn enable_xcr0_features() {
XCR0.modify(XCR0::X87::SET + XCR0::SSE::SET + XCR0::AVX::SET);
}
/// Checks for the features required by the kernel and enables them
pub fn feature_gate() {
// TODO the compiler may have generated instructions from SSE/AVX sets by now, find some way to
// perform this as early as possible
pub fn init_cpuid() {
let mut features = ProcessorFeatures::empty();
let mut data = [0; 3];
unsafe {
cpuid(1, &mut data);
cpuid(0x80000001, &mut data);
}
for (bit, name) in EAX1_ECX_REQUIRED_FEATURES {
if data[2] & bit == 0 {
panic!("Required feature not supported: {}", name);
}
}
for (bit, name) in EAX1_EDX_REQUIRED_FEATURES {
if data[1] & bit == 0 {
panic!("Required feature not supported: {}", name);
}
if data[1] & (1 << 26) != 0 {
features |= ProcessorFeatures::PDPE1GB;
}
// Enable the SSE/AVX features
enable_cr4_features();
enable_xcr0_features();
PROCESSOR_FEATURES.init(features);
}

318
src/arch/x86_64/mem/mod.rs Normal file
View File

@ -0,0 +1,318 @@
use core::{
alloc::Layout,
ops::{Deref, DerefMut},
};
use abi::error::Error;
use kernel_util::util::OneTimeInit;
use memtables::FixedTables;
use static_assertions::{const_assert_eq, const_assert_ne};
pub mod table;
use crate::{
arch::x86_64::mem::table::PageAttributes,
mem::{
address::{FromRaw, IntoRaw, KernelImageObject},
device::RawDeviceMemoryMapping,
table::EntryLevel,
PhysicalAddress, KERNEL_VIRT_OFFSET,
},
};
use self::table::{PageEntry, PageTable, L0, L1, L2, L3};
const CANONICAL_ADDRESS_MASK: usize = 0xFFFF000000000000;
const KERNEL_PHYS_BASE: usize = 0x400000;
// Mapped at compile time
const KERNEL_L0_INDEX: usize = L0::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
// Must not be zero, should be at 4MiB
const_assert_ne!(KERNEL_START_L2_INDEX, 0);
// From static mapping
const_assert_eq!(KERNEL_L0_INDEX, 511);
const_assert_eq!(KERNEL_L1_INDEX, 0);
// Mapped at boot
const EARLY_MAPPING_L2I: usize = KERNEL_START_L2_INDEX - 1;
const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1;
const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
const RAM_MAPPING_L0I: usize = KERNEL_L0_INDEX - 1;
const DEVICE_MAPPING_L3_COUNT: usize = 4;
#[link_section = ".data.tables"]
pub static mut KERNEL_TABLES: KernelImageObject<FixedTables> =
unsafe { KernelImageObject::new(FixedTables::zeroed()) };
// 2MiB for early mappings
const EARLY_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK
| (KERNEL_L0_INDEX * L0::SIZE)
| (KERNEL_L1_INDEX * L1::SIZE)
| (EARLY_MAPPING_L2I * L2::SIZE);
static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
// 1GiB for heap mapping
pub(super) const HEAP_MAPPING_OFFSET: usize =
CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (HEAP_MAPPING_L1I * L1::SIZE);
pub(super) static mut HEAP_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
// 1GiB for device MMIO mapping
const DEVICE_MAPPING_OFFSET: usize =
CANONICAL_ADDRESS_MASK | (KERNEL_L0_INDEX * L0::SIZE) | (DEVICE_MAPPING_L1I * L1::SIZE);
static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
[PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
// 512GiB for whole RAM mapping
pub(super) const RAM_MAPPING_OFFSET: usize = CANONICAL_ADDRESS_MASK | (RAM_MAPPING_L0I * L0::SIZE);
pub(super) static MEMORY_LIMIT: OneTimeInit<usize> = OneTimeInit::new();
pub(super) static mut RAM_MAPPING_L1: PageTable<L1> = PageTable::zeroed();
// Global limits
pub(super) const HEAP_SIZE_LIMIT: usize = L1::SIZE;
// Early mappings
unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
for l3i in 0..512 {
let mut taken = false;
for i in 0..count {
if EARLY_MAPPING_L3[i + l3i].is_present() {
taken = true;
break;
}
}
if taken {
continue;
}
for i in 0..count {
// TODO NX, NC
EARLY_MAPPING_L3[i + l3i] =
PageEntry::page(physical.add(i * L3::SIZE), PageAttributes::WRITABLE);
}
return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
}
loop {}
}
unsafe fn unmap_early_page(address: usize) {
if address < EARLY_MAPPING_OFFSET || address >= EARLY_MAPPING_OFFSET + L2::SIZE {
loop {}
}
let l3i = L3::index(address - EARLY_MAPPING_OFFSET);
assert!(EARLY_MAPPING_L3[l3i].is_present());
EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
}
// Device mappings
unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result<usize, Error> {
// TODO don't map pages if already mapped
'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
for j in 0..count {
let l2i = (i + j) / 512;
let l3i = (i + j) % 512;
if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
continue 'l0;
}
}
for j in 0..count {
let l2i = (i + j) / 512;
let l3i = (i + j) % 512;
// TODO NX, NC
DEVICE_MAPPING_L3S[l2i][l3i] =
PageEntry::page(base.add(j * L3::SIZE), PageAttributes::WRITABLE);
}
return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
}
loop {}
}
unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result<usize, Error> {
'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
for j in 0..count {
if DEVICE_MAPPING_L2[i + j].is_present() {
continue 'l0;
}
}
for j in 0..count {
DEVICE_MAPPING_L2[i + j] =
PageEntry::<L2>::block(base.add(j * L2::SIZE), PageAttributes::WRITABLE);
}
debugln!(
"map l2s: base={:#x}, count={} -> {:#x}",
base,
count,
DEVICE_MAPPING_OFFSET + i * L2::SIZE
);
return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
}
loop {}
}
pub(super) unsafe fn map_device_memory(
base: PhysicalAddress,
size: usize,
) -> Result<RawDeviceMemoryMapping, Error> {
debugln!("Map {}B @ {:#x}", size, base);
let l3_aligned = base.page_align_down::<L3>();
let l3_offset = L3::page_offset(base.into_raw());
let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE;
if page_count > 256 {
// Large mapping, use L2 mapping instead
let l2_aligned = base.page_align_down::<L2>();
let l2_offset = L2::page_offset(base.into_raw());
let page_count = (l2_offset + size + L2::SIZE - 1) / L2::SIZE;
let base_address = map_device_memory_l2(l2_aligned, page_count)?;
let address = base_address + l2_offset;
Ok(RawDeviceMemoryMapping {
address,
base_address,
page_count,
page_size: L2::SIZE,
})
} else {
let page_size = L3::SIZE;
// Just map the pages directly
let base_address = map_device_memory_l3(l3_aligned, page_count)?;
let address = base_address + l3_offset;
Ok(RawDeviceMemoryMapping {
address,
base_address,
page_count,
page_size: L3::SIZE,
})
}
}
pub(super) unsafe fn unmap_device_memory(map: RawDeviceMemoryMapping) {
loop {}
}
pub(super) unsafe fn map_heap_block(index: usize, page: PhysicalAddress) {
if L2::page_offset(page.into_raw()) != 0 {
loop {}
}
assert!(index < 512);
if HEAP_MAPPING_L2[index].is_present() {
loop {}
}
// TODO NX
HEAP_MAPPING_L2[index] = PageEntry::<L2>::block(page, PageAttributes::WRITABLE);
}
pub struct EarlyMapping<'a, T: ?Sized> {
value: &'a mut T,
page_count: usize,
}
impl<'a, T: Sized> EarlyMapping<'a, T> {
pub unsafe fn map_slice(
physical: PhysicalAddress,
len: usize,
) -> Result<EarlyMapping<'a, [T]>, Error> {
let layout = Layout::array::<T>(len).unwrap();
let aligned = physical.page_align_down::<L3>();
let offset = physical.page_offset::<L3>();
let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE;
let virt = map_early_pages(aligned, page_count)?;
let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
Ok(EarlyMapping { value, page_count })
}
}
impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.value
}
}
impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.value
}
}
impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> {
fn drop(&mut self) {
let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
for i in 0..self.page_count {
let page = address + i * L3::SIZE;
unsafe {
unmap_early_page(page);
}
}
}
}
fn clone_kernel_tables(dst: &mut PageTable<L0>) {
unsafe {
dst[KERNEL_L0_INDEX] = PageEntry::from_raw(KERNEL_TABLES.l0.data[KERNEL_L0_INDEX]);
dst[RAM_MAPPING_L0I] = PageEntry::from_raw(KERNEL_TABLES.l0.data[RAM_MAPPING_L0I]);
}
}
/// Sets up the following memory map:
/// ...: KERNEL_TABLES.l0:
/// * 0xFFFFFF0000000000 .. 0xFFFFFFFF8000000000 : RAM_MAPPING_L1
/// * 0xFFFFFF8000000000 .. ... : KERNEL_TABLES.kernel_l1:
/// * 0xFFFFFF8000000000 .. 0xFFFFFF8040000000 : KERNEL_TABLES.kernel_l2
/// * 0xFFFFFF8000000000 .. 0xFFFFFF8000200000 : ---
/// * 0xFFFFFF8000200000 .. 0xFFFFFF8000400000 : EARLY_MAPPING_L3
/// * 0xFFFFFF8000400000 .. ... : KERNEL_TABLES.kernel_l3s
/// * 0xFFFFFF8040000000 .. 0xFFFFFF8080000000 : HEAP_MAPPING_L2
/// * 0xFFFFFF8080000000 .. 0xFFFFFF8100000000 : DEVICE_MAPPING_L2
/// * 0xFFFFFF8080000000 .. 0xFFFFFF8080800000 : DEVICE_MAPPING_L3S
/// * 0xFFFFFF8080800000 .. 0xFFFFFF8100000000 : ...
pub unsafe fn init_fixed_tables() {
let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET;
let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET;
let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET;
let ram_mapping_l1_phys = &RAM_MAPPING_L1 as *const _ as usize - KERNEL_VIRT_OFFSET;
for i in 0..DEVICE_MAPPING_L3_COUNT {
let device_mapping_l3_phys = PhysicalAddress::from_raw(
&DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET,
);
DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::WRITABLE);
}
KERNEL_TABLES.kernel_l2.data[EARLY_MAPPING_L2I] = (early_mapping_l3_phys as u64)
| (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
KERNEL_TABLES.kernel_l1.data[HEAP_MAPPING_L1I] =
(heap_mapping_l2_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
KERNEL_TABLES.kernel_l1.data[DEVICE_MAPPING_L1I] = (device_mapping_l2_phys as u64)
| (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
KERNEL_TABLES.l0.data[RAM_MAPPING_L0I] =
(ram_mapping_l1_phys as u64) | (PageAttributes::WRITABLE | PageAttributes::PRESENT).bits();
let cr3 = &KERNEL_TABLES.l0 as *const _ as usize - KERNEL_VIRT_OFFSET;
core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
}

159
src/arch/x86_64/table/mod.rs → src/arch/x86_64/mem/table.rs
View File

@ -1,5 +1,3 @@
//! x86-64 virtual memory management implementation
use core::{
marker::PhantomData,
ops::{Index, IndexMut},
@ -8,21 +6,24 @@ use core::{
use abi::error::Error;
use bitflags::bitflags;
mod fixed;
pub use fixed::{init_fixed_tables, KERNEL_TABLES};
use crate::mem::{
phys::{self, PageUsage},
table::{
EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager,
use crate::{
mem::{
address::{AsPhysicalAddress, FromRaw},
phys,
pointer::{PhysicalRef, PhysicalRefMut},
table::{
EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager,
},
PhysicalAddress,
},
ConvertAddress,
sync::IrqSafeSpinlock,
};
use super::{clone_kernel_tables, KERNEL_TABLES};
bitflags! {
/// Describes how each page table entry is mapped
struct PageAttributes: u64 {
pub struct PageAttributes: u64 {
/// When set, the mapping is considered valid and pointing somewhere
const PRESENT = 1 << 0;
/// For tables, allows writes to further translation levels, for pages/blocks, allows
@ -41,7 +42,7 @@ bitflags! {
/// TTBR0 and TTBR1, all address spaces are initially cloned from the kernel space.
#[repr(C)]
pub struct AddressSpace {
l0: *mut PageTable<L0>,
inner: IrqSafeSpinlock<PhysicalRefMut<'static, PageTable<L0>>>,
}
/// Represents a single virtual address space mapping depending on its translation level
@ -80,6 +81,8 @@ impl NonTerminalEntryLevel for L2 {
}
impl const EntryLevel for L0 {
const SIZE: usize = 1 << 39;
fn index(addr: usize) -> usize {
(addr >> 39) & 0x1FF
}
@ -90,6 +93,8 @@ impl const EntryLevel for L0 {
}
impl const EntryLevel for L1 {
const SIZE: usize = 1 << 30;
fn index(addr: usize) -> usize {
(addr >> 30) & 0x1FF
}
@ -100,6 +105,8 @@ impl const EntryLevel for L1 {
}
impl const EntryLevel for L2 {
const SIZE: usize = 1 << 21;
fn index(addr: usize) -> usize {
(addr >> 21) & 0x1FF
}
@ -110,6 +117,8 @@ impl const EntryLevel for L2 {
}
impl const EntryLevel for L3 {
const SIZE: usize = 1 << 12;
fn index(addr: usize) -> usize {
(addr >> 12) & 0x1FF
}
@ -121,18 +130,18 @@ impl const EntryLevel for L3 {
impl PageEntry<L3> {
/// Constructs a mapping which points to a 4KiB page
fn page(phys: usize, attrs: PageAttributes) -> Self {
pub fn page(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
Self(
(phys as u64) | (attrs | PageAttributes::PRESENT | PageAttributes::USER).bits(),
u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::USER).bits(),
PhantomData,
)
}
/// Returns the physical address of the page this entry refers to, returning None if it does
/// not
pub fn as_page(self) -> Option<usize> {
pub fn as_page(self) -> Option<PhysicalAddress> {
if self.0 & PageAttributes::PRESENT.bits() != 0 {
Some((self.0 & !0xFFF) as usize)
Some(PhysicalAddress::from_raw(self.0 & !0xFFF))
} else {
None
}
@ -141,11 +150,18 @@ impl PageEntry<L3> {
impl PageEntry<L2> {
/// Constructs a mapping which points to a 2MiB block
fn block(phys: usize, attrs: PageAttributes) -> Self {
pub fn block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
Self(
(phys as u64)
| (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK | PageAttributes::USER)
.bits(),
u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(),
PhantomData,
)
}
}
impl PageEntry<L1> {
pub unsafe fn block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
Self(
u64::from(phys) | (attrs | PageAttributes::PRESENT | PageAttributes::BLOCK).bits(),
PhantomData,
)
}
@ -153,9 +169,9 @@ impl PageEntry<L2> {
impl<L: NonTerminalEntryLevel> PageEntry<L> {
/// Constructs a mapping which points to a next-level table
fn table(phys: usize, attrs: PageAttributes) -> Self {
pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
Self(
(phys as u64)
u64::from(phys)
| (attrs
| PageAttributes::PRESENT
| PageAttributes::WRITABLE
@ -167,11 +183,11 @@ impl<L: NonTerminalEntryLevel> PageEntry<L> {
/// Returns the physical address of the table this entry refers to, returning None if it
/// does not
pub fn as_table(self) -> Option<usize> {
pub fn as_table(self) -> Option<PhysicalAddress> {
if self.0 & PageAttributes::PRESENT.bits() != 0
&& self.0 & PageAttributes::BLOCK.bits() == 0
{
Some((self.0 & !0xFFF) as usize)
Some(PhysicalAddress::from_raw(self.0 & !0xFFF))
} else {
None
}
@ -182,6 +198,10 @@ impl<L: EntryLevel> PageEntry<L> {
/// An entry that is not mapped
pub const INVALID: Self = Self(0, PhantomData);
pub const unsafe fn from_raw(raw: u64) -> Self {
Self(raw, PhantomData)
}
/// Returns `true` if the entry contains a valid mapping to either a table or to a page/block
pub fn is_present(&self) -> bool {
self.0 & PageAttributes::PRESENT.bits() != 0
@ -197,41 +217,49 @@ impl<L: EntryLevel> PageTable<L> {
}
/// Allocates a new page table, filling it with non-preset entries
pub fn new_zeroed() -> Result<&'static mut Self, Error> {
let page = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() };
let table = unsafe { &mut *(page as *mut Self) };
pub fn new_zeroed<'a>() -> Result<PhysicalRefMut<'a, Self>, Error> {
let physical = phys::alloc_page()?;
let mut table = unsafe { PhysicalRefMut::<'a, Self>::map(physical) };
for i in 0..512 {
table[i] = PageEntry::INVALID;
}
Ok(table)
}
/// Returns the physical address of this table
pub fn physical_address(&self) -> usize {
unsafe { (self.data.as_ptr() as usize).physicalize() }
}
// /// Returns the physical address of this table
// pub fn physical_address(&self) -> usize {
// unsafe { (self.data.as_ptr() as usize).physicalize() }
// }
}
impl<L: NonTerminalEntryLevel> NextPageTable for PageTable<L> {
impl<L: NonTerminalEntryLevel + 'static> NextPageTable for PageTable<L> {
type NextLevel = PageTable<L::NextLevel>;
type TableRef = PhysicalRef<'static, Self::NextLevel>;
type TableRefMut = PhysicalRefMut<'static, Self::NextLevel>;
fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel> {
let entry = self[index];
entry
fn get(&self, index: usize) -> Option<Self::TableRef> {
self[index]
.as_table()
.map(|addr| unsafe { &mut *(addr.virtualize() as *mut Self::NextLevel) })
.map(|addr| unsafe { PhysicalRef::map(addr) })
}
fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error> {
fn get_mut(&mut self, index: usize) -> Option<Self::TableRefMut> {
self[index]
.as_table()
.map(|addr| unsafe { PhysicalRefMut::map(addr) })
}
fn get_mut_or_alloc(&mut self, index: usize) -> Result<Self::TableRefMut, Error> {
let entry = self[index];
if let Some(table) = entry.as_table() {
Ok(unsafe { &mut *(table.virtualize() as *mut Self::NextLevel) })
Ok(unsafe { PhysicalRefMut::map(table) })
} else {
let table = PageTable::new_zeroed()?;
self[index] = PageEntry::<L>::table(
table.physical_address(),
unsafe { table.as_physical_address() },
PageAttributes::WRITABLE | PageAttributes::USER,
);
Ok(table)
@ -285,7 +313,7 @@ impl VirtualMemoryManager for AddressSpace {
}
for i in 0..len {
let page = phys::alloc_page(PageUsage::Used)?;
let page = phys::alloc_page()?;
self.map_page(base + i * 0x1000, page, attrs)?;
}
@ -295,7 +323,12 @@ impl VirtualMemoryManager for AddressSpace {
Err(Error::OutOfMemory)
}
fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> {
fn map_page(
&self,
virt: usize,
phys: PhysicalAddress,
attrs: MapAttributes,
) -> Result<(), Error> {
self.write_entry(virt, PageEntry::page(phys, attrs.into()), true)
}
@ -318,49 +351,50 @@ impl VirtualMemoryManager for AddressSpace {
impl AddressSpace {
/// Allocates an empty address space with all entries marked as non-present
pub fn new_empty() -> Result<Self, Error> {
let l0 = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() as *mut PageTable<L0> };
let mut l0 = unsafe { PhysicalRefMut::<'static, PageTable<L0>>::map(phys::alloc_page()?) };
for i in 0..512 {
unsafe {
(*l0)[i] = PageEntry::INVALID;
l0[i] = PageEntry::INVALID;
}
}
unsafe {
KERNEL_TABLES.clone_into(&mut *l0);
}
Ok(Self { l0 })
}
clone_kernel_tables(&mut *l0);
unsafe fn as_mut(&self) -> &'static mut PageTable<L0> {
self.l0.as_mut().unwrap()
Ok(Self {
inner: IrqSafeSpinlock::new(l0),
})
}
// TODO return page size and attributes
/// Returns the physical address to which the `virt` address is mapped
pub fn translate(&self, virt: usize) -> Option<usize> {
pub fn translate(&self, virt: usize) -> Option<PhysicalAddress> {
let l0 = self.inner.lock();
let l0i = L0::index(virt);
let l1i = L1::index(virt);
let l2i = L2::index(virt);
let l3i = L3::index(virt);
let l1 = unsafe { self.as_mut().get_mut(l0i) }?;
let l2 = l1.get_mut(l1i)?;
let l3 = l2.get_mut(l2i)?;
let l1 = l0.get(l0i)?;
let l2 = l1.get(l1i)?;
let l3 = l2.get(l2i)?;
l3[l3i].as_page()
}
// Write a single 4KiB entry
fn write_entry(&self, virt: usize, entry: PageEntry<L3>, overwrite: bool) -> Result<(), Error> {
let mut l0 = self.inner.lock();
let l0i = L0::index(virt);
let l1i = L1::index(virt);
let l2i = L2::index(virt);
let l3i = L3::index(virt);
let l1 = unsafe { self.as_mut().get_mut_or_alloc(l0i) }?;
let l2 = l1.get_mut_or_alloc(l1i)?;
let l3 = l2.get_mut_or_alloc(l2i)?;
let mut l1 = l0.get_mut_or_alloc(l0i)?;
let mut l2 = l1.get_mut_or_alloc(l1i)?;
let mut l3 = l2.get_mut_or_alloc(l2i)?;
if l3[l3i].is_present() && !overwrite {
todo!();
@ -376,10 +410,7 @@ impl AddressSpace {
}
/// Returns the physical address of the root table
pub fn physical_address(&self) -> usize {
unsafe { (self.l0 as usize).physicalize() }
pub fn physical_address(&self) -> PhysicalAddress {
unsafe { self.inner.lock().as_physical_address() }
}
}
unsafe impl Send for AddressSpace {}
unsafe impl Sync for AddressSpace {}

687
src/arch/x86_64/mod.rs
View File

@ -1,176 +1,113 @@
//! x86-64 architecture and platform implementation
use core::sync::atomic::Ordering;
use core::{mem::size_of, sync::atomic::Ordering};
use abi::error::Error;
use acpi_lib::{mcfg::Mcfg, AcpiTables, InterruptModel};
use acpi_lib::{AcpiHandler, AcpiTable, AcpiTables, InterruptModel};
use alloc::boxed::Box;
use cpu::Cpu;
use device_api::{
input::KeyboardProducer,
interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController},
timer::MonotonicTimestampProviderDevice,
Device,
input::KeyboardProducer, interrupt::ExternalInterruptController,
timer::MonotonicTimestampProviderDevice, Device,
};
use git_version::git_version;
use kernel_util::util::OneTimeInit;
use yboot_proto::{AvailableRegion, IterableMemoryMap};
use yboot_proto::{v1::AvailableMemoryRegion, LoadProtocolV1};
mod acpi;
mod apic;
mod boot;
pub mod context;
pub mod cpu;
mod cpuid;
mod exception;
mod gdt;
mod intrinsics;
pub mod mem;
mod peripherals;
mod registers;
mod smp;
mod syscall;
use crate::{
arch::x86_64::{
apic::local::LocalApic,
peripherals::serial::ComPort,
table::{init_fixed_tables, KERNEL_TABLES},
intrinsics::{IoPort, IoPortAccess},
mem::{map_heap_block, table::L2, HEAP_MAPPING_OFFSET},
},
debug::{self, LogLevel},
device::{
self,
bus::pci::PciBusManager,
display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer},
tty::combined::CombinedTerminal,
tty::CombinedTerminal,
},
fs::{
devfs::{self, CharDeviceType},
Initrd, INITRD_DATA,
},
mem::{
address::{FromRaw, IntoRaw},
device::RawDeviceMemoryMapping,
heap,
phys::{self, reserved::reserve_region, PhysicalMemoryRegion},
ConvertAddress,
table::EntryLevel,
PhysicalAddress,
},
panic,
sync::SpinFence,
CPU_INIT_FENCE,
};
use self::{
acpi::{AcpiAllocator, AcpiHandlerImpl},
apic::ioapic::IoApic,
intrinsics::{IoPort, IoPortAccess},
peripherals::{i8253::I8253, ps2::PS2Controller},
apic::{ioapic::IoApic, local::LocalApic},
boot::BootData,
cpu::Cpu,
cpuid::{ProcessorFeatures, PROCESSOR_FEATURES},
mem::{
init_fixed_tables,
table::{PageAttributes, PageEntry, L1, L3},
EarlyMapping, MEMORY_LIMIT, RAM_MAPPING_L1, RAM_MAPPING_OFFSET,
},
peripherals::{i8253::I8253, ps2::PS2Controller, serial::ComPort},
smp::CPU_COUNT,
};
use super::{Architecture, CpuMessage};
#[macro_use]
pub mod intrinsics;
pub mod acpi;
pub mod apic;
pub mod boot;
pub mod context;
pub mod cpu;
pub mod cpuid;
pub mod exception;
pub mod gdt;
pub mod peripherals;
pub mod registers;
pub mod smp;
pub mod syscall;
pub mod table;
/// x86-64 interrupt number wrapper
#[derive(Clone, Copy, PartialEq, Debug)]
#[derive(Clone, Copy, PartialEq, Eq, Debug, Hash)]
pub enum IrqNumber {
/// Legacy (ISA) interrupt. Can have value in range 0..16.
Isa(u8),
/// Global system interrupt. Means an external interrupt for I/O APIC.
Gsi(u8),
}
/// Helper trait to provide abstract access to available memory regions
pub trait AbstractAvailableRegion {
/// Returns page-aligned physical start address of the region
fn start_address(&self) -> usize;
/// Returns the page count (rounded down) of this region
fn page_count(&self) -> usize;
}
/// Helper trait to provide abstract access to memory maps
pub trait AbstractMemoryMap<'a>: 'a {
/// Available memory region type contained within this memory map
type AvailableRegion: AbstractAvailableRegion;
/// Iterator type returned by [Self::iter]
type Iter: Iterator<Item = &'a Self::AvailableRegion> + Clone;
/// Returns the physical memory range which contains this memory map
fn reserved_range(&self) -> PhysicalMemoryRegion;
/// Returns an iterator over the available memory regions
fn iter(&self) -> Self::Iter;
}
impl<T: AvailableRegion> AbstractAvailableRegion for T {
fn start_address(&self) -> usize {
<T as AvailableRegion>::start_address(self)
}
fn page_count(&self) -> usize {
<T as AvailableRegion>::page_count(self)
}
}
impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T {
type AvailableRegion = T::Entry;
type Iter = T::Iter;
fn reserved_range(&self) -> PhysicalMemoryRegion {
PhysicalMemoryRegion {
base: self.data_physical_base(),
size: (self.data_size() + 0xFFF) & !0xFFF,
}
}
fn iter(&self) -> Self::Iter {
<T as IterableMemoryMap>::iter_with_offset(self, X86_64::KERNEL_VIRT_OFFSET)
}
}
/// Describes which kind of bootloader data was provided to the kernel
pub enum BootData {
/// [yboot_proto::LoadProtocolV1]
YBoot(&'static yboot_proto::LoadProtocolV1),
}
/// x86-64 architecture + platform implementation
pub struct X86_64 {
boot_data: OneTimeInit<BootData>,
acpi: OneTimeInit<AcpiTables<AcpiHandlerImpl>>,
// Display
framebuffer: OneTimeInit<LinearFramebuffer>,
fb_console: OneTimeInit<FramebufferConsole>,
combined_terminal: OneTimeInit<CombinedTerminal>,
fbconsole: OneTimeInit<FramebufferConsole>,
tty: OneTimeInit<CombinedTerminal>,
ioapic: OneTimeInit<IoApic>,
timer: OneTimeInit<I8253>,
}
/// x86-64 architecture implementation
static SHUTDOWN_FENCE: SpinFence = SpinFence::new();
pub static ARCHITECTURE: X86_64 = X86_64 {
boot_data: OneTimeInit::new(),
acpi: OneTimeInit::new(),
framebuffer: OneTimeInit::new(),
fb_console: OneTimeInit::new(),
combined_terminal: OneTimeInit::new(),
fbconsole: OneTimeInit::new(),
tty: OneTimeInit::new(),
// Devices
ioapic: OneTimeInit::new(),
timer: OneTimeInit::new(),
};
static SHUTDOWN_FENCE: SpinFence = SpinFence::new();
impl Architecture for X86_64 {
const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000;
type IrqNumber = IrqNumber;
unsafe fn init_mmu(&self, bsp: bool) {
if bsp {
init_fixed_tables();
}
let cr3 = KERNEL_TABLES.physical_address();
core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
fn cpu_count() -> usize {
CPU_COUNT.load(Ordering::Acquire)
}
unsafe fn set_interrupt_mask(mask: bool) {
@ -196,91 +133,76 @@ impl Architecture for X86_64 {
}
}
// CPU management
unsafe fn reset(&self) -> ! {
Self::set_interrupt_mask(true);
loop {
Self::wait_for_interrupt();
#[inline]
unsafe fn map_device_memory(
&self,
base: PhysicalAddress,
size: usize,
) -> Result<RawDeviceMemoryMapping, Error> {
mem::map_device_memory(base, size)
}
#[inline]
unsafe fn unmap_device_memory(&self, map: RawDeviceMemoryMapping) {
mem::unmap_device_memory(map)
}
fn map_physical_memory<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
&self,
it: I,
_memory_start: PhysicalAddress,
memory_end: PhysicalAddress,
) -> Result<(), Error> {
let end_l1i = (IntoRaw::<usize>::into_raw(memory_end) + (1 << 30) - 1) >> 30;
if end_l1i > 512 {
loop {}
}
MEMORY_LIMIT.init(memory_end.into_raw());
// Check if 1GiB pages are supported
if PROCESSOR_FEATURES
.get()
.contains(ProcessorFeatures::PDPE1GB)
{
// Just map gigabytes of RAM
for l1i in 0..end_l1i {
// TODO NX
unsafe {
RAM_MAPPING_L1[l1i] = PageEntry::<L1>::block(
PhysicalAddress::from_raw(l1i << 30),
PageAttributes::WRITABLE,
);
}
}
Ok(())
} else {
loop {}
}
}
unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> {
if !CPU_INIT_FENCE.try_wait_all(1) {
// Don't send an IPI: SMP not initialized yet
return Ok(());
}
let Some(local_apic) = Cpu::get_local().map(|cpu| cpu.local_apic()) else {
panic!("Local APIC has not been initialized yet");
};
local_apic.send_ipi(target, msg)
}
unsafe fn start_application_processors(&self) {
if let Some(acpi) = self.acpi.try_get() {
let Some(pinfo) = acpi
.platform_info_in(AcpiAllocator)
.ok()
.and_then(|p| p.processor_info)
else {
return;
};
smp::start_ap_cores(&pinfo);
#[inline]
fn virtualize(address: PhysicalAddress) -> Result<usize, Error> {
let raw: usize = address.into_raw();
if raw < *mem::MEMORY_LIMIT.get() {
Ok(raw + RAM_MAPPING_OFFSET)
} else {
errorln!("Invalid physical address: {:#x}", address);
Err(Error::InvalidMemoryOperation)
}
}
fn cpu_count() -> usize {
CPU_COUNT.load(Ordering::Acquire)
}
#[inline]
fn physicalize(address: usize) -> Result<PhysicalAddress, Error> {
if address < RAM_MAPPING_OFFSET || address - RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get()
{
errorln!("Not a virtualized physical address: {:#x}", address);
return Err(Error::InvalidMemoryOperation);
}
// Memory
fn map_device_pages(&self, phys: usize, count: usize) -> Result<usize, Error> {
unsafe { KERNEL_TABLES.map_device_pages(phys, count) }
}
// Devices
fn register_monotonic_timer(
&self,
_timer: &'static dyn device_api::timer::MonotonicTimestampProviderDevice,
) -> Result<(), Error> {
todo!()
}
fn register_local_interrupt_controller(
&self,
_intc: &'static dyn device_api::interrupt::LocalInterruptController<
IpiMessage = CpuMessage,
>,
) -> Result<(), Error> {
todo!()
}
fn register_external_interrupt_controller(
&self,
_intc: &'static dyn device_api::interrupt::ExternalInterruptController<
IrqNumber = Self::IrqNumber,
>,
) -> Result<(), Error> {
todo!()
}
fn register_reset_device(
&self,
_reset: &'static dyn device_api::ResetDevice,
) -> Result<(), Error> {
todo!()
}
fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
self.timer.get()
}
fn local_interrupt_controller(
&'static self,
) -> &'static dyn LocalInterruptController<IpiMessage = CpuMessage> {
todo!()
Ok(PhysicalAddress::from_raw(address - RAM_MAPPING_OFFSET))
}
fn external_interrupt_controller(
@ -288,35 +210,220 @@ impl Architecture for X86_64 {
) -> &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber> {
self.ioapic.get()
}
fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
self.timer.get()
}
}
impl X86_64 {
fn set_boot_data(&self, boot_data: BootData) {
match &boot_data {
unsafe fn handle_ipi(&self, msg: CpuMessage) {
loop {}
}
fn set_boot_data(&self, data: BootData) {
match data {
BootData::YBoot(data) => {
// Setup initrd
Self::init_initrd(
data.initrd_address as usize,
(data.initrd_address + data.initrd_size) as usize,
// Reserve the memory map
unsafe {
reserve_region(
"mmap",
PhysicalMemoryRegion {
base: PhysicalAddress::from_raw(data.memory_map.address),
size: data.memory_map.len as usize * size_of::<AvailableMemoryRegion>(),
},
);
}
// Reserve initrd, if not NULL
if data.initrd_address != 0 && data.initrd_size != 0 {
let aligned_start = data.initrd_address & !0xFFF;
let aligned_end = (data.initrd_address + data.initrd_size + 0xFFF) & !0xFFF;
unsafe {
reserve_region(
"initrd",
PhysicalMemoryRegion {
base: PhysicalAddress::from_raw(aligned_start),
size: (aligned_end - aligned_start) as usize,
},
);
}
}
}
}
self.boot_data.init(data);
}
unsafe fn init_physical_memory_from_yboot(data: &LoadProtocolV1) {
let mmap = EarlyMapping::<AvailableMemoryRegion>::map_slice(
PhysicalAddress::from_raw(data.memory_map.address),
data.memory_map.len as usize,
)
.unwrap();
phys::init_from_iter(mmap.as_ref().iter().map(|reg| PhysicalMemoryRegion {
base: PhysicalAddress::from_raw(reg.start_address),
size: reg.page_count as usize * 0x1000,
}));
}
unsafe fn init_memory_management(&self) {
const HEAP_PAGES: usize = 16;
init_fixed_tables();
// Reserve lower 4MiB just in case
reserve_region(
"lowmem",
PhysicalMemoryRegion {
base: PhysicalAddress::ZERO,
size: 4 * 1024 * 1024,
},
);
match self.boot_data.get() {
&BootData::YBoot(data) => Self::init_physical_memory_from_yboot(data),
}
// Setup heap
for i in 0..HEAP_PAGES {
// Allocate in 2MiB chunks
let l2_page = phys::alloc_2m_page().unwrap();
map_heap_block(i, l2_page);
}
heap::init_heap(HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE);
}
unsafe fn init_platform(&'static self, cpu_id: usize) {
Cpu::init_local(LocalApic::new(), cpu_id as _);
if cpu_id == 0 {
match self.boot_data.get() {
&BootData::YBoot(data) => {
let start = PhysicalAddress::from_raw(data.initrd_address);
Self::init_initrd(start, start.add(data.initrd_size as usize));
}
}
self.init_acpi_from_boot_data();
Self::disable_8259();
self.timer.init(I8253::new());
let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4))));
debug::add_sink(com1_3.port_a(), LogLevel::Debug);
self.init_framebuffer();
debug::init();
let ps2 = Box::leak(Box::new(PS2Controller::new(
IrqNumber::Isa(1),
IrqNumber::Isa(12),
0x64,
0x60,
)));
ps2.init().unwrap();
infoln!(
"Yggdrasil v{} ({})",
env!("CARGO_PKG_VERSION"),
git_version!()
);
if let Some(acpi) = self.acpi.try_get() {
self.init_platform_from_acpi(acpi);
}
self.timer.get().init_irq().unwrap();
ps2.connect(self.tty.get());
ps2.init_irq().unwrap();
device::register_device(self.ioapic.get());
device::register_device(ps2);
// TODO setup PCI devices
} else {
loop {}
}
}
unsafe fn init_acpi_from_boot_data(&self) {
match self.boot_data.get() {
&BootData::YBoot(data) => self.init_acpi_from_rsdp(data.rsdp_address as usize),
}
}
unsafe fn init_acpi_from_rsdp(&self, rsdp: usize) {
let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).unwrap();
self.acpi.init(acpi_tables);
}
unsafe fn init_platform_from_acpi(&self, acpi: &'static AcpiTables<AcpiHandlerImpl>) {
let platform_info = acpi.platform_info_in(AcpiAllocator).unwrap();
let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else {
panic!("The processor does not support APIC");
};
self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap());
// TODO ACPI init
// acpi::init_acpi(acpi).unwrap();
// TODO MCFG
// if let Ok(mcfg) = acpi.find_table::<Mcfg>() {
// for entry in mcfg.entries() {
// PciBusManager::add_segment_from_mcfg(entry).unwrap();
// }
// }
}
unsafe fn init_framebuffer(&'static self) {
match self.boot_data.get() {
&BootData::YBoot(data) => {
let info = &data.opt_framebuffer;
self.framebuffer.init(
LinearFramebuffer::from_physical_bits(
PhysicalAddress::from_raw(info.res_address),
info.res_size as usize,
info.res_stride as usize,
info.res_width,
info.res_height,
)
.unwrap(),
);
}
}
self.boot_data.init(boot_data);
self.fbconsole
.init(FramebufferConsole::from_framebuffer(self.framebuffer.get(), None).unwrap());
debug::add_sink(self.fbconsole.get(), LogLevel::Info);
self.tty.init(CombinedTerminal::new(self.fbconsole.get()));
devfs::add_char_device(self.tty.get(), CharDeviceType::TtyRegular).unwrap();
console::add_console_autoflush(self.fbconsole.get());
}
fn init_initrd(initrd_start: usize, initrd_end: usize) {
if initrd_start == 0 || initrd_end <= initrd_start {
fn init_initrd(initrd_start: PhysicalAddress, initrd_end: PhysicalAddress) {
if initrd_start.is_zero() || initrd_end <= initrd_start {
infoln!("No initrd loaded");
return;
}
let start_aligned = initrd_start & !0xFFF;
let end_aligned = initrd_end & !0xFFF;
let start_aligned = initrd_start.page_align_down::<L3>();
let end_aligned = initrd_start.page_align_up::<L3>();
let data = unsafe {
core::slice::from_raw_parts(
initrd_start.virtualize() as *const _,
start_aligned.virtualize_raw() as *const u8,
initrd_end - initrd_start,
)
};
@ -330,167 +437,6 @@ impl X86_64 {
INITRD_DATA.init(initrd);
}
fn init_acpi_from_boot_data(&self) {
match *self.boot_data.get() {
BootData::YBoot(data) => {
self.init_acpi_from_rsdp(data.rsdp_address as usize);
}
}
}
fn init_acpi_from_rsdp(&self, address: usize) {
let acpi_tables = unsafe { AcpiTables::from_rsdp(AcpiHandlerImpl, address).unwrap() };
self.acpi.init(acpi_tables);
}
unsafe fn init_physical_memory(&self) -> Result<(), Error> {
// Reserve the lower 8MiB of memory
reserve_region(
"lower-memory",
PhysicalMemoryRegion {
base: 0,
size: 8 << 21,
},
);
// Reserve initrd
if let Some(initrd) = INITRD_DATA.try_get() {
reserve_region(
"initrd",
PhysicalMemoryRegion {
base: initrd.phys_page_start,
size: initrd.phys_page_len,
},
);
}
match *self.boot_data.get() {
BootData::YBoot(data) => {
let memory_map = &data.memory_map;
reserve_region("memory-map", memory_map.reserved_range());
phys::init_from_iter(IterableMemoryMap::iter(memory_map).map(|r| {
PhysicalMemoryRegion {
base: AbstractAvailableRegion::start_address(r),
size: AbstractAvailableRegion::page_count(r) * 0x1000,
}
}))
.expect("Failed to initialize the physical memory manager");
}
}
Ok(())
}
unsafe fn init_platform_from_acpi(&self, acpi: &'static AcpiTables<AcpiHandlerImpl>) {
let platform_info = acpi.platform_info_in(AcpiAllocator).unwrap();
let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else {
panic!("Processor does not have an APIC");
};
self.ioapic.init(IoApic::from_acpi(&apic_info).unwrap());
acpi::init_acpi(acpi).unwrap();
// Enumerate PCIe root devices
// TODO can there be multiple MCFGs?
if let Ok(mcfg) = acpi.find_table::<Mcfg>() {
for entry in mcfg.entries() {
PciBusManager::add_segment_from_mcfg(entry).unwrap();
}
}
}
unsafe fn init_framebuffer(&'static self) {
match *self.boot_data.get() {
BootData::YBoot(data) => {
let fb = &data.opt_framebuffer;
self.framebuffer.init(
LinearFramebuffer::from_physical_bits(
fb.res_address as _,
fb.res_size as _,
fb.res_stride as _,
fb.res_width as _,
fb.res_height as _,
)
.unwrap(),
);
}
}
self.fb_console
.init(FramebufferConsole::from_framebuffer(self.framebuffer.get(), None).unwrap());
debug::add_sink(self.fb_console.get(), LogLevel::Info);
// Add a terminal to the devfs
// TODO this is ugly
let combined_terminal = CombinedTerminal::new(self.fb_console.get());
self.combined_terminal.init(combined_terminal);
devfs::add_char_device(self.combined_terminal.get(), CharDeviceType::TtyRegular).unwrap();
console::add_console_autoflush(self.fb_console.get());
}
unsafe fn init_platform(&'static self, cpu_id: usize) {
Cpu::init_local(LocalApic::new(), cpu_id as _);
if cpu_id == 0 {
self.init_acpi_from_boot_data();
Self::disable_8259();
self.timer.init(I8253::new());
// Initialize debug output as soon as possible
let com1_3 = Box::leak(Box::new(ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4))));
debug::add_sink(com1_3.port_a(), LogLevel::Debug);
// devfs::add_char_device(com1_3.port_a(), CharDeviceType::TtySerial).unwrap();
self.init_framebuffer();
debug::init();
let ps2 = Box::leak(Box::new(PS2Controller::new(
IrqNumber::Isa(1),
IrqNumber::Isa(12),
0x64,
0x60,
)));
ps2.init().unwrap();
// Print some stuff now that the output is initialized
infoln!(
"Yggdrasil v{} ({})",
env!("CARGO_PKG_VERSION"),
git_version!()
);
infoln!("Initializing x86_64 platform");
if let Some(acpi) = self.acpi.try_get() {
self.init_platform_from_acpi(acpi);
}
// Enable IRQs for the devices
self.timer.get().init_irq().unwrap();
ps2.connect(self.combined_terminal.get());
ps2.init_irq().unwrap();
device::register_device(self.ioapic.get());
// device::register_device(self.timer.get());
device::register_device(ps2);
// Initialize devices from PCI bus
PciBusManager::setup_bus_devices().unwrap();
infoln!("Device list:");
for device in device::manager_lock().devices() {
infoln!("* {}", device.display_name());
}
}
}
unsafe fn disable_8259() {
infoln!("Disabling i8259 PIC");
// TODO should I make a module for 8259 if I don't even use it?
@ -512,21 +458,4 @@ impl X86_64 {
pic_master_cmd.write(0x20);
pic_slave_cmd.write(0x20);
}
unsafe fn handle_ipi(&self, msg: CpuMessage) {
match msg {
CpuMessage::Panic => panic::panic_secondary(),
CpuMessage::Shutdown => {
Self::set_interrupt_mask(true);
let id = Cpu::local_id();
infoln!("cpu{} shutdown", id);
SHUTDOWN_FENCE.signal();
loop {
Self::wait_for_interrupt();
}
}
}
}
}

33
src/arch/x86_64/smp.rs
View File

@ -7,15 +7,18 @@ use core::{
use acpi_lib::platform::{ProcessorInfo, ProcessorState};
use crate::{
arch::{x86_64::boot::__x86_64_ap_entry, Architecture, ArchitectureImpl},
arch::{
x86_64::{boot::__x86_64_ap_entry, mem::KERNEL_TABLES},
Architecture, ArchitectureImpl,
},
mem::{
phys::{self, PageUsage},
ConvertAddress,
address::{AsPhysicalAddress, IntoRaw},
phys,
},
task::Cpu,
};
use super::{acpi::AcpiAllocator, table::KERNEL_TABLES};
use super::acpi::AcpiAllocator;
/// The number of CPUs present in the system
pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
@ -46,10 +49,11 @@ unsafe fn load_ap_bootstrap_code() {
"Invalid bootstrap code placement: is not below 1MiB"
);
let src_slice = core::slice::from_raw_parts(src_ptr, size);
let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size);
todo!();
// let src_slice = core::slice::from_raw_parts(src_ptr, size);
// let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size);
dst_slice.copy_from_slice(src_slice);
// dst_slice.copy_from_slice(src_slice);
}
unsafe fn load_ap_bootstrap_data(src: &ApBootstrapData) {
@ -62,11 +66,12 @@ unsafe fn load_ap_bootstrap_data(src: &ApBootstrapData) {
"Invalid bootstrap data placement: is not below 1MiB"
);
let src_slice = core::slice::from_raw_parts(src_ptr, size);
let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size);
todo!()
// let src_slice = core::slice::from_raw_parts(src_ptr, size);
// let dst_slice = core::slice::from_raw_parts_mut(dst_ptr.virtualize(), size);
dst_slice.copy_from_slice(src_slice);
core::arch::asm!("wbinvd");
// dst_slice.copy_from_slice(src_slice);
// core::arch::asm!("wbinvd");
}
unsafe fn start_ap_core(apic_id: u32) {
@ -75,10 +80,10 @@ unsafe fn start_ap_core(apic_id: u32) {
let bsp_cpu = Cpu::local();
let bsp_apic = bsp_cpu.local_apic();
let cr3 = KERNEL_TABLES.physical_address();
let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used)
let cr3 = KERNEL_TABLES.as_physical_address().into_raw();
let stack_base = phys::alloc_pages_contiguous(AP_STACK_PAGES)
.unwrap()
.virtualize();
.virtualize_raw();
let stack_size = AP_STACK_PAGES * 0x1000;
let data = ApBootstrapData {

161
src/arch/x86_64/table/fixed.rs
View File

@ -1,161 +0,0 @@
use abi::error::Error;
use crate::{
arch::x86_64::table::{PageAttributes, PageEntry, PageTable, L0, L1, L2, L3},
mem::KERNEL_VIRT_OFFSET,
};
// Means 4 lower GiB are mapped
const KERNEL_PD_COUNT: usize = 4;
// Leave 1GiB gap just for fool safety
const DEVICE_MAPPING_L1I: usize = KERNEL_PD_COUNT + 1;
const DEVICE_VIRT_OFFSET: usize = (DEVICE_MAPPING_L1I << 30) + KERNEL_VIRT_OFFSET;
/// Fixed tables for x86-64. Provide device mappings and static kernel mapping.
pub struct FixedTables {
// Common
l0: PageTable<L0>,
l1: PageTable<L1>,
// Kernel mapping
kernel_l2: [PageTable<L2>; KERNEL_PD_COUNT],
// Device mapping
// 511 entries
device_l2: PageTable<L2>,
// 512 entries
device_l3: PageTable<L3>,
device_l2i: usize,
device_l3i: usize,
}
impl FixedTables {
/// Constructs a set of empty translation tables
pub const fn zeroed() -> Self {
Self {
// Global
l0: PageTable::zeroed(),
// Higher-half common
l1: PageTable::zeroed(),
// Kernel
kernel_l2: [PageTable::zeroed(); KERNEL_PD_COUNT],
// Device
device_l2: PageTable::zeroed(),
device_l3: PageTable::zeroed(),
device_l2i: 1,
device_l3i: 0,
}
}
/// Maps a specified count of physical memory pages to the device virtual address space
pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result<usize, Error> {
if count > 512 * 512 {
panic!("Unsupported device memory mapping size");
} else if count > 512 {
let count = (count + 511) / 512;
// 2MiB mappings
if self.device_l2i + count > 512 {
return Err(Error::OutOfMemory);
}
let virt = DEVICE_VIRT_OFFSET + (self.device_l2i << 21);
for i in 0..count {
self.device_l2[self.device_l2i + i] =
PageEntry::block(phys, PageAttributes::WRITABLE);
}
self.device_l2i += count;
Ok(virt)
} else {
// 4KiB mappings
// Check if a mapping to that address already exists
if self.device_l3i >= count {
for i in 0..self.device_l3i {
let mut matches = true;
for j in 0..count {
let page = phys + j * 0x1000;
let existing = self.device_l3[i].as_page().unwrap();
if page != existing {
matches = false;
break;
}
}
if matches {
let virt = DEVICE_VIRT_OFFSET + (i << 12);
return Ok(virt);
}
}
}
if self.device_l3i + count > 512 {
return Err(Error::OutOfMemory);
}
let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12);
for i in 0..count {
self.device_l3[self.device_l3i + i] =
PageEntry::page(phys + i * 0x1000, PageAttributes::WRITABLE);
}
self.device_l3i += count;
Ok(virt)
}
}
/// Returns the physical address of the fixed PML4
pub fn physical_address(&self) -> usize {
self.l0.physical_address()
}
pub fn clone_into(&self, target: &mut PageTable<L0>) {
target[511] = self.l0[511];
}
}
/// Instance of fixed translation tables
pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed();
/// Initializes the fixed translation tables.
///
/// # Safety
///
/// Only meant to be called by BSP during early init.
pub unsafe fn init_fixed_tables() {
// Kernel L2
for i in 0..512 * KERNEL_PD_COUNT {
let table_index = i / 512;
let table_offset = i % 512;
KERNEL_TABLES.kernel_l2[table_index][table_offset] =
PageEntry::block(i << 21, PageAttributes::WRITABLE);
}
// Device L2
let addr = KERNEL_TABLES.device_l3.physical_address();
KERNEL_TABLES.device_l2[0] = PageEntry::table(addr, PageAttributes::empty());
// Higher-half L1
// Map kernel nGiB
for i in 0..KERNEL_PD_COUNT {
let addr = KERNEL_TABLES.kernel_l2[i].physical_address();
KERNEL_TABLES.l1[i] = PageEntry::table(addr, PageAttributes::empty());
}
// Map device tables
let addr = KERNEL_TABLES.device_l2.physical_address();
KERNEL_TABLES.l1[DEVICE_MAPPING_L1I] = PageEntry::table(addr, PageAttributes::empty());
// Global L0
let addr = KERNEL_TABLES.l1.physical_address();
// Keep the lower mapping for AP bootstrapping
KERNEL_TABLES.l0[0] = PageEntry::table(addr, PageAttributes::empty());
KERNEL_TABLES.l0[511] = PageEntry::table(addr, PageAttributes::empty());
}

2
src/debug.rs
View File

@ -52,7 +52,7 @@ macro_rules! log_print_raw {
macro_rules! log_print {
($level:expr, $($args:tt)+) => {
log_print_raw!($level, "cpu{}:{}:{}: {}", $crate::task::Cpu::local_id(), file!(), line!(), format_args!($($args)+))
log_print_raw!($level, "cpu{}:{}:{}: {}", /* $crate::task::Cpu::local_id() */ 0, file!(), line!(), format_args!($($args)+))
};
}

62
src/device/display/console.rs
View File

@ -1,20 +1,13 @@
//! Console device interfaces
use core::{mem::size_of, time::Duration};
use core::time::Duration;
use abi::{error::Error, primitive_enum};
use alloc::vec::Vec;
use alloc::{boxed::Box, vec, vec::Vec};
use bitflags::bitflags;
use kernel_util::util::StaticVector;
use crate::{
debug::DebugSink,
mem::{
phys::{self, PageUsage},
ConvertAddress,
},
sync::IrqSafeSpinlock,
task::runtime,
};
use crate::{debug::DebugSink, sync::IrqSafeSpinlock, task::runtime};
const CONSOLE_ROW_LEN: usize = 80;
const MAX_CSI_ARGS: usize = 8;
@ -93,7 +86,7 @@ pub struct ConsoleRow {
/// Buffer that contains text rows of the console with their attributes + tracks dirty rows which
/// need to be flushed to the display
pub struct ConsoleBuffer {
rows: &'static mut [ConsoleRow],
rows: Vec<ConsoleRow>,
height: u32,
}
@ -253,48 +246,27 @@ impl ConsoleRow {
impl ConsoleBuffer {
/// Constructs a fixed-size console buffer
pub fn new(height: u32) -> Result<Self, Error> {
let size = size_of::<ConsoleRow>() * (height as usize);
let page_count = (size + 0xFFF) / 0x1000;
let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?;
let rows = unsafe {
core::slice::from_raw_parts_mut(pages.virtualize() as *mut ConsoleRow, height as usize)
};
// let size = size_of::<ConsoleRow>() * (height as usize);
let mut rows = vec![ConsoleRow::zeroed(); height as usize];
for row in rows.iter_mut() {
row.clear(DEFAULT_BG_COLOR);
}
Ok(Self { rows, height })
}
// let size = size_of::<ConsoleRow>() * (height as usize);
// let page_count = (size + 0xFFF) / 0x1000;
// let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?;
/// Reallocates the internal buffer with a new size
pub fn reallocate(&mut self, new_height: u32) -> Result<(), Error> {
// TODO suppress debugging output here
if new_height <= self.height {
// Keep using the old buffer
return Ok(());
}
// let rows = unsafe {
// core::slice::from_raw_parts_mut(pages.virtualize() as *mut ConsoleRow, height as usize)
// };
let size = size_of::<ConsoleRow>() * (new_height as usize);
let page_count = (size + 0xFFF) / 0x1000;
let pages = phys::alloc_pages_contiguous(page_count, PageUsage::Used)?;
// for row in rows.iter_mut() {
// row.clear(DEFAULT_BG_COLOR);
// }
let data = unsafe {
core::slice::from_raw_parts_mut(
pages.virtualize() as *mut ConsoleRow,
new_height as usize,
)
};
// Copy rows from the old buffer
data[0..self.height as usize].copy_from_slice(self.rows);
data[self.height as usize..].fill(ConsoleRow::zeroed());
self.rows = data;
self.height = new_height;
Ok(())
// Ok(Self { rows, height })
}
#[inline(never)]

12
src/device/display/linear_fb.rs
View File

@ -5,7 +5,10 @@ use core::ops::{Index, IndexMut};
use abi::error::Error;
use device_api::Device;
use crate::{mem::device::DeviceMemory, sync::IrqSafeSpinlock};
use crate::{
mem::{device::RawDeviceMemoryMapping, PhysicalAddress},
sync::IrqSafeSpinlock,
};
use super::{DisplayDevice, DisplayDimensions};
@ -34,18 +37,17 @@ impl LinearFramebuffer {
///
/// Unsafe: the caller must ensure the validity of all the arguments.
pub unsafe fn from_physical_bits(
base: usize,
base: PhysicalAddress,
size: usize,
stride: usize,
width: u32,
height: u32,
) -> Result<Self, Error> {
// TODO this may get Dropped later
let mmio = unsafe { DeviceMemory::map("framebuffer", base, size) }?;
let base = unsafe { RawDeviceMemoryMapping::map(base, size) }?.leak();
let inner = Inner {
dimensions: DisplayDimensions { width, height },
base: mmio.base(),
base,
stride,
};

4
src/device/mod.rs
View File

@ -7,9 +7,9 @@ use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard};
#[cfg(target_arch = "aarch64")]
pub mod devtree;
pub mod bus;
// pub mod bus;
pub mod display;
pub mod power;
// pub mod power;
pub mod serial;
pub mod timer;
pub mod tty;

16
src/fs/mod.rs
View File

@ -7,18 +7,14 @@ use memfs::block::{self, BlockAllocator};
use vfs::VnodeRef;
use yggdrasil_abi::{error::Error, io::MountOptions};
use crate::mem::{
self,
phys::{self, PageUsage},
ConvertAddress,
};
use crate::mem::{self, phys, PhysicalAddress};
pub mod devfs;
/// Describes in-memory filesystem image used as initial root
pub struct Initrd {
/// Page-aligned start address of the initrd
pub phys_page_start: usize,
pub phys_page_start: PhysicalAddress,
/// Page-aligned length
pub phys_page_len: usize,
/// Safe reference to the initrd data slice
@ -35,14 +31,14 @@ unsafe impl BlockAllocator for FileBlockAllocator {
fn alloc() -> Result<NonNull<u8>, Error> {
// TODO make this a static assertion
assert_eq!(block::SIZE, 4096);
let page = phys::alloc_page(PageUsage::Used)?;
Ok(unsafe { NonNull::new_unchecked(page.virtualize() as *mut _) })
let page = phys::alloc_page()?;
Ok(unsafe { NonNull::new_unchecked(page.virtualize_raw() as *mut _) })
}
unsafe fn dealloc(block: NonNull<u8>) {
let page = block.as_ptr() as usize;
assert!(page > mem::KERNEL_VIRT_OFFSET);
phys::free_page(page.physicalize());
let physical = PhysicalAddress::from_virtualized(page);
phys::free_page(physical);
}
}

9
src/main.rs
View File

@ -1,5 +1,6 @@
//! osdev-x kernel crate
#![feature(
step_trait,
decl_macro,
naked_functions,
asm_const,
@ -13,7 +14,8 @@
linked_list_cursors,
rustc_private,
allocator_api,
async_fn_in_trait
async_fn_in_trait,
strict_provenance
)]
#![allow(clippy::new_without_default, clippy::fn_to_numeric_cast)]
// #![warn(missing_docs)]
@ -21,11 +23,12 @@
#![no_std]
#![no_main]
use sync::SpinFence;
use arch::Architecture;
use crate::{
arch::{Architecture, ArchitectureImpl, ARCHITECTURE},
arch::{ArchitectureImpl, ARCHITECTURE},
mem::heap,
sync::SpinFence,
task::{spawn_kernel_closure, Cpu},
};

185
src/mem/address.rs Normal file
View File

@ -0,0 +1,185 @@
use core::{
fmt,
iter::Step,
marker::PhantomData,
ops::{Add, Deref, DerefMut, Sub},
};
use crate::arch::{Architecture, ArchitectureImpl, ARCHITECTURE};
use super::{pointer::PhysicalPointer, table::EntryLevel, KERNEL_VIRT_OFFSET};
#[repr(transparent)]
pub struct KernelImageObject<T> {
inner: T,
}
#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Debug, Hash)]
#[repr(transparent)]
pub struct PhysicalAddress(u64);
#[const_trait]
pub trait FromRaw<T> {
fn from_raw(value: T) -> Self;
}
#[const_trait]
pub trait IntoRaw<T> {
fn into_raw(self) -> T;
}
pub trait AsPhysicalAddress {
unsafe fn as_physical_address(&self) -> PhysicalAddress;
}
// KernelImageObject wrapper for objects inside the kernel
impl<T> KernelImageObject<T> {
pub const unsafe fn new(inner: T) -> Self {
Self { inner }
}
}
impl<T> AsPhysicalAddress for KernelImageObject<T> {
unsafe fn as_physical_address(&self) -> PhysicalAddress {
PhysicalAddress::from_raw(&self.inner as *const _ as usize - KERNEL_VIRT_OFFSET)
}
}
impl<T> Deref for KernelImageObject<T> {
type Target = T;
fn deref(&self) -> &Self::Target {
&self.inner
}
}
impl<T> DerefMut for KernelImageObject<T> {
fn deref_mut(&mut self) -> &mut Self::Target {
&mut self.inner
}
}
//
impl PhysicalAddress {
pub const ZERO: Self = Self(0);
pub const MAX: Self = Self(u64::MAX);
pub const MIN: Self = Self(u64::MIN);
pub const fn add(self, offset: usize) -> Self {
Self(self.0 + offset as u64)
}
#[inline(always)]
pub const fn is_zero(self) -> bool {
self.0 == 0
}
pub const fn page_offset<L: ~const EntryLevel>(self) -> usize {
L::page_offset(self.0 as usize)
}
pub const fn page_align_down<L: ~const EntryLevel>(self) -> Self {
Self(self.0 & !(L::SIZE as u64 - 1))
}
pub const fn page_align_up<L: ~const EntryLevel>(self) -> Self {
Self((self.0 + L::SIZE as u64 - 1) & !(L::SIZE as u64 - 1))
}
pub unsafe fn from_virtualized(address: usize) -> Self {
ArchitectureImpl::physicalize(address).unwrap()
}
pub fn virtualize_raw(self) -> usize {
ArchitectureImpl::virtualize(self).unwrap()
}
pub fn virtualize<T>(self) -> PhysicalPointer<T> {
loop {}
}
pub fn virtualize_slice<T>(self, len: usize) -> PhysicalPointer<[T]> {
loop {}
}
}
impl Add for PhysicalAddress {
type Output = Self;
fn add(self, rhs: Self) -> Self::Output {
Self(self.0 + rhs.0)
}
}
impl Sub for PhysicalAddress {
type Output = usize;
fn sub(self, rhs: Self) -> Self::Output {
(self.0 - rhs.0) as usize
}
}
// Conversions
impl const FromRaw<u64> for PhysicalAddress {
fn from_raw(value: u64) -> Self {
Self(value)
}
}
impl const FromRaw<usize> for PhysicalAddress {
fn from_raw(value: usize) -> Self {
Self(value as u64)
}
}
impl const IntoRaw<u64> for PhysicalAddress {
fn into_raw(self) -> u64 {
self.0
}
}
impl const IntoRaw<usize> for PhysicalAddress {
fn into_raw(self) -> usize {
self.0 as usize
}
}
impl From<PhysicalAddress> for u64 {
fn from(addr: PhysicalAddress) -> u64 {
addr.0
}
}
impl From<PhysicalAddress> for usize {
fn from(addr: PhysicalAddress) -> usize {
addr.0 as usize
}
}
// Ranges
impl Step for PhysicalAddress {
fn steps_between(start: &Self, end: &Self) -> Option<usize> {
loop {}
}
fn forward_checked(start: Self, count: usize) -> Option<Self> {
start.0.checked_add(count as u64).map(Self)
}
fn backward_checked(start: Self, count: usize) -> Option<Self> {
loop {}
}
}
// fmt
impl fmt::LowerHex for PhysicalAddress {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::LowerHex::fmt(&self.0, f)
}
}

119
src/mem/device.rs
View File

@ -1,85 +1,80 @@
//! Facilities for mapping devices to virtual address space
use core::{marker::PhantomData, mem::size_of, ops::Deref};
use core::{marker::PhantomData, mem::size_of, ops::Deref, sync::atomic::AtomicUsize};
use abi::error::Error;
use alloc::sync::Arc;
use crate::arch::{Architecture, ARCHITECTURE};
/// Generic MMIO access mapping
use super::PhysicalAddress;
#[derive(Debug)]
pub struct RawDeviceMemoryMapping {
pub address: usize,
pub base_address: usize,
pub page_size: usize,
pub page_count: usize,
}
#[derive(Clone, Debug)]
#[allow(unused)]
pub struct DeviceMemory {
name: &'static str,
base: usize,
size: usize,
pub struct DeviceMemoryMapping {
inner: Arc<RawDeviceMemoryMapping>,
address: usize,
}
/// MMIO wrapper for `T`
pub struct DeviceMemoryIo<T> {
mmio: DeviceMemory,
_pd: PhantomData<T>,
#[derive(Clone, Debug)]
pub struct DeviceMemoryIo<'a, T: ?Sized> {
inner: Arc<RawDeviceMemoryMapping>,
value: &'a T,
}
impl DeviceMemory {
/// Maps the device to some virtual memory address and constructs a wrapper for that range.
///
/// # Safety
///
/// The caller is responsible for making sure the (phys, size) range is valid and actually
/// points to some device's MMIO. The caller must also make sure no aliasing for that range is
/// possible.
pub unsafe fn map(name: &'static str, phys: usize, size: usize) -> Result<Self, Error> {
let aligned_base = phys & !0xFFF;
let base_offset = phys & 0xFFF;
let aligned_size = (size + 0xFFF) & !0xFFF;
let base = ARCHITECTURE.map_device_pages(aligned_base, aligned_size / 0x1000)?;
let base = base + base_offset;
debugln!("Mapped {}@{:#x} to {:#x}", name, phys, base);
Ok(Self { name, base, size })
}
/// Returns the base address of this mapping
impl RawDeviceMemoryMapping {
#[inline]
pub fn base(&self) -> usize {
self.base
pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result<Self, Error> {
ARCHITECTURE.map_device_memory(base, size)
}
pub fn leak(self) -> usize {
let address = self.address;
core::mem::forget(self);
address
}
}
impl<T> DeviceMemoryIo<T> {
/// Maps the `T` struct at `phys` to some virtual memory address and provides a [Deref]able
/// wrapper to it.
///
/// # Safety
///
/// The caller is responsible for making sure the `phys` address points to a MMIO region which
/// is at least `size_of::<T>()` and no aliasing for that region is possible.
pub unsafe fn map(name: &'static str, phys: usize) -> Result<Self, Error> {
DeviceMemory::map(name, phys, size_of::<T>()).map(|t| Self::new(t))
}
/// Constructs a device MMIO wrapper from given [DeviceMemory] mapping.
///
/// # Safety
///
/// The caller must ensure `mmio` actually points to a device of type `T`.
pub unsafe fn new(mmio: DeviceMemory) -> Self {
assert!(mmio.size >= size_of::<T>());
// TODO check align
Self {
mmio,
_pd: PhantomData,
}
impl Drop for RawDeviceMemoryMapping {
fn drop(&mut self) {
loop {}
}
}
impl<T> Deref for DeviceMemoryIo<T> {
impl DeviceMemoryMapping {
pub unsafe fn map(base: PhysicalAddress, size: usize) -> Result<Self, Error> {
let inner = RawDeviceMemoryMapping::map(base, size)?;
loop {}
}
}
impl<'a, T: Sized> DeviceMemoryIo<'a, T> {
pub unsafe fn from_raw(raw: DeviceMemoryMapping) -> DeviceMemoryIo<'a, T> {
// TODO
loop {}
}
pub unsafe fn map(base: PhysicalAddress) -> Result<DeviceMemoryIo<'a, T>, Error> {
let inner = RawDeviceMemoryMapping::map(base, size_of::<T>())?;
let value = &*(inner.address as *const T);
Ok(DeviceMemoryIo {
inner: Arc::new(inner),
value,
})
}
}
impl<'a, T: ?Sized> Deref for DeviceMemoryIo<'a, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
unsafe { &*(self.mmio.base as *const T) }
self.value
}
}

2
src/mem/heap.rs
View File

@ -35,7 +35,7 @@ unsafe impl GlobalAlloc for KernelAllocator {
match self.inner.lock().allocate_first_fit(layout) {
Ok(v) => v.as_ptr(),
Err(e) => {
errorln!("Failed to allocate {:?}: {:?}", layout, e);
// errorln!("Failed to allocate {:?}: {:?}", layout, e);
null_mut()
}
}

150
src/mem/mod.rs
View File

@ -1,57 +1,75 @@
//! Memory management utilities and types
// use core::{alloc::Layout, mem::size_of};
// //! Memory management utilities and types
// // use core::{alloc::Layout, mem::size_of};
//
// use core::{alloc::Layout, ffi::c_void, mem::size_of};
//
// use abi::error::Error;
//
// // use abi::error::Error;
// //
// use crate::arch::{Architecture, ArchitectureImpl /*, PlatformImpl*/};
//
// use self::table::AddressSpace;
// //
// // use self::table::AddressSpace;
//
// pub mod device;
use core::{alloc::Layout, ffi::c_void, mem::size_of};
use core::{alloc::Layout, ffi::c_void, mem::size_of, ops::Add};
use abi::error::Error;
// use abi::error::Error;
//
use crate::arch::{Architecture, ArchitectureImpl /*, PlatformImpl*/};
use self::table::AddressSpace;
//
// use self::table::AddressSpace;
use crate::arch::{Architecture, ArchitectureImpl};
pub mod address;
pub mod device;
pub mod heap;
pub mod phys;
pub mod pointer;
pub mod table;
/// Kernel's physical load address
// pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE;
/// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] +
/// [KERNEL_VIRT_OFFSET])
pub use address::PhysicalAddress;
use self::table::AddressSpace;
pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET;
/// Interface for converting between address spaces.
///
/// # Safety
///
/// An incorrect implementation can produce invalid address.
pub unsafe trait ConvertAddress {
/// Convert the address into a virtual one
///
/// # Panics
///
/// Panics if the address is already a virtual one
///
/// # Safety
///
/// An incorrect implementation can produce invalid address.
unsafe fn virtualize(self) -> Self;
/// Convert the address into a physical one
///
/// # Panics
///
/// Panics if the address is already a physical one
///
/// # Safety
///
/// An incorrect implementation can produce invalid address.
unsafe fn physicalize(self) -> Self;
}
// pub mod phys;
//
// /// Kernel's physical load address
// // pub const KERNEL_PHYS_BASE: usize = PlatformImpl::KERNEL_PHYS_BASE;
// /// Kernel's virtual memory mapping offset (i.e. kernel's virtual address is [KERNEL_PHYS_BASE] +
// /// [KERNEL_VIRT_OFFSET])
// pub const KERNEL_VIRT_OFFSET: usize = ArchitectureImpl::KERNEL_VIRT_OFFSET;
//
// /// Interface for converting between address spaces.
// ///
// /// # Safety
// ///
// /// An incorrect implementation can produce invalid address.
// pub unsafe trait ConvertAddress {
// /// Convert the address into a virtual one
// ///
// /// # Panics
// ///
// /// Panics if the address is already a virtual one
// ///
// /// # Safety
// ///
// /// An incorrect implementation can produce invalid address.
// unsafe fn virtualize(self) -> Self;
// /// Convert the address into a physical one
// ///
// /// # Panics
// ///
// /// Panics if the address is already a physical one
// ///
// /// # Safety
// ///
// /// An incorrect implementation can produce invalid address.
// unsafe fn physicalize(self) -> Self;
// }
//
/// Helper trait to allow cross-address space access to pointers
pub trait ForeignPointer: Sized {
/// Perform a volatile pointer write without dropping the old value.
@ -120,52 +138,6 @@ pub trait ForeignPointer: Sized {
) -> Result<&'a mut [Self], Error>;
}
unsafe impl ConvertAddress for usize {
#[inline(always)]
unsafe fn virtualize(self) -> Self {
#[cfg(debug_assertions)]
if self > KERNEL_VIRT_OFFSET {
todo!();
}
self + KERNEL_VIRT_OFFSET
}
#[inline(always)]
unsafe fn physicalize(self) -> Self {
#[cfg(debug_assertions)]
if self < KERNEL_VIRT_OFFSET {
todo!();
}
self - KERNEL_VIRT_OFFSET
}
}
unsafe impl<T> ConvertAddress for *mut T {
#[inline(always)]
unsafe fn virtualize(self) -> Self {
(self as usize).virtualize() as Self
}
#[inline(always)]
unsafe fn physicalize(self) -> Self {
(self as usize).physicalize() as Self
}
}
unsafe impl<T> ConvertAddress for *const T {
#[inline(always)]
unsafe fn virtualize(self) -> Self {
(self as usize).virtualize() as Self
}
#[inline(always)]
unsafe fn physicalize(self) -> Self {
(self as usize).physicalize() as Self
}
}
impl<T> ForeignPointer for T {
unsafe fn write_foreign_volatile(self: *mut Self, space: &AddressSpace, value: T) {
// TODO check align
@ -182,7 +154,7 @@ impl<T> ForeignPointer for T {
.translate(start_page)
.expect("Address is not mapped in the target address space");
let virt_ptr = (phys_page + page_offset).virtualize() as *mut T;
let virt_ptr = phys_page.add(page_offset).virtualize_raw() as *mut T;
virt_ptr.write_volatile(value);
}

190
src/mem/phys/manager.rs
View File

@ -3,89 +3,132 @@ use core::mem::size_of;
use abi::error::Error;
use super::{Page, PageUsage, PhysicalMemoryStats};
use crate::mem::{
address::{FromRaw, IntoRaw},
pointer::{PhysicalRef, PhysicalRefMut},
PhysicalAddress,
};
pub type BitmapWord = u64;
pub(super) const BITMAP_WORD_SIZE: usize = size_of::<BitmapWord>() * 8;
pub(super) const BITMAP_PAGE_COUNT: usize = 256;
const HUGE_PAGE_WORD_COUNT: usize = 512 / BITMAP_WORD_SIZE;
pub(super) const TRACKED_PAGE_LIMIT: usize = (BITMAP_PAGE_COUNT * 4096) * 8;
/// Physical memory management interface
pub struct PhysicalMemoryManager {
pages: &'static mut [Page],
offset: usize,
stats: PhysicalMemoryStats,
bitmap: PhysicalRefMut<'static, [u64]>,
last_free_bit: usize,
page_count: usize,
}
impl PhysicalMemoryManager {
/// Constructs a [PhysicalMemoryManager] with page tracking array placed at given
/// `base`..`base+size` range. Physical addresses allocated are offset by the given value.
///
/// # Safety
///
/// Addresses are not checked. The caller is responsible for making sure (base, size) ranges do
/// not alias/overlap, they're accessible through virtual memory and that the offset is a
/// meaningful value.
pub unsafe fn new(offset: usize, base: usize, size: usize) -> PhysicalMemoryManager {
// TODO check alignment
let page_count = size / size_of::<Page>();
let pages = core::slice::from_raw_parts_mut(base as *mut _, page_count);
pub unsafe fn new(
bitmap_phys_base: PhysicalAddress,
page_count: usize,
) -> PhysicalMemoryManager {
// let bitmap_addr = bitmap_phys_base.virtualize();
let bitmap_len = (page_count + (BITMAP_WORD_SIZE - 1)) / BITMAP_WORD_SIZE;
let mut bitmap = PhysicalRefMut::<'static, u64>::map_slice(bitmap_phys_base, bitmap_len);
// let bitmap = core::slice::from_raw_parts_mut(bitmap_addr as *mut BitmapWord, bitmap_len);
for page in pages.iter_mut() {
*page = Page {
usage: PageUsage::Reserved,
refcount: 0,
};
}
bitmap.fill(BitmapWord::MAX);
PhysicalMemoryManager {
pages,
offset,
stats: PhysicalMemoryStats {
available_pages: 0,
used_pages: 0,
},
Self {
bitmap,
page_count,
last_free_bit: 0,
}
}
#[inline]
fn mark_alloc(&mut self, index: usize) {
self.bitmap[index / BITMAP_WORD_SIZE] |= 1 << (index & (BITMAP_WORD_SIZE - 1));
}
#[inline]
fn mark_free(&mut self, index: usize) {
self.bitmap[index / BITMAP_WORD_SIZE] &= !(1 << (index & (BITMAP_WORD_SIZE - 1)));
}
#[inline(always)]
fn is_alloc(&self, index: usize) -> bool {
self.bitmap[index / BITMAP_WORD_SIZE] & (1 << (index & (BITMAP_WORD_SIZE - 1))) != 0
}
/// Allocates a single page, marking it as used with `usage`
pub fn alloc_page(&mut self, usage: PageUsage) -> Result<usize, Error> {
assert_ne!(usage, PageUsage::Available);
assert_ne!(usage, PageUsage::Reserved);
for index in 0..self.pages.len() {
if self.pages[index].usage == PageUsage::Available {
self.pages[index].usage = PageUsage::Used;
self.stats.add_allocated_pages(1, usage);
return Ok(index * 4096 + self.offset);
pub fn alloc_page(&mut self) -> Result<PhysicalAddress, Error> {
for i in self.last_free_bit..self.page_count {
if self.is_alloc(i) {
continue;
}
self.last_free_bit = i + 1;
self.mark_alloc(i);
return Ok(PhysicalAddress::from_raw(i * 0x1000));
}
Err(Error::OutOfMemory)
if self.last_free_bit != 0 {
self.last_free_bit = 0;
self.alloc_page()
} else {
loop {}
}
}
/// Allocates a contiguous range of physical pages, marking it as used with `usage`
pub fn alloc_contiguous_pages(
&mut self,
count: usize,
usage: PageUsage,
) -> Result<usize, Error> {
assert_ne!(usage, PageUsage::Available);
assert_ne!(usage, PageUsage::Reserved);
assert_ne!(count, 0);
pub fn alloc_2m_page(&mut self) -> Result<PhysicalAddress, Error> {
let aligned_bit = self.last_free_bit & !511;
'l0: for i in 0..self.pages.len() {
for j in 0..count {
if self.pages[i + j].usage != PageUsage::Available {
'l0: for i in (aligned_bit..self.page_count).step_by(512) {
for j in 0..HUGE_PAGE_WORD_COUNT {
if self.bitmap[i / BITMAP_WORD_SIZE] != 0 {
continue 'l0;
}
}
for j in 0..count {
let page = &mut self.pages[i + j];
assert!(page.usage == PageUsage::Available);
page.usage = usage;
page.refcount = 1;
for j in 0..HUGE_PAGE_WORD_COUNT {
self.bitmap[i / BITMAP_WORD_SIZE + j] = BitmapWord::MAX;
}
self.stats.add_allocated_pages(count, usage);
return Ok(self.offset + i * 0x1000);
self.last_free_bit = i + 512;
return Ok(PhysicalAddress::from_raw(i * 0x1000));
}
Err(Error::OutOfMemory)
if self.last_free_bit != 0 {
self.last_free_bit = 0;
self.alloc_2m_page()
} else {
loop {}
}
}
/// Allocates a contiguous range of physical pages, marking it as used with `usage`
pub fn alloc_contiguous_pages(&mut self, count: usize) -> Result<PhysicalAddress, Error> {
'l0: for i in self.last_free_bit..self.page_count {
for j in 0..count {
if self.is_alloc(i + j) {
continue 'l0;
}
}
for j in 0..count {
self.mark_alloc(i + j);
}
self.last_free_bit = i + count;
return Ok(PhysicalAddress::from_raw(i * 0x1000));
}
if self.last_free_bit != 0 {
self.last_free_bit = 0;
self.alloc_contiguous_pages(count)
} else {
loop {}
}
}
/// Deallocates a physical memory page.
@ -93,15 +136,11 @@ impl PhysicalMemoryManager {
/// # Safety
///
/// `addr` must be a page-aligned physical address previously allocated by this implementation.
pub unsafe fn free_page(&mut self, addr: usize) {
assert!(addr > self.offset);
let index = (addr - self.offset) / 0x1000;
let page = &mut self.pages[index];
assert_eq!(page.usage, PageUsage::Used);
pub unsafe fn free_page(&mut self, page: PhysicalAddress) {
let index = IntoRaw::<usize>::into_raw(page) / 0x1000;
self.stats.add_freed_pages(1, page.usage);
page.usage = PageUsage::Available;
assert!(self.is_alloc(index));
self.mark_free(index);
}
/// Marks a previously reserved page as available.
@ -109,19 +148,10 @@ impl PhysicalMemoryManager {
/// # Panics
///
/// Will panic if the address does not point to a valid, reserved (and unallocated) page.
pub fn add_available_page(&mut self, addr: usize) {
assert!(addr >= self.offset);
let index = (addr - self.offset) / 4096;
pub fn add_available_page(&mut self, page: PhysicalAddress) {
let index = IntoRaw::<usize>::into_raw(page) / 0x1000;
assert_eq!(self.pages[index].usage, PageUsage::Reserved);
assert_eq!(self.pages[index].refcount, 0);
self.stats.add_available_pages(1);
self.pages[index].usage = PageUsage::Available;
}
/// Returns a reference to physical memory stats info
pub fn stats(&self) -> &PhysicalMemoryStats {
&self.stats
assert!(self.is_alloc(index));
self.mark_free(index);
}
}

347
src/mem/phys/mod.rs
View File

@ -1,135 +1,164 @@
//! Physical memory management facilities
use core::{iter::StepBy, mem::size_of, ops::Range};
use core::{iter::StepBy, ops::Range};
use abi::error::Error;
use kernel_util::util::OneTimeInit;
use crate::{
debug::LogLevel,
mem::{
phys::reserved::{is_reserved, reserve_region},
ConvertAddress, /*, KERNEL_PHYS_BASE */
},
arch::{Architecture, ARCHITECTURE},
mem::phys::reserved::is_reserved,
sync::IrqSafeSpinlock,
};
use self::manager::PhysicalMemoryManager;
use self::{
manager::{PhysicalMemoryManager, BITMAP_PAGE_COUNT, BITMAP_WORD_SIZE, TRACKED_PAGE_LIMIT},
reserved::reserve_region,
};
// Enumerating lots of pages is slow and I'm too lazy right now to write a better algorithm, so
// capping the page count helps
const PHYS_MEMORY_PAGE_CAP: usize = 65536;
use super::{address::FromRaw, PhysicalAddress};
pub mod manager;
// //! Physical memory management facilities
// use core::{iter::StepBy, mem::size_of, ops::Range};
//
// use abi::error::Error;
// use kernel_util::util::OneTimeInit;
//
// use crate::{
// debug::LogLevel,
// mem::{
// phys::reserved::{is_reserved, reserve_region},
// ConvertAddress, /*, KERNEL_PHYS_BASE */
// },
// sync::IrqSafeSpinlock,
// };
//
// use self::manager::PhysicalMemoryManager;
//
// // Enumerating lots of pages is slow and I'm too lazy right now to write a better algorithm, so
// // capping the page count helps
// const PHYS_MEMORY_PAGE_CAP: usize = 65536;
//
// 8 * 4096 bits per page, 1 page per bit
const MEMORY_UPPER_LIMIT: PhysicalAddress = PhysicalAddress::from_raw(TRACKED_PAGE_LIMIT * 4096);
mod manager;
pub mod reserved;
/// Contains information about the physical memory usage
#[derive(Clone, Copy, Debug)]
pub struct PhysicalMemoryStats {
/// Number of pages available for allocation
pub available_pages: usize,
/// Number of pages being used
pub used_pages: usize,
}
/// Represents the way in which the page is used (or not)
#[derive(PartialEq, Clone, Copy, Debug)]
#[repr(u32)]
pub enum PageUsage {
/// Page is not available for allocation or use
Reserved = 0,
/// Regular page available for allocation
Available,
/// Page is used by some kernel facility
Used,
}
/// Page descriptor structure for the page management array
#[repr(C)]
pub struct Page {
usage: PageUsage,
refcount: u32,
}
//
// /// Contains information about the physical memory usage
// #[derive(Clone, Copy, Debug)]
// pub struct PhysicalMemoryStats {
// /// Number of pages available for allocation
// pub available_pages: usize,
// /// Number of pages being used
// pub used_pages: usize,
// }
//
// /// Represents the way in which the page is used (or not)
// #[derive(PartialEq, Clone, Copy, Debug)]
// #[repr(u32)]
// pub enum PageUsage {
// /// Page is not available for allocation or use
// Reserved = 0,
// /// Regular page available for allocation
// Available,
// /// Page is used by some kernel facility
// Used,
// }
//
// /// Page descriptor structure for the page management array
// #[repr(C)]
// pub struct Page {
// usage: PageUsage,
// refcount: u32,
// }
//
/// Defines an usable memory region
#[derive(Clone, Copy, Debug)]
pub struct PhysicalMemoryRegion {
/// Start of the region
pub base: usize,
pub base: PhysicalAddress,
/// Length of the region
pub size: usize,
}
impl PhysicalMemoryRegion {
/// Returns the end address of the region
pub const fn end(&self) -> usize {
self.base + self.size
pub const fn end(&self) -> PhysicalAddress {
self.base.add(self.size)
}
/// Returns an address range covered by the region
pub fn range(&self) -> Range<usize> {
pub fn range(&self) -> Range<PhysicalAddress> {
self.base..self.end()
}
/// Provides an iterator over the pages in the region
pub fn pages(&self) -> StepBy<Range<usize>> {
self.range().step_by(0x1000)
pub fn clamp(self) -> Option<(PhysicalAddress, PhysicalAddress)> {
let start = self.base.min(MEMORY_UPPER_LIMIT);
let end = self.end().min(MEMORY_UPPER_LIMIT);
if start < end {
Some((start, end))
} else {
None
}
}
}
impl PhysicalMemoryStats {
/// Handles "alloc" cases of the memory manager
pub fn add_allocated_pages(&mut self, count: usize, _usage: PageUsage) {
assert!(self.available_pages >= count);
self.available_pages -= count;
self.used_pages += count;
}
/// Handles "free" cases of the memory manager
pub fn add_freed_pages(&mut self, count: usize, _usage: PageUsage) {
assert!(self.used_pages >= count);
self.used_pages -= count;
self.available_pages += count;
}
/// Increases the available pages counter
pub fn add_available_pages(&mut self, count: usize) {
self.available_pages += count;
}
/// Prints out the statistics into specified log level
pub fn dump(&self, level: LogLevel) {
log_print_raw!(level, "+++ Physical memory stats +++\n");
log_print_raw!(
level,
"Available: {}K ({} pages)\n",
self.available_pages * 4,
self.available_pages
);
log_print_raw!(
level,
"Used: {}K ({} pages)\n",
self.used_pages * 4,
self.used_pages
);
log_print_raw!(level, "-----------------------------\n");
}
}
//
// impl PhysicalMemoryStats {
// /// Handles "alloc" cases of the memory manager
// pub fn add_allocated_pages(&mut self, count: usize, _usage: PageUsage) {
// assert!(self.available_pages >= count);
// self.available_pages -= count;
// self.used_pages += count;
// }
//
// /// Handles "free" cases of the memory manager
// pub fn add_freed_pages(&mut self, count: usize, _usage: PageUsage) {
// assert!(self.used_pages >= count);
// self.used_pages -= count;
// self.available_pages += count;
// }
//
// /// Increases the available pages counter
// pub fn add_available_pages(&mut self, count: usize) {
// self.available_pages += count;
// }
//
// /// Prints out the statistics into specified log level
// pub fn dump(&self, level: LogLevel) {
// log_print_raw!(level, "+++ Physical memory stats +++\n");
// log_print_raw!(
// level,
// "Available: {}K ({} pages)\n",
// self.available_pages * 4,
// self.available_pages
// );
// log_print_raw!(
// level,
// "Used: {}K ({} pages)\n",
// self.used_pages * 4,
// self.used_pages
// );
// log_print_raw!(level, "-----------------------------\n");
// }
// }
//
/// Global physical memory manager
pub static PHYSICAL_MEMORY: OneTimeInit<IrqSafeSpinlock<PhysicalMemoryManager>> =
OneTimeInit::new();
/// Allocates a single physical page from the global manager
pub fn alloc_page(usage: PageUsage) -> Result<usize, Error> {
PHYSICAL_MEMORY.get().lock().alloc_page(usage)
pub fn alloc_page() -> Result<PhysicalAddress, Error> {
PHYSICAL_MEMORY.get().lock().alloc_page()
}
/// Allocates a contiguous range of physical pages from the global manager
pub fn alloc_pages_contiguous(count: usize, usage: PageUsage) -> Result<usize, Error> {
PHYSICAL_MEMORY
.get()
.lock()
.alloc_contiguous_pages(count, usage)
pub fn alloc_pages_contiguous(count: usize) -> Result<PhysicalAddress, Error> {
PHYSICAL_MEMORY.get().lock().alloc_contiguous_pages(count)
}
pub fn alloc_2m_page() -> Result<PhysicalAddress, Error> {
PHYSICAL_MEMORY.get().lock().alloc_2m_page()
}
/// Deallocates a physical memory page.
@ -137,26 +166,26 @@ pub fn alloc_pages_contiguous(count: usize, usage: PageUsage) -> Result<usize, E
/// # Safety
///
/// `addr` must be a page-aligned physical address previously allocated by this implementation.
pub unsafe fn free_page(addr: usize) {
pub unsafe fn free_page(addr: PhysicalAddress) {
PHYSICAL_MEMORY.get().lock().free_page(addr)
}
fn physical_memory_range<I: Iterator<Item = PhysicalMemoryRegion>>(
it: I,
) -> Option<(usize, usize)> {
let mut start = usize::MAX;
let mut end = usize::MIN;
) -> Option<(PhysicalAddress, PhysicalAddress)> {
let mut start = PhysicalAddress::MAX;
let mut end = PhysicalAddress::MIN;
for reg in it {
if reg.base < start {
start = reg.base;
for (reg_start, reg_end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) {
if reg_start < start {
start = reg_start;
}
if reg.base + reg.size > end {
end = reg.base + reg.size;
if reg_end > end {
end = reg_end;
}
}
if start == usize::MAX || end == usize::MIN {
if start == PhysicalAddress::MAX || end == PhysicalAddress::MIN {
None
} else {
Some((start, end))
@ -166,12 +195,12 @@ fn physical_memory_range<I: Iterator<Item = PhysicalMemoryRegion>>(
fn find_contiguous_region<I: Iterator<Item = PhysicalMemoryRegion>>(
it: I,
count: usize,
) -> Option<usize> {
for region in it {
) -> Option<PhysicalAddress> {
for (reg_start, reg_end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) {
let mut collected = 0;
let mut base_addr = None;
for addr in region.pages() {
for addr in (reg_start..reg_end).step_by(0x1000) {
if is_reserved(addr) {
collected = 0;
base_addr = None;
@ -188,7 +217,7 @@ fn find_contiguous_region<I: Iterator<Item = PhysicalMemoryRegion>>(
}
todo!()
}
//
/// Initializes physical memory manager from given available memory region iterator.
///
/// 1. Finds a non-reserved range to place the page tracking array.
@ -201,68 +230,102 @@ fn find_contiguous_region<I: Iterator<Item = PhysicalMemoryRegion>>(
pub unsafe fn init_from_iter<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
it: I,
) -> Result<(), Error> {
// Map the physical memory
let (phys_start, phys_end) = physical_memory_range(it.clone()).unwrap();
ARCHITECTURE.map_physical_memory(it.clone(), phys_start, phys_end)?;
let total_count = (phys_end - phys_start) / 0x1000;
let pages_array_size = total_count * size_of::<Page>();
let page_bitmap_size = (total_count + BITMAP_WORD_SIZE - 1) / BITMAP_WORD_SIZE;
let page_bitmap_page_count = (page_bitmap_size + 0xFFF) / 0x1000;
debugln!("Initializing physical memory manager");
debugln!("Total tracked pages: {}", total_count);
// Reserve memory regions from which allocation is forbidden
reserve_region("kernel", kernel_physical_memory_region());
let pages_array_base = find_contiguous_region(it.clone(), (pages_array_size + 0xFFF) / 0x1000)
.ok_or(Error::OutOfMemory)?;
debugln!(
"Placing page tracking at {:#x}",
pages_array_base.virtualize()
);
let page_bitmap_phys_base = find_contiguous_region(it.clone(), page_bitmap_page_count).unwrap();
reserve_region(
"pages",
"page-bitmap",
PhysicalMemoryRegion {
base: pages_array_base,
size: (pages_array_size + 0xFFF) & !0xFFF,
base: page_bitmap_phys_base,
size: page_bitmap_page_count * 0x1000,
},
);
let mut manager =
PhysicalMemoryManager::new(phys_start, pages_array_base.virtualize(), pages_array_size);
let mut page_count = 0;
let mut manager = PhysicalMemoryManager::new(page_bitmap_phys_base, total_count);
for region in it {
if page_count >= PHYS_MEMORY_PAGE_CAP {
break;
}
for page in region.pages() {
for (start, end) in it.into_iter().filter_map(PhysicalMemoryRegion::clamp) {
for page in (start..end).step_by(0x1000) {
if is_reserved(page) {
continue;
}
manager.add_available_page(page);
page_count += 1;
if page_count >= PHYS_MEMORY_PAGE_CAP {
break;
}
}
}
infoln!("{} available pages ({}KiB)", page_count, page_count * 4);
PHYSICAL_MEMORY.init(IrqSafeSpinlock::new(manager));
Ok(())
}
//
// debugln!("Initializing physical memory manager");
// debugln!("Total tracked pages: {}", total_count);
//
// // Reserve memory regions from which allocation is forbidden
// reserve_region("kernel", kernel_physical_memory_region());
//
// let pages_array_base = find_contiguous_region(it.clone(), (pages_array_size + 0xFFF) / 0x1000)
// .ok_or(Error::OutOfMemory)?;
//
// debugln!(
// "Placing page tracking at {:#x}",
// pages_array_base.virtualize()
// );
//
// reserve_region(
// "pages",
// PhysicalMemoryRegion {
// base: pages_array_base,
// size: (pages_array_size + 0xFFF) & !0xFFF,
// },
// );
//
// let mut manager =
// PhysicalMemoryManager::new(phys_start, pages_array_base.virtualize(), pages_array_size);
// let mut page_count = 0;
//
// for region in it {
// if page_count >= PHYS_MEMORY_PAGE_CAP {
// break;
// }
//
// for page in region.pages() {
// if is_reserved(page) {
// continue;
// }
//
// manager.add_available_page(page);
// page_count += 1;
//
// if page_count >= PHYS_MEMORY_PAGE_CAP {
// break;
// }
// }
// }
//
// infoln!("{} available pages ({}KiB)", page_count, page_count * 4);
//
// PHYSICAL_MEMORY.init(IrqSafeSpinlock::new(manager));
// Ok(())
// }
//
fn kernel_physical_memory_region() -> PhysicalMemoryRegion {
extern "C" {
static __kernel_phys_start: u8;
static __kernel_size: u8;
}
let base = absolute_address!(__kernel_phys_start);
let base = PhysicalAddress::from_raw(absolute_address!(__kernel_phys_start));
let size = absolute_address!(__kernel_size);
PhysicalMemoryRegion { base, size }

16
src/mem/phys/reserved.rs
View File

@ -2,6 +2,8 @@
use kernel_util::util::StaticVector;
use crate::mem::PhysicalAddress;
use super::PhysicalMemoryRegion;
static mut RESERVED_MEMORY: StaticVector<PhysicalMemoryRegion, 8> = StaticVector::new();
@ -12,18 +14,18 @@ static mut RESERVED_MEMORY: StaticVector<PhysicalMemoryRegion, 8> = StaticVector
///
/// Can only be called from initialization code **before** physical memory manager is initialized.
pub unsafe fn reserve_region(reason: &str, region: PhysicalMemoryRegion) {
debugln!(
"Reserve {:?} memory: {:#x}..{:#x}",
reason,
region.base,
region.end()
);
// debugln!(
// "Reserve {:?} memory: {:#x}..{:#x}",
// reason,
// region.base,
// region.end()
// );
RESERVED_MEMORY.push(region);
}
/// Returns `true` if `addr` refers to any reserved memory region
pub fn is_reserved(addr: usize) -> bool {
pub fn is_reserved(addr: PhysicalAddress) -> bool {
for region in unsafe { RESERVED_MEMORY.iter() } {
if region.range().contains(&addr) {
return true;

132
src/mem/pointer.rs Normal file
View File

@ -0,0 +1,132 @@
use core::{
alloc::Layout,
fmt,
ops::{Deref, DerefMut},
};
use super::{address::AsPhysicalAddress, PhysicalAddress};
#[derive(Clone, Copy, PartialEq, PartialOrd, Debug, Hash)]
#[repr(transparent)]
pub struct PhysicalPointer<T: ?Sized> {
pointer: *mut T,
}
#[repr(transparent)]
pub struct PhysicalRef<'a, T: ?Sized> {
value: &'a T,
}
#[repr(transparent)]
pub struct PhysicalRefMut<'a, T: ?Sized> {
value: &'a mut T,
}
// PhysicalPointer<T> wrapper for direct access to any memory location
impl<T: ?Sized> PhysicalPointer<T> {
pub fn into_address(self) -> usize {
self.pointer.addr()
}
}
impl<T: Sized> PhysicalPointer<T> {
pub unsafe fn write_unaligned(self, value: T) {
self.write_unaligned(value);
}
}
impl<T: ?Sized> AsPhysicalAddress for PhysicalPointer<T> {
unsafe fn as_physical_address(&self) -> PhysicalAddress {
todo!()
}
}
// PhysicalRefMut<T> wrapper for safe mutable access to physical addresses
impl<'a, T: Sized> PhysicalRefMut<'a, T> {
pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRefMut<'a, T> {
let value = virtualize_raw(physical);
PhysicalRefMut { value }
}
pub unsafe fn map_slice(physical: PhysicalAddress, len: usize) -> PhysicalRefMut<'a, [T]> {
let value = virtualize_slice_raw(physical, len);
PhysicalRefMut { value }
}
}
impl<T: ?Sized> PhysicalRefMut<'_, T> {
#[inline]
pub fn as_address(&self) -> usize {
(self.value as *const T).addr()
}
}
impl<T: ?Sized> AsPhysicalAddress for PhysicalRefMut<'_, T> {
unsafe fn as_physical_address(&self) -> PhysicalAddress {
PhysicalAddress::from_virtualized(self.as_address())
}
}
impl<T: ?Sized> Deref for PhysicalRefMut<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.value
}
}
impl<T: ?Sized> DerefMut for PhysicalRefMut<'_, T> {
fn deref_mut(&mut self) -> &mut Self::Target {
self.value
}
}
impl<T: ?Sized> fmt::Pointer for PhysicalRefMut<'_, T> {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
fmt::Pointer::fmt(&self.value, f)
}
}
// PhysicalRef<T>: same as PhysicalRefMut<T>, except immutable
impl<'a, T: Sized> PhysicalRef<'a, T> {
pub unsafe fn map(physical: PhysicalAddress) -> PhysicalRef<'a, T> {
let value = virtualize_raw(physical);
PhysicalRef { value }
}
}
impl<T: ?Sized> PhysicalRef<'_, T> {
#[inline]
pub fn as_address(&self) -> usize {
(self.value as *const T).addr()
}
}
impl<T: ?Sized> AsPhysicalAddress for PhysicalRef<'_, T> {
unsafe fn as_physical_address(&self) -> PhysicalAddress {
PhysicalAddress::from_virtualized(self.as_address())
}
}
impl<T: ?Sized> Deref for PhysicalRef<'_, T> {
type Target = T;
fn deref(&self) -> &Self::Target {
self.value
}
}
unsafe fn virtualize_raw<'a, T: Sized>(physical: PhysicalAddress) -> &'a mut T {
// TODO check align
let address = physical.virtualize_raw();
&mut *(address as *mut T)
}
unsafe fn virtualize_slice_raw<'a, T: Sized>(physical: PhysicalAddress, len: usize) -> &'a mut [T] {
// TODO check align
let address = physical.virtualize_raw();
core::slice::from_raw_parts_mut(address as *mut T, len)
}

23
src/mem/table.rs
View File

@ -1,13 +1,17 @@
//! Virtual memory table interface
use core::ops::{Deref, DerefMut};
use abi::error::Error;
use bitflags::bitflags;
use cfg_if::cfg_if;
use super::PhysicalAddress;
cfg_if! {
if #[cfg(target_arch = "aarch64")] {
pub use crate::arch::aarch64::table::{AddressSpace, PageAttributes, PageEntry, PageTable};
} else if #[cfg(target_arch = "x86_64")] {
pub use crate::arch::x86_64::table::{AddressSpace, PageEntry, PageTable};
pub use crate::arch::x86_64::mem::table::{AddressSpace, PageEntry, PageTable};
}
}
@ -36,7 +40,12 @@ pub trait VirtualMemoryManager {
) -> Result<usize, Error>;
/// Insert a single 4KiB-page translation mapping into the table
fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error>;
fn map_page(
&self,
virt: usize,
phys: PhysicalAddress,
attrs: MapAttributes,
) -> Result<(), Error>;
/// Releases the virtual memory region from the address space and the pages it refers to
fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error>;
@ -46,17 +55,23 @@ pub trait VirtualMemoryManager {
pub trait NextPageTable {
/// Type for the next-level page table
type NextLevel;
type TableRef: Deref<Target = Self::NextLevel>;
type TableRefMut: DerefMut<Target = Self::NextLevel>;
/// Tries looking up a next-level table at given index, allocating and mapping one if it is not
/// present there
fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error>;
fn get_mut_or_alloc(&mut self, index: usize) -> Result<Self::TableRefMut, Error>;
/// Returns a mutable reference to a next-level table at `index`, if present
fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel>;
fn get_mut(&mut self, index: usize) -> Option<Self::TableRefMut>;
fn get(&self, index: usize) -> Option<Self::TableRef>;
}
/// Interface for a single level of address translation
#[const_trait]
pub trait EntryLevel: Copy {
const SIZE: usize;
/// Returns the index into a page table for a given address
fn index(addr: usize) -> usize;
/// Returns the offset of an address from the page start at current level

31
src/proc/elf.rs
View File

@ -1,4 +1,6 @@
//! ELF binary format support
use core::ops::DerefMut;
use elf::{
abi::{PF_W, PF_X, PT_LOAD},
endian::AnyEndian,
@ -8,9 +10,9 @@ use vfs::{FileRef, Read, Seek};
use yggdrasil_abi::{error::Error, io::SeekFrom};
use crate::mem::{
phys::{self, PageUsage},
phys,
pointer::PhysicalRefMut,
table::{AddressSpace, MapAttributes, VirtualMemoryManager},
ConvertAddress,
};
#[derive(Clone, Copy)]
@ -64,7 +66,7 @@ fn load_bytes<F>(
elf_attrs: u32,
) -> Result<(), Error>
where
F: FnMut(usize, &mut [u8]) -> Result<(), Error>,
F: FnMut(usize, PhysicalRefMut<'_, [u8]>) -> Result<(), Error>,
{
// TODO check for crazy addresses here
@ -87,19 +89,22 @@ where
let virt_page = dst_page_aligned + page_idx * 0x1000;
assert_eq!(virt_page & 0xFFF, 0);
if space.translate(virt_page).is_some() {
// Handle these cases
if let Some(page) = space.translate(virt_page) {
// TODO Handle these cases
warnln!("Page {:#x} is already mapped to {:#x}", virt_page, page);
todo!();
}
let phys_page = phys::alloc_page(PageUsage::Used)?;
let phys_page = phys::alloc_page()?;
space.map_page(virt_page, phys_page, attrs)?;
debugln!("Map {:#x} -> {:#x}", virt_page, phys_page);
let dst_slice = unsafe {
let addr = (phys_page + page_off).virtualize();
let dst_slice = unsafe { PhysicalRefMut::map_slice(phys_page.add(page_off), count) };
// let dst_slice = unsafe {
// let addr = (phys_page + page_off).virtualize();
core::slice::from_raw_parts_mut(addr as *mut u8, count)
};
// core::slice::from_raw_parts_mut(addr as *mut u8, count)
// };
src(off, dst_slice)?;
@ -127,10 +132,10 @@ pub fn load_elf_from_file(space: &AddressSpace, file: FileRef) -> Result<usize,
load_bytes(
space,
phdr.p_vaddr as usize,
|off, dst| {
|off, mut dst| {
let mut source = file.file.borrow_mut();
source.seek(SeekFrom::Start(phdr.p_offset + off as u64))?;
source.read_exact(dst)
source.read_exact(dst.deref_mut())
},
phdr.p_filesz as usize,
phdr.p_flags,
@ -144,7 +149,7 @@ pub fn load_elf_from_file(space: &AddressSpace, file: FileRef) -> Result<usize,
load_bytes(
space,
addr,
|_, dst| {
|_, mut dst| {
dst.fill(0);
Ok(())
},

12
src/proc/exec.rs
View File

@ -7,9 +7,9 @@ use vfs::FileRef;
use crate::{
mem::{
phys::{self, PageUsage},
phys,
table::{AddressSpace, MapAttributes, VirtualMemoryManager},
ConvertAddress, ForeignPointer,
ForeignPointer,
},
proc,
task::{context::TaskContextImpl, process::Process, TaskContext},
@ -29,15 +29,15 @@ fn setup_args(space: &AddressSpace, virt: usize, args: &[&str]) -> Result<(), Er
debugln!("arg data size = {}", args_size);
let phys_page = phys::alloc_page(PageUsage::Used)?;
let phys_page = phys::alloc_page()?;
// TODO check if this doesn't overwrite anything
space.map_page(
virt,
phys_page,
MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, // PageAttributes::AP_BOTH_READWRITE | PageAttributes::NON_GLOBAL,
MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL,
)?;
let write = unsafe { phys_page.virtualize() };
let write = phys_page.virtualize_raw();
let mut offset = args_ptr_size;
@ -83,7 +83,7 @@ fn setup_binary<S: Into<String>>(
let virt_args_base = virt_stack_base + (USER_STACK_PAGES + 1) * 0x1000;
for i in 0..USER_STACK_PAGES {
let phys = phys::alloc_page(PageUsage::Used)?;
let phys = phys::alloc_page()?;
space.map_page(
virt_stack_base + i * 0x1000,
phys,

9
src/task/context.rs
View File

@ -4,7 +4,7 @@ use abi::{arch::SavedFrame, error::Error, process::ExitCode};
use alloc::boxed::Box;
use cfg_if::cfg_if;
use crate::task::process::Process;
use crate::{mem::PhysicalAddress, task::process::Process};
cfg_if! {
if #[cfg(target_arch = "aarch64")] {
@ -56,7 +56,12 @@ pub trait TaskContextImpl: Sized {
/// Constructs a user thread context. The caller is responsible for allocating the userspace
/// stack and setting up a valid address space for the context.
fn user(entry: usize, arg: usize, cr3: usize, user_stack_sp: usize) -> Result<Self, Error>;
fn user(
entry: usize,
arg: usize,
cr3: PhysicalAddress,
user_stack_sp: usize,
) -> Result<Self, Error>;
/// Performs an entry into a context.
///

9
src/task/process.rs
View File

@ -82,7 +82,7 @@ pub struct Process {
}
/// Guard type that provides [Process] operations only available for current processes
pub struct CurrentProcess(Arc<Process>);
pub struct CurrentProcess(Arc<Process>, IrqGuard);
impl Process {
/// Creates a process from raw architecture-specific [TaskContext].
@ -447,11 +447,8 @@ impl CurrentProcess {
/// # Safety
///
/// Only meant to be called from [Process::current] or [CpuQueue::current_process].
pub unsafe fn new(inner: Arc<Process>) -> Self {
// XXX
// assert_eq!(DAIF.read(DAIF::I), 1);
assert!(ArchitectureImpl::interrupt_mask());
Self(inner)
pub unsafe fn new(inner: Arc<Process>, guard: IrqGuard) -> Self {
Self(inner, guard)
}
/// Configures signal entry information for the process

7
src/task/runtime/task_queue.rs
View File

@ -3,7 +3,11 @@ use alloc::sync::Arc;
use crossbeam_queue::ArrayQueue;
use kernel_util::util::OneTimeInit;
use crate::{sync::IrqGuard, task::process::Process};
use crate::{
arch::{Architecture, ArchitectureImpl},
sync::IrqGuard,
task::process::Process,
};
use super::task::Task;
@ -41,6 +45,7 @@ impl TaskQueue {
pub fn dequeue(&self) -> Result<Arc<Task>, Error> {
let process = Process::current();
assert!(ArchitectureImpl::interrupt_mask());
loop {
if let Some(task) = self.task_queue.pop() {
return Ok(task);

5
src/task/sched.rs
View File

@ -8,7 +8,7 @@ use kernel_util::util::OneTimeInit;
use crate::{
// arch::aarch64::{context::TaskContext, cpu::Cpu},
arch::{Architecture, ArchitectureImpl},
sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard},
sync::{IrqGuard, IrqSafeSpinlock, IrqSafeSpinlockGuard},
};
use super::{
@ -276,11 +276,12 @@ impl CpuQueue {
/// will remain valid until the end of the interrupt or until [CpuQueue::yield_cpu]
/// is called.
pub fn current_process(&self) -> Option<CurrentProcess> {
let guard = IrqGuard::acquire();
self.inner
.lock()
.current
.clone()
.map(|p| unsafe { CurrentProcess::new(p) })
.map(|p| unsafe { CurrentProcess::new(p, guard) })
}
/// Returns a queue for given CPU index

15
tools/gentables/Cargo.toml Normal file
View File

@ -0,0 +1,15 @@
[package]
name = "gentables"
version = "0.1.0"
edition = "2021"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
[dependencies]
memtables = { path = "../../lib/memtables" }
bytemuck = "1.14.0"
elf = "0.7.2"
thiserror = "1.0.48"
clap = { version = "4.4.2", features = ["derive"] }
bitflags = "2.4.0"

186
tools/gentables/src/main.rs Normal file
View File

@ -0,0 +1,186 @@
#![feature(offset_of)]
use std::{
fs::OpenOptions,
io::{Read, Seek, SeekFrom, Write},
ops::Range,
path::{Path, PathBuf},
process::ExitCode,
};
use clap::Parser;
use elf::{
abi::{EM_X86_64, PT_LOAD},
endian::AnyEndian,
ElfStream,
};
use memtables::FixedTables;
use thiserror::Error;
use crate::x86_64::X8664Builder;
mod x86_64;
#[derive(Error, Debug)]
pub enum GenError {
#[error("I/O error: {0}")]
IoError(#[from] std::io::Error),
#[error("ELF parse error: {0}")]
ElfParseError(#[from] elf::ParseError),
#[error("Image's arhitecture is not supported")]
UnsupportedArchitecture,
#[error("Could not determine the kernel image address range (possibly incorrect segments?)")]
NoKernelImageRange,
#[error("Kernel image is too large: {0:#x?} ({1}B). Maximum size: {2}B")]
KernelTooLarge(Range<u64>, u64, u64),
#[error("Kernel image is missing a required symbol: {0:?}")]
MissingSymbol(&'static str),
#[error("Kernel image is missing a required section: {0:?}")]
MissingSection(&'static str),
#[error("Incorrect tables section placement: {0:#x}")]
IncorrectTablesPlacement(u64),
}
#[derive(Parser)]
struct Args {
image: PathBuf,
}
pub struct GenData {
pub kernel_start: u64,
pub kernel_end: u64,
pub table_offset: u64,
pub table_physical_address: u64,
pub kernel_virt_offset: u64,
}
fn kernel_image_range<F: Read + Seek>(
elf: &mut ElfStream<AnyEndian, F>,
kernel_virt_offset: u64,
) -> Result<(u64, u64), GenError> {
let mut start = u64::MAX;
let mut end = u64::MIN;
for segment in elf.segments() {
if segment.p_type != PT_LOAD || segment.p_vaddr != segment.p_paddr + kernel_virt_offset {
continue;
}
let aligned_start = segment.p_vaddr & !0xFFF;
let aligned_end = (segment.p_vaddr + segment.p_memsz + 0xFFF) & !0xFFF;
if aligned_end > end {
end = aligned_end;
}
if aligned_start < start {
start = aligned_start;
}
}
if start < end {
Ok((start, end))
} else {
Err(GenError::NoKernelImageRange)
}
}
fn kernel_virt_offset<F: Read + Seek>(elf: &mut ElfStream<AnyEndian, F>) -> Result<u64, GenError> {
let (symtab, symstrtab) = elf
.symbol_table()?
.ok_or_else(|| GenError::MissingSection(".symtab"))?;
for sym in symtab {
let name = symstrtab.get(sym.st_name as _)?;
if name == "KERNEL_VIRT_OFFSET" {
// TODO symbol checks
return Ok(sym.st_value);
}
}
Err(GenError::MissingSymbol("KERNEL_VIRT_OFFSET"))
}
fn find_tables<F: Read + Seek>(elf: &mut ElfStream<AnyEndian, F>) -> Result<(u64, u64), GenError> {
let (shdrs, strtab) = elf.section_headers_with_strtab()?;
let strtab = strtab.ok_or_else(|| GenError::MissingSection(".strtab"))?;
for shdr in shdrs {
let name = strtab.get(shdr.sh_name as _)?;
if name == ".data.tables" {
// TODO section checks
return Ok((shdr.sh_offset, shdr.sh_addr));
}
}
Err(GenError::MissingSection(".data.tables"))
}
fn build_tables<F: Read + Seek>(file: F) -> Result<(FixedTables, u64), GenError> {
let mut elf = ElfStream::<AnyEndian, F>::open_stream(file)?;
let kernel_virt_offset = kernel_virt_offset(&mut elf)?;
let (kernel_start, kernel_end) = kernel_image_range(&mut elf, kernel_virt_offset)?;
let (table_offset, table_virt_addr) = find_tables(&mut elf)?;
let table_physical_address = table_virt_addr
.checked_sub(kernel_virt_offset)
.ok_or_else(|| GenError::IncorrectTablesPlacement(table_virt_addr))?;
println!("Kernel image range: {:#x?}", kernel_start..kernel_end);
println!("KERNEL_VIRT_OFFSET = {:#x}", kernel_virt_offset);
match elf.ehdr.e_machine {
EM_X86_64 => X8664Builder::new(
elf,
GenData {
kernel_virt_offset,
kernel_start,
kernel_end,
table_offset,
table_physical_address,
},
)?
.build(),
_ => todo!(),
}
}
fn write_tables<F: Write + Seek>(
mut file: F,
offset: u64,
tables: FixedTables,
) -> Result<(), GenError> {
let bytes = bytemuck::bytes_of(&tables);
file.seek(SeekFrom::Start(offset))?;
file.write_all(bytes)?;
Ok(())
}
fn gentables<P: AsRef<Path>>(image: P) -> Result<(), GenError> {
let mut file = OpenOptions::new()
.read(true)
.write(true)
.truncate(false)
.open(image)?;
let (tables, file_offset) = build_tables(&mut file)?;
write_tables(file, file_offset, tables)?;
Ok(())
}
fn main() -> ExitCode {
let args = Args::parse();
match gentables(&args.image) {
Ok(()) => ExitCode::SUCCESS,
Err(err) => {
eprintln!("{}: {}", args.image.display(), err);
ExitCode::FAILURE
}
}
}

189
tools/gentables/src/x86_64.rs Normal file
View File

@ -0,0 +1,189 @@
use core::fmt;
use std::{
io::{Read, Seek},
mem::offset_of,
};
use bitflags::bitflags;
use bytemuck::Zeroable;
use elf::{
abi::{PF_W, PF_X, PT_LOAD},
endian::AnyEndian,
ElfStream,
};
use memtables::{FixedTables, KERNEL_L3_COUNT};
use crate::{GenData, GenError};
bitflags! {
#[derive(Clone, Copy)]
struct PageFlags: u64 {
const PRESENT = 1 << 0;
const WRITABLE = 1 << 1;
const NX = 1 << 63;
}
}
pub struct X8664Builder<F: Seek + Read> {
elf: ElfStream<AnyEndian, F>,
data: GenData,
tables: FixedTables,
l0i: usize,
l1i: usize,
start_l2i: usize,
end_l2i: usize,
}
impl PageFlags {
fn from_elf(flags: u32) -> Self {
let mut out = Self::empty();
if flags & PF_W != 0 {
out |= Self::WRITABLE;
}
if flags & PF_X == 0 {
out |= Self::NX;
}
out
}
}
impl fmt::Display for PageFlags {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(
f,
"r{}{}",
if self.contains(Self::WRITABLE) {
'w'
} else {
'-'
},
if self.contains(Self::NX) { '-' } else { 'x' }
)
}
}
impl<F: Seek + Read> X8664Builder<F> {
pub fn new(elf: ElfStream<AnyEndian, F>, data: GenData) -> Result<Self, GenError> {
let l2_aligned_start = data.kernel_start & !0x1FFFFF;
let l2_aligned_end = (data.kernel_end + 0x1FFFFF) & !0x1FFFFF;
if l2_aligned_end <= l2_aligned_start {
todo!();
}
if (l2_aligned_end - l2_aligned_start) as usize >= KERNEL_L3_COUNT * 0x200000 {
return Err(GenError::KernelTooLarge(
l2_aligned_start..l2_aligned_end,
l2_aligned_end - l2_aligned_start,
(KERNEL_L3_COUNT * 0x20000) as u64,
));
}
let l0i = (data.kernel_start >> 39) as usize & 0x1FF;
let l1i = (data.kernel_start >> 30) as usize & 0x1FF;
let start_l2i = (l2_aligned_start >> 21) as usize & 0x1FF;
let end_l2i = (l2_aligned_end >> 21) as usize & 0x1FF;
Ok(Self {
elf,
data,
tables: FixedTables::zeroed(),
l0i,
l1i,
start_l2i,
end_l2i,
})
}
pub fn build(mut self) -> Result<(FixedTables, u64), GenError> {
// L0 -> L1
let l1_physical_address =
self.data.table_physical_address + offset_of!(FixedTables, kernel_l1) as u64;
self.tables.l0.data[self.l0i] =
l1_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits();
// L1 -> L2
let l2_physical_address =
self.data.table_physical_address + offset_of!(FixedTables, kernel_l2) as u64;
self.tables.kernel_l1.data[self.l1i] =
l2_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits();
// L2 -> L3s
for i in 0..KERNEL_L3_COUNT {
let l3_physical_address = self.data.table_physical_address
+ (offset_of!(FixedTables, kernel_l3s) + 0x1000 * i) as u64;
self.tables.kernel_l2.data[i + self.start_l2i] =
l3_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits();
}
for (i, segment) in self.elf.segments().into_iter().enumerate() {
if segment.p_type != PT_LOAD
|| segment.p_vaddr != segment.p_paddr + self.data.kernel_virt_offset
{
continue;
}
let aligned_virt_start = segment.p_vaddr & !0xFFF;
let aligned_virt_end = (segment.p_vaddr + segment.p_memsz + 0xFFF) & !0xFFF;
let aligned_phys_start = segment.p_paddr & !0xFFF;
let count = (aligned_virt_end - aligned_virt_start) / 0x1000;
let flags = PageFlags::from_elf(segment.p_flags);
println!(
"{}: {:#x?} -> {:#x} {}",
i,
aligned_virt_start..aligned_virt_end,
aligned_phys_start,
flags
);
Self::map_segment(
self.start_l2i,
&mut self.tables,
aligned_virt_start,
aligned_phys_start,
count as usize,
flags,
)?;
}
Ok((self.tables, self.data.table_offset))
}
fn map_segment(
l2i_offset: usize,
tables: &mut FixedTables,
vaddr: u64,
paddr: u64,
count: usize,
flags: PageFlags,
) -> Result<(), GenError> {
for index in 0..count {
let address = vaddr + index as u64 * 0x1000;
let page = paddr + index as u64 * 0x1000;
let entry = page | (PageFlags::PRESENT | flags).bits();
let l2i = (address >> 21) as usize & 0x1FF - l2i_offset;
let l3i = (address >> 12) as usize & 0x1FF;
let l3 = &mut tables.kernel_l3s[l2i];
if l3.data[l3i] != 0 {
if l3.data[l3i] != entry {
todo!();
} else {
continue;
}
}
l3.data[l3i] = entry;
}
Ok(())
}
}