diff --git a/lib/memtables/Cargo.toml b/lib/memtables/Cargo.toml index 948e37e0..2697e627 100644 --- a/lib/memtables/Cargo.toml +++ b/lib/memtables/Cargo.toml @@ -7,3 +7,7 @@ edition = "2021" [dependencies] bytemuck = { version = "1.14.0", features = ["derive"] } + +[features] +default = [] +all = [] diff --git a/lib/memtables/src/aarch64.rs b/lib/memtables/src/aarch64.rs new file mode 100644 index 00000000..5adab8bc --- /dev/null +++ b/lib/memtables/src/aarch64.rs @@ -0,0 +1,26 @@ +use bytemuck::{Pod, Zeroable}; + +use crate::RawTable; + +pub const KERNEL_L3_COUNT: usize = 4; + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub struct FixedTables { + // 1GiB entries + pub l1: RawTable, + + // 2MiB entries + pub l2: RawTable, + pub l3s: [RawTable; KERNEL_L3_COUNT], +} + +impl FixedTables { + pub const fn zeroed() -> Self { + Self { + l1: RawTable::zeroed(), + l2: RawTable::zeroed(), + l3s: [RawTable::zeroed(); KERNEL_L3_COUNT], + } + } +} diff --git a/lib/memtables/src/any.rs b/lib/memtables/src/any.rs new file mode 100644 index 00000000..1dcbcfe4 --- /dev/null +++ b/lib/memtables/src/any.rs @@ -0,0 +1,27 @@ +use crate::{aarch64, x86_64}; + +pub enum AnyTables { + X86_64(x86_64::FixedTables), + AArch64(aarch64::FixedTables), +} + +impl AnyTables { + pub fn as_bytes(&self) -> &[u8] { + match self { + Self::X86_64(tables) => bytemuck::bytes_of(tables), + Self::AArch64(tables) => bytemuck::bytes_of(tables), + } + } +} + +impl From for AnyTables { + fn from(value: x86_64::FixedTables) -> Self { + Self::X86_64(value) + } +} + +impl From for AnyTables { + fn from(value: aarch64::FixedTables) -> Self { + Self::AArch64(value) + } +} diff --git a/lib/memtables/src/lib.rs b/lib/memtables/src/lib.rs index 79caa00c..fa3334e1 100644 --- a/lib/memtables/src/lib.rs +++ b/lib/memtables/src/lib.rs @@ -2,7 +2,20 @@ use bytemuck::{Pod, Zeroable}; -pub const KERNEL_L3_COUNT: usize = 16; +// AArch64 +#[cfg(any(feature = "all", target_arch = "aarch64"))] +pub mod aarch64; +#[cfg(all(not(feature = "all"), target_arch = "aarch64"))] +pub use aarch64::FixedTables; + +// x86-64 +#[cfg(any(feature = "all", target_arch = "x86_64"))] +pub mod x86_64; +#[cfg(all(not(feature = "all"), target_arch = "x86_64"))] +pub use x86_64::FixedTables; + +#[cfg(feature = "all")] +pub mod any; #[derive(Clone, Copy, Pod, Zeroable)] #[repr(C, align(0x1000))] @@ -10,30 +23,8 @@ pub struct RawTable { pub data: [u64; 512], } -#[derive(Clone, Copy, Pod, Zeroable)] -#[repr(C)] -pub struct FixedTables { - pub l0: RawTable, - - pub kernel_l1: RawTable, - pub kernel_l2: RawTable, - pub kernel_l3s: [RawTable; KERNEL_L3_COUNT], -} - impl RawTable { pub const fn zeroed() -> Self { Self { data: [0; 512] } } } - -impl FixedTables { - pub const fn zeroed() -> Self { - Self { - l0: RawTable::zeroed(), - - kernel_l1: RawTable::zeroed(), - kernel_l2: RawTable::zeroed(), - kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT], - } - } -} diff --git a/lib/memtables/src/x86_64.rs b/lib/memtables/src/x86_64.rs new file mode 100644 index 00000000..cea2608a --- /dev/null +++ b/lib/memtables/src/x86_64.rs @@ -0,0 +1,27 @@ +use bytemuck::{Pod, Zeroable}; + +use crate::RawTable; + +pub const KERNEL_L3_COUNT: usize = 16; + +#[derive(Clone, Copy, Pod, Zeroable)] +#[repr(C)] +pub struct FixedTables { + pub l0: RawTable, + + pub kernel_l1: RawTable, + pub kernel_l2: RawTable, + pub kernel_l3s: [RawTable; KERNEL_L3_COUNT], +} + +impl FixedTables { + pub const fn zeroed() -> Self { + Self { + l0: RawTable::zeroed(), + + kernel_l1: RawTable::zeroed(), + kernel_l2: RawTable::zeroed(), + kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT], + } + } +} diff --git a/src/arch/aarch64/boot/entry.S b/src/arch/aarch64/boot/entry.S index 7cdfcc48..c41548c7 100644 --- a/src/arch/aarch64/boot/entry.S +++ b/src/arch/aarch64/boot/entry.S @@ -13,6 +13,12 @@ movk \reg, #((\value) >> 16), lsl #16 .endm +.macro MOV_ABS reg, sym + movz \reg, #:abs_g2:\sym + movk \reg, #:abs_g1_nc:\sym + movk \reg, #:abs_g0_nc:\sym +.endm + .macro LEAVE_EL2, ret_label mrs x8, CNTHCTL_EL2 orr x8, x8, #(CNTHCTL_EL2_EL1PCTEN | CNTHCTL_EL2_EL1PCEN) @@ -61,8 +67,9 @@ __aarch64_entry: isb // Zero .bss - adr x8, __bss_start_phys - adr x9, __bss_end_phys + MOV_ABS x8, __bss_start_phys + MOV_ABS x9, __bss_end_phys + // Zero .bss 1: cmp x8, x9 beq 2f @@ -84,7 +91,7 @@ __aarch64_entry: .section .text __aarch64_ap_entry: - // x0 -- stack pointer (lower address space) + // x0 -- physical sp mrs x8, CurrentEL lsr x8, x8, #2 diff --git a/src/arch/aarch64/boot/mod.rs b/src/arch/aarch64/boot/mod.rs index c209e74b..338f0910 100644 --- a/src/arch/aarch64/boot/mod.rs +++ b/src/arch/aarch64/boot/mod.rs @@ -1,135 +1,168 @@ -//! Main entry point for the AArch64 platforms -use core::{ - arch::{asm, global_asm}, - sync::atomic::Ordering, -}; +use core::{arch::global_asm, sync::atomic::Ordering}; use aarch64_cpu::{ asm::barrier, - registers::{CPACR_EL1, TTBR0_EL1}, + registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1}, }; -use tock_registers::interfaces::{ReadWriteable, Writeable}; +use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; -use super::{cpu::Cpu, exception, AArch64, KernelStack, ARCHITECTURE, BOOT_STACK_SIZE}; +use super::{ + exception, mem::load_fixed_tables, smp::CPU_COUNT, AArch64, BootStack, ARCHITECTURE, + BOOT_STACK_SIZE, +}; use crate::{ - absolute_address, - arch::{aarch64::smp::CPU_COUNT, Architecture, ArchitectureImpl}, + arch::{aarch64::mem::table::L3, Architecture}, fs::devfs, kernel_main, kernel_secondary_main, - mem::{ - heap, - phys::{self, PageUsage}, - ConvertAddress, KERNEL_VIRT_OFFSET, - }, + mem::{address::IntoRaw, phys, table::EntryLevel, PhysicalAddress, KERNEL_VIRT_OFFSET}, + task::runtime, }; -extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! { - // Unmask FP operations - CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); - - unsafe { - ARCHITECTURE.init_mmu(true); +unsafe fn pre_init_mmu() { + if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) { + // TODO early panic + loop {} } - let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE).virtualize() }; - let elr = absolute_address!(__aarch64_bsp_upper_entry); + MAIR_EL1.write( + //// Attribute 0 -- normal memory + // Inner + MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + + // Outer + MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + + //// Attribute 1 -- device memory + MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck, + ); - barrier::dsb(barrier::SY); - barrier::isb(barrier::SY); - enter_higher_half(sp as usize, elr, dtb_phys); + TCR_EL1.modify( + // General + TCR_EL1::IPS::Bits_48 + + // TTBR0 + TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner + + // TTBR1 + TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer, + ); } -unsafe extern "C" fn el1_ap_lower_entry(sp: usize) -> ! { - ArchitectureImpl::set_interrupt_mask(true); +unsafe fn enable_mmu() { + barrier::dmb(barrier::ISH); + + SCTLR_EL1.modify( + // Enable translation + SCTLR_EL1::M::Enable + + // (TODO) Disable I + D caches + SCTLR_EL1::I::NonCacheable + SCTLR_EL1::C::NonCacheable, + ); + + barrier::isb(barrier::SY); +} + +unsafe fn enter_higher_half(sp: usize, elr: usize, x0: usize) -> ! { + unsafe { + core::arch::asm!(r#" + mov sp, {sp} + mov x0, {x0} + mov lr, xzr + br {elr} + "#, elr = in(reg) elr, sp = in(reg) sp, x0 = in(reg) x0, options(noreturn)); + } +} + +// NOTE executes in "lower-half" address space, MMU not yet enabled +unsafe extern "C" fn __aarch64_el1_bsp_lower_entry(dtb: PhysicalAddress) -> ! { + AArch64::set_interrupt_mask(true); + + // Don't trap FP operations + CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); + + // Setup MMU to jump to "higher-half" address space + pre_init_mmu(); + load_fixed_tables(); + enable_mmu(); + + // Safety: SP points to the .bss section, so it's +offset mapped + let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE) as usize } + KERNEL_VIRT_OFFSET; + let elr = absolute_address!(__aarch64_bsp_upper_entry); + + // TODO pass dtb + enter_higher_half(sp, elr, dtb.into_raw()); +} + +unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! { + // Remove the "lower-half" mapping, no longer needed + TTBR0_EL1.set(0); + + // Setup the "runtime" part of the kernel tables + ARCHITECTURE + .init_memory_management(dtb) + .expect("Could not initialize memory management"); + barrier::isb(barrier::SY); + + exception::init_exceptions(); + + // // Setup initrd + // super::setup_initrd(); + + devfs::init(); + + runtime::init_task_queue(); + + // Initialize the BSP CPU + the devices + ARCHITECTURE + .init_platform(true) + .expect("Could not initialize the platform"); + + kernel_main() +} + +unsafe extern "C" fn __aarch64_el1_ap_lower_entry(sp: PhysicalAddress) -> ! { + const AP_STACK_PAGES: usize = 8; + AArch64::set_interrupt_mask(true); // Unmask FP operations CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing); - unsafe { - ARCHITECTURE.init_mmu(false); - } + pre_init_mmu(); + load_fixed_tables(); + enable_mmu(); + + let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES).unwrap(); + let stack_base = stack_pages.virtualize_raw(); + let sp = stack_base + L3::SIZE * AP_STACK_PAGES; - let sp = sp.virtualize(); let elr = absolute_address!(__aarch64_ap_upper_entry); enter_higher_half(sp, elr, 0); } -fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! { - unsafe { - asm!(r#" - mov sp, {sp} - mov x0, {arg} - br {entry} - "#, entry = in(reg) elr, arg = in(reg) arg, sp = in(reg) sp, options(noreturn)); - } -} - -unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! { - // NOTE it is critical that the code does not panic until the debug is set up, otherwise no - // message will be displayed - - // Unmap TTBR0 - TTBR0_EL1.set(0); +extern "C" fn __aarch64_ap_upper_entry() -> ! { + barrier::dmb(barrier::ISH); barrier::isb(barrier::SY); - ARCHITECTURE.init_device_tree(dtb_phys); - - AArch64::set_interrupt_mask(true); - - exception::init_exceptions(); - - // Setup initrd - super::setup_initrd(); - - ARCHITECTURE - .init_physical_memory(dtb_phys) - .expect("Failed to initialize the physical memory manager"); - - // Setup heap - let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used) - .expect("Could not allocate a block for heap"); - - heap::init_heap(heap_base.virtualize(), 16 * 0x1000); - - devfs::init(); - - // Enumerate the device tree - ARCHITECTURE.init_platform(true); - - Cpu::init_local(); - - kernel_main() -} - -extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! { - assert!(ArchitectureImpl::interrupt_mask()); - - // Signal to BSP that we're up - CPU_COUNT.fetch_add(1, Ordering::SeqCst); + let cpu_id = CPU_COUNT.fetch_add(1, Ordering::SeqCst); aarch64_cpu::asm::sev(); + infoln!("cpu{} initializing", cpu_id); + exception::init_exceptions(); - // Initialize CPU-local GIC and timer unsafe { ARCHITECTURE.init_platform(false); - - Cpu::init_local(); - - kernel_secondary_main() } + + kernel_secondary_main() } -static BSP_STACK: KernelStack = KernelStack { +#[link_section = ".bss"] +static BSP_STACK: BootStack = BootStack { data: [0; BOOT_STACK_SIZE], }; global_asm!( include_str!("entry.S"), - kernel_lower_entry = sym el1_bsp_lower_entry, - kernel_ap_lower_entry = sym el1_ap_lower_entry, + kernel_lower_entry = sym __aarch64_el1_bsp_lower_entry, + kernel_ap_lower_entry = sym __aarch64_el1_ap_lower_entry, stack_bottom = sym BSP_STACK, - stack_size = const BOOT_STACK_SIZE, - kernel_virt_offset = const KERNEL_VIRT_OFFSET + kernel_virt_offset = const KERNEL_VIRT_OFFSET, + stack_size = const BOOT_STACK_SIZE ); diff --git a/src/arch/aarch64/context.rs b/src/arch/aarch64/context.rs index 1da9a20d..508e4dbb 100644 --- a/src/arch/aarch64/context.rs +++ b/src/arch/aarch64/context.rs @@ -4,10 +4,7 @@ use core::{arch::global_asm, cell::UnsafeCell}; use abi::error::Error; use crate::{ - mem::{ - phys::{self, PageUsage}, - ConvertAddress, - }, + mem::{address::IntoRaw, phys, PhysicalAddress}, task::context::TaskContextImpl, }; @@ -64,8 +61,8 @@ impl StackBuilder { self.sp } - fn init_common(&mut self, entry: usize, ttbr0: usize) { - self.push(ttbr0); // ttbr0_el1 + fn init_common(&mut self, entry: usize, ttbr0: u64) { + self.push(ttbr0 as _); // ttbr0_el1 self.push(0); // tpidr_el0 self.push(entry); // x30/lr @@ -89,9 +86,8 @@ impl TaskContextImpl for TaskContext { fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 8; - let stack_base = unsafe { - phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize() - }; + let stack_base = + unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw() }; let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000); @@ -112,10 +108,9 @@ impl TaskContextImpl for TaskContext { }) } - fn user(entry: usize, arg: usize, ttbr0: usize, user_stack_sp: usize) -> Result { + fn user(entry: usize, arg: usize, ttbr0: u64, user_stack_sp: usize) -> Result { const USER_TASK_PAGES: usize = 16; - let stack_base = - unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() }; + let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw() }; let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); diff --git a/src/arch/aarch64/cpu.rs b/src/arch/aarch64/cpu.rs index 8ab4463a..2ef75b42 100644 --- a/src/arch/aarch64/cpu.rs +++ b/src/arch/aarch64/cpu.rs @@ -122,6 +122,7 @@ impl Cpu { match msg { CpuMessage::Panic => panic::panic_secondary(), + CpuMessage::Shutdown => todo!(), } } } diff --git a/src/arch/aarch64/devtree.rs b/src/arch/aarch64/devtree.rs deleted file mode 100644 index e69de29b..00000000 diff --git a/src/arch/aarch64/exception.rs b/src/arch/aarch64/exception.rs index 4bd89603..e82417c5 100644 --- a/src/arch/aarch64/exception.rs +++ b/src/arch/aarch64/exception.rs @@ -16,10 +16,7 @@ use abi::{ use tock_registers::interfaces::{Readable, Writeable}; use crate::{ - arch::{ - aarch64::{cpu::Cpu, table::AddressSpace}, - Architecture, ArchitectureImpl, - }, + arch::{aarch64::cpu::Cpu, Architecture, ArchitectureImpl}, debug::LogLevel, syscall::raw_syscall_handler, task::{context::TaskFrame, process::Process}, @@ -179,14 +176,14 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { _ => (), } - unsafe { - let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _); - let far = FAR_EL1.get() as usize; - space.walk(far, |level, raw| { - log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw); - true - }); - } + // unsafe { + // let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _); + // let far = FAR_EL1.get() as usize; + // space.walk(far, |level, raw| { + // log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw); + // true + // }); + // } log_print_raw!(LogLevel::Fatal, "System register dump:\n"); log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get()); diff --git a/src/arch/aarch64/gic/gicc.rs b/src/arch/aarch64/gic/gicc.rs index cd55b3a6..1f11d485 100644 --- a/src/arch/aarch64/gic/gicc.rs +++ b/src/arch/aarch64/gic/gicc.rs @@ -36,11 +36,11 @@ register_structs! { } pub(super) struct Gicc { - regs: DeviceMemoryIo, + regs: DeviceMemoryIo<'static, GiccRegs>, } impl Gicc { - pub const fn new(regs: DeviceMemoryIo) -> Self { + pub const fn new(regs: DeviceMemoryIo<'static, GiccRegs>) -> Self { Self { regs } } diff --git a/src/arch/aarch64/gic/gicd.rs b/src/arch/aarch64/gic/gicd.rs index 373839b5..60314cc8 100644 --- a/src/arch/aarch64/gic/gicd.rs +++ b/src/arch/aarch64/gic/gicd.rs @@ -65,8 +65,8 @@ register_structs! { } pub(super) struct Gicd { - shared_regs: Spinlock>, - banked_regs: DeviceMemoryIo, + shared_regs: Spinlock>, + banked_regs: DeviceMemoryIo<'static, GicdBankedRegs>, } impl GicdSharedRegs { @@ -85,8 +85,8 @@ impl GicdSharedRegs { impl Gicd { pub const fn new( - shared_regs: DeviceMemoryIo, - banked_regs: DeviceMemoryIo, + shared_regs: DeviceMemoryIo<'static, GicdSharedRegs>, + banked_regs: DeviceMemoryIo<'static, GicdBankedRegs>, ) -> Self { let shared_regs = Spinlock::new(shared_regs); Self { diff --git a/src/arch/aarch64/gic/mod.rs b/src/arch/aarch64/gic/mod.rs index 0bb60616..4b528b5d 100644 --- a/src/arch/aarch64/gic/mod.rs +++ b/src/arch/aarch64/gic/mod.rs @@ -4,7 +4,7 @@ use core::sync::atomic::Ordering; use aarch64_cpu::asm::barrier; use abi::error::Error; -use alloc::boxed::Box; +use alloc::{boxed::Box, sync::Arc}; use device_api::{ interrupt::{ ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable, @@ -18,7 +18,11 @@ use crate::{ arch::{aarch64::IrqNumber, Architecture, CpuMessage}, device::devtree::{self, DevTreeIndexPropExt}, device_tree_driver, - mem::device::{DeviceMemory, DeviceMemoryIo}, + mem::{ + address::FromRaw, + device::{DeviceMemoryIo, DeviceMemoryMapping, RawDeviceMemoryMapping}, + PhysicalAddress, + }, sync::IrqSafeSpinlock, }; @@ -36,8 +40,8 @@ pub mod gicd; pub struct Gic { gicc: OneTimeInit, gicd: OneTimeInit, - gicd_base: usize, - gicc_base: usize, + gicd_base: PhysicalAddress, + gicc_base: PhysicalAddress, table: IrqSafeSpinlock>, } @@ -47,10 +51,10 @@ impl Device for Gic { } unsafe fn init(&'static self) -> Result<(), Error> { - let gicd_mmio = DeviceMemory::map("GICv2 Distributor registers", self.gicd_base, 0x1000)?; - let gicd_mmio_shared = DeviceMemoryIo::new(gicd_mmio.clone()); - let gicd_mmio_banked = DeviceMemoryIo::new(gicd_mmio); - let gicc_mmio = DeviceMemoryIo::map("GICv2 CPU registers", self.gicc_base)?; + let gicd_mmio = Arc::new(RawDeviceMemoryMapping::map(self.gicd_base, 0x1000)?); + let gicd_mmio_shared = DeviceMemoryIo::from_raw(gicd_mmio.clone())?; + let gicd_mmio_banked = DeviceMemoryIo::from_raw(gicd_mmio)?; + let gicc_mmio = DeviceMemoryIo::map(self.gicc_base)?; let gicd = Gicd::new(gicd_mmio_shared, gicd_mmio_banked); let gicc = Gicc::new(gicc_mmio); @@ -169,7 +173,7 @@ impl Gic { /// # Safety /// /// The caller must ensure the addresses actually point to the GIC components. - pub const unsafe fn new(gicd_base: usize, gicc_base: usize) -> Self { + pub const unsafe fn new(gicd_base: PhysicalAddress, gicc_base: PhysicalAddress) -> Self { Self { gicc: OneTimeInit::new(), gicd: OneTimeInit::new(), @@ -188,6 +192,9 @@ device_tree_driver! { let (gicc_base, _) = reg.cell2_array_item(0, dt.address_cells, dt.size_cells)?; let (gicd_base, _) = reg.cell2_array_item(1, dt.address_cells, dt.size_cells)?; - Some(Box::new(unsafe { Gic::new(gicc_base as usize, gicd_base as usize) })) + Some(Box::new(unsafe { Gic::new( + PhysicalAddress::from_raw(gicc_base), + PhysicalAddress::from_raw(gicd_base), + )})) } } diff --git a/src/arch/aarch64/mem/mod.rs b/src/arch/aarch64/mem/mod.rs new file mode 100644 index 00000000..8ef804f2 --- /dev/null +++ b/src/arch/aarch64/mem/mod.rs @@ -0,0 +1,374 @@ +use core::{ + alloc::Layout, + ops::{Deref, DerefMut}, +}; + +use abi::error::Error; +use cfg_if::cfg_if; +use kernel_util::util::OneTimeInit; +use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; + +use aarch64_cpu::registers::{TTBR0_EL1, TTBR1_EL1}; +use static_assertions::const_assert_eq; +use tock_registers::interfaces::Writeable; + +use crate::mem::{ + address::{FromRaw, IntoRaw, KernelImageObject}, + device::RawDeviceMemoryMapping, + table::EntryLevel, + PhysicalAddress, KERNEL_VIRT_OFFSET, +}; + +use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3}; + +pub mod process; +pub(super) mod table; + +// TODO eliminate this requirement by using precomputed indices +const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET; +cfg_if! { + if #[cfg(feature = "aarch64_qemu")] { + const KERNEL_PHYS_BASE: usize = 0x40080000; + } else { + const KERNEL_PHYS_BASE: usize = 0x40080000; + } +} + +// Precomputed mappings +const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE); +const KERNEL_END_L2_INDEX: usize = + L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE) + KERNEL_L3_COUNT; + +// Must not be zero, should be at 4MiB +const_assert_eq!(KERNEL_START_L2_INDEX, 0); +// From static mapping +const_assert_eq!(KERNEL_L1_INDEX, 1); + +// Runtime mappings +// 2MiB max +const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1; +// 1GiB max +const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1; +// 1GiB max +const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2; +// 16GiB max +pub(super) const RAM_MAPPING_L1_COUNT: usize = 16; +const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3; +const RAM_MAPPING_END_L1I: usize = RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT; + +const DEVICE_MAPPING_L3_COUNT: usize = 4; + +// 2MiB for early mappings +const EARLY_MAPPING_OFFSET: usize = + MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE); +static mut EARLY_MAPPING_L3: PageTable = PageTable::zeroed(); +// 1GiB for heap mapping +pub(super) const HEAP_MAPPING_OFFSET: usize = MAPPING_OFFSET | (HEAP_MAPPING_L1I * L1::SIZE); +pub(super) static mut HEAP_MAPPING_L2: PageTable = PageTable::zeroed(); +// 1GiB for device MMIO mapping +const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE); +static mut DEVICE_MAPPING_L2: PageTable = PageTable::zeroed(); +static mut DEVICE_MAPPING_L3S: [PageTable; DEVICE_MAPPING_L3_COUNT] = + [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT]; +// 16GiB for RAM mapping +pub(super) const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE); +pub(super) static MEMORY_LIMIT: OneTimeInit = OneTimeInit::new(); + +#[link_section = ".data.tables"] +pub static mut KERNEL_TABLES: KernelImageObject = + unsafe { KernelImageObject::new(FixedTables::zeroed()) }; + +pub struct EarlyMapping<'a, T: ?Sized> { + value: &'a mut T, + page_count: usize, +} + +impl<'a, T: Sized> EarlyMapping<'a, T> { + pub unsafe fn map(physical: PhysicalAddress) -> Result, Error> { + let layout = Layout::new::(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = &mut *((virt + offset) as *mut T); + + Ok(EarlyMapping { value, page_count }) + } + + pub unsafe fn map_slice( + physical: PhysicalAddress, + len: usize, + ) -> Result, Error> { + let layout = Layout::array::(len).unwrap(); + let aligned = physical.page_align_down::(); + let offset = physical.page_offset::(); + let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE; + + let virt = map_early_pages(aligned, page_count)?; + let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len); + + Ok(EarlyMapping { value, page_count }) + } +} + +impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> { + type Target = T; + + fn deref(&self) -> &Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> { + fn deref_mut(&mut self) -> &mut Self::Target { + self.value + } +} + +impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> { + fn drop(&mut self) { + let address = (self.value as *mut T).addr() & !(L3::SIZE - 1); + + for i in 0..self.page_count { + let page = address + i * L3::SIZE; + + unsafe { + unmap_early_page(page); + } + } + } +} + +fn kernel_table_flags() -> PageAttributes { + PageAttributes::TABLE + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | PageAttributes::PRESENT +} + +fn ram_block_flags() -> PageAttributes { + // TODO UXN, PXN + PageAttributes::BLOCK + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | PageAttributes::PRESENT +} + +// Early mappings +unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result { + for l3i in 0..512 { + let mut taken = false; + for i in 0..count { + if EARLY_MAPPING_L3[i + l3i].is_present() { + taken = true; + break; + } + } + + if taken { + continue; + } + + for i in 0..count { + let page = physical.add(i * L3::SIZE); + // TODO NX, NC + EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty()); + } + + return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn unmap_early_page(address: usize) { + if address < EARLY_MAPPING_OFFSET || address >= EARLY_MAPPING_OFFSET + L2::SIZE { + panic!("Tried to unmap invalid early mapping: {:#x}", address); + } + + let l3i = L3::index(address - EARLY_MAPPING_OFFSET); + + assert!(EARLY_MAPPING_L3[l3i].is_present()); + EARLY_MAPPING_L3[l3i] = PageEntry::INVALID; + + // TODO invalidate tlb +} + +pub(super) unsafe fn map_ram_l1(index: usize) { + if index >= RAM_MAPPING_L1_COUNT { + loop {} + } + assert_eq!(KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I], 0); + + KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I] = + ((index * L1::SIZE) as u64) | ram_block_flags().bits(); +} + +pub(super) unsafe fn map_heap_l2(index: usize, page: PhysicalAddress) { + if index >= 512 { + loop {} + } + assert!(!HEAP_MAPPING_L2[index].is_present()); + // TODO UXN, PXN + HEAP_MAPPING_L2[index] = PageEntry::normal_block(page, PageAttributes::empty()); +} + +// Device mappings +unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result { + // TODO don't map pages if already mapped + + 'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 { + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + if DEVICE_MAPPING_L3S[l2i][l3i].is_present() { + continue 'l0; + } + } + + for j in 0..count { + let l2i = (i + j) / 512; + let l3i = (i + j) % 512; + + // TODO NX, NC + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE)); + } + + return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE); + } + + Err(Error::OutOfMemory) +} + +unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result { + 'l0: for i in DEVICE_MAPPING_L3_COUNT..512 { + for j in 0..count { + if DEVICE_MAPPING_L2[i + j].is_present() { + continue 'l0; + } + } + + for j in 0..count { + DEVICE_MAPPING_L2[i + j] = PageEntry::::device_block(base.add(j * L2::SIZE)); + } + + debugln!( + "map l2s: base={:#x}, count={} -> {:#x}", + base, + count, + DEVICE_MAPPING_OFFSET + i * L2::SIZE + ); + return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE); + } + + Err(Error::OutOfMemory) +} + +pub(super) unsafe fn map_device_memory( + base: PhysicalAddress, + size: usize, +) -> Result { + // debugln!("Map {}B @ {:#x}", size, base); + let l3_aligned = base.page_align_down::(); + let l3_offset = L3::page_offset(base.into_raw()); + let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE; + + if page_count > 256 { + // Large mapping, use L2 mapping instead + let l2_aligned = base.page_align_down::(); + let l2_offset = L2::page_offset(base.into_raw()); + let page_count = (l2_offset + size + L2::SIZE - 1) / L2::SIZE; + + let base_address = map_device_memory_l2(l2_aligned, page_count)?; + let address = base_address + l2_offset; + + Ok(RawDeviceMemoryMapping { + address, + base_address, + page_count, + page_size: L2::SIZE, + }) + } else { + // Just map the pages directly + let base_address = map_device_memory_l3(l3_aligned, page_count)?; + let address = base_address + l3_offset; + + Ok(RawDeviceMemoryMapping { + address, + base_address, + page_count, + page_size: L3::SIZE, + }) + } +} + +pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) { + // debugln!( + // "Unmap {}B @ {:#x}", + // map.page_count * map.page_size, + // map.base_address + // ); + match map.page_size { + L3::SIZE => { + for i in 0..map.page_count { + let page = map.base_address + i * L3::SIZE; + let l2i = L2::index(page); + let l3i = L3::index(page); + assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present()); + DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID; + // TODO flush the TLB entry + loop {} + // intrinsics::flush_tlb_entry(page); + } + } + L2::SIZE => todo!(), + _ => unimplemented!(), + } +} + +/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half" +/// +/// # Safety +/// +/// Unsafe, must only be called by BSP during its early init while still in "lower-half" +pub(super) unsafe fn load_fixed_tables() { + let ttbr0 = KERNEL_TABLES.l1.data.as_ptr() as u64; + TTBR0_EL1.set(ttbr0); + TTBR1_EL1.set(ttbr0); +} + +/// Sets up additional translation tables for kernel usage +/// +/// # Safety +/// +/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half" +pub(super) unsafe fn init_fixed_tables() { + // TODO this could be built in compile-time too? + let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET; + let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; + let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET; + + for i in 0..DEVICE_MAPPING_L3_COUNT { + let device_mapping_l3_phys = PhysicalAddress::from_raw( + &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET, + ); + DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty()); + } + + assert_eq!(KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I], 0); + KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I] = + (early_mapping_l3_phys as u64) | kernel_table_flags().bits(); + + assert_eq!(KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I], 0); + KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I] = + (heap_mapping_l2_phys as u64) | kernel_table_flags().bits(); + + assert_eq!(KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I], 0); + KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I] = + (device_mapping_l2_phys as u64) | kernel_table_flags().bits(); +} diff --git a/src/arch/aarch64/mem/process.rs b/src/arch/aarch64/mem/process.rs new file mode 100644 index 00000000..0f9370c2 --- /dev/null +++ b/src/arch/aarch64/mem/process.rs @@ -0,0 +1,130 @@ +use core::sync::atomic::{AtomicU8, Ordering}; + +use abi::error::Error; + +use crate::mem::{ + address::{AsPhysicalAddress, IntoRaw}, + phys, + pointer::PhysicalRefMut, + process::ProcessAddressSpaceManager, + table::{EntryLevel, MapAttributes, NextPageTable}, + PhysicalAddress, +}; + +use super::table::{PageEntry, PageTable, L1, L2, L3}; + +#[repr(C)] +pub struct ProcessAddressSpaceImpl { + l1: PhysicalRefMut<'static, PageTable>, + asid: u8, +} + +impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { + const PAGE_SIZE: usize = L3::SIZE; + const LOWER_LIMIT_PFN: usize = 8; + // 16GiB VM limit + const UPPER_LIMIT_PFN: usize = (16 << 30) / Self::PAGE_SIZE; + + fn new() -> Result { + static LAST_ASID: AtomicU8 = AtomicU8::new(1); + + let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel); + + let mut l1 = unsafe { PhysicalRefMut::<'static, PageTable>::map(phys::alloc_page()?) }; + + for i in 0..512 { + l1[i] = PageEntry::INVALID; + } + + Ok(Self { l1, asid }) + } + + fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error> { + self.read_l3_entry(address).ok_or(Error::DoesNotExist) + } + + unsafe fn map_page( + &mut self, + address: usize, + physical: PhysicalAddress, + flags: MapAttributes, + ) -> Result<(), Error> { + self.write_l3_entry( + address, + PageEntry::normal_page(physical, flags.into()), + false, + ) + } + + unsafe fn unmap_page(&mut self, address: usize) -> Result { + self.pop_l3_entry(address) + } + + fn as_address_with_asid(&self) -> u64 { + unsafe { u64::from(self.l1.as_physical_address()) | ((self.asid as u64) << 48) } + } +} + +impl ProcessAddressSpaceImpl { + // Write a single 4KiB entry + fn write_l3_entry( + &mut self, + virt: usize, + entry: PageEntry, + overwrite: bool, + ) -> Result<(), Error> { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let mut l2 = self.l1.get_mut_or_alloc(l1i)?; + let mut l3 = l2.get_mut_or_alloc(l2i)?; + + if l3[l3i].is_present() && !overwrite { + todo!(); + } + + l3[l3i] = entry; + tlb_flush_vaae1(virt); + + Ok(()) + } + + fn pop_l3_entry(&mut self, virt: usize) -> Result { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + // TODO somehow drop tables if they're known to be empty? + let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; + let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; + + let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?; + + l3[l3i] = PageEntry::INVALID; + tlb_flush_vaae1(virt); + + Ok(page) + } + + fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { + let l1i = L1::index(virt); + let l2i = L2::index(virt); + let l3i = L3::index(virt); + + let l2 = self.l1.get(l1i)?; + let l3 = l2.get(l2i)?; + + let page = l3[l3i].as_page()?; + + Some((page, l3[l3i].attributes().into())) + } +} + +#[inline] +fn tlb_flush_vaae1(mut page: usize) { + page >>= 12; + unsafe { + core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); + } +} diff --git a/src/arch/aarch64/mem/table.rs b/src/arch/aarch64/mem/table.rs new file mode 100644 index 00000000..36daff94 --- /dev/null +++ b/src/arch/aarch64/mem/table.rs @@ -0,0 +1,295 @@ +use core::{ + marker::PhantomData, + ops::{Index, IndexMut}, +}; + +use abi::error::Error; +use bitflags::bitflags; + +use crate::mem::{ + address::{AsPhysicalAddress, FromRaw, IntoRaw}, + phys, + pointer::{PhysicalRef, PhysicalRefMut}, + table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel}, + PhysicalAddress, +}; + +bitflags! { + #[derive(Clone, Copy, PartialEq, Eq)] + pub struct PageAttributes: u64 { + const PRESENT = 1 << 0; + + const TABLE = 1 << 1; + const PAGE = 1 << 1; + const BLOCK = 0 << 1; + + const ACCESS = 1 << 10; + + const AP_KERNEL_READWRITE = 0 << 6; + const AP_BOTH_READWRITE = 1 << 6; + const AP_KERNEL_READONLY = 2 << 6; + const AP_BOTH_READONLY = 3 << 6; + const AP_ACCESS_MASK = 3 << 6; + + const SH_OUTER = 2 << 8; + const SH_INNER = 3 << 8; + + const PAGE_ATTR_NORMAL = 0 << 2; + const PAGE_ATTR_DEVICE = 1 << 2; + + const NON_GLOBAL = 1 << 11; + + const PXN = 1 << 53; + const UXN = 1 << 54; + } +} + +#[derive(Clone, Copy)] +#[repr(C, align(0x1000))] +pub struct PageTable { + entries: [PageEntry; 512], +} + +#[derive(Clone, Copy)] +pub struct PageEntry(u64, PhantomData); + +#[derive(Clone, Copy)] +pub struct L1; +#[derive(Clone, Copy)] +pub struct L2; +#[derive(Clone, Copy)] +pub struct L3; + +impl const EntryLevel for L1 { + const SHIFT: usize = 30; +} + +impl NonTerminalEntryLevel for L1 { + type NextLevel = L2; +} + +impl const EntryLevel for L2 { + const SHIFT: usize = 21; +} + +impl NonTerminalEntryLevel for L2 { + type NextLevel = L3; +} + +impl const EntryLevel for L3 { + const SHIFT: usize = 12; +} + +impl PageTable { + pub const fn zeroed() -> Self { + Self { + entries: [PageEntry::INVALID; 512], + } + } + + pub fn new_zeroed<'a>() -> Result, Error> { + let physical = phys::alloc_page()?; + let mut table = unsafe { PhysicalRefMut::<'a, Self>::map(physical) }; + + for i in 0..512 { + table[i] = PageEntry::INVALID; + } + + Ok(table) + } +} + +impl PageEntry { + pub const INVALID: Self = Self(0, PhantomData); + + pub const fn is_present(self) -> bool { + self.0 & PageAttributes::PRESENT.bits() != 0 + } + + pub fn attributes(self) -> PageAttributes { + PageAttributes::from_bits_retain(self.0) + } +} + +impl NextPageTable for PageTable { + type NextLevel = PageTable; + type TableRef = PhysicalRef<'static, PageTable>; + type TableRefMut = PhysicalRefMut<'static, PageTable>; + + fn get(&self, index: usize) -> Option { + self[index] + .as_table() + .map(|phys| unsafe { PhysicalRef::map(phys) }) + } + + fn get_mut(&mut self, index: usize) -> Option { + self[index] + .as_table() + .map(|phys| unsafe { PhysicalRefMut::map(phys) }) + } + + fn get_mut_or_alloc(&mut self, index: usize) -> Result { + let entry = self[index]; + + if let Some(table) = entry.as_table() { + Ok(unsafe { PhysicalRefMut::map(table) }) + } else { + let table = PageTable::new_zeroed()?; + self[index] = PageEntry::::table( + unsafe { table.as_physical_address() }, + PageAttributes::empty(), + ); + Ok(table) + } + } +} + +impl PageEntry { + pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(), + PhantomData, + ) + } + + pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::BLOCK + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | attrs) + .bits(), + PhantomData, + ) + } + + pub fn device_block(phys: PhysicalAddress) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::BLOCK + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_OUTER + | PageAttributes::PAGE_ATTR_DEVICE + | PageAttributes::UXN + | PageAttributes::PXN) + .bits(), + PhantomData, + ) + } + + /// Returns the physical address of the table this entry refers to, returning None if it + /// does not + pub fn as_table(self) -> Option { + if self.0 & PageAttributes::PRESENT.bits() != 0 + && self.0 & PageAttributes::BLOCK.bits() == 0 + { + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) + } else { + None + } + } +} + +impl PageEntry { + pub fn normal_page(phys: PhysicalAddress, attrs: PageAttributes) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_INNER + | PageAttributes::PAGE_ATTR_NORMAL + | attrs) + .bits(), + PhantomData, + ) + } + + pub fn device_page(phys: PhysicalAddress) -> Self { + Self( + IntoRaw::::into_raw(phys) + | (PageAttributes::PAGE + | PageAttributes::PRESENT + | PageAttributes::ACCESS + | PageAttributes::SH_OUTER + | PageAttributes::PAGE_ATTR_DEVICE + | PageAttributes::UXN + | PageAttributes::PXN) + .bits(), + PhantomData, + ) + } + + pub fn as_page(&self) -> Option { + let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); + if self.0 & mask == mask { + Some(PhysicalAddress::from_raw(self.0 & !0xFFF)) + } else { + None + } + } +} + +impl Index for PageTable { + type Output = PageEntry; + + fn index(&self, index: usize) -> &Self::Output { + &self.entries[index] + } +} + +impl IndexMut for PageTable { + fn index_mut(&mut self, index: usize) -> &mut Self::Output { + &mut self.entries[index] + } +} + +impl From for PageAttributes { + fn from(value: MapAttributes) -> Self { + let mut out = PageAttributes::empty(); + // TODO kernel cannot write el0 readonly pages + if value.contains(MapAttributes::USER_WRITE) { + // Read/write + out |= PageAttributes::AP_BOTH_READWRITE; + } else if value.contains(MapAttributes::USER_READ) { + // Read only + out |= PageAttributes::AP_BOTH_READONLY; + } else { + // No read/write + out |= PageAttributes::AP_KERNEL_READONLY; + } + + if value.contains(MapAttributes::NON_GLOBAL) { + out |= PageAttributes::NON_GLOBAL; + } + + out + } +} + +impl From for MapAttributes { + fn from(value: PageAttributes) -> Self { + let mut out = MapAttributes::empty(); + + out |= match value.intersection(PageAttributes::AP_ACCESS_MASK) { + PageAttributes::AP_BOTH_READWRITE => { + MapAttributes::USER_WRITE | MapAttributes::USER_READ + } + PageAttributes::AP_BOTH_READONLY => MapAttributes::USER_READ, + PageAttributes::AP_KERNEL_READONLY => MapAttributes::empty(), + PageAttributes::AP_KERNEL_READWRITE => panic!("This variant cannot be constructed"), + _ => unreachable!(), + }; + + if value.contains(PageAttributes::NON_GLOBAL) { + out |= MapAttributes::NON_GLOBAL; + } + + out + } +} diff --git a/src/arch/aarch64/mod.rs b/src/arch/aarch64/mod.rs index 407924fe..71c3d84b 100644 --- a/src/arch/aarch64/mod.rs +++ b/src/arch/aarch64/mod.rs @@ -1,49 +1,47 @@ //! AArch64 architecture and platforms implementation +pub mod mem; + use core::sync::atomic::Ordering; -use aarch64_cpu::{ - asm::barrier, - registers::{ - CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, - TTBR0_EL1, TTBR1_EL1, - }, -}; +use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF}; use abi::error::Error; use device_api::{ interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, ResetDevice, }; -use fdt_rs::prelude::PropReader; +use fdt_rs::base::DevTree; use git_version::git_version; use kernel_util::util::OneTimeInit; use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ - arch::Architecture, + arch::aarch64::{ + cpu::Cpu, + mem::table::{L2, L3}, + }, debug, device::{ self, - devtree::{ - self, DevTreeIndexNodePropGet, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, - FdtMemoryRegionIter, - }, + devtree::{self, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter}, power::arm_psci::Psci, }, fs::{Initrd, INITRD_DATA}, mem::{ + address::{FromRaw, IntoRaw}, + device::RawDeviceMemoryMapping, + heap, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, - ConvertAddress, + pointer::PhysicalRef, + table::EntryLevel, + PhysicalAddress, }, }; -use self::{ - smp::CPU_COUNT, - table::{init_fixed_tables, KERNEL_TABLES}, -}; +use self::mem::{table::L1, EarlyMapping}; -use super::CpuMessage; +use super::{Architecture, CpuMessage}; pub mod boot; pub mod context; @@ -51,111 +49,44 @@ pub mod cpu; pub mod exception; pub mod gic; pub mod smp; -pub mod table; pub mod timer; -const BOOT_STACK_SIZE: usize = 65536; +const BOOT_STACK_SIZE: usize = 4096 * 32; #[derive(Clone, Copy)] #[repr(C, align(0x20))] -struct KernelStack { +struct BootStack { data: [u8; BOOT_STACK_SIZE], } -#[derive(Clone, Copy, Debug, PartialEq)] -pub enum IrqNumber { - Private(u32), - Shared(u32), -} - -/// AArch64 platform interface pub struct AArch64 { dt: OneTimeInit>, - ext_intc: OneTimeInit<&'static dyn ExternalInterruptController>, - local_intc: OneTimeInit<&'static dyn LocalInterruptController>, - mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, + + pub psci: OneTimeInit<&'static Psci>, reset: OneTimeInit<&'static dyn ResetDevice>, - // ARM-only devices - /// ARM PSCI instance on this system (there may not be one) - pub psci: OneTimeInit<&'static Psci>, + lintc: OneTimeInit<&'static dyn LocalInterruptController>, + xintc: OneTimeInit<&'static dyn ExternalInterruptController>, + + mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>, + + initrd: OneTimeInit>, } -/// Global platform handle -pub static ARCHITECTURE: AArch64 = AArch64 { - dt: OneTimeInit::new(), - ext_intc: OneTimeInit::new(), - local_intc: OneTimeInit::new(), - mtimer: OneTimeInit::new(), - reset: OneTimeInit::new(), - - // ARM-only devices - psci: OneTimeInit::new(), -}; - impl Architecture for AArch64 { - type IrqNumber = IrqNumber; - const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000; - unsafe fn init_mmu(&self, bsp: bool) { - if bsp { - init_fixed_tables(); - } - - let tables_phys = absolute_address!(KERNEL_TABLES).physicalize() as u64; - - if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) { - todo!(); - } - - MAIR_EL1.write( - // Attribute 0 -- normal memory - MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc + - MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc + - // Attribute 1 -- device memory - MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck, - ); - - TCR_EL1.modify( - // General - TCR_EL1::IPS::Bits_48 + - // TTBR0 - TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner + - // TTBR1 - TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer, - ); - - barrier::dmb(barrier::ISH); - - TTBR0_EL1.set_baddr(tables_phys); - TTBR1_EL1.set_baddr(tables_phys); - - barrier::isb(barrier::SY); - - // Enable instruction cache, data cache and translation - SCTLR_EL1 - .modify(SCTLR_EL1::M::Enable + SCTLR_EL1::I::NonCacheable + SCTLR_EL1::C::NonCacheable); - - barrier::isb(barrier::SY); - } + type IrqNumber = IrqNumber; unsafe fn start_application_processors(&self) { let dt = self.dt.get(); - if let Err(e) = smp::start_ap_cores(dt) { - errorln!( - "Could not initialize AP CPUs: {:?}. Will continue with one CPU.", - e - ); + if let Err(error) = smp::start_ap_cores(dt) { + errorln!("Could not initialize AP CPUs: {:?}", error); } } - fn map_device_pages(&self, phys: usize, count: usize) -> Result { - unsafe { KERNEL_TABLES.map_device_pages(phys, count) } - } - - fn wait_for_interrupt() { - aarch64_cpu::asm::wfi(); + fn cpu_count() -> usize { + smp::CPU_COUNT.load(Ordering::Acquire) } unsafe fn set_interrupt_mask(mask: bool) { @@ -170,26 +101,98 @@ impl Architecture for AArch64 { DAIF.read(DAIF::I) != 0 } - fn cpu_count() -> usize { - CPU_COUNT.load(Ordering::Acquire) + fn wait_for_interrupt() { + aarch64_cpu::asm::wfi(); } - fn register_external_interrupt_controller( + unsafe fn map_device_memory( &self, - intc: &'static dyn ExternalInterruptController, + base: PhysicalAddress, + size: usize, + ) -> Result { + mem::map_device_memory(base, size) + } + + unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) { + mem::unmap_device_memory(map) + } + + fn map_physical_memory + Clone>( + &self, + _it: I, + _memory_start: PhysicalAddress, + memory_end: PhysicalAddress, ) -> Result<(), Error> { - self.ext_intc.init(intc); + let end_l1i = L1::index(memory_end.page_align_up::().into_raw()); + if end_l1i > mem::RAM_MAPPING_L1_COUNT { + loop {} + } + + // Map 1GiB chunks + for index in 0..end_l1i { + unsafe { + mem::map_ram_l1(index); + } + } + + mem::MEMORY_LIMIT.init(memory_end.into_raw()); + Ok(()) } + fn virtualize(address: PhysicalAddress) -> Result { + let raw: usize = address.into_raw(); + if raw < *mem::MEMORY_LIMIT.get() { + Ok(raw + mem::RAM_MAPPING_OFFSET) + } else { + errorln!("Invalid physical address: {:#x}", address); + Err(Error::InvalidMemoryOperation) + } + } + + fn physicalize(address: usize) -> Result { + if address < mem::RAM_MAPPING_OFFSET + || address - mem::RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get() + { + errorln!("Not a virtualized physical address: {:#x}", address); + return Err(Error::InvalidMemoryOperation); + } + + Ok(PhysicalAddress::from_raw(address - mem::RAM_MAPPING_OFFSET)) + } + + fn local_interrupt_controller( + &'static self, + ) -> &'static dyn LocalInterruptController { + *self.lintc.get() + } + + fn external_interrupt_controller( + &'static self, + ) -> &'static dyn ExternalInterruptController { + *self.xintc.get() + } + fn register_local_interrupt_controller( &self, intc: &'static dyn LocalInterruptController, ) -> Result<(), Error> { - self.local_intc.init(intc); + self.lintc.init(intc); Ok(()) } + fn register_external_interrupt_controller( + &self, + intc: &'static dyn ExternalInterruptController, + ) -> Result<(), Error> { + self.xintc.init(intc); + Ok(()) + } + + fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice { + *self.mtimer.get() + } + fn register_monotonic_timer( &self, timer: &'static dyn MonotonicTimestampProviderDevice, @@ -198,107 +201,143 @@ impl Architecture for AArch64 { Ok(()) } - fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { - self.reset.init(reset); - Ok(()) - } - - fn external_interrupt_controller( - &self, - ) -> &'static dyn ExternalInterruptController { - *self.ext_intc.get() - } - - fn local_interrupt_controller( - &self, - ) -> &'static dyn LocalInterruptController { - *self.local_intc.get() - } - - fn monotonic_timer(&self) -> &'static dyn MonotonicTimestampProviderDevice { - *self.mtimer.get() - } - unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> { - if let Some(local_intc) = self.local_intc.try_get() { + if let Some(local_intc) = self.lintc.try_get() { local_intc.send_ipi(target, msg) } else { Ok(()) } } + fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> { + self.reset.init(reset); + Ok(()) + } + unsafe fn reset(&self) -> ! { if let Some(reset) = self.reset.try_get() { - reset.reset(); + reset.reset() } else { let psci = self.psci.get(); - psci.reset(); + psci.reset() } } } impl AArch64 { - /// Initializes the architecture's device tree - /// - /// # Safety - /// - /// Only makes sense to call during the early initialization, once. - pub unsafe fn init_device_tree(&self, dtb_phys: usize) { - let dt = DeviceTree::from_addr(dtb_phys.virtualize()); - self.dt.init(dt); + fn extract_initrd_from_dt( + &self, + dt: &DeviceTree, + ) -> Option<(PhysicalAddress, PhysicalAddress)> { + let chosen = dt.node_by_path("/chosen")?; + let initrd_start = devtree::find_prop(&chosen, "linux,initrd-start")?; + let initrd_end = devtree::find_prop(&chosen, "linux,initrd-end")?; + + let address_cells = dt.address_cells(); + + let initrd_start = initrd_start.cell1_array_item(0, address_cells)?; + let initrd_end = initrd_end.cell1_array_item(0, address_cells)?; + + let initrd_start = PhysicalAddress::from_raw(initrd_start); + let initrd_end = PhysicalAddress::from_raw(initrd_end); + + Some((initrd_start, initrd_end)) } - /// Returns the device tree - /// - /// # Panics - /// - /// Will panic if the device tree has not yet been initialized - pub fn device_tree(&self) -> &DeviceTree { - self.dt.get() - } + unsafe fn init_memory_management(&'static self, dtb: PhysicalAddress) -> Result<(), Error> { + // 16x2MiB + const HEAP_PAGES: usize = 16; - unsafe fn init_physical_memory(&self, dtb_phys: usize) -> Result<(), Error> { - let dt = self.device_tree(); + // Initialize the runtime mappings + mem::init_fixed_tables(); - if let Some(initrd) = INITRD_DATA.try_get() { - reserve_region( - "initrd", - PhysicalMemoryRegion { - base: initrd.phys_page_start, - size: initrd.phys_page_len, - }, - ); - } + // Extract the size of the device tree + let dtb_size = { + let dtb_header = EarlyMapping::::map_slice(dtb, DevTree::MIN_HEADER_SIZE)?; + DevTree::read_totalsize(dtb_header.as_ref()).unwrap() + }; reserve_region( "dtb", PhysicalMemoryRegion { - base: dtb_phys, - size: (dt.size() + 0xFFF) & !0xFFF, + base: dtb, + size: (dtb_size + 0xFFF) & !0xFFF, }, ); - let regions = FdtMemoryRegionIter::new(dt); + let dtb_slice = EarlyMapping::::map_slice(dtb, dtb_size)?; - phys::init_from_iter(regions) + let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize); + + // Setup initrd from the dt + let initrd = self.extract_initrd_from_dt(&dt); + + if let Some((start, end)) = initrd { + let aligned_start = start.page_align_down::(); + let aligned_end = end.page_align_up::(); + + let size = aligned_end - aligned_start; + reserve_region( + "initrd", + PhysicalMemoryRegion { + base: aligned_start, + size, + }, + ); + } + + // Initialize the physical memory + let regions = FdtMemoryRegionIter::new(&dt); + + phys::init_from_iter(regions)?; + + // Setup the heap + for i in 0..HEAP_PAGES { + let l2_page = phys::alloc_2m_page()?; + mem::map_heap_l2(i, l2_page); + } + + heap::init_heap(mem::HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE); + + // EarlyMapping for DTB no longer needed, it lives in physical memory and can be obtained + // through PhysicalRef + let dtb_slice: PhysicalRef<'static, [u8]> = PhysicalRef::map_slice(dtb, dtb_size); + let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize); + + self.dt.init(dt); + + // Setup initrd + if let Some((initrd_start, initrd_end)) = initrd { + let aligned_start = initrd_start.page_align_down::(); + let aligned_end = initrd_end.page_align_up::(); + let len = initrd_end - initrd_start; + + let data = unsafe { PhysicalRef::map_slice(initrd_start, len) }; + self.initrd.init(data); + + INITRD_DATA.init(Initrd { + phys_page_start: aligned_start, + phys_page_len: aligned_end - aligned_start, + data: self.initrd.get().as_ref(), + }); + } + + Ok(()) } - fn chosen_stdout_path<'a>(dt: &'a DeviceTree) -> Option<&'a str> { - let chosen = dt.node_by_path("/chosen")?; - chosen.prop("stdout-path") - } + unsafe fn init_platform(&self, is_bsp: bool) -> Result<(), Error> { + Cpu::init_local(); - fn init_platform(&self, bsp: bool) { - if bsp { - let dt = self.device_tree(); + if is_bsp { + let dt = self.dt.get(); let address_cells = dt.address_cells(); let size_cells = dt.size_cells(); - let chosen_stdout_path = Self::chosen_stdout_path(dt); + // Setup /chosen.stdout-path to get early debug printing + let chosen_stdout_path = dt.chosen_stdout_path(); let chosen_stdout = chosen_stdout_path.and_then(|path| dt.node_by_path(path)); - // Probe and initialize the /chosen.stdout-path device first if let Some(node) = chosen_stdout.clone() { let probe = DevTreeNodeInfo { address_cells, @@ -307,15 +346,12 @@ impl AArch64 { }; if let Some((device, _)) = devtree::probe_dt_node(&probe) { - unsafe { - device.init().unwrap(); - } + device.init()?; } - }; + } - debug::reset(); + debug::init(); - // Print some stuff now that the output is initialized infoln!( "Yggdrasil v{} ({})", env!("CARGO_PKG_VERSION"), @@ -323,22 +359,19 @@ impl AArch64 { ); infoln!("Initializing aarch64 platform"); - // Probe and initialize the rest of devices let nodes = dt.root().children(); if let Err(error) = devtree::enumerate_dt( address_cells, size_cells, nodes, |_, probe| { - // Ignore /chosen/stdout-path node + // Skip chosen-stdout, already initialized if let Some(ref chosen_stdout) = chosen_stdout && chosen_stdout.name() == probe.node.name() { return Ok(()); } if let Some((device, _)) = devtree::probe_dt_node(&probe) { - unsafe { - device.init()?; - } + device.init()?; } Ok(()) @@ -353,15 +386,14 @@ impl AArch64 { // Initialize IRQs for the devices device::manager_lock().devices().for_each(|dev| unsafe { if let Err(error) = dev.init_irq() { - errorln!( - "Could not init interrupts for {:?}: {:?}", + warnln!( + "Could not init IRQs for {:?}: {:?}", dev.display_name(), error ); } }); - // Print the device list infoln!("Enumerated devices:"); device::manager_lock().devices().for_each(|dev| { infoln!("* {:?}", dev.display_name()); @@ -369,7 +401,7 @@ impl AArch64 { } else { // BSP already initialized everything needed // Setup timer and local interrupt controller - let intc = self.local_intc.get(); + let intc = self.lintc.get(); unsafe { intc.init_ap().unwrap(); @@ -378,57 +410,28 @@ impl AArch64 { // TODO device-tree initialization for this CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR); CNTP_TVAL_EL0.set(10000000); - self.ext_intc - .get() - .enable_irq(IrqNumber::Private(14)) - .unwrap(); + self.xintc.get().enable_irq(IrqNumber::Private(14)).unwrap(); } + + Ok(()) } } -fn setup_initrd() { - let dt = ARCHITECTURE.device_tree(); +pub static ARCHITECTURE: AArch64 = AArch64 { + dt: OneTimeInit::new(), + initrd: OneTimeInit::new(), - let Some(chosen) = dt.node_by_path("/chosen") else { - return; - }; + psci: OneTimeInit::new(), + reset: OneTimeInit::new(), - let Some(initrd_start) = devtree::find_prop(&chosen, "linux,initrd-start") else { - return; - }; - let Some(initrd_end) = devtree::find_prop(&chosen, "linux,initrd-end") else { - return; - }; + lintc: OneTimeInit::new(), + xintc: OneTimeInit::new(), - let address_cells = dt.address_cells(); + mtimer: OneTimeInit::new(), +}; - let Some(initrd_start) = initrd_start.cell1_array_item(0, address_cells) else { - infoln!("No initrd specified"); - return; - }; - let Some(initrd_end) = initrd_end.cell1_array_item(0, address_cells) else { - infoln!("No initrd specified"); - return; - }; - - let initrd_start = initrd_start as usize; - let initrd_end = initrd_end as usize; - - let start_aligned = initrd_start & !0xFFF; - let end_aligned = initrd_end & !0xFFF; - - let data = unsafe { - core::slice::from_raw_parts( - initrd_start.virtualize() as *const _, - initrd_end - initrd_start, - ) - }; - - let initrd = Initrd { - phys_page_start: start_aligned, - phys_page_len: end_aligned - start_aligned, - data, - }; - - INITRD_DATA.init(initrd); +#[derive(Clone, Copy, Debug, PartialEq)] +pub enum IrqNumber { + Private(u32), + Shared(u32), } diff --git a/src/arch/aarch64/plat_orange_pi3/mod.rs b/src/arch/aarch64/plat_orange_pi3/mod.rs deleted file mode 100644 index 8b650c3d..00000000 --- a/src/arch/aarch64/plat_orange_pi3/mod.rs +++ /dev/null @@ -1,103 +0,0 @@ -//! Orange Pi 3 (Allwinner H6/sun50i-h6 SoC) -//! -//! # Booting using u-boot -//! -//! > fatload mmc 0:1 0x40000000 uRamdisk -//! > fatload mmc 0:1 0x4d000000 sun50i-h6-orangepi-3.dtb -//! > loady 0x44000000 -//! ... -//! > bootm 0x44000000 0x40000000 0x4d000000 -//! -use abi::error::Error; - -use crate::{ - arch::CpuMessage, - debug::{self, LogLevel}, - device::{ - interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget}, - platform::Platform, - timer::TimestampSource, - InitializableDevice, - }, - fs::devfs::{self, CharDeviceType}, -}; - -use self::{r_wdog::RWatchdog, serial::Serial}; - -use super::{ - gic::{Gic, IrqNumber}, - timer::ArmTimer, -}; - -pub mod r_wdog; -pub mod serial; - -/// Orange Pi 3 implementation -pub struct PlatformImpl { - uart0: Serial, - r_wdog: RWatchdog, - /// ... - pub gic: Gic, - timer: ArmTimer, -} - -impl Platform for PlatformImpl { - type IrqNumber = IrqNumber; - - const KERNEL_PHYS_BASE: usize = 0x50000000; - - unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> { - if is_bsp { - self.gic.init(())?; - - self.timer.init(())?; - self.r_wdog.init(())?; - - self.timer.init_irq()?; - self.uart0.init_irq()?; - - devfs::add_char_device(&self.uart0, CharDeviceType::TtySerial)?; - } else { - todo!(); - } - - Ok(()) - } - - unsafe fn init_debug(&'static self) { - self.uart0.init(()).unwrap(); - debug::add_sink(&self.uart0, LogLevel::Debug); - } - - fn interrupt_controller( - &self, - ) -> &dyn ExternalInterruptController { - &self.gic - } - - fn timestamp_source(&self) -> &dyn TimestampSource { - &self.timer - } - - unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> { - todo!(); - } - - fn name(&self) -> &'static str { - "Orange Pi 3" - } - - unsafe fn reset(&self) -> ! { - self.r_wdog.reset_board(); - } -} - -/// Orange Pi 3 platform implementation -pub static PLATFORM: PlatformImpl = unsafe { - PlatformImpl { - uart0: Serial::new(0x05000000, IrqNumber::new(32)), - r_wdog: RWatchdog::new(0x07020400), - gic: Gic::new(0x03021000, 0x03022000), - timer: ArmTimer::new(IrqNumber::new(30)), - } -}; diff --git a/src/arch/aarch64/plat_orange_pi3/r_wdog.rs b/src/arch/aarch64/plat_orange_pi3/r_wdog.rs deleted file mode 100644 index 5deac6a9..00000000 --- a/src/arch/aarch64/plat_orange_pi3/r_wdog.rs +++ /dev/null @@ -1,90 +0,0 @@ -//! Allwinner H6 R Watchdog driver -use abi::error::Error; -use tock_registers::{ - interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite, -}; - -use crate::{ - device::InitializableDevice, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock, - util::OneTimeInit, -}; - -register_bitfields! { - u32, - CTRL [ - KEY OFFSET(1) NUMBITS(12) [ - Value = 0xA57 - ], - RESTART OFFSET(0) NUMBITS(1) [] - ], - CFG [ - CONFIG OFFSET(0) NUMBITS(2) [ - System = 1, - ] - ], - MODE [ - EN OFFSET(0) NUMBITS(1) [], - ] -} - -register_structs! { - #[allow(non_snake_case)] - Regs { - (0x00 => IRQ_EN: ReadWrite), - (0x04 => IRQ_STA: ReadWrite), - (0x08 => _0), - (0x10 => CTRL: ReadWrite), - (0x14 => CFG: ReadWrite), - (0x18 => MODE: ReadWrite), - (0x1C => @END), - } -} - -/// Allwinner H6 R Watchdog -pub struct RWatchdog { - inner: OneTimeInit>>, - base: usize, -} - -impl InitializableDevice for RWatchdog { - type Options = (); - - unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { - let regs = DeviceMemoryIo::map("r_wdog", self.base)?; - - self.inner.init(IrqSafeSpinlock::new(regs)); - - Ok(()) - } -} - -impl RWatchdog { - /// Performs a reset through watchdog. - /// - /// # Safety - /// - /// Only meant to be called by platform reset code. - pub unsafe fn reset_board(&self) -> ! { - let regs = self.inner.get().lock(); - - regs.CFG.write(CFG::CONFIG::System); - regs.MODE.write(MODE::EN::SET); - regs.CTRL.write(CTRL::KEY::Value + CTRL::RESTART::SET); - - loop { - core::arch::asm!("wfe"); - } - } - - /// Constructs an instance of the device at `base`. - /// - /// # Safety - /// - /// The caller must ensure the address is valid. - pub const unsafe fn new(base: usize) -> Self { - Self { - inner: OneTimeInit::new(), - base, - } - } -} diff --git a/src/arch/aarch64/plat_orange_pi3/serial.rs b/src/arch/aarch64/plat_orange_pi3/serial.rs deleted file mode 100644 index 864189a0..00000000 --- a/src/arch/aarch64/plat_orange_pi3/serial.rs +++ /dev/null @@ -1,160 +0,0 @@ -//! Allwinner H6 UART driver -use abi::{error::Error, io::DeviceRequest}; -use tock_registers::{ - interfaces::{Readable, Writeable}, - register_bitfields, register_structs, - registers::{ReadOnly, ReadWrite}, -}; -use vfs::CharDevice; - -use crate::{ - arch::aarch64::gic::IrqNumber, - arch::PLATFORM, - debug::DebugSink, - device::{ - interrupt::InterruptSource, - platform::Platform, - serial::SerialDevice, - tty::{CharRing, TtyDevice}, - Device, InitializableDevice, - }, - mem::device::DeviceMemoryIo, - sync::IrqSafeSpinlock, - util::OneTimeInit, -}; - -register_bitfields! { - u32, - USR [ - TFE OFFSET(2) NUMBITS(1) [], - TFNF OFFSET(1) NUMBITS(1) [] - ] -} - -register_structs! { - #[allow(non_snake_case)] - Regs { - (0x00 => DLL: ReadWrite), - (0x04 => _0), - (0x7C => USR: ReadOnly), - (0x80 => @END), - } -} - -struct Inner { - regs: DeviceMemoryIo, -} - -/// Allwinner H6 UART -pub struct Serial { - inner: OneTimeInit>, - ring: CharRing<16>, - base: usize, - irq: IrqNumber, -} - -impl CharDevice for Serial { - fn read(&'static self, _blocking: bool, data: &mut [u8]) -> Result { - self.line_read(data) - } - - fn write(&self, _blocking: bool, data: &[u8]) -> Result { - self.line_write(data) - } - - fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { - match req { - &mut DeviceRequest::SetTerminalGroup(id) => { - self.set_signal_group(id as _); - Ok(()) - } - _ => Err(Error::InvalidArgument), - } - } -} - -impl TtyDevice<16> for Serial { - fn ring(&self) -> &CharRing<16> { - &self.ring - } -} - -impl InterruptSource for Serial { - unsafe fn init_irq(&'static self) -> Result<(), Error> { - let intc = PLATFORM.interrupt_controller(); - - intc.register_handler(self.irq, self)?; - intc.enable_irq(self.irq)?; - - Ok(()) - } - - fn handle_irq(&self) -> Result { - let byte = self.inner.get().lock().regs.DLL.get(); - - self.recv_byte(byte as u8); - - Ok(true) - } -} - -impl DebugSink for Serial { - fn putc(&self, c: u8) -> Result<(), Error> { - self.send(c).ok(); - Ok(()) - } -} - -impl SerialDevice for Serial { - fn send(&self, byte: u8) -> Result<(), Error> { - let inner = self.inner.get().lock(); - if byte == b'\n' { - while inner.regs.USR.matches_all(USR::TFE::CLEAR) { - core::hint::spin_loop(); - } - inner.regs.DLL.set(b'\r' as u32); - } - while inner.regs.USR.matches_all(USR::TFE::CLEAR) { - core::hint::spin_loop(); - } - inner.regs.DLL.set(byte as u32); - Ok(()) - } - - fn receive(&self, _blocking: bool) -> Result { - todo!() - } -} - -impl InitializableDevice for Serial { - type Options = (); - - unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> { - let regs = DeviceMemoryIo::::map("h6-uart", self.base)?; - self.inner.init(IrqSafeSpinlock::new(Inner { regs })); - Ok(()) - } -} - -impl Device for Serial { - fn name(&self) -> &'static str { - "Allwinner H6 UART" - } -} - -impl Serial { - /// Constructs an instance of the device at `base`. - /// - /// # Safety - /// - /// The caller must ensure the address is valid. - pub const unsafe fn new(base: usize, irq: IrqNumber) -> Self { - Self { - inner: OneTimeInit::new(), - ring: CharRing::new(), - - base, - irq, - } - } -} diff --git a/src/arch/aarch64/smp.rs b/src/arch/aarch64/smp.rs index 424a8cf7..ee01418f 100644 --- a/src/arch/aarch64/smp.rs +++ b/src/arch/aarch64/smp.rs @@ -5,17 +5,18 @@ use abi::error::Error; use device_api::CpuBringupDevice; use fdt_rs::prelude::PropReader; +use crate::arch::Architecture; +use crate::mem::address::IntoRaw; +use crate::mem::KERNEL_VIRT_OFFSET; use crate::{ - absolute_address, - arch::ARCHITECTURE, - mem::{ - phys::{self, PageUsage}, - ConvertAddress, - }, + arch::{ArchitectureImpl, ARCHITECTURE}, + mem::phys, }; use crate::device::devtree::{self, DevTreeIndexNodePropGet, DeviceTree}; +use super::{BootStack, BOOT_STACK_SIZE}; + #[derive(Debug)] enum CpuEnableMethod { Psci, @@ -74,6 +75,12 @@ impl CpuEnableMethod { /// Number of online CPUs, initially set to 1 (BSP processor is up) pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1); +// TODO can be made smaller +#[link_section = ".bss"] +static AP_TRAMPOLINE_STACK: BootStack = BootStack { + data: [0; BOOT_STACK_SIZE], +}; + /// Starts application processors using the method specified in the device tree. /// /// TODO: currently does not handle systems where APs are already started before entry. @@ -87,6 +94,9 @@ pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> { fn __aarch64_ap_entry(); } + // Safety: safe, the stack is inside the kernel + let sp = AP_TRAMPOLINE_STACK.data.as_ptr() as usize - KERNEL_VIRT_OFFSET + BOOT_STACK_SIZE; + for cpu in enumerate_cpus(dt).filter(|cpu| cpu.id != 0) { debugln!( "cpu{}: enable-method={:?}, compatible={:?}", @@ -95,19 +105,20 @@ pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> { cpu.compatible ); - const AP_STACK_PAGES: usize = 4; - let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used)?; - debugln!( - "cpu{} stack: {:#x}..{:#x}", - cpu.id, - stack_pages, - stack_pages + AP_STACK_PAGES * 0x1000 - ); + // const AP_STACK_PAGES: usize = 4; + // let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES)?; + // debugln!( + // "cpu{} stack: {:#x}..{:#x}", + // cpu.id, + // stack_pages, + // stack_pages.add(AP_STACK_PAGES * 0x1000) + // ); // Wait for the CPU to come up let old_count = CPU_COUNT.load(Ordering::Acquire); - let ip = absolute_address!(__aarch64_ap_entry).physicalize(); - let sp = stack_pages + AP_STACK_PAGES * 0x1000; + // Safety: safe, the function is inside the kernel + let ip = __aarch64_ap_entry as usize - KERNEL_VIRT_OFFSET; + // let sp = stack_pages.add(AP_STACK_PAGES * 0x1000); if let Err(error) = cpu.enable_method.start_cpu(cpu.id as usize, ip, sp) { errorln!("Couldn't start cpu{} up: {:?}", cpu.id, error); continue; diff --git a/src/arch/aarch64/table.rs b/src/arch/aarch64/table.rs deleted file mode 100644 index 1690ea38..00000000 --- a/src/arch/aarch64/table.rs +++ /dev/null @@ -1,553 +0,0 @@ -//! AArch64 virtual memory management facilities -use core::{ - marker::PhantomData, - ops::{Index, IndexMut}, - sync::atomic::{AtomicU8, Ordering}, -}; - -use aarch64_cpu::registers::DAIF; -use abi::error::Error; -use bitflags::bitflags; -use tock_registers::interfaces::Readable; - -use crate::mem::{ - phys::{self, PageUsage}, - table::{ - EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager, - }, - ConvertAddress, KERNEL_VIRT_OFFSET, -}; - -/// TODO -#[derive(Clone)] -#[repr(C)] -pub struct AddressSpace { - l1: *mut PageTable, - asid: u8, -} - -/// Page table representing a single level of address translation -#[derive(Clone)] -#[repr(C, align(0x1000))] -pub struct PageTable { - data: [PageEntry; 512], -} - -/// Translation level 1: Entry is 1GiB page/table -#[derive(Clone, Copy)] -pub struct L1; -/// Translation level 2: Entry is 2MiB page/table -#[derive(Clone, Copy)] -pub struct L2; -/// Translation level 3: Entry is 4KiB page -#[derive(Clone, Copy)] -pub struct L3; - -impl NonTerminalEntryLevel for L1 { - type NextLevel = L2; -} -impl NonTerminalEntryLevel for L2 { - type NextLevel = L3; -} - -bitflags! { - /// Describes how each page is mapped: access, presence, type of the mapping. - #[derive(Clone, Copy)] - pub struct PageAttributes: u64 { - /// When set, the mapping is considered valid and assumed to point to a page/table - const PRESENT = 1 << 0; - - /// For L1/L2 mappings, indicates that the mapping points to the next-level translation - /// table - const TABLE = 1 << 1; - /// (Must be set) For L3 mappings, indicates that the mapping points to a page - const PAGE = 1 << 1; - /// For L1/L2 mappings, indicates that the mapping points to a page of given level's size - const BLOCK = 0 << 1; - - /// (Must be set) For page/block mappings, indicates to the hardware that the page is - /// accessed - const ACCESS = 1 << 10; - - /// For page/block mappings, allows both user and kernel code to read/write to the page - const AP_BOTH_READWRITE = 1 << 6; - /// For page/block mappings, only allows read access for EL0/EL1 - const AP_BOTH_READONLY = 3 << 6; - - /// Indicates outer shareability domain - const SH_OUTER = 2 << 8; - /// Indicates inner shareability domain - const SH_INNER = 3 << 8; - - /// For page/block mappings, indicates regular physical memory - const PAGE_ATTR_NORMAL = 0 << 2; - /// For page/block mappings, indicates device memory - const PAGE_ATTR_DEVICE = 1 << 2; - - /// Indicates the mapping is unique to a specific ASID (important for proper TLB - /// maintenance) - const NON_GLOBAL = 1 << 11; - } -} - -impl const EntryLevel for L1 { - fn index(addr: usize) -> usize { - (addr >> 30) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x3FFFFFFF - } -} -impl const EntryLevel for L2 { - fn index(addr: usize) -> usize { - (addr >> 21) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x1FFFFF - } -} -impl const EntryLevel for L3 { - fn index(addr: usize) -> usize { - (addr >> 12) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0xFFF - } -} - -/// Represents a single entry in a translation table -#[derive(Clone, Copy)] -#[repr(transparent)] -pub struct PageEntry(u64, PhantomData); - -/// Fixed-layout kernel-space address mapping tables -pub struct FixedTables { - l1: PageTable, - device_l2: PageTable, - device_l3: PageTable, - - device_l3i: usize, -} - -impl PageEntry { - /// Creates a 4KiB normal memory page mapping - pub fn normal_page(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) - | (PageAttributes::PAGE - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | attrs) - .bits(), - PhantomData, - ) - } - - /// Creates a 4KiB device memory page mapping - pub fn device_page(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) - | (PageAttributes::PAGE - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_OUTER - | PageAttributes::PAGE_ATTR_DEVICE - | attrs) - .bits(), - PhantomData, - ) - } - - /// Returns the physical address of the page this entry refers to, returning None if it does - /// not - pub fn as_page(self) -> Option { - let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits(); - - if self.0 & mask == mask { - Some((self.0 & !0xFFF) as usize) - } else { - None - } - } -} - -impl PageEntry { - /// Creates a 2MiB page mapping - pub fn normal_block(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) - | (PageAttributes::BLOCK - | PageAttributes::PRESENT - | PageAttributes::ACCESS - | PageAttributes::SH_INNER - | PageAttributes::PAGE_ATTR_NORMAL - | attrs) - .bits(), - PhantomData, - ) - } - - /// Creates a mapping pointing to the next-level translation table - pub fn table(phys: usize, attrs: PageAttributes) -> Self { - Self( - (phys as u64) | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(), - PhantomData, - ) - } - - /// Returns the physical address of the table this entry refers to, returning None if it - /// does not - pub fn as_table(self) -> Option { - if self.0 & (PageAttributes::TABLE | PageAttributes::PRESENT).bits() - == (PageAttributes::TABLE | PageAttributes::PRESENT).bits() - { - Some((self.0 & !0xFFF) as usize) - } else { - None - } - } -} - -impl PageEntry { - /// Represents an absent/invalid mapping in the table - pub const INVALID: Self = Self(0, PhantomData); - - /// Converts a raw mapping value into this wrapper type - /// - /// # Safety - /// - /// The caller is responsible for making sure that `raw` is a valid mapping value for the - /// current translation level. - pub unsafe fn from_raw(raw: u64) -> Self { - Self(raw, PhantomData) - } - - /// Returns `true` if the entry refers to some table/block/page - pub fn is_present(&self) -> bool { - self.0 & PageAttributes::PRESENT.bits() != 0 - } -} - -impl NextPageTable for PageTable { - type NextLevel = PageTable; - - fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel> { - let entry = self[index]; - - entry - .as_table() - .map(|addr| unsafe { &mut *(addr.virtualize() as *mut Self::NextLevel) }) - } - - fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error> { - let entry = self[index]; - - if let Some(table) = entry.as_table() { - Ok(unsafe { &mut *(table.virtualize() as *mut Self::NextLevel) }) - } else { - let table = PageTable::new_zeroed()?; - self[index] = PageEntry::::table(table.physical_address(), PageAttributes::empty()); - Ok(table) - } - } -} - -impl PageTable { - /// Constructs a page table with all entries marked as invalid - pub const fn zeroed() -> Self { - Self { - data: [PageEntry::INVALID; 512], - } - } - - /// Allocates a new page table, filling it with non-preset entries - pub fn new_zeroed() -> Result<&'static mut Self, Error> { - let page = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() }; - let table = unsafe { &mut *(page as *mut Self) }; - for i in 0..512 { - table[i] = PageEntry::INVALID; - } - Ok(table) - } - - /// Returns a physical address pointing to this page table - pub fn physical_address(&self) -> usize { - // &self may already by a physical address - let addr = self.data.as_ptr() as usize; - if addr < KERNEL_VIRT_OFFSET { - addr - } else { - unsafe { addr.physicalize() } - } - } -} - -impl Index for PageTable { - type Output = PageEntry; - - fn index(&self, index: usize) -> &Self::Output { - &self.data[index] - } -} - -impl IndexMut for PageTable { - fn index_mut(&mut self, index: usize) -> &mut Self::Output { - &mut self.data[index] - } -} - -impl FixedTables { - /// Constructs an empty table group - pub const fn zeroed() -> Self { - Self { - l1: PageTable::zeroed(), - device_l2: PageTable::zeroed(), - device_l3: PageTable::zeroed(), - - device_l3i: 0, - } - } - - /// Maps a physical memory region as device memory and returns its allocated base address - pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result { - if count > 512 * 512 { - panic!("Unsupported device memory mapping size"); - } else if count > 512 { - // 2MiB mappings - todo!(); - } else { - // 4KiB mappings - if self.device_l3i + count > 512 { - return Err(Error::OutOfMemory); - } - - let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12); - for i in 0..count { - self.device_l3[self.device_l3i + i] = - PageEntry::device_page(phys + i * 0x1000, PageAttributes::empty()); - } - self.device_l3i += count; - - Ok(virt) - } - } -} - -impl From for PageAttributes { - fn from(value: MapAttributes) -> Self { - let mut res = Self::empty(); - if value.contains(MapAttributes::USER_WRITE) { - res |= PageAttributes::AP_BOTH_READWRITE; - } else { - res |= PageAttributes::AP_BOTH_READONLY; - } - if value.contains(MapAttributes::NON_GLOBAL) { - res |= PageAttributes::NON_GLOBAL; - } - res - } -} - -impl VirtualMemoryManager for AddressSpace { - fn allocate( - &self, - hint: Option, - len: usize, - attrs: MapAttributes, - ) -> Result { - assert_eq!(DAIF.read(DAIF::I), 1); - - if hint.is_some() { - todo!(); - } - - const TRY_ALLOC_START: usize = 0x100000000; - const TRY_ALLOC_END: usize = 0xF00000000; - - 'l0: for base in (TRY_ALLOC_START..TRY_ALLOC_END - len * 0x1000).step_by(0x1000) { - for i in 0..len { - if self.translate(base + i * 0x1000).is_some() { - continue 'l0; - } - } - - for i in 0..len { - let page = phys::alloc_page(PageUsage::Used)?; - self.map_page(base + i * 0x1000, page, attrs)?; - } - - return Ok(base); - } - - Err(Error::OutOfMemory) - } - - fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error> { - assert_eq!(DAIF.read(DAIF::I), 1); - - for page in (addr..addr + len).step_by(0x1000) { - let Some(phys) = self.translate(page) else { - todo!( - "Tried to deallocate address not present in the table: {:#x}", - addr - ); - }; - - self.write_entry(page, PageEntry::INVALID, true)?; - unsafe { - phys::free_page(phys); - } - } - - Ok(()) - } - - fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> { - self.write_entry(virt, PageEntry::normal_page(phys, attrs.into()), true) - } -} - -impl AddressSpace { - /// Allocates an empty address space with all entries marked as non-present - pub fn new_empty() -> Result { - static LAST_ASID: AtomicU8 = AtomicU8::new(1); - - let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel); - - let l1 = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() as *mut PageTable }; - - for i in 0..512 { - unsafe { - (*l1)[i] = PageEntry::INVALID; - } - } - - Ok(Self { l1, asid }) - } - - /// Interprets a physical address as an address space structure pointer. - /// - /// # Safety - /// - /// Unsafe: accepts arbitrary addresses and ignores ASIDs. - pub unsafe fn from_phys_raw(value: usize) -> Self { - let value = value.virtualize(); - Self { - l1: value as *mut PageTable, - asid: 0, - } - } - - /// Iterates through all the translation levels for a given address, invoking the callback - /// function on each of them - pub fn walk bool>(&self, vaddr: usize, f: F) { - let l1i = L1::index(vaddr); - let l2i = L2::index(vaddr); - let l3i = L3::index(vaddr); - - let l1 = unsafe { self.as_mut() }; - let l1e = l1[l1i]; - - let Some(l2) = l1.get_mut(l1i) else { - f(1, 0); - return; - }; - if !f(1, l1e.0 as _) { - return; - } - let l2e = l2[l2i]; - - let Some(l3) = l2.get_mut(l2i) else { - f(2, 0); - return; - }; - if !f(2, l2e.0 as _) { - return; - } - - let l3e = l3[l3i]; - if !f(3, l3e.0 as _) { - return; - } - } - - unsafe fn as_mut(&self) -> &'static mut PageTable { - self.l1.as_mut().unwrap() - } - - // TODO return page size and attributes - /// Returns the physical address to which the `virt` address is mapped - pub fn translate(&self, virt: usize) -> Option { - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); - - let l2 = unsafe { self.as_mut().get_mut(l1i) }?; - let l3 = l2.get_mut(l2i)?; - - l3[l3i].as_page() - } - - // Write a single 4KiB entry - fn write_entry(&self, virt: usize, entry: PageEntry, overwrite: bool) -> Result<(), Error> { - let l1i = L1::index(virt); - let l2i = L2::index(virt); - let l3i = L3::index(virt); - - let l2 = unsafe { self.as_mut().get_mut_or_alloc(l1i) }?; - let l3 = l2.get_mut_or_alloc(l2i)?; - - if l3[l3i].is_present() && !overwrite { - todo!() - } - l3[l3i] = entry; - - tlb_flush_vaae1(virt); - - Ok(()) - } - - /// Returns the physical address of the address space (to be used in a TTBRn_ELx) - pub fn physical_address(&self) -> usize { - unsafe { (self.l1 as usize).physicalize() | ((self.asid as usize) << 48) } - } -} - -/// Flush a virtual address from EL1/EL0 TLB for all ASIDs -pub fn tlb_flush_vaae1(mut page: usize) { - page >>= 12; - unsafe { - core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page); - } -} - -/// Initializes mappings for the kernel and device memory tables. -/// -/// # Safety -/// -/// Only allowed to be called once during lower-half part of the initialization process. -pub unsafe fn init_fixed_tables() { - // Map first 256GiB - for i in 0..256 { - KERNEL_TABLES.l1[i] = PageEntry::::normal_block(i << 30, PageAttributes::empty()); - } - - KERNEL_TABLES.l1[256] = PageEntry::::table( - KERNEL_TABLES.device_l2.physical_address(), - PageAttributes::empty(), - ); - KERNEL_TABLES.device_l2[0] = PageEntry::::table( - KERNEL_TABLES.device_l3.physical_address(), - PageAttributes::empty(), - ); -} - -/// Offset applied to device virtual memory mappings -pub const DEVICE_VIRT_OFFSET: usize = KERNEL_VIRT_OFFSET + (256 << 30); -/// Global kernel address space translation tables -pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed(); diff --git a/src/arch/aarch64/timer.rs b/src/arch/aarch64/timer.rs index e4cc7c8f..9a7fe43e 100644 --- a/src/arch/aarch64/timer.rs +++ b/src/arch/aarch64/timer.rs @@ -11,8 +11,7 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, device_tree_driver, - proc::wait, - task::tasklet, + task::runtime, }; use super::cpu::Cpu; @@ -30,8 +29,7 @@ impl InterruptHandler for ArmTimer { CNTP_TVAL_EL0.set(TICK_INTERVAL); let now = self.monotonic_timestamp().unwrap(); - wait::tick(now); - tasklet::tick(now); + runtime::tick(now); unsafe { Cpu::local().queue().yield_cpu(); diff --git a/src/arch/mod.rs b/src/arch/mod.rs index 956f15c4..47c20ead 100644 --- a/src/arch/mod.rs +++ b/src/arch/mod.rs @@ -20,7 +20,6 @@ macro_rules! absolute_address { } use cfg_if::cfg_if; -// pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE}; use device_api::{ interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController}, timer::MonotonicTimestampProviderDevice, diff --git a/src/arch/x86_64/context.rs b/src/arch/x86_64/context.rs index e26c13f6..2c4458ea 100644 --- a/src/arch/x86_64/context.rs +++ b/src/arch/x86_64/context.rs @@ -65,11 +65,11 @@ impl StackBuilder { self.sp } - fn init_common(&mut self, entry: usize, cr3: PhysicalAddress) { + fn init_common(&mut self, entry: usize, cr3: u64) { self.push(entry); // address for ret // End of common context - self.push(cr3.into_raw()); // %cr3 + self.push(cr3 as _); // %cr3 self.push(0); // %rbp self.push(0); // %fs (TODO) @@ -99,7 +99,7 @@ impl TaskContextImpl for TaskContext { stack.push(arg); stack.init_common(__x86_64_task_enter_kernel as _, unsafe { - KERNEL_TABLES.as_physical_address() + KERNEL_TABLES.as_physical_address().into_raw() }); let sp = stack.build(); @@ -114,12 +114,7 @@ impl TaskContextImpl for TaskContext { }) } - fn user( - entry: usize, - arg: usize, - cr3: PhysicalAddress, - user_stack_sp: usize, - ) -> Result { + fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result { const USER_TASK_PAGES: usize = 8; let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw(); diff --git a/src/arch/x86_64/mem/process.rs b/src/arch/x86_64/mem/process.rs index cde7e096..48963678 100644 --- a/src/arch/x86_64/mem/process.rs +++ b/src/arch/x86_64/mem/process.rs @@ -3,7 +3,7 @@ use yggdrasil_abi::error::Error; use crate::{ arch::x86_64::intrinsics, mem::{ - address::AsPhysicalAddress, + address::{AsPhysicalAddress, IntoRaw}, phys, pointer::PhysicalRefMut, process::ProcessAddressSpaceManager, @@ -61,6 +61,11 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { self.read_l3_entry(address) .ok_or(Error::InvalidMemoryOperation) } + + fn as_address_with_asid(&self) -> u64 { + // TODO x86-64 PCID/ASID? + unsafe { self.l0.as_physical_address().into_raw() } + } } impl ProcessAddressSpaceImpl { @@ -128,9 +133,3 @@ impl ProcessAddressSpaceImpl { Some((page, l3[l3i].attributes().into())) } } - -impl AsPhysicalAddress for ProcessAddressSpaceImpl { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - self.l0.as_physical_address() - } -} diff --git a/src/arch/x86_64/mem/table.rs b/src/arch/x86_64/mem/table.rs index c8e1234d..c4adfa93 100644 --- a/src/arch/x86_64/mem/table.rs +++ b/src/arch/x86_64/mem/table.rs @@ -67,51 +67,19 @@ impl NonTerminalEntryLevel for L2 { } impl const EntryLevel for L0 { - const SIZE: usize = 1 << 39; - - fn index(addr: usize) -> usize { - (addr >> 39) & 0x1FF - } - - fn page_offset(_addr: usize) -> usize { - todo!() - } + const SHIFT: usize = 39; } impl const EntryLevel for L1 { - const SIZE: usize = 1 << 30; - - fn index(addr: usize) -> usize { - (addr >> 30) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x3FFFFFFF - } + const SHIFT: usize = 30; } impl const EntryLevel for L2 { - const SIZE: usize = 1 << 21; - - fn index(addr: usize) -> usize { - (addr >> 21) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0x1FFFFF - } + const SHIFT: usize = 21; } impl const EntryLevel for L3 { - const SIZE: usize = 1 << 12; - - fn index(addr: usize) -> usize { - (addr >> 12) & 0x1FF - } - - fn page_offset(addr: usize) -> usize { - addr & 0xFFF - } + const SHIFT: usize = 12; } impl PageEntry { diff --git a/src/device/devtree.rs b/src/device/devtree.rs index 8882fb1e..d85e7d3a 100644 --- a/src/device/devtree.rs +++ b/src/device/devtree.rs @@ -10,7 +10,10 @@ use fdt_rs::{ prelude::PropReader, }; -use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion}; +use crate::{ + debug::LogLevel, + mem::{address::FromRaw, phys::PhysicalMemoryRegion, PhysicalAddress}, +}; use super::register_device; @@ -162,6 +165,7 @@ impl<'a> DeviceTree<'a> { /// /// The caller must ensure the validity of the address. pub unsafe fn from_addr(virt: usize) -> Self { + FDT_INDEX_BUFFER.0.fill(0); let tree = DevTree::from_raw_pointer(virt as _).unwrap(); let index = DevTreeIndex::new(tree, &mut FDT_INDEX_BUFFER.0).unwrap(); Self { tree, index } @@ -198,6 +202,12 @@ impl<'a> DeviceTree<'a> { pub fn root(&self) -> DevTreeIndexNode { self.index.root() } + + // Commonly used functions for convenience + pub fn chosen_stdout_path(&self) -> Option<&str> { + let chosen = self.node_by_path("/chosen")?; + chosen.prop("stdout-path") + } } impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> { @@ -314,10 +324,10 @@ impl Iterator for FdtMemoryRegionIter<'_> { .cell2_array_item(0, self.address_cells, self.size_cells) .unwrap(); - break Some(PhysicalMemoryRegion { - base: base as usize, - size: size as usize, - }); + let base = PhysicalAddress::from_raw(base); + let size = size as usize; + + break Some(PhysicalMemoryRegion { base, size }); } } } diff --git a/src/device/mod.rs b/src/device/mod.rs index 1dbd0a6f..032006a3 100644 --- a/src/device/mod.rs +++ b/src/device/mod.rs @@ -7,9 +7,12 @@ use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard}; #[cfg(target_arch = "aarch64")] pub mod devtree; +// TODO bus device support on aarch64 +#[cfg(not(target_arch = "aarch64"))] pub mod bus; + pub mod display; -// pub mod power; +pub mod power; pub mod serial; pub mod timer; pub mod tty; diff --git a/src/device/power/mod.rs b/src/device/power/mod.rs index 7cf0ce21..8b996e54 100644 --- a/src/device/power/mod.rs +++ b/src/device/power/mod.rs @@ -5,6 +5,6 @@ use cfg_if::cfg_if; cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod arm_psci; - pub mod sunxi_rwdog; + // pub mod sunxi_rwdog; } } diff --git a/src/device/serial/mod.rs b/src/device/serial/mod.rs index 7f672fe6..4a364aa0 100644 --- a/src/device/serial/mod.rs +++ b/src/device/serial/mod.rs @@ -5,6 +5,6 @@ use cfg_if::cfg_if; cfg_if! { if #[cfg(target_arch = "aarch64")] { pub mod pl011; - pub mod sunxi_uart; + // pub mod sunxi_uart; } } diff --git a/src/device/serial/pl011.rs b/src/device/serial/pl011.rs index 40ed12e6..e90bbc31 100644 --- a/src/device/serial/pl011.rs +++ b/src/device/serial/pl011.rs @@ -12,14 +12,15 @@ use vfs::CharDevice; use crate::{ arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE}, + block, debug::{self, DebugSink, LogLevel}, device::{ devtree::{self, DevTreeIndexPropExt}, - tty::{CharRing, TtyDevice}, + tty::{TtyContext, TtyDevice}, }, device_tree_driver, fs::devfs::{self, CharDeviceType}, - mem::device::DeviceMemoryIo, + mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress}, sync::IrqSafeSpinlock, }; @@ -62,15 +63,15 @@ register_structs! { } struct Pl011Inner { - regs: DeviceMemoryIo, + regs: DeviceMemoryIo<'static, Regs>, } /// PL011 device instance pub struct Pl011 { inner: OneTimeInit>, - base: usize, + base: PhysicalAddress, irq: IrqNumber, - ring: CharRing<16>, + context: TtyContext, } impl Pl011Inner { @@ -101,9 +102,9 @@ impl DebugSink for Pl011 { } } -impl TtyDevice<16> for Pl011 { - fn ring(&self) -> &CharRing<16> { - &self.ring +impl TtyDevice for Pl011 { + fn context(&self) -> &TtyContext { + &self.context } } @@ -115,7 +116,12 @@ impl CharDevice for Pl011 { fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result { assert!(blocking); - self.line_read(data) + match block! { + self.line_read(data).await + } { + Ok(res) => res, + Err(err) => Err(err), + } } fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> { @@ -174,7 +180,7 @@ impl Device for Pl011 { unsafe fn init(&'static self) -> Result<(), Error> { let mut inner = Pl011Inner { - regs: DeviceMemoryIo::map("pl011 UART", self.base)?, + regs: DeviceMemoryIo::map(self.base)?, }; inner.init(); @@ -207,8 +213,8 @@ device_tree_driver! { inner: OneTimeInit::new(), // TODO obtain IRQ from dt irq: IrqNumber::Shared(1), - ring: CharRing::new(), - base: base as usize + context: TtyContext::new(), + base: PhysicalAddress::from_raw(base) })) } } diff --git a/src/mem/device.rs b/src/mem/device.rs index df7f1b9f..dcba8ea0 100644 --- a/src/mem/device.rs +++ b/src/mem/device.rs @@ -67,9 +67,15 @@ impl DeviceMemoryMapping { } impl<'a, T: Sized> DeviceMemoryIo<'a, T> { - pub unsafe fn from_raw(raw: DeviceMemoryMapping) -> DeviceMemoryIo<'a, T> { - // TODO - loop {} + pub unsafe fn from_raw( + inner: Arc, + ) -> Result, Error> { + if size_of::() > inner.page_size * inner.page_count { + todo!(); + } + // TODO check align + let value = &*(inner.address as *const T); + Ok(DeviceMemoryIo { inner, value }) } pub unsafe fn map(base: PhysicalAddress) -> Result, Error> { diff --git a/src/mem/phys/manager.rs b/src/mem/phys/manager.rs index 1f22832f..f98cbc7f 100644 --- a/src/mem/phys/manager.rs +++ b/src/mem/phys/manager.rs @@ -22,12 +22,14 @@ pub(super) const TRACKED_PAGE_LIMIT: usize = (BITMAP_PAGE_COUNT * 4096) * 8; pub struct PhysicalMemoryManager { bitmap: PhysicalRefMut<'static, [u64]>, last_free_bit: usize, + offset: usize, page_count: usize, } impl PhysicalMemoryManager { pub unsafe fn new( bitmap_phys_base: PhysicalAddress, + offset: usize, page_count: usize, ) -> PhysicalMemoryManager { let bitmap_len = (page_count + (BITMAP_WORD_SIZE - 1)) / BITMAP_WORD_SIZE; @@ -38,6 +40,7 @@ impl PhysicalMemoryManager { Self { bitmap, page_count, + offset, last_free_bit: 0, } } @@ -67,7 +70,7 @@ impl PhysicalMemoryManager { self.last_free_bit = i + 1; self.mark_alloc(i); - return Ok(PhysicalAddress::from_raw(i * 0x1000)); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } if self.last_free_bit != 0 { @@ -93,7 +96,7 @@ impl PhysicalMemoryManager { } self.last_free_bit = i + 512; - return Ok(PhysicalAddress::from_raw(i * 0x1000)); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } if self.last_free_bit != 0 { @@ -118,7 +121,7 @@ impl PhysicalMemoryManager { } self.last_free_bit = i + count; - return Ok(PhysicalAddress::from_raw(i * 0x1000)); + return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset)); } if self.last_free_bit != 0 { @@ -135,7 +138,9 @@ impl PhysicalMemoryManager { /// /// `addr` must be a page-aligned physical address previously allocated by this implementation. pub unsafe fn free_page(&mut self, page: PhysicalAddress) { - let index = IntoRaw::::into_raw(page) / 0x1000; + let page: usize = page.into_raw(); + assert!(page >= self.offset); + let index = (page - self.offset) / 0x1000; assert!(self.is_alloc(index)); self.mark_free(index); @@ -147,7 +152,9 @@ impl PhysicalMemoryManager { /// /// Will panic if the address does not point to a valid, reserved (and unallocated) page. pub fn add_available_page(&mut self, page: PhysicalAddress) { - let index = IntoRaw::::into_raw(page) / 0x1000; + let page: usize = page.into_raw(); + assert!(page >= self.offset); + let index = (page - self.offset) / 0x1000; assert!(self.is_alloc(index)); self.mark_free(index); diff --git a/src/mem/phys/mod.rs b/src/mem/phys/mod.rs index e1328d03..59476452 100644 --- a/src/mem/phys/mod.rs +++ b/src/mem/phys/mod.rs @@ -5,7 +5,10 @@ use kernel_util::util::OneTimeInit; use crate::{ arch::{Architecture, ARCHITECTURE}, - mem::phys::reserved::is_reserved, + mem::{ + address::IntoRaw, + phys::{self, reserved::is_reserved}, + }, sync::IrqSafeSpinlock, }; @@ -162,7 +165,12 @@ pub unsafe fn init_from_iter + Clone>( }, ); - let mut manager = PhysicalMemoryManager::new(page_bitmap_phys_base, total_count); + if IntoRaw::::into_raw(phys_start) & 0x1FFFFFF != 0 { + loop {} + } + + let mut manager = + PhysicalMemoryManager::new(page_bitmap_phys_base, phys_start.into_raw(), total_count); let mut collected = 0; const MAX_MEMORY: usize = 16 * 1024; diff --git a/src/mem/process.rs b/src/mem/process.rs index a6727de6..f4855656 100644 --- a/src/mem/process.rs +++ b/src/mem/process.rs @@ -2,20 +2,20 @@ use abi::error::Error; use cfg_if::cfg_if; use vmalloc::VirtualMemoryAllocator; -use crate::{arch::x86_64::mem::table::L3, mem::phys, sync::IrqSafeSpinlock}; +use crate::{mem::phys, sync::IrqSafeSpinlock}; use super::{address::AsPhysicalAddress, table::MapAttributes, PhysicalAddress}; cfg_if! { if #[cfg(target_arch = "aarch64")] { - use crate::arch::aarch64::table::AddressSpace; + use crate::arch::aarch64::mem::process::ProcessAddressSpaceImpl; } else if #[cfg(target_arch = "x86_64")] { use crate::arch::x86_64::mem::process::ProcessAddressSpaceImpl; } } /// Interface for virtual memory address space management -pub trait ProcessAddressSpaceManager: Sized + AsPhysicalAddress { +pub trait ProcessAddressSpaceManager: Sized { const PAGE_SIZE: usize; const LOWER_LIMIT_PFN: usize; const UPPER_LIMIT_PFN: usize; @@ -32,6 +32,8 @@ pub trait ProcessAddressSpaceManager: Sized + AsPhysicalAddress { unsafe fn unmap_page(&mut self, address: usize) -> Result; fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; + + fn as_address_with_asid(&self) -> u64; } struct Inner { @@ -186,7 +188,8 @@ impl ProcessAddressSpace { attributes: MapAttributes, ) -> Result<(), Error> { assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0); - assert!(physical.is_aligned_for::()); + // XXX + // assert!(physical.is_aligned_for::()); self.inner .lock() @@ -211,22 +214,7 @@ impl ProcessAddressSpace { ) } - pub fn debug_dump(&self) { - let lock = self.inner.lock(); - - debugln!("Address space @ {:#x}", unsafe { - lock.table.as_physical_address() - }); - for (used, range) in lock.allocator.ranges() { - let start = range.start_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE; - let end = range.end_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE; - debugln!("{:#x?}: {}", start..end, used); - } - } -} - -impl AsPhysicalAddress for ProcessAddressSpace { - unsafe fn as_physical_address(&self) -> PhysicalAddress { - self.inner.lock().table.as_physical_address() + pub fn as_address_with_asid(&self) -> u64 { + self.inner.lock().table.as_address_with_asid() } } diff --git a/src/mem/table.rs b/src/mem/table.rs index f7dc9493..057946ad 100644 --- a/src/mem/table.rs +++ b/src/mem/table.rs @@ -4,6 +4,7 @@ use core::ops::{Deref, DerefMut}; use abi::error::Error; use bitflags::bitflags; +// TODO EXECUTABLE bitflags! { /// Describes how a page translation mapping should behave #[derive(Clone, Copy)] @@ -36,12 +37,29 @@ pub trait NextPageTable { /// Interface for a single level of address translation #[const_trait] pub trait EntryLevel: Copy { - const SIZE: usize; + const SHIFT: usize; + const SIZE: usize = 1 << Self::SHIFT; /// Returns the index into a page table for a given address - fn index(addr: usize) -> usize; + #[inline] + fn index(addr: usize) -> usize { + (addr >> Self::SHIFT) & 0x1FF + } /// Returns the offset of an address from the page start at current level - fn page_offset(addr: usize) -> usize; + #[inline] + fn page_offset(addr: usize) -> usize { + addr & (Self::SIZE - 1) + } + + #[inline] + fn align_up(addr: usize) -> usize { + (addr + Self::SIZE - 1) & !(Self::SIZE - 1) + } + + #[inline] + fn page_count(addr: usize) -> usize { + (addr + Self::SIZE - 1) / Self::SIZE + } } /// Tag trait to mark that the page table level may point to a next-level table diff --git a/src/proc/elf.rs b/src/proc/elf.rs index 63f14c97..06afd6aa 100644 --- a/src/proc/elf.rs +++ b/src/proc/elf.rs @@ -98,8 +98,6 @@ pub fn load_elf_from_file(space: &ProcessAddressSpace, file: FileRef) -> Result< let elf = ElfStream::::open_stream(file).map_err(from_parse_error)?; - space.debug_dump(); - for phdr in elf.segments() { if phdr.p_type != PT_LOAD { continue; diff --git a/src/proc/exec.rs b/src/proc/exec.rs index 653c930e..e40a2b96 100644 --- a/src/proc/exec.rs +++ b/src/proc/exec.rs @@ -174,7 +174,7 @@ fn setup_binary>( let context = TaskContext::user( entry, virt_args_base, - unsafe { space.as_physical_address() }, + unsafe { space.as_address_with_asid() }, user_sp, )?; diff --git a/src/task/context.rs b/src/task/context.rs index dadfd1c1..a2f1bb07 100644 --- a/src/task/context.rs +++ b/src/task/context.rs @@ -56,12 +56,7 @@ pub trait TaskContextImpl: Sized { /// Constructs a user thread context. The caller is responsible for allocating the userspace /// stack and setting up a valid address space for the context. - fn user( - entry: usize, - arg: usize, - cr3: PhysicalAddress, - user_stack_sp: usize, - ) -> Result; + fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result; /// Performs an entry into a context. /// diff --git a/tools/gentables/Cargo.toml b/tools/gentables/Cargo.toml index ae7f7609..2e0a20e3 100644 --- a/tools/gentables/Cargo.toml +++ b/tools/gentables/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html [dependencies] -memtables = { path = "../../lib/memtables" } +memtables = { path = "../../lib/memtables", features = ["all"] } bytemuck = "1.14.0" elf = "0.7.2" diff --git a/tools/gentables/src/aarch64.rs b/tools/gentables/src/aarch64.rs new file mode 100644 index 00000000..624b61e8 --- /dev/null +++ b/tools/gentables/src/aarch64.rs @@ -0,0 +1,193 @@ +use core::fmt; +use std::{ + io::{Read, Seek}, + mem::offset_of, +}; + +use bitflags::bitflags; +use elf::{ + abi::{PF_W, PF_X, PT_LOAD}, + endian::AnyEndian, + ElfStream, +}; +use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; + +use crate::{GenData, GenError}; + +bitflags! { + #[derive(Clone, Copy)] + struct PageFlags: u64 { + const PRESENT = 1 << 0; + const ACCESS = 1 << 10; + const SH_INNER = 3 << 8; + const PAGE_ATTR_NORMAL = 0 << 2; + const AP_BOTH_READONLY = 3 << 6; + const TABLE = 1 << 1; + const PAGE = 1 << 1; + const UXN = 1 << 54; + const PXN = 1 << 53; + } +} + +impl PageFlags { + pub fn kernel_table() -> Self { + Self::PRESENT | Self::ACCESS | Self::SH_INNER | Self::PAGE_ATTR_NORMAL | Self::TABLE + } + + pub fn kernel_page() -> Self { + Self::PRESENT | Self::ACCESS | Self::SH_INNER | Self::PAGE_ATTR_NORMAL | Self::PAGE + } +} + +pub struct AArch64Builder { + elf: ElfStream, + data: GenData, + tables: FixedTables, + + l1i: usize, + l2i_start: usize, + l2i_end: usize, +} + +impl PageFlags { + fn from_elf(flags: u32) -> Self { + let mut out = Self::UXN | Self::PXN; + if flags & PF_X != 0 { + out.remove(Self::PXN); + } + if flags & PF_W == 0 { + out |= Self::AP_BOTH_READONLY; + } + out + } +} + +impl fmt::Display for PageFlags { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let x = if self.contains(Self::PXN) { '-' } else { 'x' }; + let w = if self.contains(Self::AP_BOTH_READONLY) { + '-' + } else { + 'w' + }; + write!(f, "r{}{}", w, x) + } +} + +const L1_SHIFT: u64 = 30; +const L2_SHIFT: u64 = 21; +const L3_SHIFT: u64 = 12; +const L2_ENTRY_SIZE: u64 = 1 << L2_SHIFT; +const L3_ENTRY_SIZE: u64 = 1 << L3_SHIFT; + +// TODO proper granularity +impl AArch64Builder { + pub fn new(elf: ElfStream, data: GenData) -> Result { + let l1i = (data.kernel_start >> L1_SHIFT) as usize & 0x1FF; + let l2i_start = (data.kernel_start >> L2_SHIFT) as usize & 0x1FF; + let l2i_end = ((data.kernel_end + L2_ENTRY_SIZE - 1) >> L2_SHIFT) as usize & 0x1FF; + + if l2i_end - l2i_start > KERNEL_L3_COUNT { + todo!() + } + + Ok(Self { + elf, + data, + + tables: FixedTables::zeroed(), + + l1i, + l2i_start, + l2i_end, + }) + } + + // TODO the build function is almost identical to x86-64 one, but with slight changes, so might + // wanna unify this later + pub fn build(mut self) -> Result<(FixedTables, u64), GenError> { + assert_eq!(offset_of!(FixedTables, l1), 0); + + let l2_physical_address = + self.data.table_physical_address + offset_of!(FixedTables, l2) as u64; + + // L1 -> L2 + self.tables.l1.data[self.l1i] = l2_physical_address | PageFlags::kernel_table().bits(); + + // L2 -> L3s + for l2i in self.l2i_start..self.l2i_end { + let l3_table_index = l2i - self.l2i_start; + let l3_physical_address = self.data.table_physical_address + + (offset_of!(FixedTables, l3s) + 0x1000 * l3_table_index) as u64; + + self.tables.l2.data[l2i] = l3_physical_address | PageFlags::kernel_table().bits(); + } + + for (i, segment) in self.elf.segments().into_iter().enumerate() { + if segment.p_type != PT_LOAD + || segment.p_vaddr != segment.p_paddr + self.data.kernel_virt_offset + { + continue; + } + + let aligned_virt_start = segment.p_vaddr & !(L3_ENTRY_SIZE - 1); + let aligned_virt_end = + (segment.p_vaddr + segment.p_memsz + L3_ENTRY_SIZE - 1) & !(L3_ENTRY_SIZE - 1); + let aligned_phys_start = segment.p_paddr & !(L3_ENTRY_SIZE - 1); + let count = (aligned_virt_end - aligned_virt_start) / 0x1000; + + let flags = PageFlags::from_elf(segment.p_flags); + + println!( + "{}: {:#x?} -> {:#x} {}", + i, + aligned_virt_start..aligned_virt_end, + aligned_phys_start, + flags + ); + Self::map_segment( + self.l2i_start, + &mut self.tables, + aligned_virt_start, + aligned_phys_start, + count as usize, + flags, + )?; + } + + Ok((self.tables, self.data.table_offset)) + } + + fn map_segment( + start_l2i: usize, + tables: &mut FixedTables, + vaddr_start: u64, + paddr_start: u64, + count: usize, + flags: PageFlags, + ) -> Result<(), GenError> { + for index in 0..count { + let vaddr = vaddr_start + index as u64 * L3_ENTRY_SIZE; + let paddr = paddr_start + index as u64 * L3_ENTRY_SIZE; + + let entry = paddr | (PageFlags::kernel_page() | flags).bits(); + + let l2i = (vaddr >> L2_SHIFT) as usize & 0x1FF - start_l2i; + let l3i = (vaddr >> L3_SHIFT) as usize & 0x1FF; + + let l3 = &mut tables.l3s[l2i]; + + if l3.data[l3i] != 0 { + if l3.data[l3i] != entry { + todo!(); + } else { + continue; + } + } + + l3.data[l3i] = entry; + } + + Ok(()) + } +} diff --git a/tools/gentables/src/main.rs b/tools/gentables/src/main.rs index 0ec86b02..7edfcd9c 100644 --- a/tools/gentables/src/main.rs +++ b/tools/gentables/src/main.rs @@ -10,15 +10,16 @@ use std::{ use clap::Parser; use elf::{ - abi::{EM_X86_64, PT_LOAD}, + abi::{EM_AARCH64, EM_X86_64, PT_LOAD}, endian::AnyEndian, ElfStream, }; -use memtables::FixedTables; +use memtables::any::AnyTables; use thiserror::Error; -use crate::x86_64::X8664Builder; +use crate::{aarch64::AArch64Builder, x86_64::X8664Builder}; +mod aarch64; mod x86_64; #[derive(Error, Debug)] @@ -120,7 +121,11 @@ fn find_tables(elf: &mut ElfStream) -> Result<(u64 Err(GenError::MissingSection(".data.tables")) } -fn build_tables(file: F) -> Result<(FixedTables, u64), GenError> { +fn into_any, U>((l, r): (T, U)) -> (AnyTables, U) { + (l.into(), r) +} + +fn build_tables(file: F) -> Result<(AnyTables, u64), GenError> { let mut elf = ElfStream::::open_stream(file)?; let kernel_virt_offset = kernel_virt_offset(&mut elf)?; @@ -144,7 +149,20 @@ fn build_tables(file: F) -> Result<(FixedTables, u64), GenError> table_physical_address, }, )? - .build(), + .build() + .map(into_any), + EM_AARCH64 => AArch64Builder::new( + elf, + GenData { + kernel_virt_offset, + kernel_start, + kernel_end, + table_offset, + table_physical_address, + }, + )? + .build() + .map(into_any), _ => todo!(), } } @@ -152,11 +170,10 @@ fn build_tables(file: F) -> Result<(FixedTables, u64), GenError> fn write_tables( mut file: F, offset: u64, - tables: FixedTables, + tables: AnyTables, ) -> Result<(), GenError> { - let bytes = bytemuck::bytes_of(&tables); file.seek(SeekFrom::Start(offset))?; - file.write_all(bytes)?; + file.write_all(tables.as_bytes())?; Ok(()) } diff --git a/tools/gentables/src/x86_64.rs b/tools/gentables/src/x86_64.rs index d41551a4..725636e3 100644 --- a/tools/gentables/src/x86_64.rs +++ b/tools/gentables/src/x86_64.rs @@ -11,7 +11,7 @@ use elf::{ endian::AnyEndian, ElfStream, }; -use memtables::{FixedTables, KERNEL_L3_COUNT}; +use memtables::x86_64::{FixedTables, KERNEL_L3_COUNT}; use crate::{GenData, GenError};