aarch64: new memory arch impl

lib/memtables/Cargo.toml

@@ -7,3 +7,7 @@ edition = "2021"
 
 [dependencies]
 bytemuck = { version = "1.14.0", features = ["derive"] }
+
+[features]
+default = []
+all = []

lib/memtables/src/aarch64.rs

@@ -0,0 +1,26 @@
+use bytemuck::{Pod, Zeroable};
+
+use crate::RawTable;
+
+pub const KERNEL_L3_COUNT: usize = 4;
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+pub struct FixedTables {
+    // 1GiB entries
+    pub l1: RawTable,
+
+    // 2MiB entries
+    pub l2: RawTable,
+    pub l3s: [RawTable; KERNEL_L3_COUNT],
+}
+
+impl FixedTables {
+    pub const fn zeroed() -> Self {
+        Self {
+            l1: RawTable::zeroed(),
+            l2: RawTable::zeroed(),
+            l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
+        }
+    }
+}

lib/memtables/src/any.rs

@@ -0,0 +1,27 @@
+use crate::{aarch64, x86_64};
+
+pub enum AnyTables {
+    X86_64(x86_64::FixedTables),
+    AArch64(aarch64::FixedTables),
+}
+
+impl AnyTables {
+    pub fn as_bytes(&self) -> &[u8] {
+        match self {
+            Self::X86_64(tables) => bytemuck::bytes_of(tables),
+            Self::AArch64(tables) => bytemuck::bytes_of(tables),
+        }
+    }
+}
+
+impl From<x86_64::FixedTables> for AnyTables {
+    fn from(value: x86_64::FixedTables) -> Self {
+        Self::X86_64(value)
+    }
+}
+
+impl From<aarch64::FixedTables> for AnyTables {
+    fn from(value: aarch64::FixedTables) -> Self {
+        Self::AArch64(value)
+    }
+}

lib/memtables/src/lib.rs

@@ -2,7 +2,20 @@
 
 use bytemuck::{Pod, Zeroable};
 
-pub const KERNEL_L3_COUNT: usize = 16;
+// AArch64
+#[cfg(any(feature = "all", target_arch = "aarch64"))]
+pub mod aarch64;
+#[cfg(all(not(feature = "all"), target_arch = "aarch64"))]
+pub use aarch64::FixedTables;
+
+// x86-64
+#[cfg(any(feature = "all", target_arch = "x86_64"))]
+pub mod x86_64;
+#[cfg(all(not(feature = "all"), target_arch = "x86_64"))]
+pub use x86_64::FixedTables;
+
+#[cfg(feature = "all")]
+pub mod any;
 
 #[derive(Clone, Copy, Pod, Zeroable)]
 #[repr(C, align(0x1000))]
@@ -10,30 +23,8 @@ pub struct RawTable {
     pub data: [u64; 512],
 }
 
-#[derive(Clone, Copy, Pod, Zeroable)]
-#[repr(C)]
-pub struct FixedTables {
-    pub l0: RawTable,
-
-    pub kernel_l1: RawTable,
-    pub kernel_l2: RawTable,
-    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
-}
-
 impl RawTable {
     pub const fn zeroed() -> Self {
         Self { data: [0; 512] }
     }
 }
-
-impl FixedTables {
-    pub const fn zeroed() -> Self {
-        Self {
-            l0: RawTable::zeroed(),
-
-            kernel_l1: RawTable::zeroed(),
-            kernel_l2: RawTable::zeroed(),
-            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
-        }
-    }
-}

lib/memtables/src/x86_64.rs

@@ -0,0 +1,27 @@
+use bytemuck::{Pod, Zeroable};
+
+use crate::RawTable;
+
+pub const KERNEL_L3_COUNT: usize = 16;
+
+#[derive(Clone, Copy, Pod, Zeroable)]
+#[repr(C)]
+pub struct FixedTables {
+    pub l0: RawTable,
+
+    pub kernel_l1: RawTable,
+    pub kernel_l2: RawTable,
+    pub kernel_l3s: [RawTable; KERNEL_L3_COUNT],
+}
+
+impl FixedTables {
+    pub const fn zeroed() -> Self {
+        Self {
+            l0: RawTable::zeroed(),
+
+            kernel_l1: RawTable::zeroed(),
+            kernel_l2: RawTable::zeroed(),
+            kernel_l3s: [RawTable::zeroed(); KERNEL_L3_COUNT],
+        }
+    }
+}

src/arch/aarch64/boot/entry.S

@@ -13,6 +13,12 @@
     movk \reg, #((\value) >> 16), lsl #16
 .endm
 
+.macro MOV_ABS reg, sym
+    movz \reg, #:abs_g2:\sym
+    movk \reg, #:abs_g1_nc:\sym
+    movk \reg, #:abs_g0_nc:\sym
+.endm
+
 .macro LEAVE_EL2, ret_label
     mrs x8, CNTHCTL_EL2
     orr x8, x8, #(CNTHCTL_EL2_EL1PCTEN | CNTHCTL_EL2_EL1PCEN)
@@ -61,8 +67,9 @@ __aarch64_entry:
     isb
 
     // Zero .bss
-    adr x8, __bss_start_phys
+    MOV_ABS x8, __bss_start_phys
-    adr x9, __bss_end_phys
+    MOV_ABS x9, __bss_end_phys
+    // Zero .bss
 1:
     cmp x8, x9
     beq 2f
@@ -84,7 +91,7 @@ __aarch64_entry:
 
 .section .text
 __aarch64_ap_entry:
-    // x0 -- stack pointer (lower address space)
+    // x0 -- physical sp
 
     mrs x8, CurrentEL
     lsr x8, x8, #2

src/arch/aarch64/boot/mod.rs

@@ -1,135 +1,168 @@
-//! Main entry point for the AArch64 platforms
+use core::{arch::global_asm, sync::atomic::Ordering};
-use core::{
-    arch::{asm, global_asm},
-    sync::atomic::Ordering,
-};
 
 use aarch64_cpu::{
     asm::barrier,
-    registers::{CPACR_EL1, TTBR0_EL1},
+    registers::{CPACR_EL1, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1, TTBR0_EL1},
 };
-use tock_registers::interfaces::{ReadWriteable, Writeable};
+use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 
-use super::{cpu::Cpu, exception, AArch64, KernelStack, ARCHITECTURE, BOOT_STACK_SIZE};
+use super::{
+    exception, mem::load_fixed_tables, smp::CPU_COUNT, AArch64, BootStack, ARCHITECTURE,
+    BOOT_STACK_SIZE,
+};
 use crate::{
-    absolute_address,
+    arch::{aarch64::mem::table::L3, Architecture},
-    arch::{aarch64::smp::CPU_COUNT, Architecture, ArchitectureImpl},
     fs::devfs,
     kernel_main, kernel_secondary_main,
-    mem::{
+    mem::{address::IntoRaw, phys, table::EntryLevel, PhysicalAddress, KERNEL_VIRT_OFFSET},
-        heap,
+    task::runtime,
-        phys::{self, PageUsage},
-        ConvertAddress, KERNEL_VIRT_OFFSET,
-    },
 };
 
-extern "C" fn el1_bsp_lower_entry(dtb_phys: usize) -> ! {
+unsafe fn pre_init_mmu() {
-    // Unmask FP operations
+    if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) {
-    CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing);
+        // TODO early panic
-
+        loop {}
-    unsafe {
-        ARCHITECTURE.init_mmu(true);
     }
 
-    let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE).virtualize() };
+    MAIR_EL1.write(
-    let elr = absolute_address!(__aarch64_bsp_upper_entry);
+        //// Attribute 0 -- normal memory
+        // Inner
+        MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +
+        // Outer
+        MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +
+        //// Attribute 1 -- device memory
+        MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
+    );
 
-    barrier::dsb(barrier::SY);
+    TCR_EL1.modify(
-    barrier::isb(barrier::SY);
+        // General
-    enter_higher_half(sp as usize, elr, dtb_phys);
+        TCR_EL1::IPS::Bits_48 +
+        // TTBR0
+        TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
+        // TTBR1
+        TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer,
+    );
 }
 
-unsafe extern "C" fn el1_ap_lower_entry(sp: usize) -> ! {
+unsafe fn enable_mmu() {
-    ArchitectureImpl::set_interrupt_mask(true);
+    barrier::dmb(barrier::ISH);
+
+    SCTLR_EL1.modify(
+        // Enable translation
+        SCTLR_EL1::M::Enable +
+        // (TODO) Disable I + D caches
+        SCTLR_EL1::I::NonCacheable + SCTLR_EL1::C::NonCacheable,
+    );
+
+    barrier::isb(barrier::SY);
+}
+
+unsafe fn enter_higher_half(sp: usize, elr: usize, x0: usize) -> ! {
+    unsafe {
+        core::arch::asm!(r#"
+            mov sp, {sp}
+            mov x0, {x0}
+            mov lr, xzr
+            br {elr}
+        "#, elr = in(reg) elr, sp = in(reg) sp, x0 = in(reg) x0, options(noreturn));
+    }
+}
+
+// NOTE executes in "lower-half" address space, MMU not yet enabled
+unsafe extern "C" fn __aarch64_el1_bsp_lower_entry(dtb: PhysicalAddress) -> ! {
+    AArch64::set_interrupt_mask(true);
+
+    // Don't trap FP operations
+    CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing);
+
+    // Setup MMU to jump to "higher-half" address space
+    pre_init_mmu();
+    load_fixed_tables();
+    enable_mmu();
+
+    // Safety: SP points to the .bss section, so it's +offset mapped
+    let sp = unsafe { BSP_STACK.data.as_ptr().add(BOOT_STACK_SIZE) as usize } + KERNEL_VIRT_OFFSET;
+    let elr = absolute_address!(__aarch64_bsp_upper_entry);
+
+    // TODO pass dtb
+    enter_higher_half(sp, elr, dtb.into_raw());
+}
+
+unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb: PhysicalAddress) -> ! {
+    // Remove the "lower-half" mapping, no longer needed
+    TTBR0_EL1.set(0);
+
+    // Setup the "runtime" part of the kernel tables
+    ARCHITECTURE
+        .init_memory_management(dtb)
+        .expect("Could not initialize memory management");
+    barrier::isb(barrier::SY);
+
+    exception::init_exceptions();
+
+    // // Setup initrd
+    // super::setup_initrd();
+
+    devfs::init();
+
+    runtime::init_task_queue();
+
+    // Initialize the BSP CPU + the devices
+    ARCHITECTURE
+        .init_platform(true)
+        .expect("Could not initialize the platform");
+
+    kernel_main()
+}
+
+unsafe extern "C" fn __aarch64_el1_ap_lower_entry(sp: PhysicalAddress) -> ! {
+    const AP_STACK_PAGES: usize = 8;
+    AArch64::set_interrupt_mask(true);
 
     // Unmask FP operations
     CPACR_EL1.modify(CPACR_EL1::FPEN::TrapNothing);
 
-    unsafe {
+    pre_init_mmu();
-        ARCHITECTURE.init_mmu(false);
+    load_fixed_tables();
-    }
+    enable_mmu();
+
+    let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES).unwrap();
+    let stack_base = stack_pages.virtualize_raw();
+    let sp = stack_base + L3::SIZE * AP_STACK_PAGES;
 
-    let sp = sp.virtualize();
     let elr = absolute_address!(__aarch64_ap_upper_entry);
 
     enter_higher_half(sp, elr, 0);
 }
 
-fn enter_higher_half(sp: usize, elr: usize, arg: usize) -> ! {
+extern "C" fn __aarch64_ap_upper_entry() -> ! {
-    unsafe {
+    barrier::dmb(barrier::ISH);
-        asm!(r#"
-            mov sp, {sp}
-            mov x0, {arg}
-            br {entry}
-            "#, entry = in(reg) elr, arg = in(reg) arg, sp = in(reg) sp, options(noreturn));
-    }
-}
-
-unsafe extern "C" fn __aarch64_bsp_upper_entry(dtb_phys: usize) -> ! {
-    // NOTE it is critical that the code does not panic until the debug is set up, otherwise no
-    // message will be displayed
-
-    // Unmap TTBR0
-    TTBR0_EL1.set(0);
     barrier::isb(barrier::SY);
 
-    ARCHITECTURE.init_device_tree(dtb_phys);
+    let cpu_id = CPU_COUNT.fetch_add(1, Ordering::SeqCst);
-
-    AArch64::set_interrupt_mask(true);
-
-    exception::init_exceptions();
-
-    // Setup initrd
-    super::setup_initrd();
-
-    ARCHITECTURE
-        .init_physical_memory(dtb_phys)
-        .expect("Failed to initialize the physical memory manager");
-
-    // Setup heap
-    let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used)
-        .expect("Could not allocate a block for heap");
-
-    heap::init_heap(heap_base.virtualize(), 16 * 0x1000);
-
-    devfs::init();
-
-    // Enumerate the device tree
-    ARCHITECTURE.init_platform(true);
-
-    Cpu::init_local();
-
-    kernel_main()
-}
-
-extern "C" fn __aarch64_ap_upper_entry(_x0: usize) -> ! {
-    assert!(ArchitectureImpl::interrupt_mask());
-
-    // Signal to BSP that we're up
-    CPU_COUNT.fetch_add(1, Ordering::SeqCst);
     aarch64_cpu::asm::sev();
 
+    infoln!("cpu{} initializing", cpu_id);
+
     exception::init_exceptions();
 
-    // Initialize CPU-local GIC and timer
     unsafe {
         ARCHITECTURE.init_platform(false);
-
-        Cpu::init_local();
-
-        kernel_secondary_main()
     }
+
+    kernel_secondary_main()
 }
 
-static BSP_STACK: KernelStack = KernelStack {
+#[link_section = ".bss"]
+static BSP_STACK: BootStack = BootStack {
     data: [0; BOOT_STACK_SIZE],
 };
 
 global_asm!(
     include_str!("entry.S"),
-    kernel_lower_entry = sym el1_bsp_lower_entry,
+    kernel_lower_entry = sym __aarch64_el1_bsp_lower_entry,
-    kernel_ap_lower_entry = sym el1_ap_lower_entry,
+    kernel_ap_lower_entry = sym __aarch64_el1_ap_lower_entry,
     stack_bottom = sym BSP_STACK,
-    stack_size = const BOOT_STACK_SIZE,
+    kernel_virt_offset = const KERNEL_VIRT_OFFSET,
-    kernel_virt_offset = const KERNEL_VIRT_OFFSET
+    stack_size = const BOOT_STACK_SIZE
 );

src/arch/aarch64/context.rs

@@ -4,10 +4,7 @@ use core::{arch::global_asm, cell::UnsafeCell};
 use abi::error::Error;
 
 use crate::{
-    mem::{
+    mem::{address::IntoRaw, phys, PhysicalAddress},
-        phys::{self, PageUsage},
-        ConvertAddress,
-    },
     task::context::TaskContextImpl,
 };
 
@@ -64,8 +61,8 @@ impl StackBuilder {
         self.sp
     }
 
-    fn init_common(&mut self, entry: usize, ttbr0: usize) {
+    fn init_common(&mut self, entry: usize, ttbr0: u64) {
-        self.push(ttbr0); // ttbr0_el1
+        self.push(ttbr0 as _); // ttbr0_el1
         self.push(0); // tpidr_el0
 
         self.push(entry); // x30/lr
@@ -89,9 +86,8 @@ impl TaskContextImpl for TaskContext {
 
     fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
         const KERNEL_TASK_PAGES: usize = 8;
-        let stack_base = unsafe {
+        let stack_base =
-            phys::alloc_pages_contiguous(KERNEL_TASK_PAGES, PageUsage::Used)?.virtualize()
+            unsafe { phys::alloc_pages_contiguous(KERNEL_TASK_PAGES)?.virtualize_raw() };
-        };
 
         let mut stack = StackBuilder::new(stack_base, KERNEL_TASK_PAGES * 0x1000);
 
@@ -112,10 +108,9 @@ impl TaskContextImpl for TaskContext {
         })
     }
 
-    fn user(entry: usize, arg: usize, ttbr0: usize, user_stack_sp: usize) -> Result<Self, Error> {
+    fn user(entry: usize, arg: usize, ttbr0: u64, user_stack_sp: usize) -> Result<Self, Error> {
         const USER_TASK_PAGES: usize = 16;
-        let stack_base =
+        let stack_base = unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw() };
-            unsafe { phys::alloc_pages_contiguous(USER_TASK_PAGES, PageUsage::Used)?.virtualize() };
 
         let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000);
 

src/arch/aarch64/cpu.rs

@@ -122,6 +122,7 @@ impl Cpu {
 
         match msg {
             CpuMessage::Panic => panic::panic_secondary(),
+            CpuMessage::Shutdown => todo!(),
         }
     }
 }

src/arch/aarch64/exception.rs

@@ -16,10 +16,7 @@ use abi::{
 use tock_registers::interfaces::{Readable, Writeable};
 
 use crate::{
-    arch::{
+    arch::{aarch64::cpu::Cpu, Architecture, ArchitectureImpl},
-        aarch64::{cpu::Cpu, table::AddressSpace},
-        Architecture, ArchitectureImpl,
-    },
     debug::LogLevel,
     syscall::raw_syscall_handler,
     task::{context::TaskFrame, process::Process},
@@ -179,14 +176,14 @@ fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) {
         _ => (),
     }
 
-    unsafe {
+    // unsafe {
-        let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _);
+    //     let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _);
-        let far = FAR_EL1.get() as usize;
+    //     let far = FAR_EL1.get() as usize;
-        space.walk(far, |level, raw| {
+    //     space.walk(far, |level, raw| {
-            log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw);
+    //         log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw);
-            true
+    //         true
-        });
+    //     });
-    }
+    // }
 
     log_print_raw!(LogLevel::Fatal, "System register dump:\n");
     log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get());

src/arch/aarch64/gic/gicc.rs

@@ -36,11 +36,11 @@ register_structs! {
 }
 
 pub(super) struct Gicc {
-    regs: DeviceMemoryIo<GiccRegs>,
+    regs: DeviceMemoryIo<'static, GiccRegs>,
 }
 
 impl Gicc {
-    pub const fn new(regs: DeviceMemoryIo<GiccRegs>) -> Self {
+    pub const fn new(regs: DeviceMemoryIo<'static, GiccRegs>) -> Self {
         Self { regs }
     }
 

src/arch/aarch64/gic/gicd.rs

@@ -65,8 +65,8 @@ register_structs! {
 }
 
 pub(super) struct Gicd {
-    shared_regs: Spinlock<DeviceMemoryIo<GicdSharedRegs>>,
+    shared_regs: Spinlock<DeviceMemoryIo<'static, GicdSharedRegs>>,
-    banked_regs: DeviceMemoryIo<GicdBankedRegs>,
+    banked_regs: DeviceMemoryIo<'static, GicdBankedRegs>,
 }
 
 impl GicdSharedRegs {
@@ -85,8 +85,8 @@ impl GicdSharedRegs {
 
 impl Gicd {
     pub const fn new(
-        shared_regs: DeviceMemoryIo<GicdSharedRegs>,
+        shared_regs: DeviceMemoryIo<'static, GicdSharedRegs>,
-        banked_regs: DeviceMemoryIo<GicdBankedRegs>,
+        banked_regs: DeviceMemoryIo<'static, GicdBankedRegs>,
     ) -> Self {
         let shared_regs = Spinlock::new(shared_regs);
         Self {

src/arch/aarch64/gic/mod.rs

@@ -4,7 +4,7 @@ use core::sync::atomic::Ordering;
 
 use aarch64_cpu::asm::barrier;
 use abi::error::Error;
-use alloc::boxed::Box;
+use alloc::{boxed::Box, sync::Arc};
 use device_api::{
     interrupt::{
         ExternalInterruptController, FixedInterruptTable, InterruptHandler, InterruptTable,
@@ -18,7 +18,11 @@ use crate::{
     arch::{aarch64::IrqNumber, Architecture, CpuMessage},
     device::devtree::{self, DevTreeIndexPropExt},
     device_tree_driver,
-    mem::device::{DeviceMemory, DeviceMemoryIo},
+    mem::{
+        address::FromRaw,
+        device::{DeviceMemoryIo, DeviceMemoryMapping, RawDeviceMemoryMapping},
+        PhysicalAddress,
+    },
     sync::IrqSafeSpinlock,
 };
 
@@ -36,8 +40,8 @@ pub mod gicd;
 pub struct Gic {
     gicc: OneTimeInit<Gicc>,
     gicd: OneTimeInit<Gicd>,
-    gicd_base: usize,
+    gicd_base: PhysicalAddress,
-    gicc_base: usize,
+    gicc_base: PhysicalAddress,
     table: IrqSafeSpinlock<FixedInterruptTable<MAX_IRQ>>,
 }
 
@@ -47,10 +51,10 @@ impl Device for Gic {
     }
 
     unsafe fn init(&'static self) -> Result<(), Error> {
-        let gicd_mmio = DeviceMemory::map("GICv2 Distributor registers", self.gicd_base, 0x1000)?;
+        let gicd_mmio = Arc::new(RawDeviceMemoryMapping::map(self.gicd_base, 0x1000)?);
-        let gicd_mmio_shared = DeviceMemoryIo::new(gicd_mmio.clone());
+        let gicd_mmio_shared = DeviceMemoryIo::from_raw(gicd_mmio.clone())?;
-        let gicd_mmio_banked = DeviceMemoryIo::new(gicd_mmio);
+        let gicd_mmio_banked = DeviceMemoryIo::from_raw(gicd_mmio)?;
-        let gicc_mmio = DeviceMemoryIo::map("GICv2 CPU registers", self.gicc_base)?;
+        let gicc_mmio = DeviceMemoryIo::map(self.gicc_base)?;
 
         let gicd = Gicd::new(gicd_mmio_shared, gicd_mmio_banked);
         let gicc = Gicc::new(gicc_mmio);
@@ -169,7 +173,7 @@ impl Gic {
     /// # Safety
     ///
     /// The caller must ensure the addresses actually point to the GIC components.
-    pub const unsafe fn new(gicd_base: usize, gicc_base: usize) -> Self {
+    pub const unsafe fn new(gicd_base: PhysicalAddress, gicc_base: PhysicalAddress) -> Self {
         Self {
             gicc: OneTimeInit::new(),
             gicd: OneTimeInit::new(),
@@ -188,6 +192,9 @@ device_tree_driver! {
         let (gicc_base, _) = reg.cell2_array_item(0, dt.address_cells, dt.size_cells)?;
         let (gicd_base, _) = reg.cell2_array_item(1, dt.address_cells, dt.size_cells)?;
 
-        Some(Box::new(unsafe { Gic::new(gicc_base as usize, gicd_base as usize) }))
+        Some(Box::new(unsafe { Gic::new(
+            PhysicalAddress::from_raw(gicc_base),
+            PhysicalAddress::from_raw(gicd_base),
+        )}))
     }
 }

src/arch/aarch64/mem/mod.rs

@@ -0,0 +1,374 @@
+use core::{
+    alloc::Layout,
+    ops::{Deref, DerefMut},
+};
+
+use abi::error::Error;
+use cfg_if::cfg_if;
+use kernel_util::util::OneTimeInit;
+use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT};
+
+use aarch64_cpu::registers::{TTBR0_EL1, TTBR1_EL1};
+use static_assertions::const_assert_eq;
+use tock_registers::interfaces::Writeable;
+
+use crate::mem::{
+    address::{FromRaw, IntoRaw, KernelImageObject},
+    device::RawDeviceMemoryMapping,
+    table::EntryLevel,
+    PhysicalAddress, KERNEL_VIRT_OFFSET,
+};
+
+use self::table::{PageAttributes, PageEntry, PageTable, L1, L2, L3};
+
+pub mod process;
+pub(super) mod table;
+
+// TODO eliminate this requirement by using precomputed indices
+const MAPPING_OFFSET: usize = KERNEL_VIRT_OFFSET;
+cfg_if! {
+    if #[cfg(feature = "aarch64_qemu")] {
+        const KERNEL_PHYS_BASE: usize = 0x40080000;
+    } else {
+        const KERNEL_PHYS_BASE: usize = 0x40080000;
+    }
+}
+
+// Precomputed mappings
+const KERNEL_L1_INDEX: usize = L1::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+const KERNEL_START_L2_INDEX: usize = L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE);
+const KERNEL_END_L2_INDEX: usize =
+    L2::index(KERNEL_VIRT_OFFSET + KERNEL_PHYS_BASE) + KERNEL_L3_COUNT;
+
+// Must not be zero, should be at 4MiB
+const_assert_eq!(KERNEL_START_L2_INDEX, 0);
+// From static mapping
+const_assert_eq!(KERNEL_L1_INDEX, 1);
+
+// Runtime mappings
+// 2MiB max
+const EARLY_MAPPING_L2I: usize = KERNEL_END_L2_INDEX + 1;
+// 1GiB max
+const HEAP_MAPPING_L1I: usize = KERNEL_L1_INDEX + 1;
+// 1GiB max
+const DEVICE_MAPPING_L1I: usize = KERNEL_L1_INDEX + 2;
+// 16GiB max
+pub(super) const RAM_MAPPING_L1_COUNT: usize = 16;
+const RAM_MAPPING_START_L1I: usize = KERNEL_L1_INDEX + 3;
+const RAM_MAPPING_END_L1I: usize = RAM_MAPPING_START_L1I + RAM_MAPPING_L1_COUNT;
+
+const DEVICE_MAPPING_L3_COUNT: usize = 4;
+
+// 2MiB for early mappings
+const EARLY_MAPPING_OFFSET: usize =
+    MAPPING_OFFSET | (KERNEL_L1_INDEX * L1::SIZE) | (EARLY_MAPPING_L2I * L2::SIZE);
+static mut EARLY_MAPPING_L3: PageTable<L3> = PageTable::zeroed();
+// 1GiB for heap mapping
+pub(super) const HEAP_MAPPING_OFFSET: usize = MAPPING_OFFSET | (HEAP_MAPPING_L1I * L1::SIZE);
+pub(super) static mut HEAP_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
+// 1GiB for device MMIO mapping
+const DEVICE_MAPPING_OFFSET: usize = MAPPING_OFFSET | (DEVICE_MAPPING_L1I * L1::SIZE);
+static mut DEVICE_MAPPING_L2: PageTable<L2> = PageTable::zeroed();
+static mut DEVICE_MAPPING_L3S: [PageTable<L3>; DEVICE_MAPPING_L3_COUNT] =
+    [PageTable::zeroed(); DEVICE_MAPPING_L3_COUNT];
+// 16GiB for RAM mapping
+pub(super) const RAM_MAPPING_OFFSET: usize = MAPPING_OFFSET | (RAM_MAPPING_START_L1I * L1::SIZE);
+pub(super) static MEMORY_LIMIT: OneTimeInit<usize> = OneTimeInit::new();
+
+#[link_section = ".data.tables"]
+pub static mut KERNEL_TABLES: KernelImageObject<FixedTables> =
+    unsafe { KernelImageObject::new(FixedTables::zeroed()) };
+
+pub struct EarlyMapping<'a, T: ?Sized> {
+    value: &'a mut T,
+    page_count: usize,
+}
+
+impl<'a, T: Sized> EarlyMapping<'a, T> {
+    pub unsafe fn map(physical: PhysicalAddress) -> Result<EarlyMapping<'a, T>, Error> {
+        let layout = Layout::new::<T>();
+        let aligned = physical.page_align_down::<L3>();
+        let offset = physical.page_offset::<L3>();
+        let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE;
+
+        let virt = map_early_pages(aligned, page_count)?;
+        let value = &mut *((virt + offset) as *mut T);
+
+        Ok(EarlyMapping { value, page_count })
+    }
+
+    pub unsafe fn map_slice(
+        physical: PhysicalAddress,
+        len: usize,
+    ) -> Result<EarlyMapping<'a, [T]>, Error> {
+        let layout = Layout::array::<T>(len).unwrap();
+        let aligned = physical.page_align_down::<L3>();
+        let offset = physical.page_offset::<L3>();
+        let page_count = (offset + layout.size() + L3::SIZE - 1) / L3::SIZE;
+
+        let virt = map_early_pages(aligned, page_count)?;
+        let value = core::slice::from_raw_parts_mut((virt + offset) as *mut T, len);
+
+        Ok(EarlyMapping { value, page_count })
+    }
+}
+
+impl<'a, T: ?Sized> Deref for EarlyMapping<'a, T> {
+    type Target = T;
+
+    fn deref(&self) -> &Self::Target {
+        self.value
+    }
+}
+
+impl<'a, T: ?Sized> DerefMut for EarlyMapping<'a, T> {
+    fn deref_mut(&mut self) -> &mut Self::Target {
+        self.value
+    }
+}
+
+impl<'a, T: ?Sized> Drop for EarlyMapping<'a, T> {
+    fn drop(&mut self) {
+        let address = (self.value as *mut T).addr() & !(L3::SIZE - 1);
+
+        for i in 0..self.page_count {
+            let page = address + i * L3::SIZE;
+
+            unsafe {
+                unmap_early_page(page);
+            }
+        }
+    }
+}
+
+fn kernel_table_flags() -> PageAttributes {
+    PageAttributes::TABLE
+        | PageAttributes::ACCESS
+        | PageAttributes::SH_INNER
+        | PageAttributes::PAGE_ATTR_NORMAL
+        | PageAttributes::PRESENT
+}
+
+fn ram_block_flags() -> PageAttributes {
+    // TODO UXN, PXN
+    PageAttributes::BLOCK
+        | PageAttributes::ACCESS
+        | PageAttributes::SH_INNER
+        | PageAttributes::PAGE_ATTR_NORMAL
+        | PageAttributes::PRESENT
+}
+
+// Early mappings
+unsafe fn map_early_pages(physical: PhysicalAddress, count: usize) -> Result<usize, Error> {
+    for l3i in 0..512 {
+        let mut taken = false;
+        for i in 0..count {
+            if EARLY_MAPPING_L3[i + l3i].is_present() {
+                taken = true;
+                break;
+            }
+        }
+
+        if taken {
+            continue;
+        }
+
+        for i in 0..count {
+            let page = physical.add(i * L3::SIZE);
+            // TODO NX, NC
+            EARLY_MAPPING_L3[i + l3i] = PageEntry::normal_page(page, PageAttributes::empty());
+        }
+
+        return Ok(EARLY_MAPPING_OFFSET + l3i * L3::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+unsafe fn unmap_early_page(address: usize) {
+    if address < EARLY_MAPPING_OFFSET || address >= EARLY_MAPPING_OFFSET + L2::SIZE {
+        panic!("Tried to unmap invalid early mapping: {:#x}", address);
+    }
+
+    let l3i = L3::index(address - EARLY_MAPPING_OFFSET);
+
+    assert!(EARLY_MAPPING_L3[l3i].is_present());
+    EARLY_MAPPING_L3[l3i] = PageEntry::INVALID;
+
+    // TODO invalidate tlb
+}
+
+pub(super) unsafe fn map_ram_l1(index: usize) {
+    if index >= RAM_MAPPING_L1_COUNT {
+        loop {}
+    }
+    assert_eq!(KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I], 0);
+
+    KERNEL_TABLES.l1.data[index + RAM_MAPPING_START_L1I] =
+        ((index * L1::SIZE) as u64) | ram_block_flags().bits();
+}
+
+pub(super) unsafe fn map_heap_l2(index: usize, page: PhysicalAddress) {
+    if index >= 512 {
+        loop {}
+    }
+    assert!(!HEAP_MAPPING_L2[index].is_present());
+    // TODO UXN, PXN
+    HEAP_MAPPING_L2[index] = PageEntry::normal_block(page, PageAttributes::empty());
+}
+
+// Device mappings
+unsafe fn map_device_memory_l3(base: PhysicalAddress, count: usize) -> Result<usize, Error> {
+    // TODO don't map pages if already mapped
+
+    'l0: for i in 0..DEVICE_MAPPING_L3_COUNT * 512 {
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            if DEVICE_MAPPING_L3S[l2i][l3i].is_present() {
+                continue 'l0;
+            }
+        }
+
+        for j in 0..count {
+            let l2i = (i + j) / 512;
+            let l3i = (i + j) % 512;
+
+            // TODO NX, NC
+            DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::device_page(base.add(j * L3::SIZE));
+        }
+
+        return Ok(DEVICE_MAPPING_OFFSET + i * L3::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+unsafe fn map_device_memory_l2(base: PhysicalAddress, count: usize) -> Result<usize, Error> {
+    'l0: for i in DEVICE_MAPPING_L3_COUNT..512 {
+        for j in 0..count {
+            if DEVICE_MAPPING_L2[i + j].is_present() {
+                continue 'l0;
+            }
+        }
+
+        for j in 0..count {
+            DEVICE_MAPPING_L2[i + j] = PageEntry::<L2>::device_block(base.add(j * L2::SIZE));
+        }
+
+        debugln!(
+            "map l2s: base={:#x}, count={} -> {:#x}",
+            base,
+            count,
+            DEVICE_MAPPING_OFFSET + i * L2::SIZE
+        );
+        return Ok(DEVICE_MAPPING_OFFSET + i * L2::SIZE);
+    }
+
+    Err(Error::OutOfMemory)
+}
+
+pub(super) unsafe fn map_device_memory(
+    base: PhysicalAddress,
+    size: usize,
+) -> Result<RawDeviceMemoryMapping, Error> {
+    // debugln!("Map {}B @ {:#x}", size, base);
+    let l3_aligned = base.page_align_down::<L3>();
+    let l3_offset = L3::page_offset(base.into_raw());
+    let page_count = (l3_offset + size + L3::SIZE - 1) / L3::SIZE;
+
+    if page_count > 256 {
+        // Large mapping, use L2 mapping instead
+        let l2_aligned = base.page_align_down::<L2>();
+        let l2_offset = L2::page_offset(base.into_raw());
+        let page_count = (l2_offset + size + L2::SIZE - 1) / L2::SIZE;
+
+        let base_address = map_device_memory_l2(l2_aligned, page_count)?;
+        let address = base_address + l2_offset;
+
+        Ok(RawDeviceMemoryMapping {
+            address,
+            base_address,
+            page_count,
+            page_size: L2::SIZE,
+        })
+    } else {
+        // Just map the pages directly
+        let base_address = map_device_memory_l3(l3_aligned, page_count)?;
+        let address = base_address + l3_offset;
+
+        Ok(RawDeviceMemoryMapping {
+            address,
+            base_address,
+            page_count,
+            page_size: L3::SIZE,
+        })
+    }
+}
+
+pub(super) unsafe fn unmap_device_memory(map: &RawDeviceMemoryMapping) {
+    // debugln!(
+    //     "Unmap {}B @ {:#x}",
+    //     map.page_count * map.page_size,
+    //     map.base_address
+    // );
+    match map.page_size {
+        L3::SIZE => {
+            for i in 0..map.page_count {
+                let page = map.base_address + i * L3::SIZE;
+                let l2i = L2::index(page);
+                let l3i = L3::index(page);
+                assert!(DEVICE_MAPPING_L3S[l2i][l3i].is_present());
+                DEVICE_MAPPING_L3S[l2i][l3i] = PageEntry::INVALID;
+                // TODO flush the TLB entry
+                loop {}
+                // intrinsics::flush_tlb_entry(page);
+            }
+        }
+        L2::SIZE => todo!(),
+        _ => unimplemented!(),
+    }
+}
+
+/// (BSP-early init) loads precomputed kernel mapping tables for the kernel to jump to "higher-half"
+///
+/// # Safety
+///
+/// Unsafe, must only be called by BSP during its early init while still in "lower-half"
+pub(super) unsafe fn load_fixed_tables() {
+    let ttbr0 = KERNEL_TABLES.l1.data.as_ptr() as u64;
+    TTBR0_EL1.set(ttbr0);
+    TTBR1_EL1.set(ttbr0);
+}
+
+/// Sets up additional translation tables for kernel usage
+///
+/// # Safety
+///
+/// Unsafe, must only be called by BSP during its early init, must already be in "higher-half"
+pub(super) unsafe fn init_fixed_tables() {
+    // TODO this could be built in compile-time too?
+    let early_mapping_l3_phys = &EARLY_MAPPING_L3 as *const _ as usize - KERNEL_VIRT_OFFSET;
+    let device_mapping_l2_phys = &DEVICE_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET;
+    let heap_mapping_l2_phys = &HEAP_MAPPING_L2 as *const _ as usize - KERNEL_VIRT_OFFSET;
+
+    for i in 0..DEVICE_MAPPING_L3_COUNT {
+        let device_mapping_l3_phys = PhysicalAddress::from_raw(
+            &DEVICE_MAPPING_L3S[i] as *const _ as usize - KERNEL_VIRT_OFFSET,
+        );
+        DEVICE_MAPPING_L2[i] = PageEntry::table(device_mapping_l3_phys, PageAttributes::empty());
+    }
+
+    assert_eq!(KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I], 0);
+    KERNEL_TABLES.l2.data[EARLY_MAPPING_L2I] =
+        (early_mapping_l3_phys as u64) | kernel_table_flags().bits();
+
+    assert_eq!(KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I], 0);
+    KERNEL_TABLES.l1.data[HEAP_MAPPING_L1I] =
+        (heap_mapping_l2_phys as u64) | kernel_table_flags().bits();
+
+    assert_eq!(KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I], 0);
+    KERNEL_TABLES.l1.data[DEVICE_MAPPING_L1I] =
+        (device_mapping_l2_phys as u64) | kernel_table_flags().bits();
+}

src/arch/aarch64/mem/process.rs

@@ -0,0 +1,130 @@
+use core::sync::atomic::{AtomicU8, Ordering};
+
+use abi::error::Error;
+
+use crate::mem::{
+    address::{AsPhysicalAddress, IntoRaw},
+    phys,
+    pointer::PhysicalRefMut,
+    process::ProcessAddressSpaceManager,
+    table::{EntryLevel, MapAttributes, NextPageTable},
+    PhysicalAddress,
+};
+
+use super::table::{PageEntry, PageTable, L1, L2, L3};
+
+#[repr(C)]
+pub struct ProcessAddressSpaceImpl {
+    l1: PhysicalRefMut<'static, PageTable<L1>>,
+    asid: u8,
+}
+
+impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl {
+    const PAGE_SIZE: usize = L3::SIZE;
+    const LOWER_LIMIT_PFN: usize = 8;
+    // 16GiB VM limit
+    const UPPER_LIMIT_PFN: usize = (16 << 30) / Self::PAGE_SIZE;
+
+    fn new() -> Result<Self, Error> {
+        static LAST_ASID: AtomicU8 = AtomicU8::new(1);
+
+        let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel);
+
+        let mut l1 = unsafe { PhysicalRefMut::<'static, PageTable<L1>>::map(phys::alloc_page()?) };
+
+        for i in 0..512 {
+            l1[i] = PageEntry::INVALID;
+        }
+
+        Ok(Self { l1, asid })
+    }
+
+    fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error> {
+        self.read_l3_entry(address).ok_or(Error::DoesNotExist)
+    }
+
+    unsafe fn map_page(
+        &mut self,
+        address: usize,
+        physical: PhysicalAddress,
+        flags: MapAttributes,
+    ) -> Result<(), Error> {
+        self.write_l3_entry(
+            address,
+            PageEntry::normal_page(physical, flags.into()),
+            false,
+        )
+    }
+
+    unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error> {
+        self.pop_l3_entry(address)
+    }
+
+    fn as_address_with_asid(&self) -> u64 {
+        unsafe { u64::from(self.l1.as_physical_address()) | ((self.asid as u64) << 48) }
+    }
+}
+
+impl ProcessAddressSpaceImpl {
+    // Write a single 4KiB entry
+    fn write_l3_entry(
+        &mut self,
+        virt: usize,
+        entry: PageEntry<L3>,
+        overwrite: bool,
+    ) -> Result<(), Error> {
+        let l1i = L1::index(virt);
+        let l2i = L2::index(virt);
+        let l3i = L3::index(virt);
+
+        let mut l2 = self.l1.get_mut_or_alloc(l1i)?;
+        let mut l3 = l2.get_mut_or_alloc(l2i)?;
+
+        if l3[l3i].is_present() && !overwrite {
+            todo!();
+        }
+
+        l3[l3i] = entry;
+        tlb_flush_vaae1(virt);
+
+        Ok(())
+    }
+
+    fn pop_l3_entry(&mut self, virt: usize) -> Result<PhysicalAddress, Error> {
+        let l1i = L1::index(virt);
+        let l2i = L2::index(virt);
+        let l3i = L3::index(virt);
+
+        // TODO somehow drop tables if they're known to be empty?
+        let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?;
+        let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?;
+
+        let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?;
+
+        l3[l3i] = PageEntry::INVALID;
+        tlb_flush_vaae1(virt);
+
+        Ok(page)
+    }
+
+    fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> {
+        let l1i = L1::index(virt);
+        let l2i = L2::index(virt);
+        let l3i = L3::index(virt);
+
+        let l2 = self.l1.get(l1i)?;
+        let l3 = l2.get(l2i)?;
+
+        let page = l3[l3i].as_page()?;
+
+        Some((page, l3[l3i].attributes().into()))
+    }
+}
+
+#[inline]
+fn tlb_flush_vaae1(mut page: usize) {
+    page >>= 12;
+    unsafe {
+        core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page);
+    }
+}

src/arch/aarch64/mem/table.rs

@@ -0,0 +1,295 @@
+use core::{
+    marker::PhantomData,
+    ops::{Index, IndexMut},
+};
+
+use abi::error::Error;
+use bitflags::bitflags;
+
+use crate::mem::{
+    address::{AsPhysicalAddress, FromRaw, IntoRaw},
+    phys,
+    pointer::{PhysicalRef, PhysicalRefMut},
+    table::{EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel},
+    PhysicalAddress,
+};
+
+bitflags! {
+    #[derive(Clone, Copy, PartialEq, Eq)]
+    pub struct PageAttributes: u64 {
+        const PRESENT = 1 << 0;
+
+        const TABLE = 1 << 1;
+        const PAGE = 1 << 1;
+        const BLOCK = 0 << 1;
+
+        const ACCESS = 1 << 10;
+
+        const AP_KERNEL_READWRITE = 0 << 6;
+        const AP_BOTH_READWRITE = 1 << 6;
+        const AP_KERNEL_READONLY = 2 << 6;
+        const AP_BOTH_READONLY = 3 << 6;
+        const AP_ACCESS_MASK = 3 << 6;
+
+        const SH_OUTER = 2 << 8;
+        const SH_INNER = 3 << 8;
+
+        const PAGE_ATTR_NORMAL = 0 << 2;
+        const PAGE_ATTR_DEVICE = 1 << 2;
+
+        const NON_GLOBAL = 1 << 11;
+
+        const PXN = 1 << 53;
+        const UXN = 1 << 54;
+    }
+}
+
+#[derive(Clone, Copy)]
+#[repr(C, align(0x1000))]
+pub struct PageTable<L: EntryLevel> {
+    entries: [PageEntry<L>; 512],
+}
+
+#[derive(Clone, Copy)]
+pub struct PageEntry<L: EntryLevel>(u64, PhantomData<L>);
+
+#[derive(Clone, Copy)]
+pub struct L1;
+#[derive(Clone, Copy)]
+pub struct L2;
+#[derive(Clone, Copy)]
+pub struct L3;
+
+impl const EntryLevel for L1 {
+    const SHIFT: usize = 30;
+}
+
+impl NonTerminalEntryLevel for L1 {
+    type NextLevel = L2;
+}
+
+impl const EntryLevel for L2 {
+    const SHIFT: usize = 21;
+}
+
+impl NonTerminalEntryLevel for L2 {
+    type NextLevel = L3;
+}
+
+impl const EntryLevel for L3 {
+    const SHIFT: usize = 12;
+}
+
+impl<L: EntryLevel> PageTable<L> {
+    pub const fn zeroed() -> Self {
+        Self {
+            entries: [PageEntry::INVALID; 512],
+        }
+    }
+
+    pub fn new_zeroed<'a>() -> Result<PhysicalRefMut<'a, Self>, Error> {
+        let physical = phys::alloc_page()?;
+        let mut table = unsafe { PhysicalRefMut::<'a, Self>::map(physical) };
+
+        for i in 0..512 {
+            table[i] = PageEntry::INVALID;
+        }
+
+        Ok(table)
+    }
+}
+
+impl<L: EntryLevel> PageEntry<L> {
+    pub const INVALID: Self = Self(0, PhantomData);
+
+    pub const fn is_present(self) -> bool {
+        self.0 & PageAttributes::PRESENT.bits() != 0
+    }
+
+    pub fn attributes(self) -> PageAttributes {
+        PageAttributes::from_bits_retain(self.0)
+    }
+}
+
+impl<L: NonTerminalEntryLevel + 'static> NextPageTable for PageTable<L> {
+    type NextLevel = PageTable<L::NextLevel>;
+    type TableRef = PhysicalRef<'static, PageTable<L::NextLevel>>;
+    type TableRefMut = PhysicalRefMut<'static, PageTable<L::NextLevel>>;
+
+    fn get(&self, index: usize) -> Option<Self::TableRef> {
+        self[index]
+            .as_table()
+            .map(|phys| unsafe { PhysicalRef::map(phys) })
+    }
+
+    fn get_mut(&mut self, index: usize) -> Option<Self::TableRefMut> {
+        self[index]
+            .as_table()
+            .map(|phys| unsafe { PhysicalRefMut::map(phys) })
+    }
+
+    fn get_mut_or_alloc(&mut self, index: usize) -> Result<Self::TableRefMut, Error> {
+        let entry = self[index];
+
+        if let Some(table) = entry.as_table() {
+            Ok(unsafe { PhysicalRefMut::map(table) })
+        } else {
+            let table = PageTable::new_zeroed()?;
+            self[index] = PageEntry::<L>::table(
+                unsafe { table.as_physical_address() },
+                PageAttributes::empty(),
+            );
+            Ok(table)
+        }
+    }
+}
+
+impl<L: NonTerminalEntryLevel> PageEntry<L> {
+    pub fn table(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
+        Self(
+            IntoRaw::<u64>::into_raw(phys)
+                | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(),
+            PhantomData,
+        )
+    }
+
+    pub fn normal_block(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
+        Self(
+            IntoRaw::<u64>::into_raw(phys)
+                | (PageAttributes::BLOCK
+                    | PageAttributes::PRESENT
+                    | PageAttributes::ACCESS
+                    | PageAttributes::SH_INNER
+                    | PageAttributes::PAGE_ATTR_NORMAL
+                    | attrs)
+                    .bits(),
+            PhantomData,
+        )
+    }
+
+    pub fn device_block(phys: PhysicalAddress) -> Self {
+        Self(
+            IntoRaw::<u64>::into_raw(phys)
+                | (PageAttributes::BLOCK
+                    | PageAttributes::PRESENT
+                    | PageAttributes::ACCESS
+                    | PageAttributes::SH_OUTER
+                    | PageAttributes::PAGE_ATTR_DEVICE
+                    | PageAttributes::UXN
+                    | PageAttributes::PXN)
+                    .bits(),
+            PhantomData,
+        )
+    }
+
+    /// Returns the physical address of the table this entry refers to, returning None if it
+    /// does not
+    pub fn as_table(self) -> Option<PhysicalAddress> {
+        if self.0 & PageAttributes::PRESENT.bits() != 0
+            && self.0 & PageAttributes::BLOCK.bits() == 0
+        {
+            Some(PhysicalAddress::from_raw(self.0 & !0xFFF))
+        } else {
+            None
+        }
+    }
+}
+
+impl PageEntry<L3> {
+    pub fn normal_page(phys: PhysicalAddress, attrs: PageAttributes) -> Self {
+        Self(
+            IntoRaw::<u64>::into_raw(phys)
+                | (PageAttributes::PAGE
+                    | PageAttributes::PRESENT
+                    | PageAttributes::ACCESS
+                    | PageAttributes::SH_INNER
+                    | PageAttributes::PAGE_ATTR_NORMAL
+                    | attrs)
+                    .bits(),
+            PhantomData,
+        )
+    }
+
+    pub fn device_page(phys: PhysicalAddress) -> Self {
+        Self(
+            IntoRaw::<u64>::into_raw(phys)
+                | (PageAttributes::PAGE
+                    | PageAttributes::PRESENT
+                    | PageAttributes::ACCESS
+                    | PageAttributes::SH_OUTER
+                    | PageAttributes::PAGE_ATTR_DEVICE
+                    | PageAttributes::UXN
+                    | PageAttributes::PXN)
+                    .bits(),
+            PhantomData,
+        )
+    }
+
+    pub fn as_page(&self) -> Option<PhysicalAddress> {
+        let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits();
+        if self.0 & mask == mask {
+            Some(PhysicalAddress::from_raw(self.0 & !0xFFF))
+        } else {
+            None
+        }
+    }
+}
+
+impl<L: EntryLevel> Index<usize> for PageTable<L> {
+    type Output = PageEntry<L>;
+
+    fn index(&self, index: usize) -> &Self::Output {
+        &self.entries[index]
+    }
+}
+
+impl<L: EntryLevel> IndexMut<usize> for PageTable<L> {
+    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
+        &mut self.entries[index]
+    }
+}
+
+impl From<MapAttributes> for PageAttributes {
+    fn from(value: MapAttributes) -> Self {
+        let mut out = PageAttributes::empty();
+        // TODO kernel cannot write el0 readonly pages
+        if value.contains(MapAttributes::USER_WRITE) {
+            // Read/write
+            out |= PageAttributes::AP_BOTH_READWRITE;
+        } else if value.contains(MapAttributes::USER_READ) {
+            // Read only
+            out |= PageAttributes::AP_BOTH_READONLY;
+        } else {
+            // No read/write
+            out |= PageAttributes::AP_KERNEL_READONLY;
+        }
+
+        if value.contains(MapAttributes::NON_GLOBAL) {
+            out |= PageAttributes::NON_GLOBAL;
+        }
+
+        out
+    }
+}
+
+impl From<PageAttributes> for MapAttributes {
+    fn from(value: PageAttributes) -> Self {
+        let mut out = MapAttributes::empty();
+
+        out |= match value.intersection(PageAttributes::AP_ACCESS_MASK) {
+            PageAttributes::AP_BOTH_READWRITE => {
+                MapAttributes::USER_WRITE | MapAttributes::USER_READ
+            }
+            PageAttributes::AP_BOTH_READONLY => MapAttributes::USER_READ,
+            PageAttributes::AP_KERNEL_READONLY => MapAttributes::empty(),
+            PageAttributes::AP_KERNEL_READWRITE => panic!("This variant cannot be constructed"),
+            _ => unreachable!(),
+        };
+
+        if value.contains(PageAttributes::NON_GLOBAL) {
+            out |= MapAttributes::NON_GLOBAL;
+        }
+
+        out
+    }
+}

src/arch/aarch64/mod.rs

@@ -1,49 +1,47 @@
 //! AArch64 architecture and platforms implementation
 
+pub mod mem;
+
 use core::sync::atomic::Ordering;
 
-use aarch64_cpu::{
+use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF};
-    asm::barrier,
-    registers::{
-        CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF, ID_AA64MMFR0_EL1, MAIR_EL1, SCTLR_EL1, TCR_EL1,
-        TTBR0_EL1, TTBR1_EL1,
-    },
-};
 use abi::error::Error;
 use device_api::{
     interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController},
     timer::MonotonicTimestampProviderDevice,
     ResetDevice,
 };
-use fdt_rs::prelude::PropReader;
+use fdt_rs::base::DevTree;
 use git_version::git_version;
 use kernel_util::util::OneTimeInit;
 use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 
 use crate::{
-    arch::Architecture,
+    arch::aarch64::{
+        cpu::Cpu,
+        mem::table::{L2, L3},
+    },
     debug,
     device::{
         self,
-        devtree::{
+        devtree::{self, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter},
-            self, DevTreeIndexNodePropGet, DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree,
-            FdtMemoryRegionIter,
-        },
         power::arm_psci::Psci,
     },
     fs::{Initrd, INITRD_DATA},
     mem::{
+        address::{FromRaw, IntoRaw},
+        device::RawDeviceMemoryMapping,
+        heap,
         phys::{self, reserved::reserve_region, PhysicalMemoryRegion},
-        ConvertAddress,
+        pointer::PhysicalRef,
+        table::EntryLevel,
+        PhysicalAddress,
     },
 };
 
-use self::{
+use self::mem::{table::L1, EarlyMapping};
-    smp::CPU_COUNT,
-    table::{init_fixed_tables, KERNEL_TABLES},
-};
 
-use super::CpuMessage;
+use super::{Architecture, CpuMessage};
 
 pub mod boot;
 pub mod context;
@@ -51,111 +49,44 @@ pub mod cpu;
 pub mod exception;
 pub mod gic;
 pub mod smp;
-pub mod table;
 pub mod timer;
 
-const BOOT_STACK_SIZE: usize = 65536;
+const BOOT_STACK_SIZE: usize = 4096 * 32;
 
 #[derive(Clone, Copy)]
 #[repr(C, align(0x20))]
-struct KernelStack {
+struct BootStack {
     data: [u8; BOOT_STACK_SIZE],
 }
 
-#[derive(Clone, Copy, Debug, PartialEq)]
-pub enum IrqNumber {
-    Private(u32),
-    Shared(u32),
-}
-
-/// AArch64 platform interface
 pub struct AArch64 {
     dt: OneTimeInit<DeviceTree<'static>>,
-    ext_intc: OneTimeInit<&'static dyn ExternalInterruptController<IrqNumber = IrqNumber>>,
+
-    local_intc: OneTimeInit<&'static dyn LocalInterruptController<IpiMessage = CpuMessage>>,
+    pub psci: OneTimeInit<&'static Psci>,
-    mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>,
     reset: OneTimeInit<&'static dyn ResetDevice>,
 
-    // ARM-only devices
+    lintc: OneTimeInit<&'static dyn LocalInterruptController<IpiMessage = CpuMessage>>,
-    /// ARM PSCI instance on this system (there may not be one)
+    xintc: OneTimeInit<&'static dyn ExternalInterruptController<IrqNumber = IrqNumber>>,
-    pub psci: OneTimeInit<&'static Psci>,
+
+    mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>,
+
+    initrd: OneTimeInit<PhysicalRef<'static, [u8]>>,
 }
 
-/// Global platform handle
-pub static ARCHITECTURE: AArch64 = AArch64 {
-    dt: OneTimeInit::new(),
-    ext_intc: OneTimeInit::new(),
-    local_intc: OneTimeInit::new(),
-    mtimer: OneTimeInit::new(),
-    reset: OneTimeInit::new(),
-
-    // ARM-only devices
-    psci: OneTimeInit::new(),
-};
-
 impl Architecture for AArch64 {
-    type IrqNumber = IrqNumber;
-
     const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000;
 
-    unsafe fn init_mmu(&self, bsp: bool) {
+    type IrqNumber = IrqNumber;
-        if bsp {
-            init_fixed_tables();
-        }
-
-        let tables_phys = absolute_address!(KERNEL_TABLES).physicalize() as u64;
-
-        if !ID_AA64MMFR0_EL1.matches_all(ID_AA64MMFR0_EL1::TGran4::Supported) {
-            todo!();
-        }
-
-        MAIR_EL1.write(
-            // Attribute 0 -- normal memory
-            MAIR_EL1::Attr0_Normal_Inner::WriteBack_NonTransient_ReadWriteAlloc +
-        MAIR_EL1::Attr0_Normal_Outer::WriteBack_NonTransient_ReadWriteAlloc +
-        // Attribute 1 -- device memory
-        MAIR_EL1::Attr1_Device::nonGathering_nonReordering_EarlyWriteAck,
-        );
-
-        TCR_EL1.modify(
-            // General
-            TCR_EL1::IPS::Bits_48 +
-            // TTBR0
-            TCR_EL1::TG0::KiB_4 + TCR_EL1::T0SZ.val(25) + TCR_EL1::SH0::Inner +
-            // TTBR1
-            TCR_EL1::TG1::KiB_4 + TCR_EL1::T1SZ.val(25) + TCR_EL1::SH1::Outer,
-        );
-
-        barrier::dmb(barrier::ISH);
-
-        TTBR0_EL1.set_baddr(tables_phys);
-        TTBR1_EL1.set_baddr(tables_phys);
-
-        barrier::isb(barrier::SY);
-
-        // Enable instruction cache, data cache and translation
-        SCTLR_EL1
-            .modify(SCTLR_EL1::M::Enable + SCTLR_EL1::I::NonCacheable + SCTLR_EL1::C::NonCacheable);
-
-        barrier::isb(barrier::SY);
-    }
 
     unsafe fn start_application_processors(&self) {
         let dt = self.dt.get();
-        if let Err(e) = smp::start_ap_cores(dt) {
+        if let Err(error) = smp::start_ap_cores(dt) {
-            errorln!(
+            errorln!("Could not initialize AP CPUs: {:?}", error);
-                "Could not initialize AP CPUs: {:?}. Will continue with one CPU.",
-                e
-            );
         }
     }
 
-    fn map_device_pages(&self, phys: usize, count: usize) -> Result<usize, Error> {
+    fn cpu_count() -> usize {
-        unsafe { KERNEL_TABLES.map_device_pages(phys, count) }
+        smp::CPU_COUNT.load(Ordering::Acquire)
-    }
-
-    fn wait_for_interrupt() {
-        aarch64_cpu::asm::wfi();
     }
 
     unsafe fn set_interrupt_mask(mask: bool) {
@@ -170,26 +101,98 @@ impl Architecture for AArch64 {
         DAIF.read(DAIF::I) != 0
     }
 
-    fn cpu_count() -> usize {
+    fn wait_for_interrupt() {
-        CPU_COUNT.load(Ordering::Acquire)
+        aarch64_cpu::asm::wfi();
     }
 
-    fn register_external_interrupt_controller(
+    unsafe fn map_device_memory(
         &self,
-        intc: &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber>,
+        base: PhysicalAddress,
+        size: usize,
+    ) -> Result<RawDeviceMemoryMapping, Error> {
+        mem::map_device_memory(base, size)
+    }
+
+    unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) {
+        mem::unmap_device_memory(map)
+    }
+
+    fn map_physical_memory<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
+        &self,
+        _it: I,
+        _memory_start: PhysicalAddress,
+        memory_end: PhysicalAddress,
     ) -> Result<(), Error> {
-        self.ext_intc.init(intc);
+        let end_l1i = L1::index(memory_end.page_align_up::<L1>().into_raw());
+        if end_l1i > mem::RAM_MAPPING_L1_COUNT {
+            loop {}
+        }
+
+        // Map 1GiB chunks
+        for index in 0..end_l1i {
+            unsafe {
+                mem::map_ram_l1(index);
+            }
+        }
+
+        mem::MEMORY_LIMIT.init(memory_end.into_raw());
+
         Ok(())
     }
 
+    fn virtualize(address: PhysicalAddress) -> Result<usize, Error> {
+        let raw: usize = address.into_raw();
+        if raw < *mem::MEMORY_LIMIT.get() {
+            Ok(raw + mem::RAM_MAPPING_OFFSET)
+        } else {
+            errorln!("Invalid physical address: {:#x}", address);
+            Err(Error::InvalidMemoryOperation)
+        }
+    }
+
+    fn physicalize(address: usize) -> Result<PhysicalAddress, Error> {
+        if address < mem::RAM_MAPPING_OFFSET
+            || address - mem::RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get()
+        {
+            errorln!("Not a virtualized physical address: {:#x}", address);
+            return Err(Error::InvalidMemoryOperation);
+        }
+
+        Ok(PhysicalAddress::from_raw(address - mem::RAM_MAPPING_OFFSET))
+    }
+
+    fn local_interrupt_controller(
+        &'static self,
+    ) -> &'static dyn LocalInterruptController<IpiMessage = super::CpuMessage> {
+        *self.lintc.get()
+    }
+
+    fn external_interrupt_controller(
+        &'static self,
+    ) -> &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber> {
+        *self.xintc.get()
+    }
+
     fn register_local_interrupt_controller(
         &self,
         intc: &'static dyn LocalInterruptController<IpiMessage = super::CpuMessage>,
     ) -> Result<(), Error> {
-        self.local_intc.init(intc);
+        self.lintc.init(intc);
         Ok(())
     }
 
+    fn register_external_interrupt_controller(
+        &self,
+        intc: &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber>,
+    ) -> Result<(), Error> {
+        self.xintc.init(intc);
+        Ok(())
+    }
+
+    fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
+        *self.mtimer.get()
+    }
+
     fn register_monotonic_timer(
         &self,
         timer: &'static dyn MonotonicTimestampProviderDevice,
@@ -198,107 +201,143 @@ impl Architecture for AArch64 {
         Ok(())
     }
 
-    fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> {
-        self.reset.init(reset);
-        Ok(())
-    }
-
-    fn external_interrupt_controller(
-        &self,
-    ) -> &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber> {
-        *self.ext_intc.get()
-    }
-
-    fn local_interrupt_controller(
-        &self,
-    ) -> &'static dyn LocalInterruptController<IpiMessage = super::CpuMessage> {
-        *self.local_intc.get()
-    }
-
-    fn monotonic_timer(&self) -> &'static dyn MonotonicTimestampProviderDevice {
-        *self.mtimer.get()
-    }
-
     unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> {
-        if let Some(local_intc) = self.local_intc.try_get() {
+        if let Some(local_intc) = self.lintc.try_get() {
             local_intc.send_ipi(target, msg)
         } else {
             Ok(())
         }
     }
 
+    fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> {
+        self.reset.init(reset);
+        Ok(())
+    }
+
     unsafe fn reset(&self) -> ! {
         if let Some(reset) = self.reset.try_get() {
-            reset.reset();
+            reset.reset()
         } else {
             let psci = self.psci.get();
-            psci.reset();
+            psci.reset()
         }
     }
 }
 
 impl AArch64 {
-    /// Initializes the architecture's device tree
+    fn extract_initrd_from_dt(
-    ///
+        &self,
-    /// # Safety
+        dt: &DeviceTree,
-    ///
+    ) -> Option<(PhysicalAddress, PhysicalAddress)> {
-    /// Only makes sense to call during the early initialization, once.
+        let chosen = dt.node_by_path("/chosen")?;
-    pub unsafe fn init_device_tree(&self, dtb_phys: usize) {
+        let initrd_start = devtree::find_prop(&chosen, "linux,initrd-start")?;
-        let dt = DeviceTree::from_addr(dtb_phys.virtualize());
+        let initrd_end = devtree::find_prop(&chosen, "linux,initrd-end")?;
-        self.dt.init(dt);
+
+        let address_cells = dt.address_cells();
+
+        let initrd_start = initrd_start.cell1_array_item(0, address_cells)?;
+        let initrd_end = initrd_end.cell1_array_item(0, address_cells)?;
+
+        let initrd_start = PhysicalAddress::from_raw(initrd_start);
+        let initrd_end = PhysicalAddress::from_raw(initrd_end);
+
+        Some((initrd_start, initrd_end))
     }
 
-    /// Returns the device tree
+    unsafe fn init_memory_management(&'static self, dtb: PhysicalAddress) -> Result<(), Error> {
-    ///
+        // 16x2MiB
-    /// # Panics
+        const HEAP_PAGES: usize = 16;
-    ///
-    /// Will panic if the device tree has not yet been initialized
-    pub fn device_tree(&self) -> &DeviceTree {
-        self.dt.get()
-    }
 
-    unsafe fn init_physical_memory(&self, dtb_phys: usize) -> Result<(), Error> {
+        // Initialize the runtime mappings
-        let dt = self.device_tree();
+        mem::init_fixed_tables();
 
-        if let Some(initrd) = INITRD_DATA.try_get() {
+        // Extract the size of the device tree
-            reserve_region(
+        let dtb_size = {
-                "initrd",
+            let dtb_header = EarlyMapping::<u8>::map_slice(dtb, DevTree::MIN_HEADER_SIZE)?;
-                PhysicalMemoryRegion {
+            DevTree::read_totalsize(dtb_header.as_ref()).unwrap()
-                    base: initrd.phys_page_start,
+        };
-                    size: initrd.phys_page_len,
-                },
-            );
-        }
 
         reserve_region(
             "dtb",
             PhysicalMemoryRegion {
-                base: dtb_phys,
+                base: dtb,
-                size: (dt.size() + 0xFFF) & !0xFFF,
+                size: (dtb_size + 0xFFF) & !0xFFF,
             },
         );
 
-        let regions = FdtMemoryRegionIter::new(dt);
+        let dtb_slice = EarlyMapping::<u8>::map_slice(dtb, dtb_size)?;
 
-        phys::init_from_iter(regions)
+        let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize);
+
+        // Setup initrd from the dt
+        let initrd = self.extract_initrd_from_dt(&dt);
+
+        if let Some((start, end)) = initrd {
+            let aligned_start = start.page_align_down::<L3>();
+            let aligned_end = end.page_align_up::<L3>();
+
+            let size = aligned_end - aligned_start;
+            reserve_region(
+                "initrd",
+                PhysicalMemoryRegion {
+                    base: aligned_start,
+                    size,
+                },
+            );
+        }
+
+        // Initialize the physical memory
+        let regions = FdtMemoryRegionIter::new(&dt);
+
+        phys::init_from_iter(regions)?;
+
+        // Setup the heap
+        for i in 0..HEAP_PAGES {
+            let l2_page = phys::alloc_2m_page()?;
+            mem::map_heap_l2(i, l2_page);
+        }
+
+        heap::init_heap(mem::HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE);
+
+        // EarlyMapping for DTB no longer needed, it lives in physical memory and can be obtained
+        // through PhysicalRef
+        let dtb_slice: PhysicalRef<'static, [u8]> = PhysicalRef::map_slice(dtb, dtb_size);
+        let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize);
+
+        self.dt.init(dt);
+
+        // Setup initrd
+        if let Some((initrd_start, initrd_end)) = initrd {
+            let aligned_start = initrd_start.page_align_down::<L3>();
+            let aligned_end = initrd_end.page_align_up::<L3>();
+            let len = initrd_end - initrd_start;
+
+            let data = unsafe { PhysicalRef::map_slice(initrd_start, len) };
+            self.initrd.init(data);
+
+            INITRD_DATA.init(Initrd {
+                phys_page_start: aligned_start,
+                phys_page_len: aligned_end - aligned_start,
+                data: self.initrd.get().as_ref(),
+            });
+        }
+
+        Ok(())
     }
 
-    fn chosen_stdout_path<'a>(dt: &'a DeviceTree) -> Option<&'a str> {
+    unsafe fn init_platform(&self, is_bsp: bool) -> Result<(), Error> {
-        let chosen = dt.node_by_path("/chosen")?;
+        Cpu::init_local();
-        chosen.prop("stdout-path")
-    }
 
-    fn init_platform(&self, bsp: bool) {
+        if is_bsp {
-        if bsp {
+            let dt = self.dt.get();
-            let dt = self.device_tree();
 
             let address_cells = dt.address_cells();
             let size_cells = dt.size_cells();
 
-            let chosen_stdout_path = Self::chosen_stdout_path(dt);
+            // Setup /chosen.stdout-path to get early debug printing
+            let chosen_stdout_path = dt.chosen_stdout_path();
             let chosen_stdout = chosen_stdout_path.and_then(|path| dt.node_by_path(path));
 
-            // Probe and initialize the /chosen.stdout-path device first
             if let Some(node) = chosen_stdout.clone() {
                 let probe = DevTreeNodeInfo {
                     address_cells,
@@ -307,15 +346,12 @@ impl AArch64 {
                 };
 
                 if let Some((device, _)) = devtree::probe_dt_node(&probe) {
-                    unsafe {
+                    device.init()?;
-                        device.init().unwrap();
-                    }
                 }
-            };
+            }
 
-            debug::reset();
+            debug::init();
 
-            // Print some stuff now that the output is initialized
             infoln!(
                 "Yggdrasil v{} ({})",
                 env!("CARGO_PKG_VERSION"),
@@ -323,22 +359,19 @@ impl AArch64 {
             );
             infoln!("Initializing aarch64 platform");
 
-            // Probe and initialize the rest of devices
             let nodes = dt.root().children();
             if let Err(error) = devtree::enumerate_dt(
                 address_cells,
                 size_cells,
                 nodes,
                 |_, probe| {
-                    // Ignore /chosen/stdout-path node
+                    // Skip chosen-stdout, already initialized
                     if let Some(ref chosen_stdout) = chosen_stdout && chosen_stdout.name() == probe.node.name() {
                     return Ok(());
                 }
 
                     if let Some((device, _)) = devtree::probe_dt_node(&probe) {
-                        unsafe {
+                        device.init()?;
-                            device.init()?;
-                        }
                     }
 
                     Ok(())
@@ -353,15 +386,14 @@ impl AArch64 {
             // Initialize IRQs for the devices
             device::manager_lock().devices().for_each(|dev| unsafe {
                 if let Err(error) = dev.init_irq() {
-                    errorln!(
+                    warnln!(
-                        "Could not init interrupts for {:?}: {:?}",
+                        "Could not init IRQs for {:?}: {:?}",
                         dev.display_name(),
                         error
                     );
                 }
             });
 
-            // Print the device list
             infoln!("Enumerated devices:");
             device::manager_lock().devices().for_each(|dev| {
                 infoln!("* {:?}", dev.display_name());
@@ -369,7 +401,7 @@ impl AArch64 {
         } else {
             // BSP already initialized everything needed
             // Setup timer and local interrupt controller
-            let intc = self.local_intc.get();
+            let intc = self.lintc.get();
 
             unsafe {
                 intc.init_ap().unwrap();
@@ -378,57 +410,28 @@ impl AArch64 {
             // TODO device-tree initialization for this
             CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR);
             CNTP_TVAL_EL0.set(10000000);
-            self.ext_intc
+            self.xintc.get().enable_irq(IrqNumber::Private(14)).unwrap();
-                .get()
-                .enable_irq(IrqNumber::Private(14))
-                .unwrap();
         }
+
+        Ok(())
     }
 }
 
-fn setup_initrd() {
+pub static ARCHITECTURE: AArch64 = AArch64 {
-    let dt = ARCHITECTURE.device_tree();
+    dt: OneTimeInit::new(),
+    initrd: OneTimeInit::new(),
 
-    let Some(chosen) = dt.node_by_path("/chosen") else {
+    psci: OneTimeInit::new(),
-        return;
+    reset: OneTimeInit::new(),
-    };
 
-    let Some(initrd_start) = devtree::find_prop(&chosen, "linux,initrd-start") else {
+    lintc: OneTimeInit::new(),
-        return;
+    xintc: OneTimeInit::new(),
-    };
-    let Some(initrd_end) = devtree::find_prop(&chosen, "linux,initrd-end") else {
-        return;
-    };
 
-    let address_cells = dt.address_cells();
+    mtimer: OneTimeInit::new(),
+};
 
-    let Some(initrd_start) = initrd_start.cell1_array_item(0, address_cells) else {
+#[derive(Clone, Copy, Debug, PartialEq)]
-        infoln!("No initrd specified");
+pub enum IrqNumber {
-        return;
+    Private(u32),
-    };
+    Shared(u32),
-    let Some(initrd_end) = initrd_end.cell1_array_item(0, address_cells) else {
-        infoln!("No initrd specified");
-        return;
-    };
-
-    let initrd_start = initrd_start as usize;
-    let initrd_end = initrd_end as usize;
-
-    let start_aligned = initrd_start & !0xFFF;
-    let end_aligned = initrd_end & !0xFFF;
-
-    let data = unsafe {
-        core::slice::from_raw_parts(
-            initrd_start.virtualize() as *const _,
-            initrd_end - initrd_start,
-        )
-    };
-
-    let initrd = Initrd {
-        phys_page_start: start_aligned,
-        phys_page_len: end_aligned - start_aligned,
-        data,
-    };
-
-    INITRD_DATA.init(initrd);
 }

src/arch/aarch64/plat_orange_pi3/mod.rs

@@ -1,103 +0,0 @@
-//! Orange Pi 3 (Allwinner H6/sun50i-h6 SoC)
-//!
-//! # Booting using u-boot
-//!
-//! > fatload mmc 0:1 0x40000000 uRamdisk
-//! > fatload mmc 0:1 0x4d000000 sun50i-h6-orangepi-3.dtb
-//! > loady 0x44000000
-//! ...
-//! > bootm 0x44000000 0x40000000 0x4d000000
-//!
-use abi::error::Error;
-
-use crate::{
-    arch::CpuMessage,
-    debug::{self, LogLevel},
-    device::{
-        interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget},
-        platform::Platform,
-        timer::TimestampSource,
-        InitializableDevice,
-    },
-    fs::devfs::{self, CharDeviceType},
-};
-
-use self::{r_wdog::RWatchdog, serial::Serial};
-
-use super::{
-    gic::{Gic, IrqNumber},
-    timer::ArmTimer,
-};
-
-pub mod r_wdog;
-pub mod serial;
-
-/// Orange Pi 3 implementation
-pub struct PlatformImpl {
-    uart0: Serial,
-    r_wdog: RWatchdog,
-    /// ...
-    pub gic: Gic,
-    timer: ArmTimer,
-}
-
-impl Platform for PlatformImpl {
-    type IrqNumber = IrqNumber;
-
-    const KERNEL_PHYS_BASE: usize = 0x50000000;
-
-    unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> {
-        if is_bsp {
-            self.gic.init(())?;
-
-            self.timer.init(())?;
-            self.r_wdog.init(())?;
-
-            self.timer.init_irq()?;
-            self.uart0.init_irq()?;
-
-            devfs::add_char_device(&self.uart0, CharDeviceType::TtySerial)?;
-        } else {
-            todo!();
-        }
-
-        Ok(())
-    }
-
-    unsafe fn init_debug(&'static self) {
-        self.uart0.init(()).unwrap();
-        debug::add_sink(&self.uart0, LogLevel::Debug);
-    }
-
-    fn interrupt_controller(
-        &self,
-    ) -> &dyn ExternalInterruptController<IrqNumber = Self::IrqNumber> {
-        &self.gic
-    }
-
-    fn timestamp_source(&self) -> &dyn TimestampSource {
-        &self.timer
-    }
-
-    unsafe fn send_ipi(&self, _target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> {
-        todo!();
-    }
-
-    fn name(&self) -> &'static str {
-        "Orange Pi 3"
-    }
-
-    unsafe fn reset(&self) -> ! {
-        self.r_wdog.reset_board();
-    }
-}
-
-/// Orange Pi 3 platform implementation
-pub static PLATFORM: PlatformImpl = unsafe {
-    PlatformImpl {
-        uart0: Serial::new(0x05000000, IrqNumber::new(32)),
-        r_wdog: RWatchdog::new(0x07020400),
-        gic: Gic::new(0x03021000, 0x03022000),
-        timer: ArmTimer::new(IrqNumber::new(30)),
-    }
-};

src/arch/aarch64/plat_orange_pi3/r_wdog.rs

@@ -1,90 +0,0 @@
-//! Allwinner H6 R Watchdog driver
-use abi::error::Error;
-use tock_registers::{
-    interfaces::Writeable, register_bitfields, register_structs, registers::ReadWrite,
-};
-
-use crate::{
-    device::InitializableDevice, mem::device::DeviceMemoryIo, sync::IrqSafeSpinlock,
-    util::OneTimeInit,
-};
-
-register_bitfields! {
-    u32,
-    CTRL [
-        KEY OFFSET(1) NUMBITS(12) [
-            Value = 0xA57
-        ],
-        RESTART OFFSET(0) NUMBITS(1) []
-    ],
-    CFG [
-        CONFIG OFFSET(0) NUMBITS(2) [
-            System = 1,
-        ]
-    ],
-    MODE [
-        EN OFFSET(0) NUMBITS(1) [],
-    ]
-}
-
-register_structs! {
-    #[allow(non_snake_case)]
-    Regs {
-        (0x00 => IRQ_EN: ReadWrite<u32>),
-        (0x04 => IRQ_STA: ReadWrite<u32>),
-        (0x08 => _0),
-        (0x10 => CTRL: ReadWrite<u32, CTRL::Register>),
-        (0x14 => CFG: ReadWrite<u32, CFG::Register>),
-        (0x18 => MODE: ReadWrite<u32, MODE::Register>),
-        (0x1C => @END),
-    }
-}
-
-/// Allwinner H6 R Watchdog
-pub struct RWatchdog {
-    inner: OneTimeInit<IrqSafeSpinlock<DeviceMemoryIo<Regs>>>,
-    base: usize,
-}
-
-impl InitializableDevice for RWatchdog {
-    type Options = ();
-
-    unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> {
-        let regs = DeviceMemoryIo::map("r_wdog", self.base)?;
-
-        self.inner.init(IrqSafeSpinlock::new(regs));
-
-        Ok(())
-    }
-}
-
-impl RWatchdog {
-    /// Performs a reset through watchdog.
-    ///
-    /// # Safety
-    ///
-    /// Only meant to be called by platform reset code.
-    pub unsafe fn reset_board(&self) -> ! {
-        let regs = self.inner.get().lock();
-
-        regs.CFG.write(CFG::CONFIG::System);
-        regs.MODE.write(MODE::EN::SET);
-        regs.CTRL.write(CTRL::KEY::Value + CTRL::RESTART::SET);
-
-        loop {
-            core::arch::asm!("wfe");
-        }
-    }
-
-    /// Constructs an instance of the device at `base`.
-    ///
-    /// # Safety
-    ///
-    /// The caller must ensure the address is valid.
-    pub const unsafe fn new(base: usize) -> Self {
-        Self {
-            inner: OneTimeInit::new(),
-            base,
-        }
-    }
-}

src/arch/aarch64/plat_orange_pi3/serial.rs

@@ -1,160 +0,0 @@
-//! Allwinner H6 UART driver
-use abi::{error::Error, io::DeviceRequest};
-use tock_registers::{
-    interfaces::{Readable, Writeable},
-    register_bitfields, register_structs,
-    registers::{ReadOnly, ReadWrite},
-};
-use vfs::CharDevice;
-
-use crate::{
-    arch::aarch64::gic::IrqNumber,
-    arch::PLATFORM,
-    debug::DebugSink,
-    device::{
-        interrupt::InterruptSource,
-        platform::Platform,
-        serial::SerialDevice,
-        tty::{CharRing, TtyDevice},
-        Device, InitializableDevice,
-    },
-    mem::device::DeviceMemoryIo,
-    sync::IrqSafeSpinlock,
-    util::OneTimeInit,
-};
-
-register_bitfields! {
-    u32,
-    USR [
-        TFE OFFSET(2) NUMBITS(1) [],
-        TFNF OFFSET(1) NUMBITS(1) []
-    ]
-}
-
-register_structs! {
-    #[allow(non_snake_case)]
-    Regs {
-        (0x00 => DLL: ReadWrite<u32>),
-        (0x04 => _0),
-        (0x7C => USR: ReadOnly<u32, USR::Register>),
-        (0x80 => @END),
-    }
-}
-
-struct Inner {
-    regs: DeviceMemoryIo<Regs>,
-}
-
-/// Allwinner H6 UART
-pub struct Serial {
-    inner: OneTimeInit<IrqSafeSpinlock<Inner>>,
-    ring: CharRing<16>,
-    base: usize,
-    irq: IrqNumber,
-}
-
-impl CharDevice for Serial {
-    fn read(&'static self, _blocking: bool, data: &mut [u8]) -> Result<usize, Error> {
-        self.line_read(data)
-    }
-
-    fn write(&self, _blocking: bool, data: &[u8]) -> Result<usize, Error> {
-        self.line_write(data)
-    }
-
-    fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> {
-        match req {
-            &mut DeviceRequest::SetTerminalGroup(id) => {
-                self.set_signal_group(id as _);
-                Ok(())
-            }
-            _ => Err(Error::InvalidArgument),
-        }
-    }
-}
-
-impl TtyDevice<16> for Serial {
-    fn ring(&self) -> &CharRing<16> {
-        &self.ring
-    }
-}
-
-impl InterruptSource for Serial {
-    unsafe fn init_irq(&'static self) -> Result<(), Error> {
-        let intc = PLATFORM.interrupt_controller();
-
-        intc.register_handler(self.irq, self)?;
-        intc.enable_irq(self.irq)?;
-
-        Ok(())
-    }
-
-    fn handle_irq(&self) -> Result<bool, Error> {
-        let byte = self.inner.get().lock().regs.DLL.get();
-
-        self.recv_byte(byte as u8);
-
-        Ok(true)
-    }
-}
-
-impl DebugSink for Serial {
-    fn putc(&self, c: u8) -> Result<(), Error> {
-        self.send(c).ok();
-        Ok(())
-    }
-}
-
-impl SerialDevice for Serial {
-    fn send(&self, byte: u8) -> Result<(), Error> {
-        let inner = self.inner.get().lock();
-        if byte == b'\n' {
-            while inner.regs.USR.matches_all(USR::TFE::CLEAR) {
-                core::hint::spin_loop();
-            }
-            inner.regs.DLL.set(b'\r' as u32);
-        }
-        while inner.regs.USR.matches_all(USR::TFE::CLEAR) {
-            core::hint::spin_loop();
-        }
-        inner.regs.DLL.set(byte as u32);
-        Ok(())
-    }
-
-    fn receive(&self, _blocking: bool) -> Result<u8, Error> {
-        todo!()
-    }
-}
-
-impl InitializableDevice for Serial {
-    type Options = ();
-
-    unsafe fn init(&self, _opts: Self::Options) -> Result<(), Error> {
-        let regs = DeviceMemoryIo::<Regs>::map("h6-uart", self.base)?;
-        self.inner.init(IrqSafeSpinlock::new(Inner { regs }));
-        Ok(())
-    }
-}
-
-impl Device for Serial {
-    fn name(&self) -> &'static str {
-        "Allwinner H6 UART"
-    }
-}
-
-impl Serial {
-    /// Constructs an instance of the device at `base`.
-    ///
-    /// # Safety
-    ///
-    /// The caller must ensure the address is valid.
-    pub const unsafe fn new(base: usize, irq: IrqNumber) -> Self {
-        Self {
-            inner: OneTimeInit::new(),
-            ring: CharRing::new(),
-
-            base,
-            irq,
-        }
-    }
-}

src/arch/aarch64/smp.rs

@@ -5,17 +5,18 @@ use abi::error::Error;
 use device_api::CpuBringupDevice;
 use fdt_rs::prelude::PropReader;
 
+use crate::arch::Architecture;
+use crate::mem::address::IntoRaw;
+use crate::mem::KERNEL_VIRT_OFFSET;
 use crate::{
-    absolute_address,
+    arch::{ArchitectureImpl, ARCHITECTURE},
-    arch::ARCHITECTURE,
+    mem::phys,
-    mem::{
-        phys::{self, PageUsage},
-        ConvertAddress,
-    },
 };
 
 use crate::device::devtree::{self, DevTreeIndexNodePropGet, DeviceTree};
 
+use super::{BootStack, BOOT_STACK_SIZE};
+
 #[derive(Debug)]
 enum CpuEnableMethod {
     Psci,
@@ -74,6 +75,12 @@ impl CpuEnableMethod {
 /// Number of online CPUs, initially set to 1 (BSP processor is up)
 pub static CPU_COUNT: AtomicUsize = AtomicUsize::new(1);
 
+// TODO can be made smaller
+#[link_section = ".bss"]
+static AP_TRAMPOLINE_STACK: BootStack = BootStack {
+    data: [0; BOOT_STACK_SIZE],
+};
+
 /// Starts application processors using the method specified in the device tree.
 ///
 /// TODO: currently does not handle systems where APs are already started before entry.
@@ -87,6 +94,9 @@ pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> {
         fn __aarch64_ap_entry();
     }
 
+    // Safety: safe, the stack is inside the kernel
+    let sp = AP_TRAMPOLINE_STACK.data.as_ptr() as usize - KERNEL_VIRT_OFFSET + BOOT_STACK_SIZE;
+
     for cpu in enumerate_cpus(dt).filter(|cpu| cpu.id != 0) {
         debugln!(
             "cpu{}: enable-method={:?}, compatible={:?}",
@@ -95,19 +105,20 @@ pub unsafe fn start_ap_cores(dt: &DeviceTree) -> Result<(), Error> {
             cpu.compatible
         );
 
-        const AP_STACK_PAGES: usize = 4;
+        // const AP_STACK_PAGES: usize = 4;
-        let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES, PageUsage::Used)?;
+        // let stack_pages = phys::alloc_pages_contiguous(AP_STACK_PAGES)?;
-        debugln!(
+        // debugln!(
-            "cpu{} stack: {:#x}..{:#x}",
+        //     "cpu{} stack: {:#x}..{:#x}",
-            cpu.id,
+        //     cpu.id,
-            stack_pages,
+        //     stack_pages,
-            stack_pages + AP_STACK_PAGES * 0x1000
+        //     stack_pages.add(AP_STACK_PAGES * 0x1000)
-        );
+        // );
         // Wait for the CPU to come up
         let old_count = CPU_COUNT.load(Ordering::Acquire);
 
-        let ip = absolute_address!(__aarch64_ap_entry).physicalize();
+        // Safety: safe, the function is inside the kernel
-        let sp = stack_pages + AP_STACK_PAGES * 0x1000;
+        let ip = __aarch64_ap_entry as usize - KERNEL_VIRT_OFFSET;
+        // let sp = stack_pages.add(AP_STACK_PAGES * 0x1000);
         if let Err(error) = cpu.enable_method.start_cpu(cpu.id as usize, ip, sp) {
             errorln!("Couldn't start cpu{} up: {:?}", cpu.id, error);
             continue;

src/arch/aarch64/table.rs

@@ -1,553 +0,0 @@
-//! AArch64 virtual memory management facilities
-use core::{
-    marker::PhantomData,
-    ops::{Index, IndexMut},
-    sync::atomic::{AtomicU8, Ordering},
-};
-
-use aarch64_cpu::registers::DAIF;
-use abi::error::Error;
-use bitflags::bitflags;
-use tock_registers::interfaces::Readable;
-
-use crate::mem::{
-    phys::{self, PageUsage},
-    table::{
-        EntryLevel, MapAttributes, NextPageTable, NonTerminalEntryLevel, VirtualMemoryManager,
-    },
-    ConvertAddress, KERNEL_VIRT_OFFSET,
-};
-
-/// TODO
-#[derive(Clone)]
-#[repr(C)]
-pub struct AddressSpace {
-    l1: *mut PageTable<L1>,
-    asid: u8,
-}
-
-/// Page table representing a single level of address translation
-#[derive(Clone)]
-#[repr(C, align(0x1000))]
-pub struct PageTable<L: EntryLevel> {
-    data: [PageEntry<L>; 512],
-}
-
-/// Translation level 1: Entry is 1GiB page/table
-#[derive(Clone, Copy)]
-pub struct L1;
-/// Translation level 2: Entry is 2MiB page/table
-#[derive(Clone, Copy)]
-pub struct L2;
-/// Translation level 3: Entry is 4KiB page
-#[derive(Clone, Copy)]
-pub struct L3;
-
-impl NonTerminalEntryLevel for L1 {
-    type NextLevel = L2;
-}
-impl NonTerminalEntryLevel for L2 {
-    type NextLevel = L3;
-}
-
-bitflags! {
-    /// Describes how each page is mapped: access, presence, type of the mapping.
-    #[derive(Clone, Copy)]
-    pub struct PageAttributes: u64 {
-        /// When set, the mapping is considered valid and assumed to point to a page/table
-        const PRESENT = 1 << 0;
-
-        /// For L1/L2 mappings, indicates that the mapping points to the next-level translation
-        /// table
-        const TABLE = 1 << 1;
-        /// (Must be set) For L3 mappings, indicates that the mapping points to a page
-        const PAGE = 1 << 1;
-        /// For L1/L2 mappings, indicates that the mapping points to a page of given level's size
-        const BLOCK = 0 << 1;
-
-        /// (Must be set) For page/block mappings, indicates to the hardware that the page is
-        /// accessed
-        const ACCESS = 1 << 10;
-
-        /// For page/block mappings, allows both user and kernel code to read/write to the page
-        const AP_BOTH_READWRITE = 1 << 6;
-        /// For page/block mappings, only allows read access for EL0/EL1
-        const AP_BOTH_READONLY = 3 << 6;
-
-        /// Indicates outer shareability domain
-        const SH_OUTER = 2 << 8;
-        /// Indicates inner shareability domain
-        const SH_INNER = 3 << 8;
-
-        /// For page/block mappings, indicates regular physical memory
-        const PAGE_ATTR_NORMAL = 0 << 2;
-        /// For page/block mappings, indicates device memory
-        const PAGE_ATTR_DEVICE = 1 << 2;
-
-        /// Indicates the mapping is unique to a specific ASID (important for proper TLB
-        /// maintenance)
-        const NON_GLOBAL = 1 << 11;
-    }
-}
-
-impl const EntryLevel for L1 {
-    fn index(addr: usize) -> usize {
-        (addr >> 30) & 0x1FF
-    }
-
-    fn page_offset(addr: usize) -> usize {
-        addr & 0x3FFFFFFF
-    }
-}
-impl const EntryLevel for L2 {
-    fn index(addr: usize) -> usize {
-        (addr >> 21) & 0x1FF
-    }
-
-    fn page_offset(addr: usize) -> usize {
-        addr & 0x1FFFFF
-    }
-}
-impl const EntryLevel for L3 {
-    fn index(addr: usize) -> usize {
-        (addr >> 12) & 0x1FF
-    }
-
-    fn page_offset(addr: usize) -> usize {
-        addr & 0xFFF
-    }
-}
-
-/// Represents a single entry in a translation table
-#[derive(Clone, Copy)]
-#[repr(transparent)]
-pub struct PageEntry<L>(u64, PhantomData<L>);
-
-/// Fixed-layout kernel-space address mapping tables
-pub struct FixedTables {
-    l1: PageTable<L1>,
-    device_l2: PageTable<L2>,
-    device_l3: PageTable<L3>,
-
-    device_l3i: usize,
-}
-
-impl PageEntry<L3> {
-    /// Creates a 4KiB normal memory page mapping
-    pub fn normal_page(phys: usize, attrs: PageAttributes) -> Self {
-        Self(
-            (phys as u64)
-                | (PageAttributes::PAGE
-                    | PageAttributes::PRESENT
-                    | PageAttributes::ACCESS
-                    | PageAttributes::SH_INNER
-                    | PageAttributes::PAGE_ATTR_NORMAL
-                    | attrs)
-                    .bits(),
-            PhantomData,
-        )
-    }
-
-    /// Creates a 4KiB device memory page mapping
-    pub fn device_page(phys: usize, attrs: PageAttributes) -> Self {
-        Self(
-            (phys as u64)
-                | (PageAttributes::PAGE
-                    | PageAttributes::PRESENT
-                    | PageAttributes::ACCESS
-                    | PageAttributes::SH_OUTER
-                    | PageAttributes::PAGE_ATTR_DEVICE
-                    | attrs)
-                    .bits(),
-            PhantomData,
-        )
-    }
-
-    /// Returns the physical address of the page this entry refers to, returning None if it does
-    /// not
-    pub fn as_page(self) -> Option<usize> {
-        let mask = (PageAttributes::PRESENT | PageAttributes::PAGE).bits();
-
-        if self.0 & mask == mask {
-            Some((self.0 & !0xFFF) as usize)
-        } else {
-            None
-        }
-    }
-}
-
-impl<T: NonTerminalEntryLevel> PageEntry<T> {
-    /// Creates a 2MiB page mapping
-    pub fn normal_block(phys: usize, attrs: PageAttributes) -> Self {
-        Self(
-            (phys as u64)
-                | (PageAttributes::BLOCK
-                    | PageAttributes::PRESENT
-                    | PageAttributes::ACCESS
-                    | PageAttributes::SH_INNER
-                    | PageAttributes::PAGE_ATTR_NORMAL
-                    | attrs)
-                    .bits(),
-            PhantomData,
-        )
-    }
-
-    /// Creates a mapping pointing to the next-level translation table
-    pub fn table(phys: usize, attrs: PageAttributes) -> Self {
-        Self(
-            (phys as u64) | (PageAttributes::TABLE | PageAttributes::PRESENT | attrs).bits(),
-            PhantomData,
-        )
-    }
-
-    /// Returns the physical address of the table this entry refers to, returning None if it
-    /// does not
-    pub fn as_table(self) -> Option<usize> {
-        if self.0 & (PageAttributes::TABLE | PageAttributes::PRESENT).bits()
-            == (PageAttributes::TABLE | PageAttributes::PRESENT).bits()
-        {
-            Some((self.0 & !0xFFF) as usize)
-        } else {
-            None
-        }
-    }
-}
-
-impl<L: EntryLevel> PageEntry<L> {
-    /// Represents an absent/invalid mapping in the table
-    pub const INVALID: Self = Self(0, PhantomData);
-
-    /// Converts a raw mapping value into this wrapper type
-    ///
-    /// # Safety
-    ///
-    /// The caller is responsible for making sure that `raw` is a valid mapping value for the
-    /// current translation level.
-    pub unsafe fn from_raw(raw: u64) -> Self {
-        Self(raw, PhantomData)
-    }
-
-    /// Returns `true` if the entry refers to some table/block/page
-    pub fn is_present(&self) -> bool {
-        self.0 & PageAttributes::PRESENT.bits() != 0
-    }
-}
-
-impl<L: NonTerminalEntryLevel> NextPageTable for PageTable<L> {
-    type NextLevel = PageTable<L::NextLevel>;
-
-    fn get_mut(&mut self, index: usize) -> Option<&'static mut Self::NextLevel> {
-        let entry = self[index];
-
-        entry
-            .as_table()
-            .map(|addr| unsafe { &mut *(addr.virtualize() as *mut Self::NextLevel) })
-    }
-
-    fn get_mut_or_alloc(&mut self, index: usize) -> Result<&'static mut Self::NextLevel, Error> {
-        let entry = self[index];
-
-        if let Some(table) = entry.as_table() {
-            Ok(unsafe { &mut *(table.virtualize() as *mut Self::NextLevel) })
-        } else {
-            let table = PageTable::new_zeroed()?;
-            self[index] = PageEntry::<L>::table(table.physical_address(), PageAttributes::empty());
-            Ok(table)
-        }
-    }
-}
-
-impl<L: EntryLevel> PageTable<L> {
-    /// Constructs a page table with all entries marked as invalid
-    pub const fn zeroed() -> Self {
-        Self {
-            data: [PageEntry::INVALID; 512],
-        }
-    }
-
-    /// Allocates a new page table, filling it with non-preset entries
-    pub fn new_zeroed() -> Result<&'static mut Self, Error> {
-        let page = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() };
-        let table = unsafe { &mut *(page as *mut Self) };
-        for i in 0..512 {
-            table[i] = PageEntry::INVALID;
-        }
-        Ok(table)
-    }
-
-    /// Returns a physical address pointing to this page table
-    pub fn physical_address(&self) -> usize {
-        // &self may already by a physical address
-        let addr = self.data.as_ptr() as usize;
-        if addr < KERNEL_VIRT_OFFSET {
-            addr
-        } else {
-            unsafe { addr.physicalize() }
-        }
-    }
-}
-
-impl<L: EntryLevel> Index<usize> for PageTable<L> {
-    type Output = PageEntry<L>;
-
-    fn index(&self, index: usize) -> &Self::Output {
-        &self.data[index]
-    }
-}
-
-impl<L: EntryLevel> IndexMut<usize> for PageTable<L> {
-    fn index_mut(&mut self, index: usize) -> &mut Self::Output {
-        &mut self.data[index]
-    }
-}
-
-impl FixedTables {
-    /// Constructs an empty table group
-    pub const fn zeroed() -> Self {
-        Self {
-            l1: PageTable::zeroed(),
-            device_l2: PageTable::zeroed(),
-            device_l3: PageTable::zeroed(),
-
-            device_l3i: 0,
-        }
-    }
-
-    /// Maps a physical memory region as device memory and returns its allocated base address
-    pub fn map_device_pages(&mut self, phys: usize, count: usize) -> Result<usize, Error> {
-        if count > 512 * 512 {
-            panic!("Unsupported device memory mapping size");
-        } else if count > 512 {
-            // 2MiB mappings
-            todo!();
-        } else {
-            // 4KiB mappings
-            if self.device_l3i + count > 512 {
-                return Err(Error::OutOfMemory);
-            }
-
-            let virt = DEVICE_VIRT_OFFSET + (self.device_l3i << 12);
-            for i in 0..count {
-                self.device_l3[self.device_l3i + i] =
-                    PageEntry::device_page(phys + i * 0x1000, PageAttributes::empty());
-            }
-            self.device_l3i += count;
-
-            Ok(virt)
-        }
-    }
-}
-
-impl From<MapAttributes> for PageAttributes {
-    fn from(value: MapAttributes) -> Self {
-        let mut res = Self::empty();
-        if value.contains(MapAttributes::USER_WRITE) {
-            res |= PageAttributes::AP_BOTH_READWRITE;
-        } else {
-            res |= PageAttributes::AP_BOTH_READONLY;
-        }
-        if value.contains(MapAttributes::NON_GLOBAL) {
-            res |= PageAttributes::NON_GLOBAL;
-        }
-        res
-    }
-}
-
-impl VirtualMemoryManager for AddressSpace {
-    fn allocate(
-        &self,
-        hint: Option<usize>,
-        len: usize,
-        attrs: MapAttributes,
-    ) -> Result<usize, Error> {
-        assert_eq!(DAIF.read(DAIF::I), 1);
-
-        if hint.is_some() {
-            todo!();
-        }
-
-        const TRY_ALLOC_START: usize = 0x100000000;
-        const TRY_ALLOC_END: usize = 0xF00000000;
-
-        'l0: for base in (TRY_ALLOC_START..TRY_ALLOC_END - len * 0x1000).step_by(0x1000) {
-            for i in 0..len {
-                if self.translate(base + i * 0x1000).is_some() {
-                    continue 'l0;
-                }
-            }
-
-            for i in 0..len {
-                let page = phys::alloc_page(PageUsage::Used)?;
-                self.map_page(base + i * 0x1000, page, attrs)?;
-            }
-
-            return Ok(base);
-        }
-
-        Err(Error::OutOfMemory)
-    }
-
-    fn deallocate(&self, addr: usize, len: usize) -> Result<(), Error> {
-        assert_eq!(DAIF.read(DAIF::I), 1);
-
-        for page in (addr..addr + len).step_by(0x1000) {
-            let Some(phys) = self.translate(page) else {
-                todo!(
-                    "Tried to deallocate address not present in the table: {:#x}",
-                    addr
-                );
-            };
-
-            self.write_entry(page, PageEntry::INVALID, true)?;
-            unsafe {
-                phys::free_page(phys);
-            }
-        }
-
-        Ok(())
-    }
-
-    fn map_page(&self, virt: usize, phys: usize, attrs: MapAttributes) -> Result<(), Error> {
-        self.write_entry(virt, PageEntry::normal_page(phys, attrs.into()), true)
-    }
-}
-
-impl AddressSpace {
-    /// Allocates an empty address space with all entries marked as non-present
-    pub fn new_empty() -> Result<Self, Error> {
-        static LAST_ASID: AtomicU8 = AtomicU8::new(1);
-
-        let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel);
-
-        let l1 = unsafe { phys::alloc_page(PageUsage::Used)?.virtualize() as *mut PageTable<L1> };
-
-        for i in 0..512 {
-            unsafe {
-                (*l1)[i] = PageEntry::INVALID;
-            }
-        }
-
-        Ok(Self { l1, asid })
-    }
-
-    /// Interprets a physical address as an address space structure pointer.
-    ///
-    /// # Safety
-    ///
-    /// Unsafe: accepts arbitrary addresses and ignores ASIDs.
-    pub unsafe fn from_phys_raw(value: usize) -> Self {
-        let value = value.virtualize();
-        Self {
-            l1: value as *mut PageTable<L1>,
-            asid: 0,
-        }
-    }
-
-    /// Iterates through all the translation levels for a given address, invoking the callback
-    /// function on each of them
-    pub fn walk<F: Fn(usize, usize) -> bool>(&self, vaddr: usize, f: F) {
-        let l1i = L1::index(vaddr);
-        let l2i = L2::index(vaddr);
-        let l3i = L3::index(vaddr);
-
-        let l1 = unsafe { self.as_mut() };
-        let l1e = l1[l1i];
-
-        let Some(l2) = l1.get_mut(l1i) else {
-            f(1, 0);
-            return;
-        };
-        if !f(1, l1e.0 as _) {
-            return;
-        }
-        let l2e = l2[l2i];
-
-        let Some(l3) = l2.get_mut(l2i) else {
-            f(2, 0);
-            return;
-        };
-        if !f(2, l2e.0 as _) {
-            return;
-        }
-
-        let l3e = l3[l3i];
-        if !f(3, l3e.0 as _) {
-            return;
-        }
-    }
-
-    unsafe fn as_mut(&self) -> &'static mut PageTable<L1> {
-        self.l1.as_mut().unwrap()
-    }
-
-    // TODO return page size and attributes
-    /// Returns the physical address to which the `virt` address is mapped
-    pub fn translate(&self, virt: usize) -> Option<usize> {
-        let l1i = L1::index(virt);
-        let l2i = L2::index(virt);
-        let l3i = L3::index(virt);
-
-        let l2 = unsafe { self.as_mut().get_mut(l1i) }?;
-        let l3 = l2.get_mut(l2i)?;
-
-        l3[l3i].as_page()
-    }
-
-    // Write a single 4KiB entry
-    fn write_entry(&self, virt: usize, entry: PageEntry<L3>, overwrite: bool) -> Result<(), Error> {
-        let l1i = L1::index(virt);
-        let l2i = L2::index(virt);
-        let l3i = L3::index(virt);
-
-        let l2 = unsafe { self.as_mut().get_mut_or_alloc(l1i) }?;
-        let l3 = l2.get_mut_or_alloc(l2i)?;
-
-        if l3[l3i].is_present() && !overwrite {
-            todo!()
-        }
-        l3[l3i] = entry;
-
-        tlb_flush_vaae1(virt);
-
-        Ok(())
-    }
-
-    /// Returns the physical address of the address space (to be used in a TTBRn_ELx)
-    pub fn physical_address(&self) -> usize {
-        unsafe { (self.l1 as usize).physicalize() | ((self.asid as usize) << 48) }
-    }
-}
-
-/// Flush a virtual address from EL1/EL0 TLB for all ASIDs
-pub fn tlb_flush_vaae1(mut page: usize) {
-    page >>= 12;
-    unsafe {
-        core::arch::asm!("tlbi vaae1, {page}", page = in(reg) page);
-    }
-}
-
-/// Initializes mappings for the kernel and device memory tables.
-///
-/// # Safety
-///
-/// Only allowed to be called once during lower-half part of the initialization process.
-pub unsafe fn init_fixed_tables() {
-    // Map first 256GiB
-    for i in 0..256 {
-        KERNEL_TABLES.l1[i] = PageEntry::<L1>::normal_block(i << 30, PageAttributes::empty());
-    }
-
-    KERNEL_TABLES.l1[256] = PageEntry::<L1>::table(
-        KERNEL_TABLES.device_l2.physical_address(),
-        PageAttributes::empty(),
-    );
-    KERNEL_TABLES.device_l2[0] = PageEntry::<L2>::table(
-        KERNEL_TABLES.device_l3.physical_address(),
-        PageAttributes::empty(),
-    );
-}
-
-/// Offset applied to device virtual memory mappings
-pub const DEVICE_VIRT_OFFSET: usize = KERNEL_VIRT_OFFSET + (256 << 30);
-/// Global kernel address space translation tables
-pub static mut KERNEL_TABLES: FixedTables = FixedTables::zeroed();

src/arch/aarch64/timer.rs

@@ -11,8 +11,7 @@ use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
 use crate::{
     arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE},
     device_tree_driver,
-    proc::wait,
+    task::runtime,
-    task::tasklet,
 };
 
 use super::cpu::Cpu;
@@ -30,8 +29,7 @@ impl InterruptHandler for ArmTimer {
         CNTP_TVAL_EL0.set(TICK_INTERVAL);
         let now = self.monotonic_timestamp().unwrap();
 
-        wait::tick(now);
+        runtime::tick(now);
-        tasklet::tick(now);
 
         unsafe {
             Cpu::local().queue().yield_cpu();

src/arch/mod.rs

@@ -20,7 +20,6 @@ macro_rules! absolute_address {
 }
 
 use cfg_if::cfg_if;
-// pub use aarch64::{AArch64 as ArchitectureImpl, ARCHITECTURE};
 use device_api::{
     interrupt::{ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController},
     timer::MonotonicTimestampProviderDevice,

src/arch/x86_64/context.rs

@@ -65,11 +65,11 @@ impl StackBuilder {
         self.sp
     }
 
-    fn init_common(&mut self, entry: usize, cr3: PhysicalAddress) {
+    fn init_common(&mut self, entry: usize, cr3: u64) {
         self.push(entry); // address for ret
 
         // End of common context
-        self.push(cr3.into_raw()); // %cr3
+        self.push(cr3 as _); // %cr3
 
         self.push(0); // %rbp
         self.push(0); // %fs (TODO)
@@ -99,7 +99,7 @@ impl TaskContextImpl for TaskContext {
         stack.push(arg);
 
         stack.init_common(__x86_64_task_enter_kernel as _, unsafe {
-            KERNEL_TABLES.as_physical_address()
+            KERNEL_TABLES.as_physical_address().into_raw()
         });
 
         let sp = stack.build();
@@ -114,12 +114,7 @@ impl TaskContextImpl for TaskContext {
         })
     }
 
-    fn user(
+    fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result<Self, Error> {
-        entry: usize,
-        arg: usize,
-        cr3: PhysicalAddress,
-        user_stack_sp: usize,
-    ) -> Result<Self, Error> {
         const USER_TASK_PAGES: usize = 8;
 
         let stack_base = phys::alloc_pages_contiguous(USER_TASK_PAGES)?.virtualize_raw();

src/arch/x86_64/mem/process.rs

@@ -3,7 +3,7 @@ use yggdrasil_abi::error::Error;
 use crate::{
     arch::x86_64::intrinsics,
     mem::{
-        address::AsPhysicalAddress,
+        address::{AsPhysicalAddress, IntoRaw},
         phys,
         pointer::PhysicalRefMut,
         process::ProcessAddressSpaceManager,
@@ -61,6 +61,11 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl {
         self.read_l3_entry(address)
             .ok_or(Error::InvalidMemoryOperation)
     }
+
+    fn as_address_with_asid(&self) -> u64 {
+        // TODO x86-64 PCID/ASID?
+        unsafe { self.l0.as_physical_address().into_raw() }
+    }
 }
 
 impl ProcessAddressSpaceImpl {
@@ -128,9 +133,3 @@ impl ProcessAddressSpaceImpl {
         Some((page, l3[l3i].attributes().into()))
     }
 }
-
-impl AsPhysicalAddress for ProcessAddressSpaceImpl {
-    unsafe fn as_physical_address(&self) -> PhysicalAddress {
-        self.l0.as_physical_address()
-    }
-}

src/arch/x86_64/mem/table.rs

@@ -67,51 +67,19 @@ impl NonTerminalEntryLevel for L2 {
 }
 
 impl const EntryLevel for L0 {
-    const SIZE: usize = 1 << 39;
+    const SHIFT: usize = 39;
-
-    fn index(addr: usize) -> usize {
-        (addr >> 39) & 0x1FF
-    }
-
-    fn page_offset(_addr: usize) -> usize {
-        todo!()
-    }
 }
 
 impl const EntryLevel for L1 {
-    const SIZE: usize = 1 << 30;
+    const SHIFT: usize = 30;
-
-    fn index(addr: usize) -> usize {
-        (addr >> 30) & 0x1FF
-    }
-
-    fn page_offset(addr: usize) -> usize {
-        addr & 0x3FFFFFFF
-    }
 }
 
 impl const EntryLevel for L2 {
-    const SIZE: usize = 1 << 21;
+    const SHIFT: usize = 21;
-
-    fn index(addr: usize) -> usize {
-        (addr >> 21) & 0x1FF
-    }
-
-    fn page_offset(addr: usize) -> usize {
-        addr & 0x1FFFFF
-    }
 }
 
 impl const EntryLevel for L3 {
-    const SIZE: usize = 1 << 12;
+    const SHIFT: usize = 12;
-
-    fn index(addr: usize) -> usize {
-        (addr >> 12) & 0x1FF
-    }
-
-    fn page_offset(addr: usize) -> usize {
-        addr & 0xFFF
-    }
 }
 
 impl PageEntry<L3> {

src/device/devtree.rs

@@ -10,7 +10,10 @@ use fdt_rs::{
     prelude::PropReader,
 };
 
-use crate::{debug::LogLevel, mem::phys::PhysicalMemoryRegion};
+use crate::{
+    debug::LogLevel,
+    mem::{address::FromRaw, phys::PhysicalMemoryRegion, PhysicalAddress},
+};
 
 use super::register_device;
 
@@ -162,6 +165,7 @@ impl<'a> DeviceTree<'a> {
     ///
     /// The caller must ensure the validity of the address.
     pub unsafe fn from_addr(virt: usize) -> Self {
+        FDT_INDEX_BUFFER.0.fill(0);
         let tree = DevTree::from_raw_pointer(virt as _).unwrap();
         let index = DevTreeIndex::new(tree, &mut FDT_INDEX_BUFFER.0).unwrap();
         Self { tree, index }
@@ -198,6 +202,12 @@ impl<'a> DeviceTree<'a> {
     pub fn root(&self) -> DevTreeIndexNode {
         self.index.root()
     }
+
+    // Commonly used functions for convenience
+    pub fn chosen_stdout_path(&self) -> Option<&str> {
+        let chosen = self.node_by_path("/chosen")?;
+        chosen.prop("stdout-path")
+    }
 }
 
 impl<'a, 'i, 'dt> DevTreeIndexNodeExt for DevTreeIndexNode<'a, 'i, 'dt> {
@@ -314,10 +324,10 @@ impl Iterator for FdtMemoryRegionIter<'_> {
                     .cell2_array_item(0, self.address_cells, self.size_cells)
                     .unwrap();
 
-                break Some(PhysicalMemoryRegion {
+                let base = PhysicalAddress::from_raw(base);
-                    base: base as usize,
+                let size = size as usize;
-                    size: size as usize,
+
-                });
+                break Some(PhysicalMemoryRegion { base, size });
             }
         }
     }

src/device/mod.rs

@@ -7,9 +7,12 @@ use crate::sync::{IrqSafeSpinlock, IrqSafeSpinlockGuard};
 #[cfg(target_arch = "aarch64")]
 pub mod devtree;
 
+// TODO bus device support on aarch64
+#[cfg(not(target_arch = "aarch64"))]
 pub mod bus;
+
 pub mod display;
-// pub mod power;
+pub mod power;
 pub mod serial;
 pub mod timer;
 pub mod tty;

src/device/power/mod.rs

@@ -5,6 +5,6 @@ use cfg_if::cfg_if;
 cfg_if! {
     if #[cfg(target_arch = "aarch64")] {
         pub mod arm_psci;
-        pub mod sunxi_rwdog;
+        // pub mod sunxi_rwdog;
     }
 }

src/device/serial/mod.rs

@@ -5,6 +5,6 @@ use cfg_if::cfg_if;
 cfg_if! {
     if #[cfg(target_arch = "aarch64")] {
         pub mod pl011;
-        pub mod sunxi_uart;
+        // pub mod sunxi_uart;
     }
 }

src/device/serial/pl011.rs

@@ -12,14 +12,15 @@ use vfs::CharDevice;
 
 use crate::{
     arch::{aarch64::IrqNumber, Architecture, ARCHITECTURE},
+    block,
     debug::{self, DebugSink, LogLevel},
     device::{
         devtree::{self, DevTreeIndexPropExt},
-        tty::{CharRing, TtyDevice},
+        tty::{TtyContext, TtyDevice},
     },
     device_tree_driver,
     fs::devfs::{self, CharDeviceType},
-    mem::device::DeviceMemoryIo,
+    mem::{address::FromRaw, device::DeviceMemoryIo, PhysicalAddress},
     sync::IrqSafeSpinlock,
 };
 
@@ -62,15 +63,15 @@ register_structs! {
 }
 
 struct Pl011Inner {
-    regs: DeviceMemoryIo<Regs>,
+    regs: DeviceMemoryIo<'static, Regs>,
 }
 
 /// PL011 device instance
 pub struct Pl011 {
     inner: OneTimeInit<IrqSafeSpinlock<Pl011Inner>>,
-    base: usize,
+    base: PhysicalAddress,
     irq: IrqNumber,
-    ring: CharRing<16>,
+    context: TtyContext,
 }
 
 impl Pl011Inner {
@@ -101,9 +102,9 @@ impl DebugSink for Pl011 {
     }
 }
 
-impl TtyDevice<16> for Pl011 {
+impl TtyDevice for Pl011 {
-    fn ring(&self) -> &CharRing<16> {
+    fn context(&self) -> &TtyContext {
-        &self.ring
+        &self.context
     }
 }
 
@@ -115,7 +116,12 @@ impl CharDevice for Pl011 {
 
     fn read(&'static self, blocking: bool, data: &mut [u8]) -> Result<usize, Error> {
         assert!(blocking);
-        self.line_read(data)
+        match block! {
+            self.line_read(data).await
+        } {
+            Ok(res) => res,
+            Err(err) => Err(err),
+        }
     }
 
     fn device_request(&self, req: &mut DeviceRequest) -> Result<(), Error> {
@@ -174,7 +180,7 @@ impl Device for Pl011 {
 
     unsafe fn init(&'static self) -> Result<(), Error> {
         let mut inner = Pl011Inner {
-            regs: DeviceMemoryIo::map("pl011 UART", self.base)?,
+            regs: DeviceMemoryIo::map(self.base)?,
         };
         inner.init();
 
@@ -207,8 +213,8 @@ device_tree_driver! {
             inner: OneTimeInit::new(),
             // TODO obtain IRQ from dt
             irq: IrqNumber::Shared(1),
-            ring: CharRing::new(),
+            context: TtyContext::new(),
-            base: base as usize
+            base: PhysicalAddress::from_raw(base)
         }))
     }
 }

src/mem/device.rs

@@ -67,9 +67,15 @@ impl DeviceMemoryMapping {
 }
 
 impl<'a, T: Sized> DeviceMemoryIo<'a, T> {
-    pub unsafe fn from_raw(raw: DeviceMemoryMapping) -> DeviceMemoryIo<'a, T> {
+    pub unsafe fn from_raw(
-        // TODO
+        inner: Arc<RawDeviceMemoryMapping>,
-        loop {}
+    ) -> Result<DeviceMemoryIo<'a, T>, Error> {
+        if size_of::<T>() > inner.page_size * inner.page_count {
+            todo!();
+        }
+        // TODO check align
+        let value = &*(inner.address as *const T);
+        Ok(DeviceMemoryIo { inner, value })
     }
 
     pub unsafe fn map(base: PhysicalAddress) -> Result<DeviceMemoryIo<'a, T>, Error> {

src/mem/phys/manager.rs

@@ -22,12 +22,14 @@ pub(super) const TRACKED_PAGE_LIMIT: usize = (BITMAP_PAGE_COUNT * 4096) * 8;
 pub struct PhysicalMemoryManager {
     bitmap: PhysicalRefMut<'static, [u64]>,
     last_free_bit: usize,
+    offset: usize,
     page_count: usize,
 }
 
 impl PhysicalMemoryManager {
     pub unsafe fn new(
         bitmap_phys_base: PhysicalAddress,
+        offset: usize,
         page_count: usize,
     ) -> PhysicalMemoryManager {
         let bitmap_len = (page_count + (BITMAP_WORD_SIZE - 1)) / BITMAP_WORD_SIZE;
@@ -38,6 +40,7 @@ impl PhysicalMemoryManager {
         Self {
             bitmap,
             page_count,
+            offset,
             last_free_bit: 0,
         }
     }
@@ -67,7 +70,7 @@ impl PhysicalMemoryManager {
             self.last_free_bit = i + 1;
             self.mark_alloc(i);
 
-            return Ok(PhysicalAddress::from_raw(i * 0x1000));
+            return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset));
         }
 
         if self.last_free_bit != 0 {
@@ -93,7 +96,7 @@ impl PhysicalMemoryManager {
             }
             self.last_free_bit = i + 512;
 
-            return Ok(PhysicalAddress::from_raw(i * 0x1000));
+            return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset));
         }
 
         if self.last_free_bit != 0 {
@@ -118,7 +121,7 @@ impl PhysicalMemoryManager {
             }
             self.last_free_bit = i + count;
 
-            return Ok(PhysicalAddress::from_raw(i * 0x1000));
+            return Ok(PhysicalAddress::from_raw(i * 0x1000 + self.offset));
         }
 
         if self.last_free_bit != 0 {
@@ -135,7 +138,9 @@ impl PhysicalMemoryManager {
     ///
     /// `addr` must be a page-aligned physical address previously allocated by this implementation.
     pub unsafe fn free_page(&mut self, page: PhysicalAddress) {
-        let index = IntoRaw::<usize>::into_raw(page) / 0x1000;
+        let page: usize = page.into_raw();
+        assert!(page >= self.offset);
+        let index = (page - self.offset) / 0x1000;
 
         assert!(self.is_alloc(index));
         self.mark_free(index);
@@ -147,7 +152,9 @@ impl PhysicalMemoryManager {
     ///
     /// Will panic if the address does not point to a valid, reserved (and unallocated) page.
     pub fn add_available_page(&mut self, page: PhysicalAddress) {
-        let index = IntoRaw::<usize>::into_raw(page) / 0x1000;
+        let page: usize = page.into_raw();
+        assert!(page >= self.offset);
+        let index = (page - self.offset) / 0x1000;
 
         assert!(self.is_alloc(index));
         self.mark_free(index);

src/mem/phys/mod.rs

@@ -5,7 +5,10 @@ use kernel_util::util::OneTimeInit;
 
 use crate::{
     arch::{Architecture, ARCHITECTURE},
-    mem::phys::reserved::is_reserved,
+    mem::{
+        address::IntoRaw,
+        phys::{self, reserved::is_reserved},
+    },
     sync::IrqSafeSpinlock,
 };
 
@@ -162,7 +165,12 @@ pub unsafe fn init_from_iter<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
         },
     );
 
-    let mut manager = PhysicalMemoryManager::new(page_bitmap_phys_base, total_count);
+    if IntoRaw::<u64>::into_raw(phys_start) & 0x1FFFFFF != 0 {
+        loop {}
+    }
+
+    let mut manager =
+        PhysicalMemoryManager::new(page_bitmap_phys_base, phys_start.into_raw(), total_count);
     let mut collected = 0;
     const MAX_MEMORY: usize = 16 * 1024;
 

src/mem/process.rs

@@ -2,20 +2,20 @@ use abi::error::Error;
 use cfg_if::cfg_if;
 use vmalloc::VirtualMemoryAllocator;
 
-use crate::{arch::x86_64::mem::table::L3, mem::phys, sync::IrqSafeSpinlock};
+use crate::{mem::phys, sync::IrqSafeSpinlock};
 
 use super::{address::AsPhysicalAddress, table::MapAttributes, PhysicalAddress};
 
 cfg_if! {
     if #[cfg(target_arch = "aarch64")] {
-        use crate::arch::aarch64::table::AddressSpace;
+        use crate::arch::aarch64::mem::process::ProcessAddressSpaceImpl;
     } else if #[cfg(target_arch = "x86_64")] {
         use crate::arch::x86_64::mem::process::ProcessAddressSpaceImpl;
     }
 }
 
 /// Interface for virtual memory address space management
-pub trait ProcessAddressSpaceManager: Sized + AsPhysicalAddress {
+pub trait ProcessAddressSpaceManager: Sized {
     const PAGE_SIZE: usize;
     const LOWER_LIMIT_PFN: usize;
     const UPPER_LIMIT_PFN: usize;
@@ -32,6 +32,8 @@ pub trait ProcessAddressSpaceManager: Sized + AsPhysicalAddress {
     unsafe fn unmap_page(&mut self, address: usize) -> Result<PhysicalAddress, Error>;
 
     fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>;
+
+    fn as_address_with_asid(&self) -> u64;
 }
 
 struct Inner {
@@ -186,7 +188,8 @@ impl ProcessAddressSpace {
         attributes: MapAttributes,
     ) -> Result<(), Error> {
         assert_eq!(address & (ProcessAddressSpaceImpl::PAGE_SIZE - 1), 0);
-        assert!(physical.is_aligned_for::<L3>());
+        // XXX
+        // assert!(physical.is_aligned_for::<L3>());
 
         self.inner
             .lock()
@@ -211,22 +214,7 @@ impl ProcessAddressSpace {
         )
     }
 
-    pub fn debug_dump(&self) {
+    pub fn as_address_with_asid(&self) -> u64 {
-        let lock = self.inner.lock();
+        self.inner.lock().table.as_address_with_asid()
-
-        debugln!("Address space @ {:#x}", unsafe {
-            lock.table.as_physical_address()
-        });
-        for (used, range) in lock.allocator.ranges() {
-            let start = range.start_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE;
-            let end = range.end_pfn() * ProcessAddressSpaceImpl::PAGE_SIZE;
-            debugln!("{:#x?}: {}", start..end, used);
-        }
-    }
-}
-
-impl AsPhysicalAddress for ProcessAddressSpace {
-    unsafe fn as_physical_address(&self) -> PhysicalAddress {
-        self.inner.lock().table.as_physical_address()
     }
 }

src/mem/table.rs

@@ -4,6 +4,7 @@ use core::ops::{Deref, DerefMut};
 use abi::error::Error;
 use bitflags::bitflags;
 
+// TODO EXECUTABLE
 bitflags! {
     /// Describes how a page translation mapping should behave
     #[derive(Clone, Copy)]
@@ -36,12 +37,29 @@ pub trait NextPageTable {
 /// Interface for a single level of address translation
 #[const_trait]
 pub trait EntryLevel: Copy {
-    const SIZE: usize;
+    const SHIFT: usize;
+    const SIZE: usize = 1 << Self::SHIFT;
 
     /// Returns the index into a page table for a given address
-    fn index(addr: usize) -> usize;
+    #[inline]
+    fn index(addr: usize) -> usize {
+        (addr >> Self::SHIFT) & 0x1FF
+    }
     /// Returns the offset of an address from the page start at current level
-    fn page_offset(addr: usize) -> usize;
+    #[inline]
+    fn page_offset(addr: usize) -> usize {
+        addr & (Self::SIZE - 1)
+    }
+
+    #[inline]
+    fn align_up(addr: usize) -> usize {
+        (addr + Self::SIZE - 1) & !(Self::SIZE - 1)
+    }
+
+    #[inline]
+    fn page_count(addr: usize) -> usize {
+        (addr + Self::SIZE - 1) / Self::SIZE
+    }
 }
 
 /// Tag trait to mark that the page table level may point to a next-level table

src/proc/elf.rs

@@ -98,8 +98,6 @@ pub fn load_elf_from_file(space: &ProcessAddressSpace, file: FileRef) -> Result<
 
     let elf = ElfStream::<AnyEndian, _>::open_stream(file).map_err(from_parse_error)?;
 
-    space.debug_dump();
-
     for phdr in elf.segments() {
         if phdr.p_type != PT_LOAD {
             continue;

src/proc/exec.rs

@@ -174,7 +174,7 @@ fn setup_binary<S: Into<String>>(
     let context = TaskContext::user(
         entry,
         virt_args_base,
-        unsafe { space.as_physical_address() },
+        unsafe { space.as_address_with_asid() },
         user_sp,
     )?;
 

src/task/context.rs

@@ -56,12 +56,7 @@ pub trait TaskContextImpl: Sized {
 
     /// Constructs a user thread context. The caller is responsible for allocating the userspace
     /// stack and setting up a valid address space for the context.
-    fn user(
+    fn user(entry: usize, arg: usize, cr3: u64, user_stack_sp: usize) -> Result<Self, Error>;
-        entry: usize,
-        arg: usize,
-        cr3: PhysicalAddress,
-        user_stack_sp: usize,
-    ) -> Result<Self, Error>;
 
     /// Performs an entry into a context.
     ///

tools/gentables/Cargo.toml

@@ -6,7 +6,7 @@ edition = "2021"
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
 
 [dependencies]
-memtables = { path = "../../lib/memtables" }
+memtables = { path = "../../lib/memtables", features = ["all"] }
 
 bytemuck = "1.14.0"
 elf = "0.7.2"

tools/gentables/src/aarch64.rs

@@ -0,0 +1,193 @@
+use core::fmt;
+use std::{
+    io::{Read, Seek},
+    mem::offset_of,
+};
+
+use bitflags::bitflags;
+use elf::{
+    abi::{PF_W, PF_X, PT_LOAD},
+    endian::AnyEndian,
+    ElfStream,
+};
+use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT};
+
+use crate::{GenData, GenError};
+
+bitflags! {
+    #[derive(Clone, Copy)]
+    struct PageFlags: u64 {
+        const PRESENT = 1 << 0;
+        const ACCESS = 1 << 10;
+        const SH_INNER = 3 << 8;
+        const PAGE_ATTR_NORMAL = 0 << 2;
+        const AP_BOTH_READONLY = 3 << 6;
+        const TABLE = 1 << 1;
+        const PAGE = 1 << 1;
+        const UXN = 1 << 54;
+        const PXN = 1 << 53;
+    }
+}
+
+impl PageFlags {
+    pub fn kernel_table() -> Self {
+        Self::PRESENT | Self::ACCESS | Self::SH_INNER | Self::PAGE_ATTR_NORMAL | Self::TABLE
+    }
+
+    pub fn kernel_page() -> Self {
+        Self::PRESENT | Self::ACCESS | Self::SH_INNER | Self::PAGE_ATTR_NORMAL | Self::PAGE
+    }
+}
+
+pub struct AArch64Builder<F: Read + Seek> {
+    elf: ElfStream<AnyEndian, F>,
+    data: GenData,
+    tables: FixedTables,
+
+    l1i: usize,
+    l2i_start: usize,
+    l2i_end: usize,
+}
+
+impl PageFlags {
+    fn from_elf(flags: u32) -> Self {
+        let mut out = Self::UXN | Self::PXN;
+        if flags & PF_X != 0 {
+            out.remove(Self::PXN);
+        }
+        if flags & PF_W == 0 {
+            out |= Self::AP_BOTH_READONLY;
+        }
+        out
+    }
+}
+
+impl fmt::Display for PageFlags {
+    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+        let x = if self.contains(Self::PXN) { '-' } else { 'x' };
+        let w = if self.contains(Self::AP_BOTH_READONLY) {
+            '-'
+        } else {
+            'w'
+        };
+        write!(f, "r{}{}", w, x)
+    }
+}
+
+const L1_SHIFT: u64 = 30;
+const L2_SHIFT: u64 = 21;
+const L3_SHIFT: u64 = 12;
+const L2_ENTRY_SIZE: u64 = 1 << L2_SHIFT;
+const L3_ENTRY_SIZE: u64 = 1 << L3_SHIFT;
+
+// TODO proper granularity
+impl<F: Read + Seek> AArch64Builder<F> {
+    pub fn new(elf: ElfStream<AnyEndian, F>, data: GenData) -> Result<Self, GenError> {
+        let l1i = (data.kernel_start >> L1_SHIFT) as usize & 0x1FF;
+        let l2i_start = (data.kernel_start >> L2_SHIFT) as usize & 0x1FF;
+        let l2i_end = ((data.kernel_end + L2_ENTRY_SIZE - 1) >> L2_SHIFT) as usize & 0x1FF;
+
+        if l2i_end - l2i_start > KERNEL_L3_COUNT {
+            todo!()
+        }
+
+        Ok(Self {
+            elf,
+            data,
+
+            tables: FixedTables::zeroed(),
+
+            l1i,
+            l2i_start,
+            l2i_end,
+        })
+    }
+
+    // TODO the build function is almost identical to x86-64 one, but with slight changes, so might
+    // wanna unify this later
+    pub fn build(mut self) -> Result<(FixedTables, u64), GenError> {
+        assert_eq!(offset_of!(FixedTables, l1), 0);
+
+        let l2_physical_address =
+            self.data.table_physical_address + offset_of!(FixedTables, l2) as u64;
+
+        // L1 -> L2
+        self.tables.l1.data[self.l1i] = l2_physical_address | PageFlags::kernel_table().bits();
+
+        // L2 -> L3s
+        for l2i in self.l2i_start..self.l2i_end {
+            let l3_table_index = l2i - self.l2i_start;
+            let l3_physical_address = self.data.table_physical_address
+                + (offset_of!(FixedTables, l3s) + 0x1000 * l3_table_index) as u64;
+
+            self.tables.l2.data[l2i] = l3_physical_address | PageFlags::kernel_table().bits();
+        }
+
+        for (i, segment) in self.elf.segments().into_iter().enumerate() {
+            if segment.p_type != PT_LOAD
+                || segment.p_vaddr != segment.p_paddr + self.data.kernel_virt_offset
+            {
+                continue;
+            }
+
+            let aligned_virt_start = segment.p_vaddr & !(L3_ENTRY_SIZE - 1);
+            let aligned_virt_end =
+                (segment.p_vaddr + segment.p_memsz + L3_ENTRY_SIZE - 1) & !(L3_ENTRY_SIZE - 1);
+            let aligned_phys_start = segment.p_paddr & !(L3_ENTRY_SIZE - 1);
+            let count = (aligned_virt_end - aligned_virt_start) / 0x1000;
+
+            let flags = PageFlags::from_elf(segment.p_flags);
+
+            println!(
+                "{}: {:#x?} -> {:#x} {}",
+                i,
+                aligned_virt_start..aligned_virt_end,
+                aligned_phys_start,
+                flags
+            );
+            Self::map_segment(
+                self.l2i_start,
+                &mut self.tables,
+                aligned_virt_start,
+                aligned_phys_start,
+                count as usize,
+                flags,
+            )?;
+        }
+
+        Ok((self.tables, self.data.table_offset))
+    }
+
+    fn map_segment(
+        start_l2i: usize,
+        tables: &mut FixedTables,
+        vaddr_start: u64,
+        paddr_start: u64,
+        count: usize,
+        flags: PageFlags,
+    ) -> Result<(), GenError> {
+        for index in 0..count {
+            let vaddr = vaddr_start + index as u64 * L3_ENTRY_SIZE;
+            let paddr = paddr_start + index as u64 * L3_ENTRY_SIZE;
+
+            let entry = paddr | (PageFlags::kernel_page() | flags).bits();
+
+            let l2i = (vaddr >> L2_SHIFT) as usize & 0x1FF - start_l2i;
+            let l3i = (vaddr >> L3_SHIFT) as usize & 0x1FF;
+
+            let l3 = &mut tables.l3s[l2i];
+
+            if l3.data[l3i] != 0 {
+                if l3.data[l3i] != entry {
+                    todo!();
+                } else {
+                    continue;
+                }
+            }
+
+            l3.data[l3i] = entry;
+        }
+
+        Ok(())
+    }
+}

tools/gentables/src/main.rs

@@ -10,15 +10,16 @@ use std::{
 
 use clap::Parser;
 use elf::{
-    abi::{EM_X86_64, PT_LOAD},
+    abi::{EM_AARCH64, EM_X86_64, PT_LOAD},
     endian::AnyEndian,
     ElfStream,
 };
-use memtables::FixedTables;
+use memtables::any::AnyTables;
 use thiserror::Error;
 
-use crate::x86_64::X8664Builder;
+use crate::{aarch64::AArch64Builder, x86_64::X8664Builder};
 
+mod aarch64;
 mod x86_64;
 
 #[derive(Error, Debug)]
@@ -120,7 +121,11 @@ fn find_tables<F: Read + Seek>(elf: &mut ElfStream<AnyEndian, F>) -> Result<(u64
     Err(GenError::MissingSection(".data.tables"))
 }
 
-fn build_tables<F: Read + Seek>(file: F) -> Result<(FixedTables, u64), GenError> {
+fn into_any<T: Into<AnyTables>, U>((l, r): (T, U)) -> (AnyTables, U) {
+    (l.into(), r)
+}
+
+fn build_tables<F: Read + Seek>(file: F) -> Result<(AnyTables, u64), GenError> {
     let mut elf = ElfStream::<AnyEndian, F>::open_stream(file)?;
 
     let kernel_virt_offset = kernel_virt_offset(&mut elf)?;
@@ -144,7 +149,20 @@ fn build_tables<F: Read + Seek>(file: F) -> Result<(FixedTables, u64), GenError>
                 table_physical_address,
             },
         )?
-        .build(),
+        .build()
+        .map(into_any),
+        EM_AARCH64 => AArch64Builder::new(
+            elf,
+            GenData {
+                kernel_virt_offset,
+                kernel_start,
+                kernel_end,
+                table_offset,
+                table_physical_address,
+            },
+        )?
+        .build()
+        .map(into_any),
         _ => todo!(),
     }
 }
@@ -152,11 +170,10 @@ fn build_tables<F: Read + Seek>(file: F) -> Result<(FixedTables, u64), GenError>
 fn write_tables<F: Write + Seek>(
     mut file: F,
     offset: u64,
-    tables: FixedTables,
+    tables: AnyTables,
 ) -> Result<(), GenError> {
-    let bytes = bytemuck::bytes_of(&tables);
     file.seek(SeekFrom::Start(offset))?;
-    file.write_all(bytes)?;
+    file.write_all(tables.as_bytes())?;
     Ok(())
 }
 

tools/gentables/src/x86_64.rs

@@ -11,7 +11,7 @@ use elf::{
     endian::AnyEndian,
     ElfStream,
 };
-use memtables::{FixedTables, KERNEL_L3_COUNT};
+use memtables::x86_64::{FixedTables, KERNEL_L3_COUNT};
 
 use crate::{GenData, GenError};