yggdrasil/src/arch/aarch64/mod.rs

//! AArch64 architecture and platforms implementation

pub mod mem;

use core::sync::atomic::Ordering;

use aarch64_cpu::registers::{CNTP_CTL_EL0, CNTP_TVAL_EL0, DAIF};
use abi::error::Error;
use device_api::{
    interrupt::{
        ExternalInterruptController, IpiDeliveryTarget, LocalInterruptController,
        MessageInterruptController,
    },
    timer::MonotonicTimestampProviderDevice,
    ResetDevice,
};
use device_tree::dt::{DevTreeIndexPropExt, DevTreeNodeInfo, DeviceTree, FdtMemoryRegionIter};
use git_version::git_version;
use libk::{
    mem::{
        address::{FromRaw, IntoRaw, PhysicalAddress},
        device::{DeviceMemoryAttributes, RawDeviceMemoryMapping},
        phys::PhysicalMemoryRegion,
        pointer::PhysicalRef,
        table::{EntryLevel, EntryLevelExt},
    },
    util::OneTimeInit,
};
use tock_registers::interfaces::{ReadWriteable, Readable, Writeable};
use ygg_driver_pci::PciBusManager;

use crate::{
    arch::aarch64::{
        cpu::Cpu,
        mem::table::{L2, L3},
    },
    debug,
    device::{self, power::arm_psci::Psci},
    fs::{Initrd, INITRD_DATA},
    mem::{
        heap,
        phys::{self, reserved::reserve_region},
    },
};

use self::mem::{table::L1, EarlyMapping};

use super::{Architecture, CpuMessage};

pub mod boot;
pub mod context;
pub mod cpu;
pub mod exception;
pub mod gic;
pub mod smp;
pub mod timer;

const BOOT_STACK_SIZE: usize = 4096 * 32;

#[derive(Clone, Copy)]
#[repr(C, align(0x20))]
struct BootStack {
    data: [u8; BOOT_STACK_SIZE],
}

/// AArch64 architecture implementation
pub struct AArch64 {
    dt: OneTimeInit<DeviceTree<'static>>,

    /// Optional instance of PSCI on this platform
    pub psci: OneTimeInit<&'static Psci>,
    reset: OneTimeInit<&'static dyn ResetDevice>,

    lintc: OneTimeInit<&'static dyn LocalInterruptController<IpiMessage = CpuMessage>>,
    xintc: OneTimeInit<&'static dyn ExternalInterruptController<IrqNumber = IrqNumber>>,
    msi_intc: OneTimeInit<&'static dyn MessageInterruptController>,

    mtimer: OneTimeInit<&'static dyn MonotonicTimestampProviderDevice>,

    initrd: OneTimeInit<PhysicalRef<'static, [u8]>>,
}

impl Architecture for AArch64 {
    const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000;

    type IrqNumber = IrqNumber;

    type L3 = L3;

    unsafe fn start_application_processors(&self) {
        let dt = self.dt.get();
        if let Err(error) = smp::start_ap_cores(dt) {
            errorln!("Could not initialize AP CPUs: {:?}", error);
        }
    }

    fn cpu_count() -> usize {
        smp::CPU_COUNT.load(Ordering::Acquire)
    }

    unsafe fn set_interrupt_mask(mask: bool) {
        if mask {
            DAIF.modify(DAIF::I::SET);
        } else {
            DAIF.modify(DAIF::I::CLEAR);
        }
    }

    fn interrupt_mask() -> bool {
        DAIF.read(DAIF::I) != 0
    }

    fn wait_for_interrupt() {
        aarch64_cpu::asm::wfi();
    }

    unsafe fn map_device_memory(
        &self,
        base: PhysicalAddress,
        size: usize,
        attrs: DeviceMemoryAttributes,
    ) -> Result<RawDeviceMemoryMapping, Error> {
        mem::map_device_memory(base, size, attrs)
    }

    unsafe fn unmap_device_memory(&self, map: &RawDeviceMemoryMapping) {
        mem::unmap_device_memory(map)
    }

    fn map_physical_memory<I: Iterator<Item = PhysicalMemoryRegion> + Clone>(
        &self,
        _it: I,
        _memory_start: PhysicalAddress,
        memory_end: PhysicalAddress,
    ) -> Result<(), Error> {
        let end_l1i = memory_end.page_align_up::<L1>().page_index::<L1>();
        if end_l1i > mem::RAM_MAPPING_L1_COUNT {
            todo!()
        }

        // Map 1GiB chunks
        for index in 0..end_l1i {
            unsafe {
                mem::map_ram_l1(index);
            }
        }

        mem::MEMORY_LIMIT.init(memory_end.into_raw());

        Ok(())
    }

    fn virtualize(address: u64) -> usize {
        let address = address as usize;
        if address < *mem::MEMORY_LIMIT.get() {
            address + mem::RAM_MAPPING_OFFSET
        } else {
            panic!("Invalid physical address: {:#x}", address);
        }
    }

    fn physicalize(address: usize) -> u64 {
        if address < mem::RAM_MAPPING_OFFSET
            || address - mem::RAM_MAPPING_OFFSET >= *mem::MEMORY_LIMIT.get()
        {
            panic!("Not a virtualized physical address: {:#x}", address);
        }

        (address - mem::RAM_MAPPING_OFFSET) as _
    }

    fn local_interrupt_controller(
        &'static self,
    ) -> &'static dyn LocalInterruptController<IpiMessage = super::CpuMessage> {
        *self.lintc.get()
    }

    fn external_interrupt_controller(
        &'static self,
    ) -> &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber> {
        *self.xintc.get()
    }

    fn message_interrupt_controller(&'static self) -> &'static dyn MessageInterruptController {
        *self.msi_intc.get()
    }

    fn register_local_interrupt_controller(
        &self,
        intc: &'static dyn LocalInterruptController<IpiMessage = super::CpuMessage>,
    ) -> Result<(), Error> {
        self.lintc.init(intc);
        Ok(())
    }

    fn register_external_interrupt_controller(
        &self,
        intc: &'static dyn ExternalInterruptController<IrqNumber = Self::IrqNumber>,
    ) -> Result<(), Error> {
        self.xintc.init(intc);
        Ok(())
    }

    fn register_message_interrupt_controller(
        &self,
        intc: &'static dyn MessageInterruptController,
    ) -> Result<(), Error> {
        self.msi_intc.init(intc);
        Ok(())
    }

    fn monotonic_timer(&'static self) -> &'static dyn MonotonicTimestampProviderDevice {
        *self.mtimer.get()
    }

    fn register_monotonic_timer(
        &self,
        timer: &'static dyn MonotonicTimestampProviderDevice,
    ) -> Result<(), Error> {
        self.mtimer.init(timer);
        Ok(())
    }

    unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: CpuMessage) -> Result<(), Error> {
        if let Some(local_intc) = self.lintc.try_get() {
            local_intc.send_ipi(target, msg)
        } else {
            Ok(())
        }
    }

    fn register_reset_device(&self, reset: &'static dyn ResetDevice) -> Result<(), Error> {
        self.reset.init(reset);
        Ok(())
    }

    unsafe fn reset(&self) -> ! {
        if let Some(reset) = self.reset.try_get() {
            reset.reset()
        } else {
            let psci = self.psci.get();
            psci.reset()
        }
    }
}

impl AArch64 {
    fn extract_initrd_from_dt(
        &self,
        dt: &DeviceTree,
    ) -> Option<(PhysicalAddress, PhysicalAddress)> {
        let chosen = dt.node_by_path("/chosen")?;
        let initrd_start = device_tree::find_prop(&chosen, "linux,initrd-start")?;
        let initrd_end = device_tree::find_prop(&chosen, "linux,initrd-end")?;

        let address_cells = dt.address_cells();

        let initrd_start = initrd_start.cell1_array_item(0, address_cells)?;
        let initrd_end = initrd_end.cell1_array_item(0, address_cells)?;

        let initrd_start = PhysicalAddress::from_raw(initrd_start);
        let initrd_end = PhysicalAddress::from_raw(initrd_end);

        Some((initrd_start, initrd_end))
    }

    unsafe fn init_memory_management(&'static self, dtb: PhysicalAddress) -> Result<(), Error> {
        // 16x2MiB
        const HEAP_PAGES: usize = 16;

        // Initialize the runtime mappings
        mem::init_fixed_tables();

        // Extract the size of the device tree
        let dtb_size = {
            let dtb_header = EarlyMapping::<u8>::map_slice(dtb, DeviceTree::MIN_HEADER_SIZE)?;
            DeviceTree::read_totalsize(dtb_header.as_ref()).unwrap()
        };

        reserve_region(
            "dtb",
            PhysicalMemoryRegion {
                base: dtb,
                size: (dtb_size + 0xFFF) & !0xFFF,
            },
        );

        let dtb_slice = EarlyMapping::<u8>::map_slice(dtb, dtb_size)?;

        let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize);

        // Setup initrd from the dt
        let initrd = self.extract_initrd_from_dt(&dt);

        if let Some((start, end)) = initrd {
            let aligned_start = start.page_align_down::<L3>();
            let aligned_end = end.page_align_up::<L3>();

            let size = aligned_end - aligned_start;
            reserve_region(
                "initrd",
                PhysicalMemoryRegion {
                    base: aligned_start,
                    size,
                },
            );
        }

        // Initialize the physical memory
        let regions = FdtMemoryRegionIter::new(&dt);

        phys::init_from_iter(regions)?;

        // Setup the heap
        for i in 0..HEAP_PAGES {
            let l2_page = phys::alloc_2m_page()?;
            mem::map_heap_l2(i, l2_page);
        }

        heap::init_heap(mem::HEAP_MAPPING_OFFSET, HEAP_PAGES * L2::SIZE);

        // EarlyMapping for DTB no longer needed, it lives in physical memory and can be obtained
        // through PhysicalRef
        let dtb_slice: PhysicalRef<'static, [u8]> = PhysicalRef::map_slice(dtb, dtb_size);
        let dt = DeviceTree::from_addr(dtb_slice.as_ptr() as usize);

        self.dt.init(dt);

        // Setup initrd
        if let Some((initrd_start, initrd_end)) = initrd {
            let aligned_start = initrd_start.page_align_down::<L3>();
            let aligned_end = initrd_end.page_align_up::<L3>();
            let len = initrd_end - initrd_start;

            let data = unsafe { PhysicalRef::map_slice(initrd_start, len) };
            self.initrd.init(data);

            INITRD_DATA.init(Initrd {
                phys_page_start: aligned_start,
                phys_page_len: aligned_end - aligned_start,
                data: self.initrd.get().as_ref(),
            });
        }

        Ok(())
    }

    unsafe fn init_platform(&self, is_bsp: bool) -> Result<(), Error> {
        Cpu::init_local();

        if is_bsp {
            ygg_driver_pci::register_vendor_driver(
                "Virtio PCI Network Device",
                0x1AF4,
                0x1000,
                ygg_driver_virtio_net::probe,
            );
            ygg_driver_pci::register_class_driver(
                "AHCI SATA Controller",
                0x01,
                Some(0x06),
                Some(0x01),
                ygg_driver_ahci::probe,
            );

            let dt = self.dt.get();

            let address_cells = dt.address_cells();
            let size_cells = dt.size_cells();

            // Setup /chosen.stdout-path to get early debug printing
            let chosen_stdout_path = dt.chosen_stdout_path();
            let chosen_stdout = chosen_stdout_path.and_then(|path| dt.node_by_path(path));

            if let Some(node) = chosen_stdout.clone() {
                let probe = DevTreeNodeInfo {
                    address_cells,
                    size_cells,
                    node,
                };

                if let Some((device, _)) =
                    device_tree::driver::probe_dt_node(&probe, device::register_device)
                {
                    device.init()?;
                }
            }

            debug::init();

            infoln!(
                "Yggdrasil v{} ({})",
                env!("CARGO_PKG_VERSION"),
                git_version!()
            );
            infoln!("Initializing aarch64 platform");

            let nodes = dt.root().children();
            if let Err(error) =
                device_tree::driver::enumerate_dt(address_cells, size_cells, nodes, |_, probe| {
                    // Skip chosen-stdout, already initialized
                    if let Some(ref chosen_stdout) = chosen_stdout
                        && chosen_stdout.name() == probe.node.name()
                    {
                        return Ok(());
                    }

                    if let Some((device, _)) =
                        device_tree::driver::probe_dt_node(&probe, device::register_device)
                    {
                        device.init()?;
                    }

                    Ok(())
                })
            {
                warnln!(
                    "{} errors encountered when initializing platform devices",
                    error
                );
            }

            // Initialize IRQs for the devices
            device::manager_lock().devices().for_each(|dev| unsafe {
                if let Err(error) = dev.init_irq() {
                    warnln!(
                        "Could not init IRQs for {:?}: {:?}",
                        dev.display_name(),
                        error
                    );
                }
            });

            infoln!("Enumerated devices:");
            device::manager_lock().devices().for_each(|dev| {
                infoln!("* {:?}", dev.display_name());
            });

            PciBusManager::setup_bus_devices()?;
        } else {
            // BSP already initialized everything needed
            // Setup timer and local interrupt controller
            let intc = self.lintc.get();

            unsafe {
                intc.init_ap().unwrap();
            }

            // TODO device-tree initialization for this
            CNTP_CTL_EL0.write(CNTP_CTL_EL0::ENABLE::SET + CNTP_CTL_EL0::IMASK::CLEAR);
            CNTP_TVAL_EL0.set(10000000);
            self.xintc.get().enable_irq(IrqNumber::Private(14)).unwrap();
        }

        Ok(())
    }
}

/// AArch64 implementation value
pub static ARCHITECTURE: AArch64 = AArch64 {
    dt: OneTimeInit::new(),
    initrd: OneTimeInit::new(),

    psci: OneTimeInit::new(),
    reset: OneTimeInit::new(),

    lintc: OneTimeInit::new(),
    xintc: OneTimeInit::new(),
    msi_intc: OneTimeInit::new(),

    mtimer: OneTimeInit::new(),
};

/// AArch64-specific interrupt number
#[derive(Clone, Copy, Debug, PartialEq)]
pub enum IrqNumber {
    /// CPU-private interrupt
    Private(u32),
    /// Shared interrupt
    Shared(u32),
}