yggdrasil/src/arch/x86_64/mod.rs

//! x86-64 architecture and platform implementation
use core::{ptr::NonNull, sync::atomic::Ordering};

use abi::error::Error;
use acpi::{
    platform::ProcessorInfo, AcpiHandler, AcpiTables, HpetInfo, InterruptModel, PhysicalMapping,
};
use cpu::Cpu;
use git_version::git_version;
use yboot_proto::{v1::FramebufferOption, AvailableRegion, IterableMemoryMap, LoadProtocolV1};

use crate::{
    arch::x86_64::{
        apic::local::LocalApic,
        table::{init_fixed_tables, KERNEL_TABLES},
    },
    debug::{self, LogLevel},
    device::{
        display::{
            console::{add_console_autoflush, ConsoleBuffer, ConsoleRow, DisplayConsole},
            fb_console::FramebufferConsole,
            linear_fb::LinearFramebuffer,
        },
        input::KeyboardDevice,
        interrupt::{ExternalInterruptController, InterruptSource, IpiDeliveryTarget},
        platform::Platform,
        timer::TimestampSource,
        tty::CombinedTerminal,
        InitializableDevice,
    },
    fs::{
        devfs::{self, CharDeviceType},
        Initrd, INITRD_DATA,
    },
    mem::{
        heap,
        phys::{self, reserved::reserve_region, PageUsage, PhysicalMemoryRegion},
        ConvertAddress,
    },
    sync::SpinFence,
    task,
    util::OneTimeInit,
};

use self::{
    apic::{ioapic::IoApic, IrqNumber},
    intrinsics::IoPort,
    peripherals::{hpet::Hpet, ps2::PS2Controller, serial::ComPort},
    smp::CPU_COUNT,
};

use super::{Architecture, CpuMessage};

#[macro_use]
pub mod intrinsics;

pub mod apic;
pub mod boot;
pub mod context;
pub mod cpu;
pub mod cpuid;
pub mod exception;
pub mod gdt;
pub mod peripherals;
pub mod registers;
pub mod smp;
pub mod syscall;
pub mod table;

static CPU_INIT_FENCE: SpinFence = SpinFence::new();

/// Helper trait to provide abstract access to available memory regions
pub trait AbstractAvailableRegion {
    /// Returns page-aligned physical start address of the region
    fn start_address(&self) -> usize;
    /// Returns the page count (rounded down) of this region
    fn page_count(&self) -> usize;
}

/// Helper trait to provide abstract access to memory maps
pub trait AbstractMemoryMap<'a>: 'a {
    /// Available memory region type contained within this memory map
    type AvailableRegion: AbstractAvailableRegion;
    /// Iterator type returned by [Self::iter]
    type Iter: Iterator<Item = &'a Self::AvailableRegion> + Clone;

    /// Returns the physical memory range which contains this memory map
    fn reserved_range(&self) -> PhysicalMemoryRegion;
    /// Returns an iterator over the available memory regions
    fn iter(&self) -> Self::Iter;
}

impl<T: AvailableRegion> AbstractAvailableRegion for T {
    fn start_address(&self) -> usize {
        <T as AvailableRegion>::start_address(self)
    }

    fn page_count(&self) -> usize {
        <T as AvailableRegion>::page_count(self)
    }
}

impl<'a, T: IterableMemoryMap<'a> + 'a> AbstractMemoryMap<'a> for T {
    type AvailableRegion = T::Entry;
    type Iter = T::Iter;

    fn reserved_range(&self) -> PhysicalMemoryRegion {
        PhysicalMemoryRegion {
            base: self.data_physical_base(),
            size: (self.data_size() + 0xFFF) & !0xFFF,
        }
    }

    fn iter(&self) -> Self::Iter {
        <T as IterableMemoryMap>::iter_with_offset(self, X86_64::KERNEL_VIRT_OFFSET)
    }
}

#[derive(Clone, Copy)]
struct AcpiHandlerImpl;

impl AcpiHandler for AcpiHandlerImpl {
    // No actual address space modification is performed
    unsafe fn map_physical_region<T>(
        &self,
        physical_address: usize,
        size: usize,
    ) -> PhysicalMapping<Self, T> {
        if physical_address <= 0xFFFFFFFF {
            PhysicalMapping::new(
                physical_address,
                NonNull::new_unchecked(physical_address.virtualize() as *mut T),
                size,
                size,
                *self,
            )
        } else {
            todo!()
        }
    }

    // Unmap nothing, these addresses are "virtualized" to high address space
    fn unmap_physical_region<T>(_region: &PhysicalMapping<Self, T>) {}
}

/// x86-64 architecture + platform implementation
pub struct X86_64 {
    yboot_framebuffer: OneTimeInit<FramebufferOption>,
    rsdp_phys_base: OneTimeInit<usize>,

    acpi_processor_info: OneTimeInit<ProcessorInfo>,

    // TODO make this fully dynamic?
    combined_terminal: OneTimeInit<CombinedTerminal>,

    // Static devices with dynamic addresses
    ioapic: IoApic,
    timer: Hpet,

    // Static devices
    ps2: PS2Controller,
    com1_3: ComPort,
}

impl Architecture for X86_64 {
    const KERNEL_VIRT_OFFSET: usize = 0xFFFFFF8000000000;

    unsafe fn init_mmu(&self, bsp: bool) {
        if bsp {
            init_fixed_tables();
        }

        let cr3 = KERNEL_TABLES.physical_address();
        core::arch::asm!("wbinvd; mov {0}, %cr3", in(reg) cr3, options(att_syntax));
    }

    fn map_device_pages(&self, phys: usize, count: usize) -> Result<usize, Error> {
        unsafe { KERNEL_TABLES.map_device_pages(phys, count) }
    }

    fn wait_for_interrupt() {
        unsafe {
            core::arch::asm!("hlt");
        }
    }

    unsafe fn set_interrupt_mask(mask: bool) {
        if mask {
            core::arch::asm!("cli");
        } else {
            core::arch::asm!("sti");
        }
    }

    fn interrupt_mask() -> bool {
        let mut flags: u64;
        unsafe {
            core::arch::asm!("pushfq; pop {0}", out(reg) flags, options(att_syntax));
        }
        // If IF is zero, interrupts are disabled (masked)
        flags & (1 << 9) == 0
    }

    fn cpu_count() -> usize {
        CPU_COUNT.load(Ordering::Acquire)
    }
}

impl Platform for X86_64 {
    type IrqNumber = apic::IrqNumber;

    const KERNEL_PHYS_BASE: usize = 0x400000;

    unsafe fn init(&'static self, is_bsp: bool) -> Result<(), Error> {
        if is_bsp {
            Self::disable_8259();

            // Initialize I/O APIC from ACPI
            let rsdp = *self.rsdp_phys_base.get();
            let acpi_tables = AcpiTables::from_rsdp(AcpiHandlerImpl, rsdp).unwrap();

            let hpet = HpetInfo::new(&acpi_tables).unwrap();
            let platform_info = acpi_tables.platform_info().unwrap();

            if let Some(processor_info) = platform_info.processor_info {
                self.acpi_processor_info.init(processor_info);
            }

            let InterruptModel::Apic(apic_info) = platform_info.interrupt_model else {
                panic!("Processor does not have APIC");
            };

            self.ioapic.init(apic_info)?;
            self.timer.init(hpet).unwrap();

            // Enable IRQs for the devices
            self.ps2.init_irq()?;
            self.timer.init_irq()?;

            // Add a terminal to the devfs
            // TODO this is ugly
            let combined_terminal = CombinedTerminal::new(&self.ps2, FB_CONSOLE.get());
            self.combined_terminal.init(combined_terminal);
            self.ps2.attach(self.combined_terminal.get());

            devfs::add_char_device(self.combined_terminal.get(), CharDeviceType::TtyRegular)?;
        }

        Ok(())
    }

    unsafe fn init_debug(&'static self) {
        // Serial output
        self.com1_3.init(()).ok();
        debug::add_sink(self.com1_3.port_a(), LogLevel::Debug);

        // Graphical output
        if let Some(fb) = self.yboot_framebuffer.try_get() {
            LINEAR_FB.init(
                LinearFramebuffer::from_physical_bits(
                    fb.res_address as _,
                    fb.res_size as _,
                    fb.res_stride as _,
                    fb.res_width,
                    fb.res_height,
                )
                .unwrap(),
            );
            FB_CONSOLE.init(FramebufferConsole::from_framebuffer(
                LINEAR_FB.get(),
                &bitmap_font::tamzen::FONT_6x12,
                ConsoleBuffer::fixed(&mut EARLY_CONSOLE_BUFFER, 32),
            ));

            debug::add_sink(FB_CONSOLE.get(), LogLevel::Info);
        }

        debug::reset();
    }

    fn name(&self) -> &'static str {
        "x86-64"
    }

    fn interrupt_controller(
        &self,
    ) -> &dyn ExternalInterruptController<IrqNumber = Self::IrqNumber> {
        &self.ioapic
    }

    fn timestamp_source(&self) -> &dyn TimestampSource {
        &self.timer
    }

    unsafe fn send_ipi(&self, target: IpiDeliveryTarget, _msg: CpuMessage) -> Result<(), Error> {
        if !CPU_INIT_FENCE.try_wait_all(1) {
            // Don't send an IPI: SMP not initialized yet
            return Ok(());
        }

        let Some(local_apic) = Cpu::get_local().map(|cpu| cpu.local_apic()) else {
            panic!("Local APIC has not been initialized yet");
        };

        local_apic.send_ipi(target);

        Ok(())
    }
}

impl X86_64 {
    unsafe fn init_physical_memory<'a, M: AbstractMemoryMap<'a>>(memory_map: &M) {
        // Reserve the lower 8MiB of memory
        reserve_region(
            "lower-memory",
            PhysicalMemoryRegion {
                base: 0,
                size: 8 << 21,
            },
        );
        // Reserve memory map
        reserve_region("memory-map", memory_map.reserved_range());

        if let Some(initrd) = INITRD_DATA.try_get() {
            reserve_region(
                "initrd",
                PhysicalMemoryRegion {
                    base: initrd.phys_page_start,
                    size: initrd.phys_page_len,
                },
            );
        }

        phys::init_from_iter(memory_map.iter().map(|r| PhysicalMemoryRegion {
            base: r.start_address(),
            size: r.page_count() * 0x1000,
        }))
        .expect("Failed to initialize the physical memory manager");

        // Reallocate the console buffer
        FB_CONSOLE
            .get()
            .reallocate_buffer()
            .expect("Failed to reallocate console buffer");
    }

    fn init_rsdp(&self, phys: usize) {
        self.rsdp_phys_base.init(phys);
    }

    unsafe fn disable_8259() {
        infoln!("Disabling i8259 PIC");
        // TODO should I make a module for 8259 if I don't even use it?
        let pic_master_cmd = IoPort::new(0x20);
        let pic_master_data = IoPort::new(0x21);
        let pic_slave_cmd = IoPort::new(0xA0);
        let pic_slave_data = IoPort::new(0xA1);

        // Remap PIC IRQ vectors to 32..
        pic_master_cmd.write(0x11);
        pic_slave_cmd.write(0x11);

        pic_master_data.write(32);
        pic_slave_data.write(32 + 8);

        pic_slave_data.write(0xFF);
        pic_master_data.write(0xFF);

        pic_master_cmd.write(0x20);
        pic_slave_cmd.write(0x20);
    }
}

/// x86-64 architecture + platform implementation
pub static ARCHITECTURE: X86_64 = X86_64 {
    rsdp_phys_base: OneTimeInit::new(),
    yboot_framebuffer: OneTimeInit::new(),

    acpi_processor_info: OneTimeInit::new(),

    combined_terminal: OneTimeInit::new(),

    ioapic: IoApic::new(),
    timer: Hpet::uninit(),

    ps2: PS2Controller::new(IrqNumber::Isa(1), IrqNumber::Isa(12), 0x64, 0x60),
    com1_3: ComPort::new(0x3F8, 0x3E8, IrqNumber::Isa(4)),
};

static LINEAR_FB: OneTimeInit<LinearFramebuffer> = OneTimeInit::new();
static FB_CONSOLE: OneTimeInit<FramebufferConsole> = OneTimeInit::new();

const EARLY_BUFFER_LINES: usize = 32;
static mut EARLY_CONSOLE_BUFFER: [ConsoleRow; EARLY_BUFFER_LINES] =
    [ConsoleRow::zeroed(); EARLY_BUFFER_LINES];

fn setup_initrd(initrd_start: usize, initrd_end: usize) {
    if initrd_start == 0 || initrd_end <= initrd_start {
        infoln!("No initrd loaded");
        return;
    }

    let start_aligned = initrd_start & !0xFFF;
    let end_aligned = initrd_end & !0xFFF;

    let data = unsafe {
        core::slice::from_raw_parts(
            initrd_start.virtualize() as *const _,
            initrd_end - initrd_start,
        )
    };

    let initrd = Initrd {
        phys_page_start: start_aligned,
        phys_page_len: end_aligned - start_aligned,
        data,
    };

    INITRD_DATA.init(initrd);
}

fn kernel_main(boot_data: &LoadProtocolV1) -> ! {
    ARCHITECTURE
        .yboot_framebuffer
        .init(boot_data.opt_framebuffer);

    unsafe {
        ARCHITECTURE.init_debug();
    }

    infoln!("Yggdrasil kernel git {} starting", git_version!());

    cpuid::feature_gate();

    if boot_data.memory_map.address > 0xFFFFFFFF {
        errorln!("Unhandled case: memory map is above 4GiB");
        loop {
            X86_64::wait_for_interrupt();
        }
    }

    if boot_data.rsdp_address != 0 {
        infoln!("ACPI RSDP at {:#x}", boot_data.rsdp_address);
        ARCHITECTURE.init_rsdp(boot_data.rsdp_address as _);
    }

    // Reserve initrd
    let initrd_start = boot_data.initrd_address as usize;
    let initrd_end = initrd_start + boot_data.initrd_size as usize;

    setup_initrd(initrd_start, initrd_end);

    // Setup physical memory allocation
    unsafe {
        X86_64::init_physical_memory(&boot_data.memory_map);
    }

    // Allocate memory for the kernel heap
    let heap_base = phys::alloc_pages_contiguous(16, PageUsage::Used)
        .expect("Could not allocate a block for heap");
    unsafe {
        heap::init_heap(heap_base.virtualize(), 16 * 0x1000);
    }

    // Add console to flush list
    add_console_autoflush(FB_CONSOLE.get());

    // Also initializes local APIC
    unsafe {
        Cpu::init_local(LocalApic::new(), 0);

        syscall::init_syscall();

        // TODO per-CPU IDTs?
        exception::init_exceptions(0);

        devfs::init();

        ARCHITECTURE.init(true).unwrap();

        if let Some(info) = ARCHITECTURE.acpi_processor_info.try_get() {
            smp::start_ap_cores(info);
        }

        CPU_INIT_FENCE.signal();

        task::init().expect("Failed to initialize the scheduler");
        task::enter()
    }
}