yggdrasil/src/arch/aarch64/exception.rs

//! Exception and interrupt management functions
use core::{arch::global_asm, fmt};

use aarch64_cpu::{
    asm::barrier,
    registers::{
        ELR_EL1, ESR_EL1, FAR_EL1, SCTLR_EL1, TCR_EL1, TPIDR_EL0, TPIDR_EL1, TTBR0_EL1, TTBR1_EL1,
        VBAR_EL1,
    },
};
use abi::{
    arch::SavedFrame,
    process::{Signal, SignalEntryData},
    syscall::SyscallFunction,
};
use tock_registers::interfaces::{Readable, Writeable};

use crate::{
    arch::{Architecture, ArchitectureImpl},
    debug::LogLevel,
    syscall::raw_syscall_handler,
    task::{context::TaskFrame, thread::Thread},
};

use super::ARCHITECTURE;

/// Struct for register values saved when taking an exception
#[repr(C)]
pub struct ExceptionFrame {
    /// General-purpose registers
    pub r: [u64; 32],
    /// SPSR_EL1, userspace flags register
    pub spsr_el1: u64,
    /// ELR_EL1, userspace program counter
    pub elr_el1: u64,
    /// SP_EL0, userspace stack pointer
    pub sp_el0: u64,
    _x: u64,
    // ...
}

impl TaskFrame for ExceptionFrame {
    fn store(&self) -> SavedFrame {
        SavedFrame {
            gp_regs: self.r,
            spsr_el1: self.spsr_el1,
            elr_el1: self.elr_el1,
            sp_el0: self.sp_el0,
        }
    }

    fn restore(&mut self, saved: &SavedFrame) {
        self.r = saved.gp_regs;
        self.spsr_el1 = saved.spsr_el1;
        self.elr_el1 = saved.elr_el1;
        self.sp_el0 = saved.sp_el0;
    }

    fn argument(&self) -> u64 {
        self.r[0]
    }

    fn user_ip(&self) -> usize {
        self.elr_el1 as _
    }

    fn user_sp(&self) -> usize {
        self.sp_el0 as _
    }

    fn set_argument(&mut self, value: u64) {
        self.r[0] = value;
    }

    fn set_return_value(&mut self, value: u64) {
        self.r[0] = value;
    }

    fn set_user_ip(&mut self, value: usize) {
        self.elr_el1 = value as _;
    }

    fn set_user_sp(&mut self, value: usize) {
        self.sp_el0 = value as _;
    }
}

impl fmt::Debug for ExceptionFrame {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        for i in (0..32).step_by(2) {
            write!(
                f,
                "x{:<2} = {:#020x}\tx{:<2} = {:#020x}",
                i,
                self.r[i],
                i + 1,
                self.r[i + 1]
            )?;
            if i != 30 {
                f.write_str("\n")?;
            }
        }

        Ok(())
    }
}

/// Initializes the exception/interrupt vectors. May be called repeatedly (though that makes no
/// sense).
pub fn init_exceptions() {
    extern "C" {
        static __aarch64_el1_vectors: u8;
    }
    let vbar = unsafe { &__aarch64_el1_vectors as *const _ };
    VBAR_EL1.set(vbar as u64);
    barrier::isb(barrier::SY);
}

fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) {
    // let cpu = Cpu::get_local();

    log_print_raw!(LogLevel::Fatal, "SYNC exception:\n");
    log_print_raw!(LogLevel::Fatal, "FAR: {:#x}\n", FAR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "ELR: {:#x}\n", ELR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "ESR: {:#x}\n", ESR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TTBR0_EL1: {:#x}\n", TTBR0_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TTBR1_EL1: {:#x}\n", TTBR1_EL1.get());
    log_print_raw!(LogLevel::Fatal, "Register dump:\n");
    log_print_raw!(LogLevel::Fatal, "{:?}\n", frame);

    // XXX
    // if let Some(cpu) = cpu {
    //     let current = cpu.queue().current_process();

    //     if let Some(current) = current {
    //         log_print_raw!(LogLevel::Fatal, "In process {}\n", current.id());
    //     }
    // }

    match ec {
        // Data abort from lower level
        0b100100 => {
            log_print_raw!(LogLevel::Fatal, "Exception kind: Data Abort from EL0\n");
            let dfsc = iss & 0x3F;

            if iss & (1 << 24) != 0 {
                let access_size_str = match (iss >> 22) & 0x3 {
                    0 => "i8",
                    1 => "i16",
                    2 => "i32",
                    3 => "i64",
                    _ => unreachable!(),
                };
                let access_type_str = if iss & (1 << 6) != 0 { "write" } else { "read" };

                log_print_raw!(
                    LogLevel::Fatal,
                    "Invalid {} of a {} to/from {:#x}\n",
                    access_type_str,
                    access_size_str,
                    FAR_EL1.get()
                );
            }

            log_print_raw!(LogLevel::Fatal, "DFSC = {:#x}\n", dfsc);
        }
        // Instruction abort from lower level
        0b100000 => {
            log_print_raw!(
                LogLevel::Fatal,
                "Exception kind: Instruction Abort from EL0\n"
            );
            let ifsc = iss & 0x3F;
            log_print_raw!(LogLevel::Fatal, "IFSC = {:#x}\n", ifsc);
        }

        _ => (),
    }

    // unsafe {
    //     let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _);
    //     let far = FAR_EL1.get() as usize;
    //     space.walk(far, |level, raw| {
    //         log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw);
    //         true
    //     });
    // }

    log_print_raw!(LogLevel::Fatal, "System register dump:\n");
    log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TCR_EL1 = {:#x}\n", TCR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TPIDR_EL1 = {:#x}\n", TPIDR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TPIDR_EL0 = {:#x}\n", TPIDR_EL0.get());
}

#[no_mangle]
extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) {
    assert!(ArchitectureImpl::interrupt_mask());
    let frame = unsafe { &mut *frame };

    // let current_id = Process::get_current().map(|p| p.id());
    // log_print_raw!(
    //     LogLevel::Fatal,
    //     "sync_entry {:?}: x30 = {:#x} @ {:p}\n",
    //     current_id,
    //     frame.r[30],
    //     &frame.r[30]
    // );

    el0_sync_inner(frame);

    // let current_id = Process::get_current().map(|p| p.id());
    // log_print_raw!(
    //     LogLevel::Fatal,
    //     "sync_exit {:?}: x30 = {:#x} @ {:p}\n",
    //     current_id,
    //     frame.r[30],
    //     &frame.r[30]
    // );

    unsafe {
        let thread = Thread::current();
        thread.handle_pending_signals(frame);
    }
}

#[no_mangle]
extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) {
    assert!(ArchitectureImpl::interrupt_mask());
    let frame = unsafe { &mut *frame };

    // let current_id = Process::get_current().map(|p| p.id());
    // log_print_raw!(LogLevel::Fatal, "+++ irq_handler ENTRY +++\n");
    // log_print_raw!(
    //     LogLevel::Fatal,
    //     "{:?}: x30 = {:#x} @ {:p}\n",
    //     current_id,
    //     frame.r[30],
    //     &frame.r[30]
    // );
    // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0);

    irq_common();

    // let current_id = Process::get_current().map(|p| p.id());
    // log_print_raw!(LogLevel::Fatal, "+++ irq_handler EXIT +++\n");
    // log_print_raw!(
    //     LogLevel::Fatal,
    //     "{:?}: x30 = {:#x} @ {:p}\n",
    //     current_id,
    //     frame.r[30],
    //     &frame.r[30]
    // );
    // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0);

    unsafe {
        let thread = Thread::current();
        thread.handle_pending_signals(frame);
    }
}

#[no_mangle]
extern "C" fn __aa64_el0_fiq_handler() {
    todo!();
}

#[no_mangle]
extern "C" fn __aa64_el0_serror_handler() {
    todo!();
}

// EL1
#[no_mangle]
extern "C" fn __aa64_el1_sync_handler(frame: *mut ExceptionFrame) {
    let frame = unsafe { &*frame };
    let esr_el1 = ESR_EL1.get();
    let ec = (esr_el1 >> 26) & 0x3F;
    let iss = esr_el1 & 0x1FFFFFF;

    unsafe {
        kernel_util::sync::hack_locks();
    }

    dump_irrecoverable_exception(frame, ec, iss);

    panic!("Irrecoverable exception in kernel mode");
}

#[no_mangle]
extern "C" fn __aa64_el1_irq_handler(_frame: *mut ExceptionFrame) {
    irq_common();
}

#[no_mangle]
extern "C" fn __aa64_el1_fiq_handler() {
    todo!();
}

#[no_mangle]
extern "C" fn __aa64_el1_serror_handler() {
    todo!();
}

fn el0_sync_inner(frame: &mut ExceptionFrame) {
    let esr_el1 = ESR_EL1.get();
    let ec = (esr_el1 >> 26) & 0x3F;

    match ec {
        // SVC in AArch64
        0b010101 => {
            let func = frame.r[8];
            if func == usize::from(SyscallFunction::ExitSignal) as u64 {
                unsafe {
                    handle_signal_exit(frame);
                }
                return;
            }

            let args = &frame.r[0..6];
            let result = raw_syscall_handler(func, args);
            frame.r[0] = result;
        }
        // BRK in AArch64
        0b111100 => {
            let thread = Thread::current();
            warnln!("Thread {} {:?} hit a breakpoint", thread.id, thread.name);
            thread.raise_signal(Signal::Aborted);
        }
        _ => {
            let iss = esr_el1 & 0x1FFFFFF;
            if ec == 0b100100 {
                // Data abort from lower level
                let thread = Thread::current();
                warnln!(
                    "Data abort in {} {:?} at {:#x} with address {:#x}",
                    thread.id,
                    thread.name,
                    ELR_EL1.get(),
                    FAR_EL1.get()
                );

                thread.raise_signal(Signal::MemoryAccessViolation);
                return;
            }

            dump_irrecoverable_exception(frame, ec, iss);

            panic!("Irrecoverable exception");
        }
    }
}

fn irq_common() {
    ARCHITECTURE
        .external_interrupt_controller()
        .handle_pending_irqs();
}

unsafe fn handle_signal_exit(frame: &mut ExceptionFrame) {
    // TODO validate the argument
    let saved_data = &*(frame.r[0] as *const SignalEntryData);
    debugln!(
        "Handling signal exit to pc={:#x}, sp={:#x}",
        saved_data.frame.elr_el1,
        saved_data.frame.sp_el0
    );

    frame.restore(&saved_data.frame);
}

global_asm!(include_str!("vectors.S"));