yggdrasil/kernel/src/arch/aarch64/exception.rs

//! Exception and interrupt management functions

use core::arch::global_asm;

use aarch64_cpu::{
    asm::barrier,
    registers::{
        ELR_EL1, ESR_EL1, FAR_EL1, SCTLR_EL1, SP_EL0, TCR_EL1, TPIDR_EL0, TPIDR_EL1, TTBR0_EL1,
        TTBR1_EL1, VBAR_EL1,
    },
};
use abi::{process::Signal, SyscallFunction};
use kernel_arch::{Architecture, ArchitectureImpl};
use kernel_arch_aarch64::context::ExceptionFrame;
use libk::{arch::Cpu, device::external_interrupt_controller, task::thread::Thread};
use tock_registers::interfaces::{Readable, Writeable};

use crate::{
    debug::LogLevel,
    mem::KERNEL_VIRT_OFFSET,
    syscall::{self, raw_syscall_handler},
};

/// Initializes the exception/interrupt vectors. May be called repeatedly (though that makes no
/// sense).
pub fn init_exceptions() {
    extern "C" {
        static __aarch64_el1_vectors: u8;
    }
    let vbar = unsafe { &__aarch64_el1_vectors as *const _ };
    VBAR_EL1.set(vbar as u64);
    barrier::isb(barrier::SY);
}

fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) {
    let cpu = Cpu::try_local();
    let far = FAR_EL1.get() as usize;
    let ttbr0 = TTBR0_EL1.get();

    log_print_raw!(LogLevel::Fatal, "SYNC exception:\n");
    log_print_raw!(LogLevel::Fatal, "FAR: {:#x}\n", far);
    log_print_raw!(LogLevel::Fatal, "ELR: {:#x}\n", ELR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "ESR: {:#x}\n", ESR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "SP_EL0: {:#x}\n", SP_EL0.get());
    log_print_raw!(LogLevel::Fatal, "TTBR0_EL1: {:#x}\n", ttbr0);
    log_print_raw!(LogLevel::Fatal, "TTBR1_EL1: {:#x}\n", TTBR1_EL1.get());
    log_print_raw!(LogLevel::Fatal, "Register dump:\n");
    log_print_raw!(LogLevel::Fatal, "{:?}\n", frame);

    if let Some(cpu) = cpu {
        // let current = cpu.queue().current_process();
        let current = cpu.current_thread_id().and_then(Thread::get);

        if let Some(current) = current {
            log_print_raw!(LogLevel::Fatal, "In thread {}\n", current.id);

            let space = current.address_space();
            if far < KERNEL_VIRT_OFFSET && space.as_address_with_asid() == ttbr0 {
                match space.translate(far) {
                    Ok(phys) => log_print_raw!(
                        LogLevel::Fatal,
                        "FAR translation: {:#x} -> {:#x}\n",
                        far,
                        phys
                    ),
                    Err(_) => log_print_raw!(LogLevel::Fatal, "FAR does not translate\n"),
                }
            }
        }
    }

    match ec {
        // Data abort from lower level
        0b100100 => {
            log_print_raw!(LogLevel::Fatal, "Exception kind: Data Abort from EL0\n");
            let dfsc = iss & 0x3F;

            if iss & (1 << 24) != 0 {
                let access_size_str = match (iss >> 22) & 0x3 {
                    0 => "i8",
                    1 => "i16",
                    2 => "i32",
                    3 => "i64",
                    _ => unreachable!(),
                };
                let access_type_str = if iss & (1 << 6) != 0 { "write" } else { "read" };

                log_print_raw!(
                    LogLevel::Fatal,
                    "Invalid {} of a {} to/from {:#x}\n",
                    access_type_str,
                    access_size_str,
                    FAR_EL1.get()
                );
            }

            log_print_raw!(LogLevel::Fatal, "DFSC = {:#x}\n", dfsc);
        }
        // Instruction abort from lower level
        0b100000 => {
            log_print_raw!(
                LogLevel::Fatal,
                "Exception kind: Instruction Abort from EL0\n"
            );
            let ifsc = iss & 0x3F;
            log_print_raw!(LogLevel::Fatal, "IFSC = {:#x}\n", ifsc);
        }

        _ => (),
    }

    log_print_raw!(LogLevel::Fatal, "System register dump:\n");
    log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TCR_EL1 = {:#x}\n", TCR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TPIDR_EL1 = {:#x}\n", TPIDR_EL1.get());
    log_print_raw!(LogLevel::Fatal, "TPIDR_EL0 = {:#x}\n", TPIDR_EL0.get());
}

#[no_mangle]
extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) {
    assert!(ArchitectureImpl::interrupt_mask());
    let frame = unsafe { &mut *frame };

    el0_sync_inner(frame);

    unsafe {
        let thread = Thread::current();
        thread.handle_pending_signals(frame);
    }
}

#[no_mangle]
extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) {
    assert!(ArchitectureImpl::interrupt_mask());
    let frame = unsafe { &mut *frame };

    irq_common();

    unsafe {
        let thread = Thread::current();
        thread.handle_pending_signals(frame);
    }
}

#[no_mangle]
extern "C" fn __aa64_el0_fiq_handler() {
    unimplemented!()
}

#[no_mangle]
extern "C" fn __aa64_el0_serror_handler() {
    unimplemented!()
}

// EL1
#[no_mangle]
extern "C" fn __aa64_el1_sync_handler(frame: *mut ExceptionFrame) {
    let frame = unsafe { &*frame };
    let esr_el1 = ESR_EL1.get();
    let ec = (esr_el1 >> 26) & 0x3F;
    let iss = esr_el1 & 0x1FFFFFF;

    unsafe {
        libk_util::sync::hack_locks();
    }

    dump_irrecoverable_exception(frame, ec, iss);

    panic!("Irrecoverable exception in kernel mode");
}

#[no_mangle]
extern "C" fn __aa64_el1_irq_handler(_frame: *mut ExceptionFrame) {
    irq_common();
}

#[no_mangle]
extern "C" fn __aa64_el1_fiq_handler() {
    unimplemented!()
}

#[no_mangle]
extern "C" fn __aa64_el1_serror_handler() {
    unimplemented!()
}

fn el0_sync_inner(frame: &mut ExceptionFrame) {
    let esr_el1 = ESR_EL1.get();
    let ec = (esr_el1 >> 26) & 0x3F;
    let iss = esr_el1 & 0x1FFFFFF;

    let dump = match ec {
        // SVC in AArch64
        0b010101 => {
            let func = frame.r[8];
            if func == usize::from(SyscallFunction::ExitSignal) {
                unsafe {
                    syscall::handle_signal_exit(frame);
                }
                return;
            }

            let args = &frame.r[0..6];
            let result = raw_syscall_handler(func, args) as _;
            frame.r[0] = result;
            false
        }
        // Software Step from lower Exception Level
        0b110010 => {
            let thread = Thread::current();

            if thread.handle_single_step(frame) {
                // Make the PE actually step the instruction
                frame.spsr_el1 |= 1 << 21;
                false
            } else {
                thread.raise_signal(Signal::Aborted);
                true
            }
        }
        // BRK in AArch64
        0b111100 => {
            let thread = Thread::current();
            warnln!(
                "Thread {} {:?} hit a breakpoint",
                thread.id,
                *thread.name.read()
            );
            thread.raise_signal(Signal::Aborted);
            true
        }
        _ => {
            let thread = Thread::current();
            if ec == 0b100100 {
                // Data abort from lower level
                let thread = Thread::current();
                warnln!(
                    "Data abort in {} {:?} at {:#x} with address {:#x}",
                    thread.id,
                    *thread.name.read(),
                    ELR_EL1.get(),
                    FAR_EL1.get()
                );
            }

            thread.raise_signal(Signal::MemoryAccessViolation);
            true
        }
    };

    if dump {
        dump_irrecoverable_exception(frame, ec, iss);
    }
}

fn irq_common() {
    external_interrupt_controller().handle_pending_irqs();
}

global_asm!(include_str!("vectors.S"));