//! Exception and interrupt management functions use core::{arch::global_asm, fmt}; use aarch64_cpu::{ asm::barrier, registers::{ ELR_EL1, ESR_EL1, FAR_EL1, SCTLR_EL1, TCR_EL1, TPIDR_EL0, TPIDR_EL1, TTBR0_EL1, TTBR1_EL1, VBAR_EL1, }, }; use abi::{ arch::SavedFrame, process::{Signal, SignalEntryData}, syscall::SyscallFunction, }; use tock_registers::interfaces::{Readable, Writeable}; use crate::{ arch::{Architecture, ArchitectureImpl}, debug::LogLevel, syscall::raw_syscall_handler, task::{context::TaskFrame, thread::Thread}, }; use super::ARCHITECTURE; /// Struct for register values saved when taking an exception #[repr(C)] pub struct ExceptionFrame { /// General-purpose registers pub r: [u64; 32], /// SPSR_EL1, userspace flags register pub spsr_el1: u64, /// ELR_EL1, userspace program counter pub elr_el1: u64, /// SP_EL0, userspace stack pointer pub sp_el0: u64, _x: u64, // ... } impl TaskFrame for ExceptionFrame { fn store(&self) -> SavedFrame { SavedFrame { gp_regs: self.r, spsr_el1: self.spsr_el1, elr_el1: self.elr_el1, sp_el0: self.sp_el0, } } fn restore(&mut self, saved: &SavedFrame) { self.r = saved.gp_regs; self.spsr_el1 = saved.spsr_el1; self.elr_el1 = saved.elr_el1; self.sp_el0 = saved.sp_el0; } fn argument(&self) -> u64 { self.r[0] } fn user_ip(&self) -> usize { self.elr_el1 as _ } fn user_sp(&self) -> usize { self.sp_el0 as _ } fn set_argument(&mut self, value: u64) { self.r[0] = value; } fn set_return_value(&mut self, value: u64) { self.r[0] = value; } fn set_user_ip(&mut self, value: usize) { self.elr_el1 = value as _; } fn set_user_sp(&mut self, value: usize) { self.sp_el0 = value as _; } } impl fmt::Debug for ExceptionFrame { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { for i in (0..32).step_by(2) { write!( f, "x{:<2} = {:#020x}\tx{:<2} = {:#020x}", i, self.r[i], i + 1, self.r[i + 1] )?; if i != 30 { f.write_str("\n")?; } } Ok(()) } } /// Initializes the exception/interrupt vectors. May be called repeatedly (though that makes no /// sense). pub fn init_exceptions() { extern "C" { static __aarch64_el1_vectors: u8; } let vbar = unsafe { &__aarch64_el1_vectors as *const _ }; VBAR_EL1.set(vbar as u64); barrier::isb(barrier::SY); } fn dump_irrecoverable_exception(frame: &ExceptionFrame, ec: u64, iss: u64) { // let cpu = Cpu::get_local(); log_print_raw!(LogLevel::Fatal, "SYNC exception:\n"); log_print_raw!(LogLevel::Fatal, "FAR: {:#x}\n", FAR_EL1.get()); log_print_raw!(LogLevel::Fatal, "ELR: {:#x}\n", ELR_EL1.get()); log_print_raw!(LogLevel::Fatal, "ESR: {:#x}\n", ESR_EL1.get()); log_print_raw!(LogLevel::Fatal, "TTBR0_EL1: {:#x}\n", TTBR0_EL1.get()); log_print_raw!(LogLevel::Fatal, "TTBR1_EL1: {:#x}\n", TTBR1_EL1.get()); log_print_raw!(LogLevel::Fatal, "Register dump:\n"); log_print_raw!(LogLevel::Fatal, "{:?}\n", frame); // XXX // if let Some(cpu) = cpu { // let current = cpu.queue().current_process(); // if let Some(current) = current { // log_print_raw!(LogLevel::Fatal, "In process {}\n", current.id()); // } // } match ec { // Data abort from lower level 0b100100 => { log_print_raw!(LogLevel::Fatal, "Exception kind: Data Abort from EL0\n"); let dfsc = iss & 0x3F; if iss & (1 << 24) != 0 { let access_size_str = match (iss >> 22) & 0x3 { 0 => "i8", 1 => "i16", 2 => "i32", 3 => "i64", _ => unreachable!(), }; let access_type_str = if iss & (1 << 6) != 0 { "write" } else { "read" }; log_print_raw!( LogLevel::Fatal, "Invalid {} of a {} to/from {:#x}\n", access_type_str, access_size_str, FAR_EL1.get() ); } log_print_raw!(LogLevel::Fatal, "DFSC = {:#x}\n", dfsc); } // Instruction abort from lower level 0b100000 => { log_print_raw!( LogLevel::Fatal, "Exception kind: Instruction Abort from EL0\n" ); let ifsc = iss & 0x3F; log_print_raw!(LogLevel::Fatal, "IFSC = {:#x}\n", ifsc); } _ => (), } // unsafe { // let space = AddressSpace::from_phys_raw(TTBR0_EL1.get_baddr() as _); // let far = FAR_EL1.get() as usize; // space.walk(far, |level, raw| { // log_print_raw!(LogLevel::Fatal, "Level {}: entry={:#x}\n", level, raw); // true // }); // } log_print_raw!(LogLevel::Fatal, "System register dump:\n"); log_print_raw!(LogLevel::Fatal, "SCTLR_EL1 = {:#x}\n", SCTLR_EL1.get()); log_print_raw!(LogLevel::Fatal, "TCR_EL1 = {:#x}\n", TCR_EL1.get()); log_print_raw!(LogLevel::Fatal, "TPIDR_EL1 = {:#x}\n", TPIDR_EL1.get()); log_print_raw!(LogLevel::Fatal, "TPIDR_EL0 = {:#x}\n", TPIDR_EL0.get()); } #[no_mangle] extern "C" fn __aa64_el0_sync_handler(frame: *mut ExceptionFrame) { assert!(ArchitectureImpl::interrupt_mask()); let frame = unsafe { &mut *frame }; // let current_id = Process::get_current().map(|p| p.id()); // log_print_raw!( // LogLevel::Fatal, // "sync_entry {:?}: x30 = {:#x} @ {:p}\n", // current_id, // frame.r[30], // &frame.r[30] // ); el0_sync_inner(frame); // let current_id = Process::get_current().map(|p| p.id()); // log_print_raw!( // LogLevel::Fatal, // "sync_exit {:?}: x30 = {:#x} @ {:p}\n", // current_id, // frame.r[30], // &frame.r[30] // ); unsafe { let thread = Thread::current(); thread.handle_pending_signals(frame); } } #[no_mangle] extern "C" fn __aa64_el0_irq_handler(frame: *mut ExceptionFrame) { assert!(ArchitectureImpl::interrupt_mask()); let frame = unsafe { &mut *frame }; // let current_id = Process::get_current().map(|p| p.id()); // log_print_raw!(LogLevel::Fatal, "+++ irq_handler ENTRY +++\n"); // log_print_raw!( // LogLevel::Fatal, // "{:?}: x30 = {:#x} @ {:p}\n", // current_id, // frame.r[30], // &frame.r[30] // ); // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0); irq_common(); // let current_id = Process::get_current().map(|p| p.id()); // log_print_raw!(LogLevel::Fatal, "+++ irq_handler EXIT +++\n"); // log_print_raw!( // LogLevel::Fatal, // "{:?}: x30 = {:#x} @ {:p}\n", // current_id, // frame.r[30], // &frame.r[30] // ); // log_print_raw!(LogLevel::Fatal, "sp = {:#x}\n", frame.sp_el0); unsafe { let thread = Thread::current(); thread.handle_pending_signals(frame); } } #[no_mangle] extern "C" fn __aa64_el0_fiq_handler() { todo!(); } #[no_mangle] extern "C" fn __aa64_el0_serror_handler() { todo!(); } // EL1 #[no_mangle] extern "C" fn __aa64_el1_sync_handler(frame: *mut ExceptionFrame) { let frame = unsafe { &*frame }; let esr_el1 = ESR_EL1.get(); let ec = (esr_el1 >> 26) & 0x3F; let iss = esr_el1 & 0x1FFFFFF; unsafe { kernel_util::sync::hack_locks(); } dump_irrecoverable_exception(frame, ec, iss); panic!("Irrecoverable exception in kernel mode"); } #[no_mangle] extern "C" fn __aa64_el1_irq_handler(_frame: *mut ExceptionFrame) { irq_common(); } #[no_mangle] extern "C" fn __aa64_el1_fiq_handler() { todo!(); } #[no_mangle] extern "C" fn __aa64_el1_serror_handler() { todo!(); } fn el0_sync_inner(frame: &mut ExceptionFrame) { let esr_el1 = ESR_EL1.get(); let ec = (esr_el1 >> 26) & 0x3F; match ec { // SVC in AArch64 0b010101 => { let func = frame.r[8]; if func == usize::from(SyscallFunction::ExitSignal) as u64 { unsafe { handle_signal_exit(frame); } return; } let args = &frame.r[0..6]; let result = raw_syscall_handler(func, args); frame.r[0] = result; } // BRK in AArch64 0b111100 => { let thread = Thread::current(); warnln!("Thread {} {:?} hit a breakpoint", thread.id, thread.name); thread.raise_signal(Signal::Aborted); } _ => { let iss = esr_el1 & 0x1FFFFFF; if ec == 0b100100 { // Data abort from lower level let thread = Thread::current(); warnln!( "Data abort in {} {:?} at {:#x} with address {:#x}", thread.id, thread.name, ELR_EL1.get(), FAR_EL1.get() ); thread.raise_signal(Signal::MemoryAccessViolation); return; } dump_irrecoverable_exception(frame, ec, iss); panic!("Irrecoverable exception"); } } } fn irq_common() { ARCHITECTURE .external_interrupt_controller() .handle_pending_irqs(); } unsafe fn handle_signal_exit(frame: &mut ExceptionFrame) { // TODO validate the argument let saved_data = &*(frame.r[0] as *const SignalEntryData); debugln!( "Handling signal exit to pc={:#x}, sp={:#x}", saved_data.frame.elr_el1, saved_data.frame.sp_el0 ); frame.restore(&saved_data.frame); } global_asm!(include_str!("vectors.S"));