diff --git a/etc/riscv64-unknown-none.ld b/etc/riscv64-unknown-none.ld index 446b96b..b6a1ae5 100644 --- a/etc/riscv64-unknown-none.ld +++ b/etc/riscv64-unknown-none.ld @@ -23,6 +23,8 @@ SECTIONS { .tdata : ALIGN(4K) { PROVIDE(__tdata_start = .); + /* Storage for thread-locals used in assembly */ + KEEP(*(.tdata.assembly*)); *(.tdata*) PROVIDE(__tdata_end = .); } diff --git a/src/arch/aarch64/exception.zig b/src/arch/aarch64/exception.zig index 813be4b..43e1849 100644 --- a/src/arch/aarch64/exception.zig +++ b/src/arch/aarch64/exception.zig @@ -8,6 +8,10 @@ extern const __aa64_exception_vectors: u8; pub const ExceptionFrame = extern struct { xN: [32]usize, + spsr_el1: usize, + elr_el1: usize, + sp_el0: usize, + _0: usize, pub fn dump(self: *const ExceptionFrame, comptime level: log.Level) void { for (0..16) |i| { @@ -83,8 +87,25 @@ export fn __aa64_el1_serror_handler(frame: *ExceptionFrame) callconv(.C) void { // EL0 export fn __aa64_el0_sync_handler(frame: *ExceptionFrame) callconv(.C) void { - // TODO EL0 - _ = frame; + const esr = regs.ESR_EL1.read(); + + switch (esr.as_enum()) { + .svc => { + log.info("Syscall executed!", .{}); + log.info("args:", .{}); + for (0..6) |i| { + log.info(" x{} = 0x{x:016} ({})", .{ i, frame.xN[i], frame.xN[i] }); + } + return; + }, + else => {}, + } + + log.err("Unhandled exception in EL0:", .{}); + log.err(" EC = {s} (0b{b:06}) ISS = 0x{x}", .{ esr.EC.as_str(), @intFromEnum(esr.EC), esr.ISS }); + log.err(" ESR = 0x{x:016}", .{@as(u64, @bitCast(esr))}); + log.err(" ELR = 0x{x:016}", .{frame.elr_el1}); + frame.dump(log.Level.err); arch.halt(); } @@ -94,14 +115,12 @@ export fn __aa64_el0_irq_handler(frame: *ExceptionFrame) callconv(.C) void { export fn __aa64_el0_fiq_handler(frame: *ExceptionFrame) callconv(.C) void { _ = frame; - // TODO I've never used FIQ - arch.halt(); + @panic("__aa64_el0_fiq_handler"); } export fn __aa64_el0_serror_handler(frame: *ExceptionFrame) callconv(.C) void { _ = frame; - // TODO - arch.halt(); + @panic("__aa64_el0_serror_handler"); } comptime { diff --git a/src/arch/aarch64/regs.zig b/src/arch/aarch64/regs.zig index 21fb4c0..257e7a4 100644 --- a/src/arch/aarch64/regs.zig +++ b/src/arch/aarch64/regs.zig @@ -76,6 +76,7 @@ pub const ESR_EL1 = Register("esr_el1", packed struct(u64) { // 26..32 EC: enum(u6) { unknown = 0b000000, + svc = 0b010101, data_abort_lower_el = 0b100100, data_abort_same_el = 0b100101, sp_align = 0b100110, @@ -145,14 +146,16 @@ pub const ESR_EL1 = Register("esr_el1", packed struct(u64) { pub const AsEnum = union(enum) { data_abort: DataAbort, + svc, other, }; pub fn as_enum(self: @This()) AsEnum { - switch (self.EC) { - .data_abort_lower_el, .data_abort_same_el => return .{ .data_abort = @bitCast(self.ISS) }, - else => return .other, - } + return switch (self.EC) { + .data_abort_lower_el, .data_abort_same_el => .{ .data_abort = @bitCast(self.ISS) }, + .svc => .svc, + else => .other, + }; } }); diff --git a/src/arch/aarch64/vectors.S b/src/arch/aarch64/vectors.S index f500b45..1161ad1 100644 --- a/src/arch/aarch64/vectors.S +++ b/src/arch/aarch64/vectors.S @@ -2,11 +2,14 @@ // 32 general-purpose registers .set EXC_GP_SIZE, (32 * 8) -.set EXC_STATE_SIZE, (EXC_GP_SIZE) +// 4 special-purpose registers +.set EXC_SP_SIZE, (4 * 8) +.set EXC_STATE_SIZE, (EXC_GP_SIZE + EXC_SP_SIZE) .macro EXC_SAVE_STATE sub sp, sp, #EXC_STATE_SIZE + // General-purpose block stp x0, x1, [sp, #16 * 0] stp x2, x3, [sp, #16 * 1] stp x4, x5, [sp, #16 * 2] @@ -23,6 +26,45 @@ stp x26, x27, [sp, #16 * 13] stp x28, x29, [sp, #16 * 14] stp x30, x31, [sp, #16 * 15] + + // Special-purpose block + mrs x0, spsr_el1 + mrs x1, elr_el1 + mrs x2, sp_el0 + mov x3, xzr // Padding + + stp x0, x1, [sp, #EXC_GP_SIZE + 16 * 0] + stp x2, x3, [sp, #EXC_GP_SIZE + 16 * 1] +.endm + +.macro EXC_RESTORE_STATE + // Special-purpose block + ldp x0, x1, [sp, #EXC_GP_SIZE + 16 * 0] + ldp x2, x3, [sp, #EXC_GP_SIZE + 16 * 1] + + msr spsr_el1, x0 + msr elr_el1, x1 + msr sp_el0, x2 + + // General-purpose block + ldp x0, x1, [sp, #16 * 0] + ldp x2, x3, [sp, #16 * 1] + ldp x4, x5, [sp, #16 * 2] + ldp x6, x7, [sp, #16 * 3] + ldp x8, x9, [sp, #16 * 4] + ldp x10, x11, [sp, #16 * 5] + ldp x12, x13, [sp, #16 * 6] + ldp x14, x15, [sp, #16 * 7] + ldp x16, x17, [sp, #16 * 8] + ldp x18, x19, [sp, #16 * 9] + ldp x20, x21, [sp, #16 * 10] + ldp x22, x23, [sp, #16 * 11] + ldp x24, x25, [sp, #16 * 12] + ldp x26, x27, [sp, #16 * 13] + ldp x28, x29, [sp, #16 * 14] + ldp x30, x31, [sp, #16 * 15] + + add sp, sp, #EXC_STATE_SIZE .endm // Exception vector size is 0x80 @@ -38,13 +80,22 @@ __aa\bits\()_el\el\ht\()_\kind: // TODO taking exceptions from EL0t 32-bit b . .endif - EXC_SAVE_STATE + + dsb ish + isb sy + mov x0, sp mov lr, xzr bl __aa64_el\el\()_\kind\()_handler - // TODO exception return - b . + + EXC_RESTORE_STATE + + ic iallu + dsb ishst + isb sy + + eret .size __aa\bits\()_el\el\ht\()_\kind, . - __aa\bits\()_el\el\ht\()_\kind .endm diff --git a/src/arch/riscv64.zig b/src/arch/riscv64.zig index deef238..eb4e996 100644 --- a/src/arch/riscv64.zig +++ b/src/arch/riscv64.zig @@ -12,6 +12,12 @@ export const _ = boot.rv64_bsp_lower_entry; /// This CPU's HART (HARdware Thread) ID. pub threadlocal var t_hart_id: u32 = 0; +// Linked as .tdata.assembly to ensure `tp == &this` +pub export threadlocal var t_tdata_assembly: extern struct { + kernel_stack_pointer: usize, // tp + 0x00 + user_stack_pointer: usize, // tp + 0x08 +} linksection(".tdata.assembly") = undefined; + /// RISC-V task context pub const Context = @import("riscv64/context.zig").Context; diff --git a/src/arch/riscv64/context.S b/src/arch/riscv64/context.S index ae04e7b..449b62b 100644 --- a/src/arch/riscv64/context.S +++ b/src/arch/riscv64/context.S @@ -49,12 +49,15 @@ .set SSTATUS_SPIE, (1 << 5) __rv64_task_enter_user: + csrw sscratch, tp // TODO setup user thread pointer ld a0, (sp) // argument ld ra, 16(sp) // entry ld sp, 8(sp) // stack + mv tp, zero + // Clear SPP to zero to indicate a return to U-mode li t1, SSTATUS_SPP not t1, t1 @@ -81,6 +84,7 @@ __rv64_task_enter_kernel: ori t1, t1, SSTATUS_SPP csrw sstatus, t1 csrw sepc, t0 + csrw sscratch, zero sret diff --git a/src/arch/riscv64/context.zig b/src/arch/riscv64/context.zig index 8d8ef63..6c57008 100644 --- a/src/arch/riscv64/context.zig +++ b/src/arch/riscv64/context.zig @@ -3,6 +3,7 @@ const mem = @import("../../mem.zig"); const kernel = @import("../../kernel.zig"); const regs = @import("regs.zig"); const vmm = @import("vmm.zig"); +const riscv64 = @import("../riscv64.zig"); const ProcessAddressSpace = mem.vmm.ProcessAddressSpace; const log = kernel.log; @@ -72,6 +73,7 @@ pub const Context = extern struct { } fn load_state(self: *@This()) void { + riscv64.t_tdata_assembly.kernel_stack_pointer = @intFromPtr(self.kstack.sp); regs.SATP.set(self.satp); } diff --git a/src/arch/riscv64/exception.zig b/src/arch/riscv64/exception.zig index cfcca37..d5b3718 100644 --- a/src/arch/riscv64/exception.zig +++ b/src/arch/riscv64/exception.zig @@ -51,6 +51,13 @@ pub const ExceptionFrame = extern struct { sN: [12]usize, aN: [8]usize, + umode_sp: usize, + sstatus: usize, + sepc: usize, + stval: usize, + scause: usize, + sscratch: usize, + pub fn dump(self: *const @This(), comptime level: log.Level) void { log.writeln(level, " ra = 0x{x:016} gp = 0x{x:016}", .{ self.ra, self.gp }); log.writeln(level, " t0 = 0x{x:016} t1 = 0x{x:016}", .{ self.tN[0], self.tN[1] }); @@ -78,17 +85,38 @@ pub fn init() void { } export fn rv64SmodeTrapGeneral(frame: *ExceptionFrame) callconv(.C) void { - const scause = regs.SCAUSE.read(); + // const scause = regs.SCAUSE.read(); + const scause = @as(regs.SCAUSE.Bits, @bitCast(frame.scause)); if (scause.INTERRUPT) { return rv64SmodeTrapInterrupt(frame); } + const sstatus = @as(regs.SSTATUS.Bits, @bitCast(frame.sstatus)); const cause = @as(ExceptionCause, @enumFromInt(scause.CODE)); - const epc = regs.SEPC.get(); - const tval = regs.STVAL.get(); - log.err("S-mode exception:", .{}); + const is_umode = !sstatus.SPP; + + switch (cause) { + .ecall_umode => { + log.info("ecall from U-mode!!!", .{}); + log.info("args:", .{}); + for (0..6) |i| { + log.info(" a{} = 0x{x:016} ({})", .{ i, frame.aN[i], frame.aN[i] }); + } + + // Add size of `ecall` instruction to return epc to execute the next instruction + // instead of falling back to the same `ecall` instruction that caused this + // interrupt. + frame.sepc += 4; + return; + }, + else => {}, + } + + const mode_str = if (is_umode) "U-mode" else "S-mode"; + + log.err("{s} exception:", .{mode_str}); log.err(" Cause: {s} (0x{x})", .{ cause.name(), scause.CODE }); - log.err(" stval = 0x{x:016} sepc = 0x{x:016}", .{ tval, epc }); + log.err(" stval = 0x{x:016} sepc = 0x{x:016}", .{ frame.stval, frame.sepc }); frame.dump(.err); @panic("Unhandled exception in S-mode"); diff --git a/src/arch/riscv64/vectors.S b/src/arch/riscv64/vectors.S index 2ca96d1..9d07409 100644 --- a/src/arch/riscv64/vectors.S +++ b/src/arch/riscv64/vectors.S @@ -3,8 +3,17 @@ .extern rv64SmodeTrapInterrupt // ra, gp, 7×tN, 12×sN, 8×aN -.set GP_REGS_SIZE, (2 + 7 + 12 + 8) * 8 -.set TRAP_CONTEXT_SIZE, (GP_REGS_SIZE) +.set GP_REGS_SIZE, ((2 + 7 + 12 + 8) * 8) +// U-mode sp, sstatus, sepc, stval, scause, sscratch +.set SP_REGS_SIZE, (6 * 8) +.set TRAP_CONTEXT_SIZE, (GP_REGS_SIZE + SP_REGS_SIZE) + +.set REG_UMODE_SP, (GP_REGS_SIZE + 0 * 8) +.set REG_SSTATUS, (GP_REGS_SIZE + 1 * 8) +.set REG_SEPC, (GP_REGS_SIZE + 2 * 8) +.set REG_STVAL, (GP_REGS_SIZE + 3 * 8) +.set REG_SCAUSE, (GP_REGS_SIZE + 4 * 8) +.set REG_SSCRATCH, (GP_REGS_SIZE + 5 * 8) .macro SAVE_GP_REGS sd ra, 0 * 8(sp) @@ -41,19 +50,110 @@ sd a7, 28 * 8(sp) .endm +.macro RESTORE_GP_REGS + ld ra, 0 * 8(sp) + ld gp, 1 * 8(sp) + + ld t0, 2 * 8(sp) + ld t1, 3 * 8(sp) + ld t2, 4 * 8(sp) + ld t3, 5 * 8(sp) + ld t4, 6 * 8(sp) + ld t5, 7 * 8(sp) + ld t6, 8 * 8(sp) + + ld s0, 9 * 8(sp) + ld s1, 10 * 8(sp) + ld s2, 11 * 8(sp) + ld s3, 12 * 8(sp) + ld s4, 13 * 8(sp) + ld s5, 14 * 8(sp) + ld s6, 15 * 8(sp) + ld s7, 16 * 8(sp) + ld s8, 17 * 8(sp) + ld s9, 18 * 8(sp) + ld s10, 19 * 8(sp) + ld s11, 20 * 8(sp) + + ld a0, 21 * 8(sp) + ld a1, 22 * 8(sp) + ld a2, 23 * 8(sp) + ld a3, 24 * 8(sp) + ld a4, 25 * 8(sp) + ld a5, 26 * 8(sp) + ld a6, 27 * 8(sp) + ld a7, 28 * 8(sp) +.endm + +.set TPREL_KERNEL_STACK, (0 * 8) +.set TPREL_USER_STACK, (1 * 8) + +.set SSTATUS_SPP, (1 << 8) + .macro SMODE_TRAP n, handler .type __rv64_smode_trap_\n, @function __rv64_smode_trap_\n: - // TODO properly handle traps coming from U-mode - // TODO save CSRs - addi sp, sp, -(TRAP_CONTEXT_SIZE) - SAVE_GP_REGS - mv a0, sp + // If coming from U-mode, sscratch = kernel-mode tp + // If coming from S-mode, sscratch = 0 + csrrw tp, sscratch, tp + bnez tp, 1f + // Coming from S-mode + sd sp, TPREL_KERNEL_STACK(tp) // tdata_assembly.kernel_stack = sp + // [fallthrough] +1: + // Coming from U-mode + sd sp, TPREL_USER_STACK(tp) // tdata_assembly.user_stack = sp + ld sp, TPREL_KERNEL_STACK(tp) // sp = tdata_assembly.kernel_stack + // Store pre-trap context + addi sp, sp, -(TRAP_CONTEXT_SIZE) + + SAVE_GP_REGS + // Save special-purpose registers + ld t0, TPREL_USER_STACK(tp) + csrr t1, sstatus + csrr t2, sepc + csrr t3, stval + csrr t4, scause + csrr t5, sscratch + + sd t0, REG_UMODE_SP (sp) + sd t1, REG_SSTATUS (sp) + sd t2, REG_SEPC (sp) + sd t3, REG_STVAL (sp) + sd t4, REG_SCAUSE (sp) + sd t5, REG_SSCRATCH (sp) + + // Reset sscratch to zero to make sure a nested S-mode -> S-mode exception + // happens properly + csrw sscratch, zero + + mv a0, sp call \handler - // TODO return from exception - j . + // Return from exception + ld t0, REG_SSTATUS (sp) + andi t0, t0, SSTATUS_SPP + bnez t0, 2f + + // sstatus.SPP == 0, return to U-mode + // Restore sscratch to its proper value + csrw sscratch, tp + // [fallthrough] +2: + // sstatus.SPP == 1, return to S-mode + ld t0, REG_SSTATUS (sp) + ld t1, REG_SEPC (sp) + csrw sstatus, t0 + csrw sepc, t1 + + // Restore general-purpose registers + RESTORE_GP_REGS + + ld tp, REG_SSCRATCH (sp) + ld sp, REG_UMODE_SP (sp) + + sret .size __rv64_smode_trap_\n, . - __rv64_smode_trap_\n .endm diff --git a/src/kernel.zig b/src/kernel.zig index 7d105cc..76f3510 100644 --- a/src/kernel.zig +++ b/src/kernel.zig @@ -38,14 +38,18 @@ pub export fn kernel_main() callconv(.C) noreturn { const code = switch (comptime arch.cpu) { .riscv64 => &[_]u8{ - 0x93, 0x02, 0xB0, 0x07, // li t0, 123 0x13, 0x01, 0x81, 0xFF, // addi sp, sp, -8 + 0x73, 0x00, 0x00, 0x00, // ecall + 0x73, 0x00, 0x00, 0x00, // ecall + 0x93, 0x02, 0xB0, 0x07, // li t0, 123 0x23, 0x30, 0x51, 0x00, // sd t0, (sp) 0x6F, 0x00, 0x00, 0x00, // j . }, .aarch64 => &[_]u8{ - 0x60, 0x0F, 0x80, 0xD2, // mov x0, #123 0xFF, 0x23, 0x00, 0xD1, // sub sp, sp, #8 + 0x01, 0x00, 0x00, 0xD4, // svc #0 + 0x01, 0x00, 0x00, 0xD4, // svc #0 + 0x60, 0x0F, 0x80, 0xD2, // mov x0, #123 0xE0, 0x03, 0x00, 0xF9, // str x0, [sp] 0x00, 0x00, 0x00, 0x14, // b . },