aarch64: implement exception stubs

src/arch/aarch64.zig

@@ -37,15 +37,15 @@ pub fn spinHint() void {
 pub inline fn barrier(comptime kind: std.builtin.AtomicOrder) void {
     switch (kind) {
         .acquire => {
-            asm volatile ("dsb ishld":::"memory");
+            asm volatile ("dsb ishld" ::: "memory");
         },
         .release => {
-            asm volatile ("dsb ishst":::"memory");
+            asm volatile ("dsb ishst" ::: "memory");
         },
         .acq_rel, .seq_cst => {
-            asm volatile ("dsb ish":::"memory");
+            asm volatile ("dsb ish" ::: "memory");
         },
         .unordered, .monotonic => {},
     }
-    asm volatile ("isb sy":::"memory");
+    asm volatile ("isb sy" ::: "memory");
 }

src/arch/aarch64/boot.zig

@@ -1,6 +1,7 @@
 const kernel = @import("../../kernel.zig");
 const vmm = @import("vmm.zig");
 const dtb = @import("../../util/dtb.zig");
+const exception = @import("exception.zig");
 
 const arch = kernel.arch;
 const mem = kernel.mem;
@@ -45,13 +46,21 @@ fn aa64BspUpperEntry(realAddress: u64) callconv(.C) noreturn {
     arch.barrier(.acq_rel);
 
     log.setWriteFn(&earlyDebugPrint);
-    log.info("Hello, dtb is at 0x{x}", .{gDtbAddress});
+
+    exception.init();
 
     mem.PhysicalAddress.gVirtualizeBase = 0;
     mem.PhysicalAddress.gVirtualizeSize = 16 << 30;
 
     setupMemoryFromFdt(realAddress);
 
+    asm volatile ("" ::: "memory");
+
+    // Test exception handling
+    const p: *const u32 = @ptrFromInt(0x111122223338);
+    const v: u32 = p.*;
+    log.info("v = {}", .{v});
+
     arch.halt();
 }
 

src/arch/aarch64/exception.zig

@@ -0,0 +1,108 @@
+const regs = @import("regs.zig");
+const kernel = @import("../../kernel.zig");
+
+const arch = kernel.arch;
+const log = kernel.debug.log;
+
+extern const __aa64_exception_vectors: u8;
+
+pub const ExceptionFrame = extern struct {
+    xN: [32]usize,
+
+    pub fn dump(self: *const ExceptionFrame, comptime level: log.Level) void {
+        for (0..16) |i| {
+            log.writeln(level, " x{:<2} = 0x{x:016}  x{:<2} = 0x{x:016}", .{
+                i * 2,
+                self.xN[i * 2],
+                i * 2 + 1,
+                self.xN[i * 2 + 1],
+            });
+        }
+    }
+};
+
+pub fn init() void {
+    const vbar_el1 = @intFromPtr(&__aa64_exception_vectors);
+
+    regs.VBAR_EL1.set(vbar_el1);
+}
+
+fn commonIrqHandler(frame: *ExceptionFrame) void {
+    _ = frame;
+    @panic("TODO: IRQ");
+}
+
+// EL1
+// General exceptions
+export fn __aa64_el1_sync_handler(frame: *ExceptionFrame) callconv(.C) void {
+    const esr = regs.ESR_EL1.read();
+    const far = regs.FAR_EL1.get();
+    const elr = regs.ELR_EL1.get();
+
+    log.err("Exception in EL1:", .{});
+    log.err("  EC = {s} (0b{b:06}) ISS = 0x{x}", .{ esr.EC.asStr(), @intFromEnum(esr.EC), esr.ISS });
+    log.err("  ELR = 0x{x:016}", .{elr});
+
+    switch (esr.asEnum()) {
+        .data_abort => |abort| {
+            const faultKindStr = abort.DFSC.asStr();
+            const accessSizeStr = @tagName(abort.SAS);
+            const accessTypeStr = if (abort.WnR) "write" else "read";
+
+            log.err("  Illegal {s} of a {s}: {s}", .{ accessTypeStr, accessSizeStr, faultKindStr });
+            if (!abort.FnV) {
+                log.err("  FAR = 0x{x:016}", .{far});
+            } else {
+                log.err("  (FAR is not valid)", .{});
+            }
+        },
+        else => {},
+    }
+
+    frame.dump(log.Level.err);
+    arch.halt();
+}
+
+// IRQ
+export fn __aa64_el1_irq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    commonIrqHandler(frame);
+}
+
+export fn __aa64_el1_fiq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    // TODO I've never used FIQ
+    arch.halt();
+}
+
+export fn __aa64_el1_serror_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    // TODO
+    arch.halt();
+}
+
+// EL0
+export fn __aa64_el0_sync_handler(frame: *ExceptionFrame) callconv(.C) void {
+    // TODO EL0
+    _ = frame;
+    arch.halt();
+}
+
+export fn __aa64_el0_irq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    commonIrqHandler(frame);
+}
+
+export fn __aa64_el0_fiq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    // TODO I've never used FIQ
+    arch.halt();
+}
+
+export fn __aa64_el0_serror_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    // TODO
+    arch.halt();
+}
+
+comptime {
+    asm (@embedFile("vectors.S"));
+}

src/arch/aarch64/regs.zig

@@ -1,3 +1,5 @@
+const std = @import("std");
+
 fn Register(comptime name: []const u8, comptime bits: type) type {
     const repr = switch (@typeInfo(bits)) {
         .@"struct" => |s| s.backing_integer.?,
@@ -5,11 +7,16 @@ fn Register(comptime name: []const u8, comptime bits: type) type {
     };
     return enum(repr) {
         pub fn set(value: repr) void {
-            asm volatile ("msr " ++ name ++ ", %[value]"::[value]"r"(value));
+            asm volatile ("msr " ++ name ++ ", %[value]"
+                :
+                : [value] "r" (value),
+            );
         }
 
         pub fn get() repr {
-            return asm volatile ("mrs %[value], " ++ name:[value]"=r"(-> repr));
+            return asm volatile ("mrs %[value], " ++ name
+                : [value] "=r" (-> repr),
+            );
         }
 
         pub fn write(value: bits) void {
@@ -29,6 +36,97 @@ fn Register(comptime name: []const u8, comptime bits: type) type {
 
 pub const TTBR0_EL1 = Register("ttbr0_el1", u64);
 pub const TTBR1_EL1 = Register("ttbr1_el1", u64);
+pub const VBAR_EL1 = Register("vbar_el1", u64);
+pub const ELR_EL1 = Register("elr_el1", u64);
+pub const FAR_EL1 = Register("far_el1", u64);
+
+pub const ESR_EL1 = Register("esr_el1", packed struct(u64) {
+    // 0..25
+    ISS: u25 = 0,
+    // 25
+    IL: bool = false,
+    // 26..32
+    EC: enum(u6) {
+        unknown = 0b000000,
+        data_abort_lower_el = 0b100100,
+        data_abort_same_el = 0b100101,
+        sp_align = 0b100110,
+        _,
+
+        pub fn asStr(self: @This()) []const u8 {
+            return std.enums.tagName(@This(), self) orelse "<unknown>";
+        }
+    } = .unknown,
+    // 32..64
+    _0: u32 = 0,
+
+    // Specific variants of ESR
+    pub const DataAbort = packed struct(u25) {
+        // 0..6
+        DFSC: enum(u6) {
+            address_size_fault_l0 = 0b000000,
+            address_size_fault_l1 = 0b000001,
+            address_size_fault_l2 = 0b000010,
+            address_size_fault_l3 = 0b000011,
+            translation_fault_l0 = 0b000100,
+            translation_fault_l1 = 0b000101,
+            translation_fault_l2 = 0b000110,
+            translation_fault_l3 = 0b000111,
+            access_flag_fault_l1 = 0b001001,
+            access_flag_fault_l2 = 0b001010,
+            access_flag_fault_l3 = 0b001011,
+            permission_fault_l1 = 0b001101,
+            permission_fault_l2 = 0b001110,
+            permission_fault_l3 = 0b001111,
+            _,
+
+            pub fn asStr(self: @This()) []const u8 {
+                return std.enums.tagName(@This(), self) orelse "<other>";
+            }
+        } = .address_size_fault_l0,
+        // 6
+        WnR: bool = false,
+        // 7
+        S1PTW: bool = false,
+        // 8
+        CM: bool = false,
+        // 9
+        EA: bool = false,
+        // 10
+        FnV: bool = false,
+        // 11..14
+        _0: u3 = 0,
+        // 14
+        AR: bool = false,
+        // 15
+        SF: bool = false,
+        // 16..21
+        SRT: u5 = 0,
+        // 21
+        SSE: bool = false,
+        // 22..24
+        SAS: enum(u2) {
+            byte = 0,
+            half = 1,
+            word = 2,
+            dword = 3,
+        } = .byte,
+        // 24
+        ISV: bool = false,
+    };
+
+    pub const AsEnum = union(enum) {
+        data_abort: DataAbort,
+        other,
+    };
+
+    pub fn asEnum(self: @This()) AsEnum {
+        switch (self.EC) {
+            .data_abort_lower_el, .data_abort_same_el => return .{ .data_abort = @bitCast(self.ISS) },
+            else => return .other,
+        }
+    }
+});
 
 pub const Cacheability = enum(u2) {
     non_cacheable = 0,
@@ -37,19 +135,9 @@ pub const Cacheability = enum(u2) {
     writeback_readalloc_nowritealloc_cacheable = 3,
 };
 
-pub const Shareability = enum(u2) {
+pub const Shareability = enum(u2) { non_shareable = 0, outer_shareable = 1, inner_shareable = 2, _ };
-    non_shareable = 0,
-    outer_shareable = 1,
-    inner_shareable = 2,
-    _
-};
 
-pub const TranslationGranule = enum(u2) {
+pub const TranslationGranule = enum(u2) { kib_4 = 0, kib_64 = 1, kib_16 = 2, _ };
-    kib_4 = 0,
-    kib_64 = 1,
-    kib_16 = 2,
-    _
-};
 
 pub const TCR_EL1 = Register("tcr_el1", packed struct(u64) {
     // 0..6
@@ -84,11 +172,7 @@ pub const TCR_EL1 = Register("tcr_el1", packed struct(u64) {
     // 30..32
     TG1: TranslationGranule = .kib_4,
     // 32..35
-    IPS: enum(u3) {
+    IPS: enum(u3) { bits_32 = 0b000, bits_48 = 0b101, _ } = .bits_32,
-        bits_32 = 0b000,
-        bits_48 = 0b101,
-        _
-    } = .bits_32,
     // 35
     _1: bool = false,
     // 36
@@ -104,7 +188,7 @@ pub const TCR_EL1 = Register("tcr_el1", packed struct(u64) {
     _2: u25 = 0,
 });
 
-pub const SCTLR_EL1 = Register("sctlr_el1", packed struct (u64) {
+pub const SCTLR_EL1 = Register("sctlr_el1", packed struct(u64) {
     // 0
     M: bool = false,
     // 1

src/arch/aarch64/vectors.S

@@ -0,0 +1,96 @@
+.global __aa64_exception_vectors
+
+// 32 general-purpose registers
+.set EXC_GP_SIZE, (32 * 8)
+.set EXC_STATE_SIZE, (EXC_GP_SIZE)
+
+.macro EXC_SAVE_STATE
+    sub sp, sp, #EXC_STATE_SIZE
+
+    stp  x0,  x1, [sp, #16 *  0]
+    stp  x2,  x3, [sp, #16 *  1]
+    stp  x4,  x5, [sp, #16 *  2]
+    stp  x6,  x7, [sp, #16 *  3]
+    stp  x8,  x9, [sp, #16 *  4]
+    stp x10, x11, [sp, #16 *  5]
+    stp x12, x13, [sp, #16 *  6]
+    stp x14, x15, [sp, #16 *  7]
+    stp x16, x17, [sp, #16 *  8]
+    stp x18, x19, [sp, #16 *  9]
+    stp x20, x21, [sp, #16 * 10]
+    stp x22, x23, [sp, #16 * 11]
+    stp x24, x25, [sp, #16 * 12]
+    stp x26, x27, [sp, #16 * 13]
+    stp x28, x29, [sp, #16 * 14]
+    stp x30, x31, [sp, #16 * 15]
+.endm
+
+// Exception vector size is 0x80
+.macro EXC_VECTOR el, ht, bits, kind
+.p2align 7
+    b __aa\bits\()_el\el\ht\()_\kind
+.endm
+
+.macro EXC_HANDLER el, ht, bits, kind
+.type __aa\bits\()_el\el\ht\()_\kind, %function
+__aa\bits\()_el\el\ht\()_\kind:
+.if \bits == 32
+    // TODO taking exceptions from EL0t 32-bit
+    b .
+.endif
+
+    EXC_SAVE_STATE
+    mov x0, sp
+    mov lr, xzr
+    bl __aa64_el\el\()_\kind\()_handler
+    // TODO exception return
+    b .
+.size __aa\bits\()_el\el\ht\()_\kind, . - __aa\bits\()_el\el\ht\()_\kind
+.endm
+
+.pushsection .text
+.p2align 12
+.type __aa64_exception_vectors, %object
+__aa64_exception_vectors:
+    EXC_VECTOR 1, t, 64, sync
+    EXC_VECTOR 1, t, 64, irq
+    EXC_VECTOR 1, t, 64, fiq
+    EXC_VECTOR 1, t, 64, serror
+
+    EXC_VECTOR 1, h, 64, sync
+    EXC_VECTOR 1, h, 64, irq
+    EXC_VECTOR 1, h, 64, fiq
+    EXC_VECTOR 1, h, 64, serror
+
+    EXC_VECTOR 0, t, 64, sync
+    EXC_VECTOR 0, t, 64, irq
+    EXC_VECTOR 0, t, 64, fiq
+    EXC_VECTOR 0, t, 64, serror
+
+    EXC_VECTOR 0, t, 32, sync
+    EXC_VECTOR 0, t, 32, irq
+    EXC_VECTOR 0, t, 32, fiq
+    EXC_VECTOR 0, t, 32, serror
+.size __aa64_exception_vectors, . - __aa64_exception_vectors
+
+.p2align 7
+    EXC_HANDLER 1, t, 64, sync
+    EXC_HANDLER 1, t, 64, irq
+    EXC_HANDLER 1, t, 64, fiq
+    EXC_HANDLER 1, t, 64, serror
+
+    EXC_HANDLER 1, h, 64, sync
+    EXC_HANDLER 1, h, 64, irq
+    EXC_HANDLER 1, h, 64, fiq
+    EXC_HANDLER 1, h, 64, serror
+
+    EXC_HANDLER 0, t, 64, sync
+    EXC_HANDLER 0, t, 64, irq
+    EXC_HANDLER 0, t, 64, fiq
+    EXC_HANDLER 0, t, 64, serror
+
+    EXC_HANDLER 0, t, 32, sync
+    EXC_HANDLER 0, t, 32, irq
+    EXC_HANDLER 0, t, 32, fiq
+    EXC_HANDLER 0, t, 32, serror
+.popsection // .text

src/arch/riscv64/boot.zig

@@ -99,7 +99,7 @@ fn setupPerCpu() void {
     const tlsAddress = physMemory.alloc_pages(tlsPageCount).?.virtualize();
     const tlsData = @as([*]u8, @ptrFromInt(tlsAddress))[0..tlsSize];
 
-    log.info("Allocated TLS @ {*}", .{ tlsData });
+    log.info("Allocated TLS @ {*}", .{tlsData});
 
     @memcpy(tlsData[0..tdataSize], tdataData);
     @memset(tlsData[tdataSize..], 0);

src/arch/riscv64/exception.zig

@@ -56,7 +56,7 @@ pub const ExceptionFrame = extern struct {
         log.writeln(level, " t0 = 0x{x:016}   t1 = 0x{x:016}", .{ self.tN[0], self.tN[1] });
         log.writeln(level, " t2 = 0x{x:016}   t3 = 0x{x:016}", .{ self.tN[2], self.tN[3] });
         log.writeln(level, " t4 = 0x{x:016}   t5 = 0x{x:016}", .{ self.tN[4], self.tN[5] });
-        log.writeln(level, " t6 = 0x{x:016}", .{ self.tN[6] });
+        log.writeln(level, " t6 = 0x{x:016}", .{self.tN[6]});
         log.writeln(level, " s0 = 0x{x:016}   s1 = 0x{x:016}", .{ self.sN[0], self.sN[1] });
         log.writeln(level, " s2 = 0x{x:016}   s1 = 0x{x:016}", .{ self.sN[2], self.sN[3] });
         log.writeln(level, " s4 = 0x{x:016}   s6 = 0x{x:016}", .{ self.sN[4], self.sN[5] });
@@ -74,7 +74,7 @@ pub fn init() void {
         .BASE = @intCast(base >> 2),
     });
 
-    asm volatile ("":::"memory");
+    asm volatile ("" ::: "memory");
 }
 
 export fn rv64SmodeTrapGeneral(frame: *ExceptionFrame) callconv(.C) void {

src/arch/riscv64/sbi.zig

@@ -39,13 +39,12 @@ const SbiResult = union(enum) {
 fn sbiCall1(ext: SbiExtension, func: u64, arg0: u64) SbiResult {
     var a0: u64 = undefined;
     var a1: u64 = undefined;
-    asm volatile (
+    asm volatile ("ecall"
-        "ecall"
         : [ret0] "={a0}" (a0),
           [ret1] "={a1}" (a1),
         : [arg0] "{a0}" (arg0),
           [func] "{a6}" (func),
-          [extn] "{a7}" (ext)
+          [extn] "{a7}" (ext),
         : "a2", "a3", "a4", "a5"
     );
     return SbiResult.fromSbi(a0, a1);

src/arena.zig

@@ -9,16 +9,12 @@ pub const Arena = struct {
 
     pub fn init(cap: usize) ?Arena {
         const physBase = physMemory.alloc_pages(cap / mem.vmm.PAGE_SIZE) orelse return null;
-        return .{
+        return .{ .physBase = physBase, .capacity = cap, .len = 0 };
-            .physBase = physBase,
-            .capacity = cap,
-            .len = 0
-        };
     }
 
     pub fn create(self: *@This(), comptime T: type) *T {
         if (self.len + @sizeOf(T) > self.capacity) {
-            log.panic("Out of memory. Cannot allocate {} bytes", .{ @sizeOf(T) });
+            log.panic("Out of memory. Cannot allocate {} bytes", .{@sizeOf(T)});
         }
 
         const v = self.physBase.add(self.len).virtualize();

src/debug.zig

@@ -10,10 +10,8 @@ pub const log = struct {
         err,
     };
 
-    var writeFn: *const fn(u8) void = dummyWrite;
+    var writeFn: *const fn (u8) void = dummyWrite;
-    const writer: std.io.GenericWriter(u0, error{}, writeWrapperFn) = .{
+    const writer: std.io.GenericWriter(u0, error{}, writeWrapperFn) = .{ .context = 0 };
-        .context = 0
-    };
 
     fn writeWrapperFn(context: u0, data: []const u8) error{}!usize {
         _ = context;
@@ -23,7 +21,7 @@ pub const log = struct {
         return data.len;
     }
 
-    pub fn setWriteFn(f: *const fn(u8) void) void {
+    pub fn setWriteFn(f: *const fn (u8) void) void {
         writeFn = f;
     }
 

src/mem.zig

@@ -60,10 +60,7 @@ pub fn formatSize(buffer: []u8, size: u64) []const u8 {
 
         if (iLen < buffer.len + 1) {
             buffer[iLen] = '.';
-            fLen = 1 + std.fmt.formatIntBuf(buffer[iLen + 1..], fractional, 10, .lower, .{
+            fLen = 1 + std.fmt.formatIntBuf(buffer[iLen + 1 ..], fractional, 10, .lower, .{ .fill = '0', .width = 2 });
-                .fill = '0',
-                .width = 2
-            });
         }
     }
 

src/util/dtb.zig

@@ -404,7 +404,7 @@ pub const Fdt = struct {
                 }
             }
             if (found) |f| {
-                log.info("{s}", .{ element });
+                log.info("{s}", .{element});
                 currentNode = f;
             } else {
                 return null;