AArch64 basic boot and upper reloc

build.zig

@@ -7,14 +7,35 @@ const SupportedArch = enum {
     riscv64,
 
     fn makeTarget(self: SupportedArch, b: *std.Build) std.Build.ResolvedTarget {
+        switch (self) {
+            .riscv64 => {
                 return b.resolveTargetQuery(.{
-            .cpu_arch = switch (self) {
-                .riscv64 => .riscv64,
-                .aarch64 => .aarch64,
-            },
+                    .cpu_arch = .riscv64,
                     .os_tag = .freestanding,
                     .abi = .none,
                 });
+            },
+            .aarch64 => {
+                const T = std.Target.aarch64;
+
+                const addFeatures = T.featureSet(&.{
+                    T.Feature.v8a,
+                    T.Feature.strict_align,
+                });
+                const subFeatures = T.featureSet(&.{
+                    T.Feature.neon,
+                    T.Feature.fp_armv8,
+                });
+
+                return b.resolveTargetQuery(.{
+                    .cpu_arch = .aarch64,
+                    .os_tag = .freestanding,
+                    .abi = .none,
+                    .cpu_features_add = addFeatures,
+                    .cpu_features_sub = subFeatures,
+                });
+            },
+        }
     }
 
     fn addTargetSpecific(self: SupportedArch, b: *std.Build, kernel: *std.Build.Step.Compile) anyerror!*std.Build.Step {

etc/aarch64.lldb

@@ -3,6 +3,7 @@ gdb-remote localhost:1234
 target modules add zig-out/bin/kernel
 # target modules load -f zig-out/bin/kernel -s 0xFFFFFFF000200000
 target modules load -f zig-out/bin/kernel -s 0x40080000
+target modules load -f zig-out/bin/kernel -s 0xFFFFFF8040080000
 
 # breakpoint set -H -a 0x80200000
 # process continue

src/arch/aarch64.zig

@@ -1,3 +1,4 @@
+const std = @import("std");
 const boot = @import("aarch64/boot.zig");
 
 export const _ = boot.aa64BspLowerEntry;
@@ -32,3 +33,19 @@ pub fn halt() noreturn {
 pub fn spinHint() void {
     // TODO
 }
+
+pub inline fn barrier(comptime kind: std.builtin.AtomicOrder) void {
+    switch (kind) {
+        .acquire => {
+            asm volatile ("dsb ishld":::"memory");
+        },
+        .release => {
+            asm volatile ("dsb ishst":::"memory");
+        },
+        .acq_rel, .seq_cst => {
+            asm volatile ("dsb ish":::"memory");
+        },
+        .unordered, .monotonic => {},
+    }
+    asm volatile ("isb sy":::"memory");
+}

src/arch/aarch64/boot.zig

@@ -1,3 +1,88 @@
-pub export fn aa64BspLowerEntry() callconv(.C) noreturn {
-    while (true) {}
+const kernel = @import("../../kernel.zig");
+const vmm = @import("vmm.zig");
+
+const arch = kernel.arch;
+const log = kernel.debug.log;
+
+extern const __aa64_bsp_stack_top: u8;
+
+var gDtbAddress: u64 = undefined;
+
+fn earlyDebugPrint(byte: u8) void {
+    const address = 0x9000000;
+    @as(*volatile u32, @ptrFromInt(address)).* = byte;
+}
+
+fn relocAddressToUpper(ptr: *const anyopaque) usize {
+    const p = @intFromPtr(ptr);
+    if (p >= vmm.KERNEL_VIRTUAL_BASE) {
+        return p;
+    } else {
+        return p + vmm.KERNEL_VIRTUAL_BASE;
+    }
+}
+
+fn aa64BspUpperEntry(realAddress: u64) callconv(.C) noreturn {
+    // Relocate the kernel yet again
+    const relaStart = relocAddressToUpper(&__rela_start);
+    const relaEnd = relocAddressToUpper(&__rela_end);
+    const relOffset = vmm.KERNEL_VIRTUAL_BASE + realAddress;
+
+    arch.barrier(.acq_rel);
+    aa64RelocateKernel(relOffset, relaStart, relaEnd);
+    arch.barrier(.acq_rel);
+
+    log.setWriteFn(&earlyDebugPrint);
+    log.info("Hello, dtb is at 0x{x}", .{ gDtbAddress });
+
+    arch.halt();
+}
+
+pub export fn aa64BspLowerEntry(realAddress: u64, dtbAddress: u64) callconv(.C) noreturn {
+    gDtbAddress = dtbAddress;
+
+    vmm.mapEarly(realAddress);
+
+    const pc = @intFromPtr(&aa64BspUpperEntry) + vmm.KERNEL_VIRTUAL_BASE;
+    const sp = @intFromPtr(&__aa64_bsp_stack_top) + vmm.KERNEL_VIRTUAL_BASE;
+
+    longJump(pc, sp, realAddress);
+}
+
+// Functions used by the boot process
+
+extern const __rela_start: u8;
+extern const __rela_end: u8;
+
+export fn aa64RelocateKernel(imageBase: usize, relaStart: usize, relaEnd: usize) void {
+    const elf = @import("std").elf;
+
+    const relaTablePtr = @as([*]elf.Rela, @ptrFromInt(relaStart));
+    const relaCount = (relaEnd - relaStart) / @sizeOf(elf.Rela);
+    const relaTable = relaTablePtr[0..relaCount];
+    for (relaTable) |entry| {
+        const relaType: elf.R_AARCH64 = @enumFromInt(entry.r_type());
+        switch (relaType) {
+            .RELATIVE => {
+                const value = @as(*isize, @ptrFromInt(imageBase + entry.r_offset));
+                value.* = @as(isize, @bitCast(imageBase)) + entry.r_addend;
+            },
+            else => {
+                arch.halt();
+            },
+        }
+    }
+}
+
+inline fn longJump(pc: usize, sp: usize, a0: usize) noreturn {
+    asm volatile (
+        \\ mov sp, %[sp]
+        \\ br %[pc]
+        :
+        : [sp]"r"(sp),
+          [pc]"r"(pc),
+          [a0]"{x0}"(a0),
+        :"memory"
+    );
+    unreachable;
 }

src/arch/aarch64/entry.S

@@ -1,7 +1,11 @@
 .global __aa64_entry
+.global __aa64_bsp_stack_top
 
 .pushsection .text.entry
 __aa64_entry:
+99: adr x19, 99b
+    mov x20, x0
+
     // x0 -- DTB physical address
     mrs x1, mpidr_el1
     ands x1, x1, #0xF
@@ -34,6 +38,13 @@ __aa64_entry:
     adr x1, __aa64_bsp_stack_top
     mov sp, x1
 
+    mov x0, x19
+    adr x1, __rela_start
+    adr x2, __rela_end
+    bl aa64RelocateKernel
+
+    mov x0, x19
+    mov x1, x20
     b aa64BspLowerEntry
 
 .parking_lot:

src/arch/aarch64/regs.zig

@@ -0,0 +1,138 @@
+fn makeRegister(comptime name: []const u8, comptime bits: type) type {
+    const repr = switch (@typeInfo(bits)) {
+        .@"struct" => |s| s.backing_integer.?,
+        else => bits,
+    };
+    return enum(repr) {
+        pub fn set(value: repr) void {
+            asm volatile ("msr " ++ name ++ ", %[value]"::[value]"r"(value));
+        }
+
+        pub fn get() repr {
+            return asm volatile ("mrs %[value], " ++ name:[value]"=r"(-> repr));
+        }
+
+        pub fn write(value: bits) void {
+            set(@bitCast(value));
+        }
+
+        pub fn modify(s: bits, c: bits) void {
+            const v = get();
+            set((v & ~@as(repr, @bitCast(c))) | @as(repr, @bitCast(s)));
+        }
+
+        pub fn read() bits {
+            return @bitCast(get());
+        }
+    };
+}
+
+pub const TTBR0_EL1 = makeRegister("ttbr0_el1", u64);
+pub const TTBR1_EL1 = makeRegister("ttbr1_el1", u64);
+
+pub const Cacheability = enum(u2) {
+    non_cacheable = 0,
+    writeback_readalloc_writealloc_cacheable = 1,
+    writethrough_readalloc_nowritealloc_cacheable = 2,
+    writeback_readalloc_nowritealloc_cacheable = 3,
+};
+
+pub const Shareability = enum(u2) {
+    non_shareable = 0,
+    outer_shareable = 1,
+    inner_shareable = 2,
+    _
+};
+
+pub const TranslationGranule = enum(u2) {
+    kib_4 = 0,
+    kib_64 = 1,
+    kib_16 = 2,
+    _
+};
+
+pub const TCR_EL1 = makeRegister("tcr_el1", packed struct(u64) {
+    // 0..6
+    T0SZ: u6 = 0,
+    // 6
+    _0: bool = false,
+    // 7
+    EPD0: bool = false,
+    // 8..10
+    IRGN0: Cacheability = .non_cacheable,
+    // 10..12
+    ORGN0: Cacheability = .non_cacheable,
+    // 12..14
+    SH0: Shareability = .non_shareable,
+    // 14..16
+    TG0: TranslationGranule = .kib_4,
+    // 16..22
+    T1SZ: u6 = 0,
+    // 22
+    A1: enum(u1) {
+        ttbr0 = 0,
+        ttbr1 = 1,
+    } = .ttbr0,
+    // 23
+    EPD1: bool = false,
+    // 24..26
+    IRGN1: Cacheability = .non_cacheable,
+    // 26..28
+    ORGN1: Cacheability = .non_cacheable,
+    // 28..30
+    SH1: Shareability = .non_shareable,
+    // 30..32
+    TG1: TranslationGranule = .kib_4,
+    // 32..35
+    IPS: enum(u3) {
+        bits_32 = 0b000,
+        bits_48 = 0b101,
+        _
+    } = .bits_32,
+    // 35
+    _1: bool = false,
+    // 36
+    AS: enum(u1) {
+        asid_8bit = 0,
+        asid_16bit = 1,
+    } = .asid_8bit,
+    // 37
+    TBI0: bool = false,
+    // 38
+    TBI1: bool = false,
+    // 39..64
+    _2: u25 = 0,
+});
+
+pub const SCTLR_EL1 = makeRegister("sctlr_el1", packed struct (u64) {
+    // 0
+    M: bool = false,
+    // 1
+    A: bool = false,
+    // 2
+    C: bool = false,
+    // 3
+    SA: bool = false,
+    // 4
+    SA0: bool = false,
+    // 5
+    CP15BEN: bool = false,
+    // 6
+    nAA: bool = false,
+    // 7
+    ITD: bool = false,
+    // 8
+    SED: bool = false,
+    // 9
+    UMA: bool = false,
+    // 10
+    EnRCTX: bool = false,
+    // 11
+    EOS: bool = false,
+    // 12
+    I: bool = false,
+    // 13
+    EnDB: bool = false,
+    // 14..64
+    _0: u50 = 0,
+});

src/arch/aarch64/vmm.zig

@@ -0,0 +1,214 @@
+const mem = @import("../../mem.zig");
+const regs = @import("regs.zig");
+
+const PhysicalAddress = mem.PhysicalAddress;
+
+pub const KERNEL_VIRTUAL_BASE: usize = 0xFFFFFF8000000000;
+pub const KERNEL_L1_INDEX: usize = L1.index(KERNEL_VIRTUAL_BASE);
+
+pub const L1 = mem.translationLevel(30);
+pub const L2 = mem.translationLevel(21);
+pub const L3 = mem.translationLevel(12);
+
+pub const RawEntry = packed struct(u64) {
+    // 0
+    V: bool = false,
+    // 1, Table/page if true, block if 0
+    P: bool = false,
+    // 2..5
+    ATTR: enum(u3) { normal = 0, _ } = .normal,
+    // 5
+    NS: bool = false,
+    // 6..8
+    AP: enum(u2) {
+        kernel_readwrite = 0,
+        both_readwrite = 1,
+        kernel_readonly = 2,
+        both_readonly = 3,
+    } = .kernel_readwrite,
+    // 8..10
+    SH: enum(u2) { outer_shareable = 2, inner_shareable = 3, _ } = .outer_shareable,
+    // 10
+    AF: bool = false,
+    // 11
+    NG: bool = false,
+    // 12..48
+    PPN: u36 = 0,
+    // 48..51
+    _0: u3 = 0,
+    // 51
+    DBM: bool = false,
+    // 52
+    _1: bool = false,
+    // 53
+    PXN: bool = false,
+    // 54
+    UXN: bool = false,
+    // 55..64
+    _2: u9 = 0,
+
+    pub fn makeUnion(self: @This(), other: @This()) @This() {
+        const lhs = @as(u64, @bitCast(self));
+        const rhs = @as(u64, @bitCast(other));
+        return @as(@This(), @bitCast(lhs | rhs));
+    }
+
+    pub fn clear(self: *@This(), mask: @This()) void {
+        const lhs = @as(*u64, @bitCast(self));
+        const rhs = @as(u64, @bitCast(mask));
+        lhs.* &= ~rhs;
+    }
+};
+
+pub fn TableEntry(comptime Level: type) type {
+    _ = Level;
+    return struct {
+        raw: RawEntry,
+
+        pub const INVALID: @This() = .{ .raw = .{} };
+
+        pub fn address(self: @This()) PhysicalAddress {
+            if (self.raw.V) {
+                return .{ .raw = self.raw.PPN << 12 };
+            } else {
+                return PhysicalAddress.NULL;
+            }
+        }
+
+        pub fn bits(self: @This()) u64 {
+            return @as(u64, @bitCast(self.raw));
+        }
+
+        pub fn normal_page(addr: PhysicalAddress, flags: RawEntry) @This() {
+            return .{
+                .raw = flags.makeUnion(RawEntry{
+                    .PPN = @as(u36, @intCast(addr.raw >> 12)),
+                    .V = true,
+                    .P = true,
+                    .AF = true,
+                    .ATTR = .normal,
+                    .SH = .outer_shareable,
+                }),
+            };
+        }
+
+        pub fn device_page(addr: PhysicalAddress, flags: RawEntry) @This() {
+            return .{
+                .raw = flags.makeUnion(RawEntry{
+                    .PPN = @as(u36, @intCast(addr.raw >> 12)),
+                    .V = true,
+                    .P = true,
+                    .AF = true,
+                    .ATTR = .device,
+                    .SH = .outer_shareable,
+                    .PXN = true,
+                    .UXN = true,
+                }),
+            };
+        }
+
+        pub fn normal_block(addr: PhysicalAddress, flags: RawEntry) @This() {
+            return .{
+                .raw = flags.makeUnion(RawEntry{
+                    .PPN = @as(u36, @intCast(addr.raw >> 12)),
+                    .V = true,
+                    .AF = true,
+                    .ATTR = .normal,
+                    .SH = .outer_shareable,
+                }),
+            };
+        }
+
+        pub fn table(addr: PhysicalAddress, flags: RawEntry) @This() {
+            return .{
+                .raw = flags.makeUnion(.{
+                    .PPN = @as(u36, @intCast(addr.raw >> 12)),
+                    .V = true,
+                    .P = true,
+                }),
+            };
+        }
+    };
+}
+
+pub fn Table(comptime Level: type) type {
+    return struct {
+        pub const Entry = TableEntry(Level);
+
+        entries: [512]Entry align(4096) = [_]Entry{.INVALID} ** 512,
+
+        pub inline fn entry(self: *@This(), index: usize) *Entry {
+            return &self.entries[index];
+        }
+    };
+}
+
+// 0x0000_0000_0000_0000 .. 0x0000_0080_0000_0000
+var gFixedLow = Table(L1){};
+// 0xFFFF_FF80_0000_0000 .. 0xFFFF_FFFF_FFFF_FFFF
+var gFixedHigh = Table(L1){};
+
+pub fn mapEarly(realAddress: usize) void {
+    _ = realAddress;
+
+    for (0..16) |i| {
+        // Identity
+        gFixedLow.entry(i).* = TableEntry(L1).normal_block(
+            .{ .raw = i << L1.SHIFT },
+            .{},
+        );
+    }
+
+    for (0..16) |i| {
+        // Identity + KERNEL_VIRTUAL_BASE
+        gFixedHigh.entry(i).* = TableEntry(L1).normal_block(
+            .{ .raw = i << L1.SHIFT },
+            .{},
+        );
+    }
+
+    const ttbr0 = @intFromPtr(&gFixedLow);
+    const ttbr1 = @intFromPtr(&gFixedHigh);
+
+    regs.TTBR0_EL1.set(ttbr0);
+    regs.TTBR1_EL1.set(ttbr1);
+
+    regs.TCR_EL1.write(.{
+        .AS = .asid_8bit,
+        .A1 = .ttbr0,
+        .IPS = .bits_48,
+        .TG0 = .kib_4,
+        .T0SZ = 25,
+        .SH0 = .inner_shareable,
+        .TG1 = .kib_4,
+        .T1SZ = 25,
+        .SH1 = .inner_shareable,
+    });
+
+    asm volatile ("dsb ishst; isb sy" ::: "memory");
+
+    var sctlr_el1 = regs.SCTLR_EL1.read();
+    sctlr_el1.SA0 = true;
+    sctlr_el1.SA = true;
+    sctlr_el1.A = true;
+    // Disable caches for now
+    sctlr_el1.I = false;
+    sctlr_el1.C = false;
+    regs.SCTLR_EL1.write(sctlr_el1);
+
+    asm volatile ("dsb ish; isb sy" ::: "memory");
+
+    // Enable translation
+    sctlr_el1.M = true;
+    regs.SCTLR_EL1.write(sctlr_el1);
+
+    asm volatile ("isb sy" ::: "memory");
+    sctlr_el1.I = true;
+    regs.SCTLR_EL1.write(sctlr_el1);
+    asm volatile ("dsb ish; isb sy" ::: "memory");
+
+    asm volatile ("dsb ishst; isb sy" ::: "memory");
+    sctlr_el1.C = true;
+    regs.SCTLR_EL1.write(sctlr_el1);
+    asm volatile ("dsb ish; isb sy" ::: "memory");
+}