WIP: test objects

Use PFNs instead of raw physical addresses for more clarity.

.gitignore

@@ -1,2 +1,5 @@
 /zig-out
 /.zig-cache
+/*.dtb
+/*.dts
+/*.log

abi/abi.zig

@@ -0,0 +1,21 @@
+pub const SyscallNumber = enum(usize) {
+    SYS_send = 1,
+    SYS_recv = 2,
+    SYS_sendrecv = 3,
+    _,
+};
+
+pub const ProcessObjectAction = enum(usize) {
+    ZO_process_exit = 1,
+    _,
+};
+
+pub const PhysicalMemoryObjectAction = enum(usize) {
+    ZO_physical_memory_allocate = 1,
+    ZO_physical_memory_free = 2,
+    _,
+};
+
+pub const MAX_SYSCALL: usize = 32;
+
+pub const Handle = enum(u32) { _ };

build.zig

@@ -6,7 +6,7 @@ const SupportedArch = enum {
     aarch64,
     riscv64,
 
-    fn makeTarget(self: SupportedArch, b: *std.Build) std.Build.ResolvedTarget {
+    fn make_kernel_target(self: SupportedArch, b: *std.Build) std.Build.ResolvedTarget {
         switch (self) {
             .riscv64 => {
                 return b.resolveTargetQuery(.{
@@ -18,11 +18,11 @@ const SupportedArch = enum {
             .aarch64 => {
                 const T = std.Target.aarch64;
 
-                const addFeatures = T.featureSet(&.{
+                const add_features = T.featureSet(&.{
                     T.Feature.v8a,
                     T.Feature.strict_align,
                 });
-                const subFeatures = T.featureSet(&.{
+                const sub_features = T.featureSet(&.{
                     T.Feature.neon,
                     T.Feature.fp_armv8,
                 });
@@ -31,14 +31,54 @@ const SupportedArch = enum {
                     .cpu_arch = .aarch64,
                     .os_tag = .freestanding,
                     .abi = .none,
-                    .cpu_features_add = addFeatures,
-                    .cpu_features_sub = subFeatures,
+                    .cpu_features_add = add_features,
+                    .cpu_features_sub = sub_features,
                 });
             },
         }
     }
 
-    fn addTargetSpecific(self: SupportedArch, b: *std.Build, kernel: *std.Build.Step.Compile) anyerror!*std.Build.Step {
+    fn make_userspace_target(self: SupportedArch, b: *std.Build) std.Build.ResolvedTarget {
+        // TODO: it's the same for now, until userspace support for CPU extensions like floating
+        //       point is added.
+        return self.make_kernel_target(b);
+    }
+
+    fn configure_user(
+        self: SupportedArch,
+        b: *std.Build,
+        user: *std.Build.Step.Compile,
+    ) anyerror!*std.Build.Step {
+        switch (self) {
+            .riscv64 => {
+                user.setLinkerScript(b.path("user/riscv64.ld"));
+            },
+            .aarch64 => {
+                user.setLinkerScript(b.path("user/aarch64.ld"));
+            },
+        }
+
+        b.installArtifact(user);
+
+        const elf2bin = b.addSystemCommand(&.{
+            "llvm-objcopy",
+            "-O",
+            "binary",
+            "zig-out/bin/userspace",
+            "zig-out/bin/userspace.bin",
+        });
+
+        elf2bin.step.dependOn(&user.step);
+
+        return &elf2bin.step;
+    }
+
+    fn configure_kernel(
+        self: SupportedArch,
+        b: *std.Build,
+        kernel: *std.Build.Step.Compile,
+        user: *std.Build.Step,
+    ) anyerror!*std.Build.Step {
         switch (self) {
             .riscv64 => {
                 kernel.entry = .{ .symbol_name = "__rv64_entry" };
@@ -51,6 +91,7 @@ const SupportedArch = enum {
             },
             .aarch64 => {
                 kernel.entry = .{ .symbol_name = "__aa64_entry" };
+                kernel.link_z_max_page_size = 0x1000;
 
                 kernel.setLinkerScript(b.path("etc/aarch64-unknown-none.ld"));
                 kernel.addCSourceFiles(.{
@@ -60,15 +101,16 @@ const SupportedArch = enum {
             },
         }
 
+        kernel.step.dependOn(user);
         b.installArtifact(kernel);
 
         if (self == .riscv64 or self == .aarch64) {
-            const fakeLinuxHeader: *std.Build.Step = try b.allocator.create(std.Build.Step);
-            fakeLinuxHeader.* = std.Build.Step.init(.{
+            const fake_linux_header: *std.Build.Step = try b.allocator.create(std.Build.Step);
+            fake_linux_header.* = std.Build.Step.init(.{
                 .id = std.Build.Step.Id.custom,
                 .name = "insert fake linux header",
                 .owner = kernel.step.owner,
-                .makeFn = insertFakeLinuxImageHeader,
+                .makeFn = insert_fake_linux_image_header,
             });
 
             const elf2bin = b.addSystemCommand(&.{
@@ -79,8 +121,8 @@ const SupportedArch = enum {
                 "zig-out/bin/kernel.bin",
             });
 
-            fakeLinuxHeader.dependOn(b.getInstallStep());
-            elf2bin.step.dependOn(fakeLinuxHeader);
+            fake_linux_header.dependOn(b.getInstallStep());
+            elf2bin.step.dependOn(fake_linux_header);
 
             return &elf2bin.step;
         } else {
@@ -89,7 +131,7 @@ const SupportedArch = enum {
     }
 };
 
-fn insertFakeLinuxImageHeader(step: *std.Build.Step, opts: std.Build.Step.MakeOptions) anyerror!void {
+fn insert_fake_linux_image_header(step: *std.Build.Step, opts: std.Build.Step.MakeOptions) anyerror!void {
     const RISCV_MAGIC1 = "RISCV\x00\x00\x00";
     const RISCV_MAGIC2 = "RSC\x05";
 
@@ -101,15 +143,15 @@ fn insertFakeLinuxImageHeader(step: *std.Build.Step, opts: std.Build.Step.MakeOp
     _ = try file.readAll(std.mem.asBytes(&ehdr));
 
     // Figure out total image size
-    var imageAddrMax: u64 = 0;
+    var image_addr_max: u64 = 0;
     for (0..ehdr.e_phnum) |i| {
         var phdr: elf.Phdr = undefined;
         _ = try file.preadAll(std.mem.asBytes(&phdr), ehdr.e_phoff + i * ehdr.e_phentsize);
 
         const end = (phdr.p_vaddr + phdr.p_memsz + 0xFFF) & ~@as(u64, 0xFFF);
 
-        if (phdr.p_type == elf.PT_LOAD and end > imageAddrMax) {
-            imageAddrMax = end;
+        if (phdr.p_type == elf.PT_LOAD and end > image_addr_max) {
+            image_addr_max = end;
         }
     }
 
@@ -126,7 +168,7 @@ fn insertFakeLinuxImageHeader(step: *std.Build.Step, opts: std.Build.Step.MakeOp
         _ = try file.preadAll(&data, phdr.p_offset);
 
         if (std.mem.eql(u8, RISCV_MAGIC1, data[48..56]) and std.mem.eql(u8, RISCV_MAGIC2, data[56..60])) {
-            try file.pwriteAll(std.mem.asBytes(&imageAddrMax), phdr.p_offset + 16);
+            try file.pwriteAll(std.mem.asBytes(&image_addr_max), phdr.p_offset + 16);
             break;
         }
     }
@@ -135,40 +177,75 @@ fn insertFakeLinuxImageHeader(step: *std.Build.Step, opts: std.Build.Step.MakeOp
     _ = opts;
 }
 
-fn build_riscv64(b: *std.Build) anyerror!void {
-    _ = b;
-}
-
 pub fn build(b: *std.Build) anyerror!void {
-    const maybeArchOption = b.option(SupportedArch, "arch", "Architecture to use");
+    const maybe_arch_option = b.option(SupportedArch, "arch", "Architecture to use");
 
-    const arch = maybeArchOption orelse DEFAULT_ARCH;
-    const target = arch.makeTarget(b);
+    const arch = maybe_arch_option orelse DEFAULT_ARCH;
+    const kernel_target = arch.make_kernel_target(b);
+    const user_target = arch.make_userspace_target(b);
     const optimize = b.standardOptimizeOption(.{ .preferred_optimize_mode = .ReleaseFast });
 
-    const codeModel: std.builtin.CodeModel = switch (arch) {
+    const kernel_code_model: std.builtin.CodeModel = switch (arch) {
         .riscv64 => .medium,
         .aarch64 => .small,
     };
-    const kernelModule = b.addModule("kernel", .{
+    const user_code_model = kernel_code_model;
+
+    const kernel_module = b.addModule("kernel", .{
         .optimize = optimize,
-        .target = target,
+        .target = kernel_target,
         .pic = true,
         .red_zone = false,
-        .code_model = codeModel,
+        .code_model = kernel_code_model,
         .root_source_file = b.path("src/kernel.zig"),
     });
+    kernel_module.addAnonymousImport("userspace", .{ .root_source_file = b.path("zig-out/bin/userspace.bin") });
+
+    const abi_module = b.addModule("abi", .{
+        .optimize = optimize,
+        .red_zone = false,
+        .target = kernel_target,
+        .code_model = kernel_code_model,
+        .root_source_file = b.path("abi/abi.zig"),
+    });
+    const user_module = b.addModule("userspace", .{
+        .optimize = optimize,
+        .target = user_target,
+        .red_zone = false,
+        .code_model = user_code_model,
+        .root_source_file = b.path("user/main.zig"),
+    });
     const kernel = b.addExecutable(.{
         .name = "kernel",
-        .root_module = kernelModule,
+        .root_module = kernel_module,
         .pic = true,
+        .use_lld = true,
     });
     kernel.pie = true;
 
-    const kernelStep = try arch.addTargetSpecific(b, kernel);
+    user_module.addImport("abi", abi_module);
+    kernel_module.addImport("abi", abi_module);
+
+    const user = b.addExecutable(.{
+        .name = "userspace",
+        .root_module = user_module,
+        .use_lld = true,
+    });
+
+    const install_docs = b.addInstallDirectory(.{
+        .source_dir = kernel.getEmittedDocs(),
+        .install_dir = .prefix,
+        .install_subdir = "docs",
+    });
+
+    const docs_step = b.step("docs", "Install documentation");
+    docs_step.dependOn(&install_docs.step);
+
+    const user_step = try arch.configure_user(b, user);
+    const kernel_step = try arch.configure_kernel(b, kernel, user_step);
 
     // TODO QEMU binary override
-    const qemu_info = switch (target.result.cpu.arch) {
+    const qemu_info = switch (kernel_target.result.cpu.arch) {
         .riscv64 => .{ "qemu-system-riscv64", "rv64" },
         .aarch64 => .{ "qemu-system-aarch64", "cortex-a72" },
         else => unreachable,
@@ -190,11 +267,11 @@ pub fn build(b: *std.Build) anyerror!void {
         "none",
     });
 
-    if (target.result.cpu.arch == .riscv64) {
+    if (kernel_target.result.cpu.arch == .riscv64) {
         qemu_cmd.addArgs(&.{ "-bios", "etc/boot/rv64_fw_jump.bin" });
     }
 
-    qemu_cmd.step.dependOn(kernelStep);
+    qemu_cmd.step.dependOn(kernel_step);
     if (b.args) |args| qemu_cmd.addArgs(args);
     const run_step = b.step("run", "Start the OS in qemu");
     run_step.dependOn(&qemu_cmd.step);

etc/riscv64-unknown-none.ld

@@ -23,6 +23,8 @@ SECTIONS {
 
     .tdata : ALIGN(4K) {
         PROVIDE(__tdata_start = .);
+        /* Storage for thread-locals used in assembly */
+        KEEP(*(.tdata.assembly*));
         *(.tdata*)
         PROVIDE(__tdata_end = .);
     }

src/arch.zig

@@ -1,15 +1,73 @@
-pub fn arch() type {
-    const builtin = @import("builtin");
+//! Helper module to select architecture-specific modules depending on what platform is
+//! being targeted.
 
-    switch (comptime builtin.cpu.arch) {
-        .riscv64 => {
-            return @import("arch/riscv64.zig");
-        },
-        .aarch64 => {
-            return @import("arch/aarch64.zig");
-        },
-        else => {
-            @panic("Architecture is not supported");
-        },
+const std = @import("std");
+const builtin = @import("builtin");
+
+pub const cpu: enum {
+    riscv64,
+    aarch64,
+} = switch (builtin.cpu.arch) {
+    .riscv64 => .riscv64,
+    .aarch64 => .aarch64,
+    else => @compileError("Unsupported architecture"),
+};
+
+pub const impl = switch (cpu) {
+    .riscv64 => @import("arch/riscv64.zig"),
+    .aarch64 => @import("arch/aarch64.zig"),
+};
+
+pub const vmm = impl.vmm;
+
+pub const IrqGuard = struct {
+    state: bool,
+
+    pub fn acquire() @This() {
+        const state = set_interrupt_mask(true);
+        return .{ .state = state };
     }
+
+    pub fn release(self: @This()) void {
+        set_interrupt_mask(self.state);
+    }
+};
+
+/// Halts the CPU execution indefinitely, without ever returning.
+pub inline fn halt() noreturn {
+    impl.halt();
 }
+
+/// Returns the current state of IRQ masking.
+pub inline fn interrupt_mask() bool {
+    return impl.interrupt_mask();
+}
+
+/// Modifies the interrupt mask to either allow or block IRQs from being delivered to the CPU.
+/// Returns the old IRQ mask.
+pub inline fn set_interrupt_mask(masked: bool) bool {
+    return impl.set_interrupt_mask(masked);
+}
+
+/// Suspends the CPU until an interrupt is signalled.
+pub inline fn wait_for_interrupt() void {
+    impl.wait_for_interrupt();
+}
+
+/// Hint to the CPU that the code is executing a "busy-wait" or a "spin-wait" loop.
+pub inline fn spin_hint() void {
+    impl.spin_hint();
+}
+
+/// Set the CPU's thread pointer to some value.
+pub inline fn set_thread_pointer(value: usize) void {
+    impl.set_thread_pointer(value);
+}
+
+/// Combined memory/compiler fence to ensure specific ordering of instructions and memory accesses.
+pub inline fn barrier(ordering: std.builtin.AtomicOrder) void {
+    impl.barrier(ordering);
+}
+
+/// Platform-specific task context implementation
+pub const Context = impl.Context;

src/arch/aarch64.zig

@@ -1,51 +1,59 @@
 const std = @import("std");
+
 const boot = @import("aarch64/boot.zig");
+const regs = @import("aarch64/regs.zig");
 
-export const _ = boot.aa64BspLowerEntry;
+pub const vmm = @import("aarch64/vmm.zig");
 
-pub const Context = struct {
-    pub fn idle() Context {
-        @panic("TODO");
+export const _ = boot.aa64_bsp_lower_entry;
+
+pub const Context = @import("aarch64/context.zig").Context;
+
+pub fn set_interrupt_mask(masked: bool) bool {
+    const old = interrupt_mask();
+    if (masked) {
+        regs.DAIF.modify(.{ .I = true }, .{});
+    } else {
+        regs.DAIF.modify(.{}, .{ .I = true });
     }
-
-    pub fn kernel(pc: usize, arg: usize) Context {
-        _ = pc;
-        _ = arg;
-        @panic("TODO");
-    }
-
-    pub fn enter(self: *Context) noreturn {
-        _ = self;
-        @panic("TODO");
-    }
-
-    pub fn switchFrom(self: *Context, from: *Context) void {
-        _ = self;
-        _ = from;
-        @panic("TODO");
-    }
-};
-
-pub fn halt() noreturn {
-    while (true) {}
+    return old;
 }
 
-pub fn spinHint() void {
-    // TODO
+pub inline fn interrupt_mask() bool {
+    return regs.DAIF.read().I;
+}
+
+pub inline fn wait_for_interrupt() void {
+    asm volatile ("wfi");
+}
+
+pub fn halt() noreturn {
+    while (true) {
+        _ = set_interrupt_mask(true);
+        wait_for_interrupt();
+    }
+}
+
+pub fn spin_hint() void {
+    asm volatile ("isb sy" ::: "memory");
+}
+
+pub inline fn set_thread_pointer(tp: usize) void {
+    regs.TPIDR_EL0.set(tp);
 }
 
 pub inline fn barrier(comptime kind: std.builtin.AtomicOrder) void {
     switch (kind) {
         .acquire => {
-            asm volatile ("dsb ishld":::"memory");
+            asm volatile ("dsb ishld" ::: "memory");
         },
         .release => {
-            asm volatile ("dsb ishst":::"memory");
+            asm volatile ("dsb ishst" ::: "memory");
         },
         .acq_rel, .seq_cst => {
-            asm volatile ("dsb ish":::"memory");
+            asm volatile ("dsb ish" ::: "memory");
         },
         .unordered, .monotonic => {},
     }
-    asm volatile ("isb sy":::"memory");
+    asm volatile ("isb sy" ::: "memory");
 }

src/arch/aarch64/boot.zig

@@ -1,19 +1,26 @@
 const kernel = @import("../../kernel.zig");
 const vmm = @import("vmm.zig");
+const dtb = @import("../../util/dtb.zig");
+const exception = @import("exception.zig");
+const tls = @import("../../mem/tls.zig");
 
 const arch = kernel.arch;
+const mem = kernel.mem;
 const log = kernel.debug.log;
+const phys_memory = mem.phys;
 
 extern const __aa64_bsp_stack_top: u8;
 
-var gDtbAddress: u64 = undefined;
+var g_dtb_address: u64 = undefined;
 
-fn earlyDebugPrint(byte: u8) void {
-    const address = 0x9000000;
+fn early_debug_print_high(byte: u8) void {
+    // TODO this is incorrect: writes should come to a memory region marked as device memory,
+    //      "virtualize" range is normal memory.
+    const address = 0x9000000 + vmm.VIRTUALIZE_BASE;
     @as(*volatile u32, @ptrFromInt(address)).* = byte;
 }
 
-fn relocAddressToUpper(ptr: *const anyopaque) usize {
+fn reloc_address_to_upper(ptr: *const anyopaque) usize {
     const p = @intFromPtr(ptr);
     if (p >= vmm.KERNEL_VIRTUAL_BASE) {
         return p;
@@ -22,50 +29,70 @@ fn relocAddressToUpper(ptr: *const anyopaque) usize {
     }
 }
 
-fn aa64BspUpperEntry(realAddress: u64) callconv(.C) noreturn {
-    // Relocate the kernel yet again
-    const relaStart = relocAddressToUpper(&__rela_start);
-    const relaEnd = relocAddressToUpper(&__rela_end);
-    const relOffset = vmm.KERNEL_VIRTUAL_BASE + realAddress;
-
-    arch.barrier(.acq_rel);
-    aa64RelocateKernel(relOffset, relaStart, relaEnd);
-    arch.barrier(.acq_rel);
-
-    log.setWriteFn(&earlyDebugPrint);
-    log.info("Hello, dtb is at 0x{x}", .{ gDtbAddress });
-
-    arch.halt();
+fn reloc_address_to_lower(ptr: *const anyopaque) usize {
+    const p = @intFromPtr(ptr);
+    if (p >= vmm.KERNEL_VIRTUAL_BASE) {
+        return p - vmm.KERNEL_VIRTUAL_BASE;
+    } else {
+        return p;
+    }
 }
 
-pub export fn aa64BspLowerEntry(realAddress: u64, dtbAddress: u64) callconv(.C) noreturn {
-    gDtbAddress = dtbAddress;
+fn aa64_bsp_upper_entry(real_address: u64) callconv(.C) noreturn {
+    // Relocate the kernel yet again
+    const rela_start = reloc_address_to_upper(&__rela_start);
+    const rela_end = reloc_address_to_upper(&__rela_end);
+    const rel_offset = vmm.KERNEL_VIRTUAL_BASE + real_address;
 
-    vmm.mapEarly(realAddress);
+    arch.barrier(.acq_rel);
+    aa64_relocate_kernel(rel_offset, rela_start, rela_end);
+    vmm.unmap_early();
+    arch.barrier(.acq_rel);
 
-    const pc = @intFromPtr(&aa64BspUpperEntry) + vmm.KERNEL_VIRTUAL_BASE;
+    log.set_write_fn(&early_debug_print_high);
+
+    exception.init();
+
+    mem.PhysicalAddress.g_virtualize_base = vmm.VIRTUALIZE_BASE;
+    mem.PhysicalAddress.g_virtualize_size = 16 << vmm.L1.SHIFT;
+
+    setup_memory_from_fdt(real_address);
+
+    setup_per_cpu();
+
+    kernel.kernel_main();
+}
+
+pub export fn aa64_bsp_lower_entry(real_address: u64, dtb_address: u64) callconv(.C) noreturn {
+    g_dtb_address = dtb_address;
+
+    vmm.map_early(real_address);
+
+    const pc = @intFromPtr(&aa64_bsp_upper_entry) + vmm.KERNEL_VIRTUAL_BASE;
     const sp = @intFromPtr(&__aa64_bsp_stack_top) + vmm.KERNEL_VIRTUAL_BASE;
 
-    longJump(pc, sp, realAddress);
+    long_jump(pc, sp, real_address);
 }
 
 // Functions used by the boot process
 
 extern const __rela_start: u8;
 extern const __rela_end: u8;
+extern var __kernel_start: u8;
+extern var __kernel_end: u8;
 
-export fn aa64RelocateKernel(imageBase: usize, relaStart: usize, relaEnd: usize) void {
+export fn aa64_relocate_kernel(image_base: usize, rela_start: usize, rela_end: usize) void {
     const elf = @import("std").elf;
 
-    const relaTablePtr = @as([*]elf.Rela, @ptrFromInt(relaStart));
-    const relaCount = (relaEnd - relaStart) / @sizeOf(elf.Rela);
-    const relaTable = relaTablePtr[0..relaCount];
-    for (relaTable) |entry| {
-        const relaType: elf.R_AARCH64 = @enumFromInt(entry.r_type());
-        switch (relaType) {
+    const rela_table_ptr = @as([*]elf.Rela, @ptrFromInt(rela_start));
+    const rela_count = (rela_end - rela_start) / @sizeOf(elf.Rela);
+    const rela_table = rela_table_ptr[0..rela_count];
+    for (rela_table) |entry| {
+        const rela_type: elf.R_AARCH64 = @enumFromInt(entry.r_type());
+        switch (rela_type) {
             .RELATIVE => {
-                const value = @as(*isize, @ptrFromInt(imageBase + entry.r_offset));
-                value.* = @as(isize, @bitCast(imageBase)) + entry.r_addend;
+                const value = @as(*isize, @ptrFromInt(image_base + entry.r_offset));
+                value.* = @as(isize, @bitCast(image_base)) + entry.r_addend;
             },
             else => {
                 arch.halt();
@@ -74,15 +101,39 @@ export fn aa64RelocateKernel(imageBase: usize, relaStart: usize, relaEnd: usize)
     }
 }
 
-inline fn longJump(pc: usize, sp: usize, a0: usize) noreturn {
+inline fn long_jump(pc: usize, sp: usize, a0: usize) noreturn {
     asm volatile (
         \\ mov sp, %[sp]
         \\ br %[pc]
         :
-        : [sp]"r"(sp),
-          [pc]"r"(pc),
-          [a0]"{x0}"(a0),
-        :"memory"
+        : [sp] "r" (sp),
+          [pc] "r" (pc),
+          [a0] "{x0}" (a0),
+        : "memory"
     );
     unreachable;
 }
+
+fn setup_memory_from_fdt(real_address: usize) void {
+    _ = real_address;
+
+    const kernel_start = reloc_address_to_lower(&__kernel_start); // 0
+    const kernel_end = reloc_address_to_lower(&__kernel_end); // whatever
+
+    const fdt = dtb.Fdt.from_physical_address(.{ .raw = g_dtb_address }) catch |err| {
+        log.panic("Cannot initialize raw DTB: {}", .{err});
+    };
+    fdt.add_physical_memory_to_system();
+
+    phys_memory.add_reserved_region("kernel", kernel_start, kernel_end - kernel_start);
+    phys_memory.add_reserved_region("fdt", g_dtb_address, vmm.L3.align_up(fdt.bytes.len));
+
+    phys_memory.init();
+}
+
+fn setup_per_cpu() void {
+    const tls_data = tls.load_kernel_tls_image();
+    const tp = @intFromPtr(tls_data.ptr);
+    log.info("Set TP = 0x{x}", .{tp});
+    arch.set_thread_pointer(tp);
+}

src/arch/aarch64/context.S

@@ -0,0 +1,96 @@
+// vi:set ft=asm:
+
+.global __aa64_enter_task
+.global __aa64_switch_task
+.global __aa64_task_enter_kernel
+.global __aa64_task_enter_user
+
+.set CONTEXT_SIZE, (12 * 8)
+
+.macro SAVE_TASK_CONTEXT
+    sub sp, sp, #CONTEXT_SIZE
+    stp x19, x20, [sp, #0 * 16]
+    stp x21, x22, [sp, #1 * 16]
+    stp x23, x24, [sp, #2 * 16]
+    stp x25, x26, [sp, #3 * 16]
+    stp x27, x28, [sp, #4 * 16]
+    stp x29, x30, [sp, #5 * 16]
+.endm
+
+.macro RESTORE_TASK_CONTEXT
+    ldp x19, x20, [sp, #0 * 16]
+    ldp x21, x22, [sp, #1 * 16]
+    ldp x23, x24, [sp, #2 * 16]
+    ldp x25, x26, [sp, #3 * 16]
+    ldp x27, x28, [sp, #4 * 16]
+    ldp x29, x30, [sp, #5 * 16]
+    add sp, sp, #CONTEXT_SIZE
+.endm
+
+.pushsection .text
+
+.set SPSR_ELx_I,    (1 << 9)
+.set SPSR_ELx_EL1h, (0b0101)
+
+__aa64_task_enter_user:
+    // x0 == sp, ...
+    ldr x0, [sp, #16 * 0]
+    msr sp_el0, x0
+
+    // x0 == arg, x1 == entry
+    ldp x0, x1, [sp, #16 * 1]
+    add sp, sp, #32
+
+    msr elr_el1, x1
+
+    // SPSR_ELx_M[4:0] = 0, a return to EL0, AArch64 mode
+    mov x1, #SPSR_ELx_I
+    msr spsr_el1, x1
+
+    mov lr, xzr
+
+    dsb ish
+    isb sy
+
+    eret
+
+__aa64_task_enter_kernel:
+    // arg, entry
+    ldp x0, x1, [sp]
+    // return address
+    ldr lr, [sp, #16]
+    add sp, sp, #24
+
+    msr elr_el1, x1
+
+    // SPSR_ELx_M[4:0] = 0b100, a return to EL1t, AArch64 mode
+    mov x1, #SPSR_ELx_EL1h
+    orr x1, x1, #SPSR_ELx_I
+    msr spsr_el1, x1
+
+    mov x1, xzr
+
+    dsb ish
+    isb sy
+
+    eret
+
+__aa64_switch_task:
+    // x0 -- "dst" context
+    // x1 -- "src" context
+
+    SAVE_TASK_CONTEXT
+
+    mov x19, sp
+    str x19, [x1]
+
+__aa64_enter_task:
+    // x0 -- "dst" context
+    ldr x0, [x0]
+    mov sp, x0
+
+    RESTORE_TASK_CONTEXT
+
+    ret
+
+.popsection // .text

src/arch/aarch64/context.zig

@@ -0,0 +1,98 @@
+const thread = @import("../../thread.zig");
+const vmm = @import("vmm.zig");
+const mem = @import("../../mem.zig");
+const regs = @import("regs.zig");
+const kernel = @import("../../kernel.zig");
+
+const ProcessAddressSpace = mem.vmm.ProcessAddressSpace;
+const arch = kernel.arch;
+
+extern fn __aa64_enter_task(cx: *Context) callconv(.C) noreturn;
+extern fn __aa64_switch_task(dcx: *Context, scx: *Context) callconv(.C) void;
+extern fn __aa64_task_enter_kernel() callconv(.C) noreturn;
+extern fn __aa64_task_enter_user() callconv(.C) noreturn;
+
+pub const Context = extern struct {
+    const STACK_SIZE: usize = 16384;
+
+    kstack: thread.KStack(STACK_SIZE),
+
+    ttbr0: u64 = 0,
+
+    pub fn idle() Context {
+        return Context.kernel(&thread.idle_function, 0);
+    }
+
+    pub fn user(address_space: *const ProcessAddressSpace, pc: usize, sp: usize, arg: usize) @This() {
+        const space_physical = address_space.physical_address();
+        const space_asid = address_space.asid();
+
+        var ks = thread.KStack(STACK_SIZE).create();
+
+        const ttbr0 = @as(u64, @bitCast(regs.TTBR0_EL1.Bits{
+            .BADDR = @truncate(space_physical.raw),
+            .ASID = @truncate(space_asid),
+        }));
+
+        // Arguments to __aa64_task_enter_user
+        ks.push(pc);
+        ks.push(arg);
+        ks.push(0); // Padding
+        ks.push(sp);
+
+        setup_stack_common(&ks, @intFromPtr(&__aa64_task_enter_user));
+
+        return .{ .kstack = ks, .ttbr0 = ttbr0 };
+    }
+
+    pub fn kernel(function: *const thread.KernelThreadFn, arg: usize) Context {
+        var ks = thread.KStack(STACK_SIZE).create();
+
+        // Arguments to __aa64_task_enter_kernel
+        ks.push(@intFromPtr(&thread.kernel_return));
+        ks.push(@intFromPtr(function));
+        ks.push(arg);
+
+        setup_stack_common(&ks, @intFromPtr(&__aa64_task_enter_kernel));
+
+        return Context{ .kstack = ks };
+    }
+
+    pub fn enter(self: *Context) noreturn {
+        self.load_state();
+        __aa64_enter_task(self);
+    }
+
+    pub fn switch_from(self: *Context, from: *Context) void {
+        from.store_state();
+        self.load_state();
+        __aa64_switch_task(self, from);
+    }
+
+    pub fn load_state(self: *Context) void {
+        regs.TTBR0_EL1.set(self.ttbr0);
+    }
+
+    pub fn store_state(self: *Context) void {
+        _ = self;
+    }
+
+    fn setup_stack_common(ks: *thread.KStack(STACK_SIZE), entry: usize) void {
+        ks.push(entry); // x30/lr
+        ks.push(0); // x29
+        ks.push(0); // x28
+        ks.push(0); // x27
+        ks.push(0); // x26
+        ks.push(0); // x25
+        ks.push(0); // x24
+        ks.push(0); // x23
+        ks.push(0); // x22
+        ks.push(0); // x21
+        ks.push(0); // x20
+        ks.push(0); // x19
+    }
+};
+
+comptime {
+    asm (@embedFile("context.S"));
+}

src/arch/aarch64/entry.S

@@ -41,11 +41,11 @@ __aa64_entry:
     mov x0, x19
     adr x1, __rela_start
     adr x2, __rela_end
-    bl aa64RelocateKernel
+    bl aa64_relocate_kernel
 
     mov x0, x19
     mov x1, x20
-    b aa64BspLowerEntry
+    b aa64_bsp_lower_entry
 
 .parking_lot:
     wfe

src/arch/aarch64/exception.zig

@@ -0,0 +1,125 @@
+const regs = @import("regs.zig");
+const kernel = @import("../../kernel.zig");
+
+const arch = kernel.arch;
+const log = kernel.debug.log;
+const syscall = kernel.syscall;
+
+extern const __aa64_exception_vectors: u8;
+
+pub const ExceptionFrame = extern struct {
+    xN: [32]usize,
+    spsr_el1: usize,
+    elr_el1: usize,
+    sp_el0: usize,
+    _0: usize,
+
+    pub fn dump(self: *const ExceptionFrame, comptime level: log.Level) void {
+        for (0..16) |i| {
+            log.writeln(level, " x{:<2} = 0x{x:016}  x{:<2} = 0x{x:016}", .{
+                i * 2,
+                self.xN[i * 2],
+                i * 2 + 1,
+                self.xN[i * 2 + 1],
+            });
+        }
+    }
+};
+
+pub fn init() void {
+    const vbar_el1 = @intFromPtr(&__aa64_exception_vectors);
+
+    regs.VBAR_EL1.set(vbar_el1);
+}
+
+fn common_irq_handler(frame: *ExceptionFrame) void {
+    _ = frame;
+    @panic("TODO: IRQ");
+}
+
+// EL1
+// General exceptions
+export fn __aa64_el1_sync_handler(frame: *ExceptionFrame) callconv(.C) void {
+    const esr = regs.ESR_EL1.read();
+    const far = regs.FAR_EL1.get();
+    const elr = regs.ELR_EL1.get();
+
+    log.err("Exception in EL1:", .{});
+    log.err("  EC = {s} (0b{b:06}) ISS = 0x{x}", .{ esr.EC.as_str(), @intFromEnum(esr.EC), esr.ISS });
+    log.err("  ESR = 0x{x:016}", .{@as(u64, @bitCast(esr))});
+    log.err("  ELR = 0x{x:016}", .{elr});
+
+    switch (esr.as_enum()) {
+        .data_abort => |abort| {
+            const fault_kind_str = abort.DFSC.as_str();
+            const access_size_str = @tagName(abort.SAS);
+            const access_type_str = if (abort.WnR) "write" else "read";
+
+            log.err("  Illegal {s} of a {s}: {s}", .{ access_type_str, access_size_str, fault_kind_str });
+            if (!abort.FnV) {
+                log.err("  FAR = 0x{x:016}", .{far});
+            } else {
+                log.err("  (FAR is not valid)", .{});
+            }
+        },
+        else => {},
+    }
+
+    frame.dump(log.Level.err);
+    arch.halt();
+}
+
+// IRQ
+export fn __aa64_el1_irq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    common_irq_handler(frame);
+}
+
+export fn __aa64_el1_fiq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    // TODO I've never used FIQ
+    arch.halt();
+}
+
+export fn __aa64_el1_serror_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    // TODO
+    arch.halt();
+}
+
+// EL0
+export fn __aa64_el0_sync_handler(frame: *ExceptionFrame) callconv(.C) void {
+    const esr = regs.ESR_EL1.read();
+
+    switch (esr.as_enum()) {
+        .svc => {
+            syscall.syscall_handler(frame.xN[8], frame.xN[0..6]);
+            return;
+        },
+        else => {},
+    }
+
+    log.err("Unhandled exception in EL0:", .{});
+    log.err("  EC = {s} (0b{b:06}) ISS = 0x{x}", .{ esr.EC.as_str(), @intFromEnum(esr.EC), esr.ISS });
+    log.err("  ESR = 0x{x:016}", .{@as(u64, @bitCast(esr))});
+    log.err("  ELR = 0x{x:016}", .{frame.elr_el1});
+    frame.dump(log.Level.err);
+    arch.halt();
+}
+
+export fn __aa64_el0_irq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    common_irq_handler(frame);
+}
+
+export fn __aa64_el0_fiq_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    @panic("__aa64_el0_fiq_handler");
+}
+
+export fn __aa64_el0_serror_handler(frame: *ExceptionFrame) callconv(.C) void {
+    _ = frame;
+    @panic("__aa64_el0_serror_handler");
+}
+
+comptime {
+    asm (@embedFile("vectors.S"));
+}

src/arch/aarch64/regs.zig

@@ -1,15 +1,24 @@
+const std = @import("std");
+
 fn Register(comptime name: []const u8, comptime bits: type) type {
     const repr = switch (@typeInfo(bits)) {
         .@"struct" => |s| s.backing_integer.?,
         else => bits,
     };
     return enum(repr) {
+        pub const Bits = bits;
+
         pub fn set(value: repr) void {
-            asm volatile ("msr " ++ name ++ ", %[value]"::[value]"r"(value));
+            asm volatile ("msr " ++ name ++ ", %[value]"
+                :
+                : [value] "r" (value),
+            );
         }
 
         pub fn get() repr {
-            return asm volatile ("mrs %[value], " ++ name:[value]"=r"(-> repr));
+            return asm volatile ("mrs %[value], " ++ name
+                : [value] "=r" (-> repr),
+            );
         }
 
         pub fn write(value: bits) void {
@@ -27,8 +36,128 @@ fn Register(comptime name: []const u8, comptime bits: type) type {
     };
 }
 
-pub const TTBR0_EL1 = Register("ttbr0_el1", u64);
-pub const TTBR1_EL1 = Register("ttbr1_el1", u64);
+pub const TTBR = packed struct(u64) {
+    // 0..48
+    BADDR: u48 = 0,
+    // 48..64
+    ASID: u16 = 0,
+};
+
+pub const TTBR0_EL1 = Register("ttbr0_el1", TTBR);
+pub const TTBR1_EL1 = Register("ttbr1_el1", TTBR);
+
+// NOTE: tpidr_el0 is used until codegen can emit TLS instructions against tpidr_el1
+pub const TPIDR_EL0 = Register("tpidr_el0", u64);
+
+pub const DAIF = Register("daif", packed struct(u64) {
+    // 0..6
+    _0: u6 = 0,
+    // 6
+    F: bool = false,
+    // 7
+    I: bool = false,
+    // 8
+    A: bool = false,
+    // 9
+    D: bool = false,
+    // 10..64
+    _1: u54 = 0,
+});
+
+pub const VBAR_EL1 = Register("vbar_el1", u64);
+pub const ELR_EL1 = Register("elr_el1", u64);
+pub const FAR_EL1 = Register("far_el1", u64);
+
+pub const ESR_EL1 = Register("esr_el1", packed struct(u64) {
+    // 0..25
+    ISS: u25 = 0,
+    // 25
+    IL: bool = false,
+    // 26..32
+    EC: enum(u6) {
+        unknown = 0b000000,
+        svc = 0b010101,
+        data_abort_lower_el = 0b100100,
+        data_abort_same_el = 0b100101,
+        sp_align = 0b100110,
+        _,
+
+        pub fn as_str(self: @This()) []const u8 {
+            return std.enums.tagName(@This(), self) orelse "<unknown>";
+        }
+    } = .unknown,
+    // 32..64
+    _0: u32 = 0,
+
+    // Specific variants of ESR
+    pub const DataAbort = packed struct(u25) {
+        // 0..6
+        DFSC: enum(u6) {
+            address_size_fault_l0 = 0b000000,
+            address_size_fault_l1 = 0b000001,
+            address_size_fault_l2 = 0b000010,
+            address_size_fault_l3 = 0b000011,
+            translation_fault_l0 = 0b000100,
+            translation_fault_l1 = 0b000101,
+            translation_fault_l2 = 0b000110,
+            translation_fault_l3 = 0b000111,
+            access_flag_fault_l1 = 0b001001,
+            access_flag_fault_l2 = 0b001010,
+            access_flag_fault_l3 = 0b001011,
+            permission_fault_l1 = 0b001101,
+            permission_fault_l2 = 0b001110,
+            permission_fault_l3 = 0b001111,
+            _,
+
+            pub fn as_str(self: @This()) []const u8 {
+                return std.enums.tagName(@This(), self) orelse "<other>";
+            }
+        } = .address_size_fault_l0,
+        // 6
+        WnR: bool = false,
+        // 7
+        S1PTW: bool = false,
+        // 8
+        CM: bool = false,
+        // 9
+        EA: bool = false,
+        // 10
+        FnV: bool = false,
+        // 11..14
+        _0: u3 = 0,
+        // 14
+        AR: bool = false,
+        // 15
+        SF: bool = false,
+        // 16..21
+        SRT: u5 = 0,
+        // 21
+        SSE: bool = false,
+        // 22..24
+        SAS: enum(u2) {
+            byte = 0,
+            half = 1,
+            word = 2,
+            dword = 3,
+        } = .byte,
+        // 24
+        ISV: bool = false,
+    };
+
+    pub const AsEnum = union(enum) {
+        data_abort: DataAbort,
+        svc,
+        other,
+    };
+
+    pub fn as_enum(self: @This()) AsEnum {
+        return switch (self.EC) {
+            .data_abort_lower_el, .data_abort_same_el => .{ .data_abort = @bitCast(self.ISS) },
+            .svc => .svc,
+            else => .other,
+        };
+    }
+});
 
 pub const Cacheability = enum(u2) {
     non_cacheable = 0,
@@ -37,19 +166,9 @@ pub const Cacheability = enum(u2) {
     writeback_readalloc_nowritealloc_cacheable = 3,
 };
 
-pub const Shareability = enum(u2) {
-    non_shareable = 0,
-    outer_shareable = 1,
-    inner_shareable = 2,
-    _
-};
+pub const Shareability = enum(u2) { non_shareable = 0, outer_shareable = 1, inner_shareable = 2, _ };
 
-pub const TranslationGranule = enum(u2) {
-    kib_4 = 0,
-    kib_64 = 1,
-    kib_16 = 2,
-    _
-};
+pub const TranslationGranule = enum(u2) { kib_4 = 0, kib_64 = 1, kib_16 = 2, _ };
 
 pub const TCR_EL1 = Register("tcr_el1", packed struct(u64) {
     // 0..6
@@ -84,11 +203,7 @@ pub const TCR_EL1 = Register("tcr_el1", packed struct(u64) {
     // 30..32
     TG1: TranslationGranule = .kib_4,
     // 32..35
-    IPS: enum(u3) {
-        bits_32 = 0b000,
-        bits_48 = 0b101,
-        _
-    } = .bits_32,
+    IPS: enum(u3) { bits_32 = 0b000, bits_48 = 0b101, _ } = .bits_32,
     // 35
     _1: bool = false,
     // 36
@@ -104,7 +219,7 @@ pub const TCR_EL1 = Register("tcr_el1", packed struct(u64) {
     _2: u25 = 0,
 });
 
-pub const SCTLR_EL1 = Register("sctlr_el1", packed struct (u64) {
+pub const SCTLR_EL1 = Register("sctlr_el1", packed struct(u64) {
     // 0
     M: bool = false,
     // 1

src/arch/aarch64/vectors.S

@@ -0,0 +1,147 @@
+.global __aa64_exception_vectors
+
+// 32 general-purpose registers
+.set EXC_GP_SIZE, (32 * 8)
+// 4 special-purpose registers
+.set EXC_SP_SIZE, (4 * 8)
+.set EXC_STATE_SIZE, (EXC_GP_SIZE + EXC_SP_SIZE)
+
+.macro EXC_SAVE_STATE
+    sub sp, sp, #EXC_STATE_SIZE
+
+    // General-purpose block
+    stp  x0,  x1, [sp, #16 *  0]
+    stp  x2,  x3, [sp, #16 *  1]
+    stp  x4,  x5, [sp, #16 *  2]
+    stp  x6,  x7, [sp, #16 *  3]
+    stp  x8,  x9, [sp, #16 *  4]
+    stp x10, x11, [sp, #16 *  5]
+    stp x12, x13, [sp, #16 *  6]
+    stp x14, x15, [sp, #16 *  7]
+    stp x16, x17, [sp, #16 *  8]
+    stp x18, x19, [sp, #16 *  9]
+    stp x20, x21, [sp, #16 * 10]
+    stp x22, x23, [sp, #16 * 11]
+    stp x24, x25, [sp, #16 * 12]
+    stp x26, x27, [sp, #16 * 13]
+    stp x28, x29, [sp, #16 * 14]
+    stp x30, x31, [sp, #16 * 15]
+
+    // Special-purpose block
+    mrs x0, spsr_el1
+    mrs x1, elr_el1
+    mrs x2, sp_el0
+    mov x3, xzr     // Padding
+
+    stp x0, x1, [sp, #EXC_GP_SIZE + 16 * 0]
+    stp x2, x3, [sp, #EXC_GP_SIZE + 16 * 1]
+.endm
+
+.macro EXC_RESTORE_STATE
+    // Special-purpose block
+    ldp x0, x1, [sp, #EXC_GP_SIZE + 16 * 0]
+    ldp x2, x3, [sp, #EXC_GP_SIZE + 16 * 1]
+
+    msr spsr_el1, x0
+    msr elr_el1, x1
+    msr sp_el0, x2
+
+    // General-purpose block
+    ldp  x0,  x1, [sp, #16 *  0]
+    ldp  x2,  x3, [sp, #16 *  1]
+    ldp  x4,  x5, [sp, #16 *  2]
+    ldp  x6,  x7, [sp, #16 *  3]
+    ldp  x8,  x9, [sp, #16 *  4]
+    ldp x10, x11, [sp, #16 *  5]
+    ldp x12, x13, [sp, #16 *  6]
+    ldp x14, x15, [sp, #16 *  7]
+    ldp x16, x17, [sp, #16 *  8]
+    ldp x18, x19, [sp, #16 *  9]
+    ldp x20, x21, [sp, #16 * 10]
+    ldp x22, x23, [sp, #16 * 11]
+    ldp x24, x25, [sp, #16 * 12]
+    ldp x26, x27, [sp, #16 * 13]
+    ldp x28, x29, [sp, #16 * 14]
+    ldp x30, x31, [sp, #16 * 15]
+
+    add sp, sp, #EXC_STATE_SIZE
+.endm
+
+// Exception vector size is 0x80
+.macro EXC_VECTOR el, ht, bits, kind
+.p2align 7
+    b __aa\bits\()_el\el\ht\()_\kind
+.endm
+
+.macro EXC_HANDLER el, ht, bits, kind
+.type __aa\bits\()_el\el\ht\()_\kind, %function
+__aa\bits\()_el\el\ht\()_\kind:
+.if \bits == 32
+    // TODO taking exceptions from EL0t 32-bit
+    b .
+.endif
+    EXC_SAVE_STATE
+
+    dsb ish
+    isb sy
+
+    mov x0, sp
+    mov lr, xzr
+    bl __aa64_el\el\()_\kind\()_handler
+
+    EXC_RESTORE_STATE
+
+    ic iallu
+    dsb ishst
+    isb sy
+
+    eret
+.size __aa\bits\()_el\el\ht\()_\kind, . - __aa\bits\()_el\el\ht\()_\kind
+.endm
+
+.pushsection .text
+.p2align 12
+.type __aa64_exception_vectors, %object
+__aa64_exception_vectors:
+    EXC_VECTOR 1, t, 64, sync
+    EXC_VECTOR 1, t, 64, irq
+    EXC_VECTOR 1, t, 64, fiq
+    EXC_VECTOR 1, t, 64, serror
+
+    EXC_VECTOR 1, h, 64, sync
+    EXC_VECTOR 1, h, 64, irq
+    EXC_VECTOR 1, h, 64, fiq
+    EXC_VECTOR 1, h, 64, serror
+
+    EXC_VECTOR 0, t, 64, sync
+    EXC_VECTOR 0, t, 64, irq
+    EXC_VECTOR 0, t, 64, fiq
+    EXC_VECTOR 0, t, 64, serror
+
+    EXC_VECTOR 0, t, 32, sync
+    EXC_VECTOR 0, t, 32, irq
+    EXC_VECTOR 0, t, 32, fiq
+    EXC_VECTOR 0, t, 32, serror
+.size __aa64_exception_vectors, . - __aa64_exception_vectors
+
+.p2align 7
+    EXC_HANDLER 1, t, 64, sync
+    EXC_HANDLER 1, t, 64, irq
+    EXC_HANDLER 1, t, 64, fiq
+    EXC_HANDLER 1, t, 64, serror
+
+    EXC_HANDLER 1, h, 64, sync
+    EXC_HANDLER 1, h, 64, irq
+    EXC_HANDLER 1, h, 64, fiq
+    EXC_HANDLER 1, h, 64, serror
+
+    EXC_HANDLER 0, t, 64, sync
+    EXC_HANDLER 0, t, 64, irq
+    EXC_HANDLER 0, t, 64, fiq
+    EXC_HANDLER 0, t, 64, serror
+
+    EXC_HANDLER 0, t, 32, sync
+    EXC_HANDLER 0, t, 32, irq
+    EXC_HANDLER 0, t, 32, fiq
+    EXC_HANDLER 0, t, 32, serror
+.popsection // .text

src/arch/aarch64/vmm.zig

@@ -1,14 +1,21 @@
+const std = @import("std");
 const mem = @import("../../mem.zig");
 const regs = @import("regs.zig");
+const kernel = @import("../../kernel.zig");
 
 const PhysicalAddress = mem.PhysicalAddress;
+const AtomicU8 = std.atomic.Value(u8);
+const log = kernel.log;
 
 pub const KERNEL_VIRTUAL_BASE: usize = 0xFFFFFF8000000000;
 pub const KERNEL_L1_INDEX: usize = L1.index(KERNEL_VIRTUAL_BASE);
+pub const KERNEL_VIRTUAL_SIZE: usize = 16 * L1.SIZE;
+pub const VIRTUALIZE_BASE: usize = KERNEL_VIRTUAL_BASE + KERNEL_VIRTUAL_SIZE;
+pub const VIRTUALIZE_BASE_L1I: usize = L1.index(VIRTUALIZE_BASE);
 
-pub const L1 = mem.TranslationLevel(30);
-pub const L2 = mem.TranslationLevel(21);
-pub const L3 = mem.TranslationLevel(12);
+pub const L1 = mem.TranslationLevel(30, L2);
+pub const L2 = mem.TranslationLevel(21, L3);
+pub const L3 = mem.vmm.L3;
 
 pub const RawEntry = packed struct(u64) {
     // 0
@@ -47,7 +54,7 @@ pub const RawEntry = packed struct(u64) {
     // 55..64
     _2: u9 = 0,
 
-    pub fn makeUnion(self: @This(), other: @This()) @This() {
+    pub fn make_union(self: @This(), other: @This()) @This() {
         const lhs = @as(u64, @bitCast(self));
         const rhs = @as(u64, @bitCast(other));
         return @as(@This(), @bitCast(lhs | rhs));
@@ -81,7 +88,7 @@ pub fn TableEntry(comptime Level: type) type {
 
         pub fn normal_page(addr: PhysicalAddress, flags: RawEntry) @This() {
             return .{
-                .raw = flags.makeUnion(RawEntry{
+                .raw = flags.make_union(RawEntry{
                     .PPN = @as(u36, @intCast(addr.raw >> 12)),
                     .V = true,
                     .P = true,
@@ -94,7 +101,7 @@ pub fn TableEntry(comptime Level: type) type {
 
         pub fn device_page(addr: PhysicalAddress, flags: RawEntry) @This() {
             return .{
-                .raw = flags.makeUnion(RawEntry{
+                .raw = flags.make_union(RawEntry{
                     .PPN = @as(u36, @intCast(addr.raw >> 12)),
                     .V = true,
                     .P = true,
@@ -109,7 +116,7 @@ pub fn TableEntry(comptime Level: type) type {
 
         pub fn normal_block(addr: PhysicalAddress, flags: RawEntry) @This() {
             return .{
-                .raw = flags.makeUnion(RawEntry{
+                .raw = flags.make_union(RawEntry{
                     .PPN = @as(u36, @intCast(addr.raw >> 12)),
                     .V = true,
                     .AF = true,
@@ -121,7 +128,7 @@ pub fn TableEntry(comptime Level: type) type {
 
         pub fn table(addr: PhysicalAddress, flags: RawEntry) @This() {
             return .{
-                .raw = flags.makeUnion(.{
+                .raw = flags.make_union(.{
                     .PPN = @as(u36, @intCast(addr.raw >> 12)),
                     .V = true,
                     .P = true,
@@ -135,43 +142,172 @@ pub fn Table(comptime Level: type) type {
     return struct {
         pub const Entry = TableEntry(Level);
 
+        pub const Error = mem.vmm.AddressSpaceError;
+
         entries: [512]Entry align(4096) = [_]Entry{.INVALID} ** 512,
 
+        pub fn allocate_empty() Error!*@This() {
+            const page = mem.phys.alloc_page() orelse return error.out_of_pages;
+            const table = @as(*@This(), @ptrFromInt(page.virtualize()));
+            for (0..512) |i| {
+                table.entry(i).* = .INVALID;
+            }
+            return table;
+        }
+
+        pub fn from_physical_address(physical: PhysicalAddress) *@This() {
+            return @ptrFromInt(physical.virtualize());
+        }
+
         pub inline fn entry(self: *@This(), index: usize) *Entry {
             return &self.entries[index];
         }
+
+        pub fn physical_address(self: *const @This()) PhysicalAddress {
+            return PhysicalAddress.from_virtualized(@intFromPtr(self));
+        }
+
+        pub usingnamespace if (Level.NextLevel) |NextLevel| struct {
+            pub fn get_next_level(self: *Table(Level), index: usize) ?*Table(NextLevel) {
+                const ent = self.entry(index);
+                if (ent.raw.V and !ent.raw.P) {
+                    return Table(NextLevel).from_physical_address(ent.address());
+                }
+                return null;
+            }
+
+            pub fn get_or_create_next_level(self: *Table(Level), index: usize) Error!*Table(NextLevel) {
+                const ent = self.entry(index);
+                if (ent.raw.V) {
+                    if (!ent.raw.P) {
+                        @panic("TODO: mixed hugepages and tables");
+                    }
+
+                    // Entry is a table
+                    return Table(NextLevel).from_physical_address(ent.address());
+                } else {
+                    const table = try Table(NextLevel).allocate_empty();
+                    const physical = table.physical_address();
+                    ent.* = TableEntry(Level).table(physical, .{});
+                    return table;
+                }
+            }
+        } else struct {};
     };
 }
 
-// 0x0000_0000_0000_0000 .. 0x0000_0080_0000_0000
-var gFixedLow = Table(L1){};
+pub const ProcessAddressSpace = struct {
+    l1: *Table(L1),
+    asid: u8,
+
+    pub const Error = mem.vmm.AddressSpaceError;
+
+    var g_asid: AtomicU8 = .{ .raw = 1 };
+
+    pub fn init() Error!ProcessAddressSpace {
+        const table = try Table(L1).allocate_empty();
+        const asid = g_asid.fetchAdd(1, .seq_cst);
+        return .{ .l1 = table, .asid = asid };
+    }
+
+    pub fn physical_address(self: *const @This()) PhysicalAddress {
+        return self.l1.physical_address();
+    }
+
+    pub fn map_page(self: *@This(), virtual: usize, physical: PhysicalAddress) Error!void {
+        // TODO align check on both virtual and physical
+
+        const l1i = L1.index(virtual);
+        const l2i = L2.index(virtual);
+        const l3i = L3.index(virtual);
+
+        const l2 = try self.l1.get_or_create_next_level(l1i);
+        const l3 = try l2.get_or_create_next_level(l2i);
+
+        const entry = l3.entry(l3i);
+
+        if (entry.raw.V) {
+            @panic("TODO: handle already present");
+        }
+
+        entry.* = TableEntry(L3).normal_page(physical, RawEntry{ .AP = .both_readwrite, .NG = true });
+        tlb_flush_vma_asid(virtual, self.asid);
+
+        log.debug("Map 0x{x} -> page 0x{x}", .{ virtual, physical.raw });
+    }
+};
+
+pub inline fn tlb_flush_vma(vma: usize) void {
+    const xt = vma >> 12;
+    asm volatile (
+        \\ dsb ishst
+        \\ tlbi vaae1, %[xt]
+        \\ dsb ish
+        \\ isb sy
+        :
+        : [xt] "r" (xt),
+        : "memory"
+    );
+}
+
+pub inline fn tlb_flush_vma_asid(vma: usize, asid: usize) void {
+    const xt = (vma >> 12) | (asid << 48);
+    asm volatile (
+        \\ dsb ishst
+        \\ tlbi vae1, %[xt]
+        \\ dsb ish
+        \\ isb sy
+        :
+        : [xt] "r" (xt),
+        : "memory"
+    );
+}
+
+pub inline fn tlb_flush_asid(asid: usize) void {
+    const xt = asid << 48;
+    asm volatile (
+        \\ dsb ishst
+        \\ tlbi aside1, %[xt]
+        \\ dsb ish
+        \\ isb sy
+        :
+        : [xt] "r" (xt),
+        : "memory"
+    );
+}
+
 // 0xFFFF_FF80_0000_0000 .. 0xFFFF_FFFF_FFFF_FFFF
-var gFixedHigh = Table(L1){};
+var g_fixed_high = Table(L1){};
 
-pub fn mapEarly(realAddress: usize) void {
-    _ = realAddress;
+pub fn unmap_early() void {
+    // Flush whole ASID 0
+    tlb_flush_asid(0);
+    regs.TTBR0_EL1.set(0);
+}
 
-    for (0..16) |i| {
-        // Identity
-        gFixedLow.entry(i).* = TableEntry(L1).normal_block(
-            .{ .raw = i << L1.SHIFT },
-            .{},
-        );
-    }
+pub fn map_early(real_address: usize) void {
+    _ = real_address;
 
-    for (0..16) |i| {
+    for (0..L1.page_count(KERNEL_VIRTUAL_SIZE)) |i| {
         // Identity + KERNEL_VIRTUAL_BASE
-        gFixedHigh.entry(i).* = TableEntry(L1).normal_block(
+        g_fixed_high.entry(i).* = TableEntry(L1).normal_block(
             .{ .raw = i << L1.SHIFT },
             .{},
         );
     }
 
-    const ttbr0 = @intFromPtr(&gFixedLow);
-    const ttbr1 = @intFromPtr(&gFixedHigh);
+    for (0..16) |i| {
+        // Identity + VIRTUALIZE_BASE for "Whole RAM mapping"
+        g_fixed_high.entry(VIRTUALIZE_BASE_L1I + i).* = TableEntry(L1).normal_block(
+            .{ .raw = i << L1.SHIFT },
+            .{},
+        );
+    }
 
-    regs.TTBR0_EL1.set(ttbr0);
-    regs.TTBR1_EL1.set(ttbr1);
+    const ttbr = @intFromPtr(&g_fixed_high);
+
+    regs.TTBR0_EL1.write(.{ .BADDR = @truncate(ttbr) });
+    regs.TTBR1_EL1.write(.{ .BADDR = @truncate(ttbr) });
 
     regs.TCR_EL1.write(.{
         .AS = .asid_8bit,

src/arch/riscv64.zig

@@ -1,77 +1,35 @@
+//! RISC-V 64-bit platform-specific implementations.
+
 const boot = @import("riscv64/boot.zig");
 const regs = @import("riscv64/regs.zig");
-const thread = @import("../thread.zig");
 const std = @import("std");
 const builtin = @import("builtin");
 
-const Arena = @import("../arena.zig").Arena;
+pub const vmm = @import("riscv64/vmm.zig");
 
-export const _ = boot.rv64BspLowerEntry;
+export const _ = boot.rv64_bsp_lower_entry;
 
-extern fn __rv64_enter_task(cx: *Context) callconv(.C) noreturn;
-extern fn __rv64_switch_task(dcx: *Context, scx: *Context) callconv(.C) void;
-extern fn __rv64_task_enter_kernel() callconv(.C) noreturn;
+/// This CPU's HART (HARdware Thread) ID.
+pub threadlocal var t_hart_id: u32 = 0;
 
-fn idleFunction() callconv(.naked) noreturn {
-    asm volatile ("j .");
-}
+// Linked as .tdata.assembly to ensure `tp == &this`
+pub export threadlocal var t_tdata_assembly: extern struct {
+    kernel_stack_pointer: usize, // tp + 0x00
+    user_stack_pointer: usize, // tp + 0x08
+} linksection(".tdata.assembly") = undefined;
 
-pub threadlocal var tHartId: u32 = 0;
-
-pub const Context = extern struct {
-    const STACK_SIZE: usize = 8192;
-
-    // Has to be exactly at offset 0x00, used in assembly
-    kstack: thread.KStack(STACK_SIZE),
-
-    pub fn idle() @This() {
-        const entry = @intFromPtr(&idleFunction);
-        return Context.kernel(entry, 0);
-    }
-
-    pub fn kernel(pc: usize, arg: usize) @This() {
-        var ks = thread.KStack(STACK_SIZE).create();
-        const entry = @intFromPtr(&__rv64_task_enter_kernel);
-
-        ks.push(pc);
-        ks.push(arg);
-
-        ks.push(0); // x8/s0/fp
-        ks.push(0); // x9/s1
-        ks.push(0); // x18/s2
-        ks.push(0); // x19/s3
-        ks.push(0); // x20/s4
-        ks.push(0); // x21/s5
-        ks.push(0); // x22/s6
-        ks.push(0); // x23/s7
-        ks.push(0); // x24/s8
-        ks.push(0); // x25/s9
-        ks.push(0); // x26/s10
-        ks.push(0); // x27/s11
-        ks.push(0); // x4/gp
-        ks.push(entry); // x1/ra return address
-
-        return .{ .kstack = ks };
-    }
-
-    pub fn enter(self: *@This()) noreturn {
-        __rv64_enter_task(self);
-    }
-
-    pub fn switchFrom(self: *@This(), from: *@This()) void {
-        __rv64_switch_task(self, from);
-    }
-};
+/// RISC-V task context
+pub const Context = @import("riscv64/context.zig").Context;
 
 pub inline fn halt() noreturn {
     while (true) {
-        _ = setInterruptMask(true);
-        waitForInterrupt();
+        _ = set_interrupt_mask(true);
+        wait_for_interrupt();
     }
 }
 
-pub inline fn setInterruptMask(mask: bool) bool {
-    const old = interruptMask();
+pub inline fn set_interrupt_mask(mask: bool) bool {
+    const old = interrupt_mask();
     if (mask) {
         regs.SSTATUS.modify(.{}, .{ .SIE = true });
     } else {
@@ -80,15 +38,15 @@ pub inline fn setInterruptMask(mask: bool) bool {
     return old;
 }
 
-pub fn interruptMask() bool {
+pub fn interrupt_mask() bool {
     return regs.SSTATUS.read().SIE;
 }
 
-pub inline fn waitForInterrupt() void {
+pub inline fn wait_for_interrupt() void {
     asm volatile ("wfi");
 }
 
-pub inline fn spinHint() void {
+pub inline fn spin_hint() void {
     // Don't want to explicitly enable Zihintpause ext, so just paste this as raw opcode
     asm volatile (".word 0x0100000f");
 }
@@ -109,7 +67,7 @@ pub inline fn barrier(comptime ordering: std.builtin.AtomicOrder) void {
     asm volatile ("" ::: "memory");
 }
 
-pub inline fn setThreadPointer(tp: usize) void {
+pub inline fn set_thread_pointer(tp: usize) void {
     asm volatile ("mv tp, %[tp]"
         :
         : [tp] "r" (tp),

src/arch/riscv64/boot.zig

@@ -7,68 +7,73 @@ const dtb = @import("../../util/dtb.zig");
 const mem = @import("../../mem.zig");
 const arena = @import("../../arena.zig");
 const exception = @import("exception.zig");
+const tls = @import("../../mem/tls.zig");
 
-const physMemory = mem.phys;
+const phys_memory = mem.phys;
 const PAGE_SIZE = mem.vmm.PAGE_SIZE;
 const log = debug.log;
 const arch = kernel.arch;
 
 extern const __rv64_bsp_stack_top: u8;
-extern const __kernel_start: u8;
-extern const __kernel_end: u8;
 
-var gDtbAddress: usize = 0;
-var gBspHartId: u32 = 0;
+var g_dtb_address: usize = 0;
+var g_bsp_hart_id: u32 = 0;
 
-fn bspUpperEntry(realAddress: usize, unused: usize) callconv(.C) noreturn {
+fn bsp_upper_entry(real_address: usize, unused: usize) callconv(.C) noreturn {
     _ = unused;
 
     arch.barrier(.acq_rel);
 
     // Relocate the kernel yet again, this time to another base
-    const relaStart = @intFromPtr(&__rela_start);
-    const relaEnd = @intFromPtr(&__rela_end);
-    const relOffset = vmm.KERNEL_VIRTUAL_BASE + vmm.L1.offset(realAddress);
+    const rela_start = @intFromPtr(&__rela_start);
+    const rela_end = @intFromPtr(&__rela_end);
+    const rel_offset = vmm.KERNEL_VIRTUAL_BASE + vmm.L1.offset(real_address);
 
     arch.barrier(.acq_rel);
-    rv64RelocateKernel(relOffset, relaStart, relaEnd);
-    vmm.unmapEarly();
+    rv64_relocate_kernel(rel_offset, rela_start, rela_end);
+    vmm.unmap_early();
 
     // Setup exception handling
     exception.init();
 
-    debug.log.setWriteFn(&sbi.debugPrintByte);
-    kernel.mem.PhysicalAddress.gVirtualizeBase = 0;
-    kernel.mem.PhysicalAddress.gVirtualizeSize = vmm.virtualizeRange();
+    debug.log.set_write_fn(&sbi.debug_print_byte);
+    kernel.mem.PhysicalAddress.g_virtualize_base = vmm.VIRTUALIZE_BASE;
+    kernel.mem.PhysicalAddress.g_virtualize_size = vmm.virtualize_range();
+
+    // Enable supervisor access to user memory
+    regs.SSTATUS.modify(.{ .SUM = true }, .{});
 
     // Setup physical memory management
-    setupMemoryFromFdt(realAddress);
+    setup_memory_from_fdt(real_address);
 
-    setupPerCpu();
-    arch.tHartId = gBspHartId;
+    setup_per_cpu();
+    arch.impl.t_hart_id = g_bsp_hart_id;
 
     kernel.kernel_main();
 }
 
-pub export fn rv64BspLowerEntry(realAddress: usize, bspHartId: usize, dtbAddress: usize) callconv(.C) noreturn {
-    debug.log.setWriteFn(&sbi.debugPrintByte);
+pub export fn rv64_bsp_lower_entry(real_address: usize, bsp_hart_id: usize, dtb_address: usize) callconv(.C) noreturn {
+    debug.log.set_write_fn(&sbi.debug_print_byte);
 
-    gDtbAddress = dtbAddress;
-    gBspHartId = @truncate(bspHartId);
+    g_dtb_address = dtb_address;
+    g_bsp_hart_id = @truncate(bsp_hart_id);
 
-    vmm.mapEarly(realAddress);
+    vmm.g_kernel_real_base = real_address;
+
+    vmm.map_early(real_address);
 
     // &bspUpperEntry will yield a pointer like: X + P, where
     // * X is symbol's raw address,
     // * P is the physical load base of the image (0x80200000 on rv64 usually)
     //
     // Relocate the address to point to Y + P, where Y is the virtual load base
-    // const kernelL1Offset = realAddress & ((1 << 30) - 1);
-    const realAddressL1Offset = vmm.L1.offset(realAddress);
-    const virtualEntry = @intFromPtr(&bspUpperEntry) + vmm.KERNEL_VIRTUAL_BASE - realAddress + realAddressL1Offset;
-    const virtualSp = @intFromPtr(&__rv64_bsp_stack_top) + vmm.KERNEL_VIRTUAL_BASE - realAddress + realAddressL1Offset;
+    const real_address_l1_offset = vmm.L1.offset(real_address);
+    const virtual_entry = @intFromPtr(&bsp_upper_entry) + vmm.KERNEL_VIRTUAL_BASE //
+    - real_address + real_address_l1_offset;
+    const virtual_sp = @intFromPtr(&__rv64_bsp_stack_top) + vmm.KERNEL_VIRTUAL_BASE //
+    - real_address + real_address_l1_offset;
 
-    longJump(virtualEntry, virtualSp, realAddress, 0);
+    long_jump(virtual_entry, virtual_sp, real_address, 0);
 
     arch.halt();
 }
@@ -77,48 +82,28 @@ pub export fn rv64BspLowerEntry(realAddress: usize, bspHartId: usize, dtbAddress
 
 extern const __rela_start: u8;
 extern const __rela_end: u8;
-extern var __tdata_start: u8;
-extern var __tdata_end: u8;
-extern var __tbss_start: u8;
-extern var __tbss_end: u8;
+extern var __kernel_start: u8;
+extern var __kernel_end: u8;
 
-fn setupPerCpu() void {
-    // Assume .tbss follows .tdata
-    const tdataStart = @intFromPtr(&__tdata_start);
-    const tdataEnd = @intFromPtr(&__tdata_end);
-    const tdataSize = tdataEnd - tdataStart;
-    const tbssStart = @intFromPtr(&__tbss_start);
-    const tbssEnd = @intFromPtr(&__tbss_end);
-    const tbssSize = tbssEnd - tbssStart;
-
-    const tdataData = @as([*]u8, @ptrFromInt(tdataStart))[0..tdataSize];
-
-    const tlsSize = tdataSize + tbssSize;
-    const tlsPageCount = (tlsSize + PAGE_SIZE - 1) / PAGE_SIZE;
-    // Variant I: TLS block 0 follows TP after a certain displacement
-    const tlsAddress = physMemory.alloc_pages(tlsPageCount).?.virtualize();
-    const tlsData = @as([*]u8, @ptrFromInt(tlsAddress))[0..tlsSize];
-
-    log.info("Allocated TLS @ {*}", .{ tlsData });
-
-    @memcpy(tlsData[0..tdataSize], tdataData);
-    @memset(tlsData[tdataSize..], 0);
-
-    arch.setThreadPointer(tlsAddress);
+fn setup_per_cpu() void {
+    const tls_data = tls.load_kernel_tls_image();
+    const tp = @intFromPtr(tls_data.ptr);
+    log.info("Set TP = 0x{x}", .{tp});
+    arch.set_thread_pointer(tp);
 }
 
-export fn rv64RelocateKernel(imageBase: usize, relaStart: usize, relaEnd: usize) void {
+export fn rv64_relocate_kernel(image_base: usize, rela_start: usize, rela_end: usize) void {
     const elf = @import("std").elf;
 
-    const relaTablePtr = @as([*]elf.Rela, @ptrFromInt(relaStart));
-    const relaCount = (relaEnd - relaStart) / @sizeOf(elf.Rela);
-    const relaTable = relaTablePtr[0..relaCount];
-    for (relaTable) |entry| {
-        const relaType: elf.R_RISCV = @enumFromInt(entry.r_type());
-        switch (relaType) {
+    const rela_table_ptr = @as([*]elf.Rela, @ptrFromInt(rela_start));
+    const rela_count = (rela_end - rela_start) / @sizeOf(elf.Rela);
+    const rela_table = rela_table_ptr[0..rela_count];
+    for (rela_table) |entry| {
+        const rela_type: elf.R_RISCV = @enumFromInt(entry.r_type());
+        switch (rela_type) {
             .RELATIVE => {
-                const value = @as(*isize, @ptrFromInt(imageBase + entry.r_offset));
-                value.* = @as(isize, @bitCast(imageBase)) + entry.r_addend;
+                const value = @as(*isize, @ptrFromInt(image_base + entry.r_offset));
+                value.* = @as(isize, @bitCast(image_base)) + entry.r_addend;
             },
             else => {
                 arch.halt();
@@ -127,41 +112,27 @@ export fn rv64RelocateKernel(imageBase: usize, relaStart: usize, relaEnd: usize)
     }
 }
 
-fn setupMemoryFromFdt(realAddress: usize) void {
-    const kernelStart = @intFromPtr(&__kernel_start);
-    const kernelEnd = @intFromPtr(&__kernel_end);
+fn setup_memory_from_fdt(real_address: usize) void {
+    const kernel_start = @intFromPtr(&__kernel_start);
+    const kernel_end = @intFromPtr(&__kernel_end);
 
-    var cells: [2]u64 = undefined;
-
-    const fdt = dtb.Fdt.fromPhysicalAddress(.{ .raw = gDtbAddress }) catch |err| {
+    const fdt = dtb.Fdt.from_physical_address(.{ .raw = g_dtb_address }) catch |err| {
         log.panic("Cannot initialize raw DTB: {}", .{err});
     };
+    fdt.add_physical_memory_to_system();
 
-    var memoryRegions = fdt.memoryRegionIterator();
+    phys_memory.add_reserved_region(
+        "kernel",
+        kernel_start -
+            (vmm.KERNEL_VIRTUAL_BASE + vmm.L1.offset(real_address)) + real_address,
+        kernel_end - kernel_start,
+    );
+    phys_memory.add_reserved_region("fdt", g_dtb_address, vmm.L3.align_up(fdt.bytes.len));
 
-    while (memoryRegions.next()) |region| {
-        physMemory.addMemoryRegion(region.name, region.base, region.size);
-    }
-    const reservedRegions = fdt.rootNode().child("reserved-memory");
-    if (reservedRegions) |resv| {
-        var children = resv.children();
-        while (children.next()) |region| {
-            if (region.property("reg")) |reg| {
-                // TODO #address-cells, #size-cells
-                if (reg.readCells(0, &cells, &.{ 2, 2 })) {
-                    physMemory.addReservedRegion(region.name, cells[0], cells[1]);
-                }
-            }
-        }
-    }
-
-    physMemory.addReservedRegion("kernel", kernelStart - (vmm.KERNEL_VIRTUAL_BASE + vmm.L1.offset(realAddress)) + realAddress, kernelEnd - kernelStart);
-    physMemory.addReservedRegion("fdt", gDtbAddress, vmm.L3.align_up(fdt.bytes.len));
-
-    physMemory.init();
+    phys_memory.init();
 }
 
-inline fn longJump(pc: usize, sp: usize, a0: usize, a1: usize) noreturn {
+inline fn long_jump(pc: usize, sp: usize, a0: usize, a1: usize) noreturn {
     asm volatile (
         \\ mv sp, %[sp]
         \\ jr %[pc]

src/arch/riscv64/context.S

@@ -0,0 +1,103 @@
+.pushsection .text
+.option push
+.option norvc
+
+.global __rv64_enter_task
+.global __rv64_switch_task
+.global __rv64_task_enter_user
+.global __rv64_task_enter_kernel
+
+.macro LOAD_TASK_STATE
+    ld ra,   0 * 8(sp)
+    ld gp,   1 * 8(sp)
+    ld s11,  2 * 8(sp)
+    ld s10,  3 * 8(sp)
+    ld s9,   4 * 8(sp)
+    ld s8,   5 * 8(sp)
+    ld s7,   6 * 8(sp)
+    ld s6,   7 * 8(sp)
+    ld s5,   8 * 8(sp)
+    ld s4,   9 * 8(sp)
+    ld s3,  10 * 8(sp)
+    ld s2,  11 * 8(sp)
+    ld s1,  12 * 8(sp)
+    ld s0,  13 * 8(sp)
+
+    addi sp, sp, 14 * 8
+.endm
+
+.macro SAVE_TASK_STATE
+    addi sp, sp, -(14 * 8)
+
+    sd ra,   0 * 8(sp)
+    sd gp,   1 * 8(sp)
+    sd s11,  2 * 8(sp)
+    sd s10,  3 * 8(sp)
+    sd s9,   4 * 8(sp)
+    sd s8,   5 * 8(sp)
+    sd s7,   6 * 8(sp)
+    sd s6,   7 * 8(sp)
+    sd s5,   8 * 8(sp)
+    sd s4,   9 * 8(sp)
+    sd s3,  10 * 8(sp)
+    sd s2,  11 * 8(sp)
+    sd s1,  12 * 8(sp)
+    sd s0,  13 * 8(sp)
+.endm
+
+.set SSTATUS_SPP, (1 << 8)
+.set SSTATUS_SPIE, (1 << 5)
+
+__rv64_task_enter_user:
+    csrw sscratch, tp
+    // TODO setup user thread pointer
+
+    ld a0, (sp)     // argument
+    ld ra, 16(sp)   // entry
+    ld sp, 8(sp)    // stack
+
+    mv tp, zero
+
+    // Clear SPP to zero to indicate a return to U-mode
+    li t1, SSTATUS_SPP
+    not t1, t1
+
+    csrr t0, sstatus
+    // TODO enable interrupts via SPIE
+    // ori t0, t0, SSTATUS_SPIE
+    and t0, t0, t1
+    csrw sstatus, t0
+    csrw sepc, ra
+
+    sret
+
+__rv64_task_enter_kernel:
+    ld a0, (sp)     // argument
+    ld t0, 8(sp)    // entry
+    ld ra, 16(sp)   // return address
+    addi sp, sp, 24
+
+    // Set SPP to indicate a return to S-mode
+    csrr t1, sstatus
+    // TODO enable interrupts via SPIE
+    // ori t0, t0, SSTATUS_SPIE
+    ori t1, t1, SSTATUS_SPP
+    csrw sstatus, t1
+    csrw sepc, t0
+    csrw sscratch, zero
+
+    sret
+
+__rv64_switch_task:
+    // a0 - new context
+    // a1 - old context
+    SAVE_TASK_STATE
+    sd sp, (a1)
+__rv64_enter_task:
+    // a0 -- new context
+    ld sp, (a0)
+    LOAD_TASK_STATE
+    ret
+
+.option pop // norvc
+.popsection // .text

src/arch/riscv64/context.zig

@@ -0,0 +1,104 @@
+const thread = @import("../../thread.zig");
+const mem = @import("../../mem.zig");
+const kernel = @import("../../kernel.zig");
+const regs = @import("regs.zig");
+const vmm = @import("vmm.zig");
+const riscv64 = @import("../riscv64.zig");
+
+const ProcessAddressSpace = mem.vmm.ProcessAddressSpace;
+const log = kernel.log;
+
+extern fn __rv64_enter_task(cx: *Context) callconv(.C) noreturn;
+extern fn __rv64_switch_task(dcx: *Context, scx: *Context) callconv(.C) void;
+extern fn __rv64_task_enter_kernel() callconv(.C) noreturn;
+extern fn __rv64_task_enter_user() callconv(.C) noreturn;
+
+pub const Context = extern struct {
+    const STACK_SIZE: usize = 8192;
+
+    // Has to be exactly at offset 0x00, used in assembly.
+    kstack: thread.KStack(STACK_SIZE),
+
+    satp: u64 = 0,
+
+    /// Constructs an idle context struct.
+    pub fn idle() @This() {
+        return Context.kernel(&thread.idle_function, 0);
+    }
+
+    pub fn user(address_space: *const ProcessAddressSpace, pc: usize, sp: usize, arg: usize) @This() {
+        const space_physical = address_space.physical_address();
+        const space_asid = address_space.asid();
+
+        const satp = regs.SATP.Bits{ .PPN = @truncate(space_physical.raw >> 12), .ASID = @truncate(space_asid), .MODE = .sv39 };
+
+        var ks = thread.KStack(STACK_SIZE).create();
+
+        ks.push(pc);
+        ks.push(sp);
+        ks.push(arg);
+
+        setup_stack_common(&ks, @intFromPtr(&__rv64_task_enter_user));
+
+        return .{ .kstack = ks, .satp = @bitCast(satp) };
+    }
+
+    /// Constructs a kernel task context with entry point in `pc` and an `arg`ument.
+    pub fn kernel(function: *const thread.KernelThreadFn, arg: usize) @This() {
+        var ks = thread.KStack(STACK_SIZE).create();
+
+        const table_physical = vmm.kernel_table_physical();
+        const satp = regs.SATP.Bits{ .PPN = @truncate(table_physical >> 12), .MODE = .sv39 };
+
+        ks.push(@intFromPtr(&thread.kernel_return));
+        ks.push(@intFromPtr(function));
+        ks.push(arg);
+
+        setup_stack_common(&ks, @intFromPtr(&__rv64_task_enter_kernel));
+
+        return .{ .kstack = ks, .satp = @bitCast(satp) };
+    }
+
+    /// Low-level task context entry function.
+    pub fn enter(self: *@This()) noreturn {
+        self.load_state();
+        __rv64_enter_task(self);
+    }
+
+    /// Low-level task context switch function.
+    pub fn switch_from(self: *@This(), from: *@This()) void {
+        from.store_state();
+        self.load_state();
+        __rv64_switch_task(self, from);
+    }
+
+    fn load_state(self: *@This()) void {
+        riscv64.t_tdata_assembly.kernel_stack_pointer = @intFromPtr(self.kstack.sp);
+        regs.SATP.set(self.satp);
+    }
+
+    fn store_state(self: *@This()) void {
+        _ = self;
+    }
+
+    fn setup_stack_common(ks: *thread.KStack(STACK_SIZE), entry: usize) void {
+        ks.push(0); // x8/s0/fp
+        ks.push(0); // x9/s1
+        ks.push(0); // x18/s2
+        ks.push(0); // x19/s3
+        ks.push(0); // x20/s4
+        ks.push(0); // x21/s5
+        ks.push(0); // x22/s6
+        ks.push(0); // x23/s7
+        ks.push(0); // x24/s8
+        ks.push(0); // x25/s9
+        ks.push(0); // x26/s10
+        ks.push(0); // x27/s11
+        ks.push(0); // x4/gp
+        ks.push(entry); // x1/ra return address
+    }
+};
+
+comptime {
+    asm (@embedFile("context.S"));
+}

src/arch/riscv64/entry.S

@@ -1,5 +1,5 @@
-.set ENTRY_SYMBOL, rv64BspLowerEntry
-.set RELOC_SYMBOL, rv64RelocateKernel
+.set ENTRY_SYMBOL, rv64_bsp_lower_entry
+.set RELOC_SYMBOL, rv64_relocate_kernel
 
 .global __rv64_entry
 .global __rv64_bsp_stack_top
@@ -25,7 +25,7 @@ __rv64_entry:
         .ascii  "RISCV\x00\x00\x00" // Magic 1
         .ascii  "RSC\x05"           // Magic 2
         .long 0
-.option pop
+.option pop // rvc
 
 .option push
 .option norvc
@@ -80,80 +80,12 @@ __rv64_real_entry:
         jr t0
 .size __rv64_entry, . - __rv64_entry
 
-.option pop
-.popsection
+.option pop // norvc
+.popsection // .text.entry
 
 .pushsection .bss
 .p2align 4
 __rv64_bsp_stack_bottom:
     .skip 65536
 __rv64_bsp_stack_top:
-.popsection
-
-.pushsection .text
-.option push
-.option norvc
-
-.global __rv64_enter_task
-.global __rv64_switch_task
-.global __rv64_task_enter_kernel
-
-.macro LOAD_TASK_STATE
-    ld ra,   0 * 8(sp)
-    ld gp,   1 * 8(sp)
-    ld s11,  2 * 8(sp)
-    ld s10,  3 * 8(sp)
-    ld s9,   4 * 8(sp)
-    ld s8,   5 * 8(sp)
-    ld s7,   6 * 8(sp)
-    ld s6,   7 * 8(sp)
-    ld s5,   8 * 8(sp)
-    ld s4,   9 * 8(sp)
-    ld s3,  10 * 8(sp)
-    ld s2,  11 * 8(sp)
-    ld s1,  12 * 8(sp)
-    ld s0,  13 * 8(sp)
-
-    addi sp, sp, 14 * 8
-.endm
-
-.macro SAVE_TASK_STATE
-    addi sp, sp, -(14 * 8)
-
-    sd ra,   0 * 8(sp)
-    sd gp,   1 * 8(sp)
-    sd s11,  2 * 8(sp)
-    sd s10,  3 * 8(sp)
-    sd s9,   4 * 8(sp)
-    sd s8,   5 * 8(sp)
-    sd s7,   6 * 8(sp)
-    sd s6,   7 * 8(sp)
-    sd s5,   8 * 8(sp)
-    sd s4,   9 * 8(sp)
-    sd s3,  10 * 8(sp)
-    sd s2,  11 * 8(sp)
-    sd s1,  12 * 8(sp)
-    sd s0,  13 * 8(sp)
-.endm
-
-__rv64_task_enter_kernel:
-    ld a0, (sp)     // argument
-    ld ra, 8(sp)    // entry
-    addi sp, sp, 16
-
-    // TODO S-mode -> S-mode return via sret
-    ret
-
-__rv64_switch_task:
-    // a0 - new context
-    // a1 - old context
-    SAVE_TASK_STATE
-    sd sp, (a1)
-__rv64_enter_task:
-    // a0 -- new context
-    ld sp, (a0)
-    LOAD_TASK_STATE
-    ret
-
-.option pop
-.popsection
+.popsection // .bss

src/arch/riscv64/exception.zig

@@ -1,8 +1,9 @@
 const regs = @import("regs.zig");
-const debug = @import("../../debug.zig");
-const arch = @import("../../kernel.zig").arch;
+const kernel = @import("../../kernel.zig");
 
-const log = debug.log;
+const syscall = kernel.syscall;
+const arch = kernel.arch;
+const log = kernel.log;
 
 extern fn __rv64_exception_vectors() void;
 
@@ -51,12 +52,19 @@ pub const ExceptionFrame = extern struct {
     sN: [12]usize,
     aN: [8]usize,
 
+    umode_sp: usize,
+    sstatus: usize,
+    sepc: usize,
+    stval: usize,
+    scause: usize,
+    sscratch: usize,
+
     pub fn dump(self: *const @This(), comptime level: log.Level) void {
         log.writeln(level, " ra = 0x{x:016}   gp = 0x{x:016}", .{ self.ra, self.gp });
         log.writeln(level, " t0 = 0x{x:016}   t1 = 0x{x:016}", .{ self.tN[0], self.tN[1] });
         log.writeln(level, " t2 = 0x{x:016}   t3 = 0x{x:016}", .{ self.tN[2], self.tN[3] });
         log.writeln(level, " t4 = 0x{x:016}   t5 = 0x{x:016}", .{ self.tN[4], self.tN[5] });
-        log.writeln(level, " t6 = 0x{x:016}", .{ self.tN[6] });
+        log.writeln(level, " t6 = 0x{x:016}", .{self.tN[6]});
         log.writeln(level, " s0 = 0x{x:016}   s1 = 0x{x:016}", .{ self.sN[0], self.sN[1] });
         log.writeln(level, " s2 = 0x{x:016}   s1 = 0x{x:016}", .{ self.sN[2], self.sN[3] });
         log.writeln(level, " s4 = 0x{x:016}   s6 = 0x{x:016}", .{ self.sN[4], self.sN[5] });
@@ -74,21 +82,42 @@ pub fn init() void {
         .BASE = @intCast(base >> 2),
     });
 
-    asm volatile ("":::"memory");
+    asm volatile ("" ::: "memory");
 }
 
 export fn rv64SmodeTrapGeneral(frame: *ExceptionFrame) callconv(.C) void {
-    const scause = regs.SCAUSE.read();
+    // const scause = regs.SCAUSE.read();
+    const scause = @as(regs.SCAUSE.Bits, @bitCast(frame.scause));
     if (scause.INTERRUPT) {
         return rv64SmodeTrapInterrupt(frame);
     }
+    const sstatus = @as(regs.SSTATUS.Bits, @bitCast(frame.sstatus));
     const cause = @as(ExceptionCause, @enumFromInt(scause.CODE));
-    const epc = regs.SEPC.get();
-    const tval = regs.STVAL.get();
 
-    log.err("S-mode exception:", .{});
+    const is_umode = !sstatus.SPP;
+
+    switch (cause) {
+        .ecall_umode => {
+            // TODO assert that `is_umode`
+            const func = frame.aN[0];
+            const args = frame.aN[1..7];
+            const result = syscall.syscall_handler(func, args);
+            frame.aN[0] = result;
+
+            // Add size of `ecall` instruction to return epc to execute the next instruction
+            // instead of falling back to the same `ecall` instruction that caused this
+            // interrupt.
+            frame.sepc += 4;
+            return;
+        },
+        else => {},
+    }
+
+    const mode_str = if (is_umode) "U-mode" else "S-mode";
+
+    log.err("{s} exception:", .{mode_str});
     log.err("  Cause: {s} (0x{x})", .{ cause.name(), scause.CODE });
-    log.err("  stval = 0x{x:016}  sepc = 0x{x:016}", .{ tval, epc });
+    log.err("  stval = 0x{x:016}  sepc = 0x{x:016}", .{ frame.stval, frame.sepc });
     frame.dump(.err);
 
     @panic("Unhandled exception in S-mode");

src/arch/riscv64/regs.zig

@@ -4,6 +4,8 @@ fn Register(comptime name: []const u8, comptime bits: type) type {
         else => bits,
     };
     return enum(repr) {
+        pub const Bits = bits;
+
         pub fn set(value: repr) void {
             asm volatile ("csrw " ++ name ++ ", %[value]"
                 :

src/arch/riscv64/sbi.zig

@@ -27,7 +27,7 @@ const SbiResult = union(enum) {
     ok: u64,
     err: SbiError,
 
-    fn fromSbi(a0: u64, a1: u64) SbiResult {
+    fn from_sbi(a0: u64, a1: u64) SbiResult {
         if (a0 == 0) {
             return .{ .ok = a1 };
         } else {
@@ -36,21 +36,20 @@ const SbiResult = union(enum) {
     }
 };
 
-fn sbiCall1(ext: SbiExtension, func: u64, arg0: u64) SbiResult {
+fn sbi_call1(ext: SbiExtension, func: u64, arg0: u64) SbiResult {
     var a0: u64 = undefined;
     var a1: u64 = undefined;
-    asm volatile (
-        "ecall"
+    asm volatile ("ecall"
         : [ret0] "={a0}" (a0),
           [ret1] "={a1}" (a1),
         : [arg0] "{a0}" (arg0),
           [func] "{a6}" (func),
-          [extn] "{a7}" (ext)
+          [extn] "{a7}" (ext),
         : "a2", "a3", "a4", "a5"
     );
-    return SbiResult.fromSbi(a0, a1);
+    return SbiResult.from_sbi(a0, a1);
 }
 
-pub fn debugPrintByte(byte: u8) void {
-    _ = sbiCall1(.dbcn, 0x02, @as(u64, byte));
+pub fn debug_print_byte(byte: u8) void {
+    _ = sbi_call1(.dbcn, 0x02, @as(u64, byte));
 }

src/arch/riscv64/vectors.S

@@ -3,8 +3,17 @@
 .extern rv64SmodeTrapInterrupt
 
 // ra, gp, 7×tN, 12×sN, 8×aN
-.set GP_REGS_SIZE,      (2 + 7 + 12 + 8) * 8
-.set TRAP_CONTEXT_SIZE, (GP_REGS_SIZE)
+.set GP_REGS_SIZE,      ((2 + 7 + 12 + 8) * 8)
+// U-mode sp, sstatus, sepc, stval, scause, sscratch
+.set SP_REGS_SIZE,      (6 * 8)
+.set TRAP_CONTEXT_SIZE, (GP_REGS_SIZE + SP_REGS_SIZE)
+
+.set REG_UMODE_SP,      (GP_REGS_SIZE + 0 * 8)
+.set REG_SSTATUS,       (GP_REGS_SIZE + 1 * 8)
+.set REG_SEPC,          (GP_REGS_SIZE + 2 * 8)
+.set REG_STVAL,         (GP_REGS_SIZE + 3 * 8)
+.set REG_SCAUSE,        (GP_REGS_SIZE + 4 * 8)
+.set REG_SSCRATCH,      (GP_REGS_SIZE + 5 * 8)
 
 .macro SAVE_GP_REGS
     sd ra,   0 * 8(sp)
@@ -41,19 +50,110 @@
     sd a7,  28 * 8(sp)
 .endm
 
+.macro RESTORE_GP_REGS
+    ld ra,   0 * 8(sp)
+    ld gp,   1 * 8(sp)
+
+    ld t0,   2 * 8(sp)
+    ld t1,   3 * 8(sp)
+    ld t2,   4 * 8(sp)
+    ld t3,   5 * 8(sp)
+    ld t4,   6 * 8(sp)
+    ld t5,   7 * 8(sp)
+    ld t6,   8 * 8(sp)
+
+    ld s0,   9 * 8(sp)
+    ld s1,  10 * 8(sp)
+    ld s2,  11 * 8(sp)
+    ld s3,  12 * 8(sp)
+    ld s4,  13 * 8(sp)
+    ld s5,  14 * 8(sp)
+    ld s6,  15 * 8(sp)
+    ld s7,  16 * 8(sp)
+    ld s8,  17 * 8(sp)
+    ld s9,  18 * 8(sp)
+    ld s10, 19 * 8(sp)
+    ld s11, 20 * 8(sp)
+
+    ld a0,  21 * 8(sp)
+    ld a1,  22 * 8(sp)
+    ld a2,  23 * 8(sp)
+    ld a3,  24 * 8(sp)
+    ld a4,  25 * 8(sp)
+    ld a5,  26 * 8(sp)
+    ld a6,  27 * 8(sp)
+    ld a7,  28 * 8(sp)
+.endm
+
+.set TPREL_KERNEL_STACK,    (0 * 8)
+.set TPREL_USER_STACK,      (1 * 8)
+
+.set SSTATUS_SPP,           (1 << 8)
+
 .macro SMODE_TRAP n, handler
 .type __rv64_smode_trap_\n, @function
 __rv64_smode_trap_\n:
-    // TODO properly handle traps coming from U-mode
-    // TODO save CSRs
-    addi sp, sp, -(TRAP_CONTEXT_SIZE)
-    SAVE_GP_REGS
-    mv a0, sp
+    // If coming from U-mode, sscratch = kernel-mode tp
+    // If coming from S-mode, sscratch = 0
+    csrrw tp, sscratch, tp
+    bnez tp, 1f
+    // Coming from S-mode
+    sd sp, TPREL_KERNEL_STACK(tp)   // tdata_assembly.kernel_stack = sp
+    // [fallthrough]
+1:
+    // Coming from U-mode
+    sd sp, TPREL_USER_STACK(tp)     // tdata_assembly.user_stack = sp
+    ld sp, TPREL_KERNEL_STACK(tp)   // sp = tdata_assembly.kernel_stack
 
+    // Store pre-trap context
+    addi sp, sp, -(TRAP_CONTEXT_SIZE)
+
+    SAVE_GP_REGS
+    // Save special-purpose registers
+    ld t0, TPREL_USER_STACK(tp)
+    csrr t1, sstatus
+    csrr t2, sepc
+    csrr t3, stval
+    csrr t4, scause
+    csrr t5, sscratch
+
+    sd t0, REG_UMODE_SP (sp)
+    sd t1, REG_SSTATUS  (sp)
+    sd t2, REG_SEPC     (sp)
+    sd t3, REG_STVAL    (sp)
+    sd t4, REG_SCAUSE   (sp)
+    sd t5, REG_SSCRATCH (sp)
+
+    // Reset sscratch to zero to make sure a nested S-mode -> S-mode exception
+    // happens properly
+    csrw sscratch, zero
+
+    mv a0, sp
     call \handler
 
-    // TODO return from exception
-    j .
+    // Return from exception
+    ld t0, REG_SSTATUS  (sp)
+    andi t0, t0, SSTATUS_SPP
+    bnez t0, 2f
+
+    // sstatus.SPP == 0, return to U-mode
+    // Restore sscratch to its proper value
+    csrw sscratch, tp
+    // [fallthrough]
+2:
+    // sstatus.SPP == 1, return to S-mode
+    ld t0, REG_SSTATUS  (sp)
+    ld t1, REG_SEPC     (sp)
+    csrw sstatus, t0
+    csrw sepc, t1
+
+    // Restore general-purpose registers
+    RESTORE_GP_REGS
+
+    ld tp, REG_SSCRATCH (sp)
+    ld sp, REG_UMODE_SP (sp)
+
+    sret
 .size __rv64_smode_trap_\n, . - __rv64_smode_trap_\n
 .endm
 

src/arch/riscv64/vmm.zig

@@ -1,18 +1,24 @@
+const std = @import("std");
 const sync = @import("../../sync.zig");
 const regs = @import("regs.zig");
 const mem = @import("../../mem.zig");
-const arch = @import("../../kernel.zig").arch;
+const kernel = @import("../../kernel.zig");
 
+const log = kernel.log;
+const arch = kernel.arch;
 const PhysicalAddress = mem.PhysicalAddress;
+const AtomicU8 = std.atomic.Value(u8);
 
 pub const KERNEL_VIRTUAL_BASE: usize = 0xFFFFFFF000000000;
 pub const KERNEL_VIRTUAL_L1I: usize = (KERNEL_VIRTUAL_BASE >> L1.SHIFT) & 511;
+pub const VIRTUALIZE_BASE: usize = KERNEL_VIRTUAL_BASE + L1.SIZE;
+pub const VIRTUALIZE_BASE_L1I: usize = L1.index(VIRTUALIZE_BASE);
 // 16 GiB
 const EARLY_MAPPING_SIZE: usize = 16;
 
-pub const L1 = mem.TranslationLevel(30);
-pub const L2 = mem.TranslationLevel(21);
-pub const L3 = mem.TranslationLevel(12);
+pub const L1 = mem.TranslationLevel(30, L2);
+pub const L2 = mem.TranslationLevel(21, L3);
+pub const L3 = mem.vmm.L3;
 
 pub const RawEntry = packed struct(u64) {
     // 0: Valid
@@ -38,22 +44,26 @@ pub const RawEntry = packed struct(u64) {
     // 49..64: Unused bits
     _pad1: u15 = 0,
 
-    pub fn makeUnion(self: @This(), other: @This()) @This() {
+    pub fn make_union(self: @This(), other: @This()) @This() {
         const lhs = @as(u64, @bitCast(self));
         const rhs = @as(u64, @bitCast(other));
         return @as(@This(), @bitCast(lhs | rhs));
     }
 
     pub fn clear(self: *@This(), mask: @This()) void {
-        const lhs = @as(*u64, @bitCast(self));
+        const lhs = @as(*u64, @ptrCast(self));
         const rhs = @as(u64, @bitCast(mask));
         lhs.* &= ~rhs;
     }
+
+    pub fn is_table(self: @This()) bool {
+        return !self.r and !self.w and !self.x;
+    }
 };
 
 pub fn TableEntry(comptime Level: type) type {
     _ = Level;
-    return struct {
+    return packed struct(u64) {
         raw: RawEntry,
 
         pub const INVALID: @This() = .{ .raw = .{} };
@@ -72,7 +82,7 @@ pub fn TableEntry(comptime Level: type) type {
 
         pub fn page(addr: PhysicalAddress, flags: RawEntry) @This() {
             return .{
-                .raw = flags.makeUnion(.{
+                .raw = flags.make_union(.{
                     .address = @as(u39, @intCast(addr.raw >> 12)),
                     .r = true,
                     .v = true,
@@ -83,8 +93,9 @@ pub fn TableEntry(comptime Level: type) type {
         }
 
         pub fn table(addr: PhysicalAddress, flags: RawEntry) @This() {
-            flags.clear(.{ .r = true, .w = true, .x = true });
-            return .{ .raw = flags.makeUnion(.{
+            var f = flags;
+            f.clear(.{ .r = true, .w = true, .x = true });
+            return .{ .raw = f.make_union(.{
                 .address = @as(u39, @intCast(addr.raw >> 12)),
                 .v = true,
             }) };
@@ -93,52 +104,168 @@ pub fn TableEntry(comptime Level: type) type {
 }
 
 pub fn Table(comptime Level: type) type {
-    return struct {
+    return extern struct {
         pub const Entry = TableEntry(Level);
 
         entries: [512]Entry align(4096),
 
+        pub const Error = mem.vmm.AddressSpaceError;
+
         pub fn empty() @This() {
             return .{ .entries = [_]Entry{.INVALID} ** 512 };
         }
 
+        pub fn from_physical_address(physical: PhysicalAddress) *@This() {
+            return @ptrFromInt(physical.virtualize());
+        }
+
+        pub fn allocate_empty() Error!*@This() {
+            const page = mem.phys.alloc_page() orelse return error.out_of_pages;
+            const table = @as(*@This(), @ptrFromInt(page.virtualize()));
+            for (0..512) |i| {
+                table.entry(i).* = .INVALID;
+            }
+            return table;
+        }
+
+        pub fn physical_address(self: *const @This()) PhysicalAddress {
+            return PhysicalAddress.from_virtualized(@intFromPtr(self));
+        }
+
         pub inline fn entry(self: *@This(), index: usize) *Entry {
             return &self.entries[index];
         }
+
+        pub usingnamespace if (Level.NextLevel) |NextLevel| struct {
+            pub fn get_next_level(self: *Table(Level), index: usize) ?*Table(NextLevel) {
+                const ent = self.entry(index);
+                if (ent.raw.v and ent.raw.is_table()) {
+                    return Table(NextLevel).from_physical_address(ent.address());
+                }
+                return null;
+            }
+
+            pub fn get_or_create_next_level(self: *Table(Level), index: usize) Error!*Table(NextLevel) {
+                const ent = self.entry(index);
+
+                if (ent.raw.v) {
+                    // TODO handle mixed hugepages + tables
+                    if (!ent.raw.is_table()) {
+                        @panic("TODO: handle mixed hugepages and tables");
+                    }
+                    // It is a table
+                    return Table(NextLevel).from_physical_address(ent.address());
+                } else {
+                    // Allocate a new entry
+                    const table = try Table(NextLevel).allocate_empty();
+                    const physical = table.physical_address();
+                    ent.* = TableEntry(Level).table(physical, .{});
+                    return table;
+                }
+            }
+        } else struct {};
     };
 }
 
-var gFixed = Table(L1).empty();
-var gFixedLock: sync.IrqSafeSpinlock = .{};
+pub const ProcessAddressSpace = struct {
+    l1: *Table(L1),
+    asid: u8,
 
-pub fn virtualizeRange() usize {
+    pub const Error = mem.vmm.AddressSpaceError;
+
+    var g_asid: AtomicU8 = .{ .raw = 1 };
+
+    pub fn init() Error!ProcessAddressSpace {
+        const table = try Table(L1).allocate_empty();
+        // Copy kernel's mappings
+        for (KERNEL_VIRTUAL_L1I..512) |i| {
+            table.entry(i).* = g_fixed.entry(i).*;
+        }
+        const asid = g_asid.fetchAdd(1, .seq_cst);
+        return .{ .l1 = table, .asid = asid };
+    }
+
+    pub fn physical_address(self: *const @This()) PhysicalAddress {
+        return self.l1.physical_address();
+    }
+
+    pub fn map_page(self: *@This(), virtual: usize, physical: PhysicalAddress) Error!void {
+        // TODO align check on both virtual and physical
+
+        const l1i = L1.index(virtual);
+        const l2i = L2.index(virtual);
+        const l3i = L3.index(virtual);
+
+        const l2 = try self.l1.get_or_create_next_level(l1i);
+        const l3 = try l2.get_or_create_next_level(l2i);
+
+        const entry = l3.entry(l3i);
+
+        if (entry.raw.v) {
+            @panic("TODO: handle already present");
+        }
+
+        entry.* = TableEntry(L3).page(physical, .{
+            .r = true,
+            .w = true,
+            .x = true,
+            .u = true,
+        });
+        flush_vma_asid(virtual, self.asid);
+
+        log.debug("Map 0x{x} -> page 0x{x}", .{ virtual, physical.raw });
+    }
+};
+
+var g_fixed = Table(L1).empty();
+var g_fixed_lock: sync.Spinlock = .{};
+pub var g_kernel_real_base: u64 = undefined;
+extern var __kernel_start: u8;
+
+pub fn virtualize_range() usize {
     return EARLY_MAPPING_SIZE * L1.SIZE;
 }
 
-pub fn unmapEarly() void {
-    // Make lower half mappings non-executable
-    gFixedLock.lock();
-    defer gFixedLock.release();
+pub fn kernel_table_physical() u64 {
+    const address = @as(usize, @intFromPtr(&g_fixed));
+    const kernel_start = @intFromPtr(&__kernel_start);
+    return address - kernel_start + g_kernel_real_base;
+}
+
+pub fn unmap_early() void {
+    // Unmap lower half
+    const guard = g_fixed_lock.lock_irqsave();
+    defer guard.release();
     for (0..EARLY_MAPPING_SIZE) |i| {
-        gFixed.entry(i).* = .page(
+        g_fixed.entry(i).* = .INVALID;
+    }
+}
+
+pub fn map_early(real_address: usize) void {
+    const real_l1 = L1.index(real_address);
+
+    // Identity map first 16GiB of memory
+    for (0..EARLY_MAPPING_SIZE) |i| {
+        g_fixed.entry(i).* = .page(
+            .{ .raw = L1.address(i) },
+            .{ .r = true, .w = true, .x = true },
+        );
+    }
+
+    for (0..EARLY_MAPPING_SIZE) |i| {
+        g_fixed.entry(i + VIRTUALIZE_BASE_L1I).* = .page(
             .{ .raw = L1.address(i) },
             .{ .r = true, .w = true },
         );
     }
-}
-
-pub fn mapEarly(realAddress: usize) void {
-    const realL1 = L1.index(realAddress);
-
-    // Identity map first 16GiB of memory
-    for (0..EARLY_MAPPING_SIZE) |i| {
-        gFixed.entry(i).* = .page(.{ .raw = L1.address(i) }, .{ .r = true, .w = true, .x = true });
-    }
 
     // Map 1GiB at KERNEL_VIRTUAL_BASE -> physical 1GiB where the kernel is loaded
-    gFixed.entry(KERNEL_VIRTUAL_L1I).* = .page(.{ .raw = L1.address(realL1) }, .{ .r = true, .w = true, .x = true });
+    g_fixed.entry(KERNEL_VIRTUAL_L1I).* = .page(
+        .{ .raw = L1.address(real_l1) },
+        .{ .r = true, .w = true, .x = true },
+    );
 
-    const address = @as(usize, @intFromPtr(&gFixed));
+    const address = @intFromPtr(&g_fixed);
     regs.SATP.write(.{ .PPN = @intCast(address >> 12), .MODE = .sv39 });
 }
 

src/arena.zig

@@ -1,27 +1,34 @@
-const physMemory = @import("mem/phys.zig");
+//! Simple bump allocator arena.
+
+const phys_memory = @import("mem/phys.zig");
 const log = @import("debug.zig").log;
 const mem = @import("mem.zig");
 
+/// Bump allocator implementation.
 pub const Arena = struct {
-    physBase: mem.PhysicalAddress,
+    phys_base: mem.PhysicalAddress,
     capacity: usize,
     len: usize,
 
+    /// Creates a new `Arena` of given `cap`acity.
+    ///
+    /// Requires initialized physical memory management.
     pub fn init(cap: usize) ?Arena {
-        const physBase = physMemory.alloc_pages(cap / mem.vmm.PAGE_SIZE) orelse return null;
-        return .{
-            .physBase = physBase,
-            .capacity = cap,
-            .len = 0
-        };
+        const phys_base = phys_memory.alloc_pages(cap / mem.vmm.PAGE_SIZE) orelse return null;
+        return .{ .phys_base = phys_base, .capacity = cap, .len = 0 };
     }
 
+    /// Allocates an object of type `T` within this arena.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the arena runs out of memory.
     pub fn create(self: *@This(), comptime T: type) *T {
         if (self.len + @sizeOf(T) > self.capacity) {
-            log.panic("Out of memory. Cannot allocate {} bytes", .{ @sizeOf(T) });
+            log.panic("Out of memory. Cannot allocate {} bytes", .{@sizeOf(T)});
         }
 
-        const v = self.physBase.add(self.len).virtualize();
+        const v = self.phys_base.add(self.len).virtualize();
         const ptr = @as(*T, @ptrFromInt(v));
         self.len += @sizeOf(T);
 

src/debug.zig

@@ -1,73 +1,89 @@
+//! Kernel debug output and logging functions.
+
 const std = @import("std");
 
-fn dummyWrite(_: u8) void {}
+fn dummy_write(_: u8) void {}
 
+/// The main method of kernel logging.
 pub const log = struct {
+    /// Level to emit log records at.
     pub const Level = enum {
+        /// Debug.
         debug,
+        /// Info.
         info,
+        /// Warning.
         warn,
+        /// Error/Fatal.
         err,
     };
 
-    var writeFn: *const fn(u8) void = dummyWrite;
-    const writer: std.io.GenericWriter(u0, error{}, writeWrapperFn) = .{
-        .context = 0
-    };
+    var write_fn: *const fn (u8) void = dummy_write;
+    const writer: std.io.GenericWriter(u0, error{}, write_wrapper_fn) = .{ .context = 0 };
 
-    fn writeWrapperFn(context: u0, data: []const u8) error{}!usize {
+    fn write_wrapper_fn(context: u0, data: []const u8) error{}!usize {
         _ = context;
         for (data) |byte| {
-            writeFn(byte);
+            write_fn(byte);
         }
         return data.len;
     }
 
-    pub fn setWriteFn(f: *const fn(u8) void) void {
-        writeFn = f;
+    /// Replaces the function to print debug bytes with a new one.
+    pub fn set_write_fn(f: *const fn (u8) void) void {
+        write_fn = f;
     }
 
+    /// Emit an `info`-level log record.
     pub inline fn info(comptime format: []const u8, args: anytype) void {
         writeln(.info, format, args);
     }
 
+    /// Emit a `debug`-level log record.
     pub inline fn debug(comptime format: []const u8, args: anytype) void {
         writeln(.debug, format, args);
     }
 
+    /// Emit a `warn`-level log record.
     pub inline fn warn(comptime format: []const u8, args: anytype) void {
         writeln(.warn, format, args);
     }
 
+    /// Emit a `err`-level log record.
     pub inline fn err(comptime format: []const u8, args: anytype) void {
         writeln(.err, format, args);
     }
 
-    pub fn writeRaw(data: []const u8) void {
-        _ = writeWrapperFn(0, data) catch return;
+    /// Write raw byte data into the debugging output.
+    pub fn write_raw(data: []const u8) void {
+        _ = write_wrapper_fn(0, data) catch return;
     }
 
+    /// Write a formatted string (without logging prefix/suffix/newline) into the debugging output.
     pub fn write(comptime format: []const u8, args: anytype) void {
         writer.print(format, args) catch return;
     }
 
+    /// Write a formatted log record into the debugging output.
     pub fn writeln(comptime level: Level, comptime format: []const u8, args: anytype) void {
-        const prefix = comptime logPrefix(level);
-        const suffix = comptime logSuffix(level);
+        const prefix = comptime log_prefix(level);
+        const suffix = comptime log_suffix(level);
         writer.print(prefix ++ format ++ suffix ++ "\r\n", args) catch return;
     }
 
+    /// Helper function to emit a `Not yet implemented` message and panic.
     pub fn todo(comptime msg: []const u8, args: anytype) noreturn {
         err("Not yet implemented: " ++ msg, args);
         @panic("Not yet implemented");
     }
 
+    /// Helper function to emit a `PANIC` message and panic.
     pub fn panic(comptime msg: []const u8, args: anytype) noreturn {
         err("PANIC: " ++ msg, args);
         @panic("Explicit kernel panic");
     }
 
-    fn logPrefix(comptime level: Level) []const u8 {
+    fn log_prefix(comptime level: Level) []const u8 {
         switch (level) {
             .debug => return "",
             .info => return "\x1B[1;36m",
@@ -75,7 +91,7 @@ pub const log = struct {
             .err => return "\x1B[1;31m",
         }
     }
-    fn logSuffix(comptime level: Level) []const u8 {
+    fn log_suffix(comptime level: Level) []const u8 {
         if (level == .debug) {
             return "";
         } else {

src/kernel.zig

@@ -1,44 +1,57 @@
-// export const _ = @import("arch/riscv64/boot.zig").rv64BspLowerEntry;
-pub const arch = @import("arch.zig").arch();
+//! zing microkernel.
+
+pub const arch = @import("arch.zig");
 pub const mem = @import("mem.zig");
 pub const debug = @import("debug.zig");
 pub const arena = @import("arena.zig");
 pub const thread = @import("thread.zig");
+pub const util = @import("util.zig");
+pub const sync = @import("sync.zig");
+pub const syscall = @import("syscall.zig");
 
 pub const log = debug.log;
 pub const vmm = mem.vmm;
 
+/// If set to `true`, will emit log messages for physical page allocations/deallocations.
 pub const TRACE_PHYSICAL_ALLOCATOR: bool = false;
 
 const std = @import("std");
 
-fn f0(arg: usize) callconv(.C) noreturn {
-    var c: usize = 0;
-    while (true) {
-        f1(arg, c);
-        c += 1;
-        thread.yield();
-    }
+const userspace_code = @embedFile("userspace");
+
+fn f0(arg: usize) callconv(.C) void {
+    log.info("Argument is {}", .{arg});
 }
 
 noinline fn f1(arg: usize, c: usize) void {
     log.write("\x1B[1;{}H{}", .{ arg + 1, (c + arg) % 10 });
 }
 
+/// Platform-independent entry point for the kernel.
+///
+/// # Invariants
+///
+/// The following preconditions must be met to invoke this function:
+///
+/// * Physical memory must be initialized.
+/// * (optional) Logging should be set up.
 pub export fn kernel_main() callconv(.C) noreturn {
-    log.write("\x1B[2J", .{});
     var a = arena.Arena.init(256 * 0x1000) orelse @panic("Could not setup kernel arena");
-    thread.Queue.initThisCpu(&a);
+    thread.Queue.init_this_cpu(&a);
 
-    const pc = @intFromPtr(&f0);
-    for (0..4) |i| {
-        const t = thread.Thread.create(&a, pc, i);
-        thread.enqueue(t);
-    }
+    log.info("Userspace code size: {} bytes", .{userspace_code.len});
+    const t1 = thread.test_create_user_from_code(&a, userspace_code) catch @panic("Could not create test thread");
+
+    const t = thread.Thread.create_kernel(&a, &f0, 1234);
+    thread.enqueue(t);
+    thread.enqueue(t1);
+
+    log.info("Test", .{});
 
     thread.enter();
 }
 
+/// Kernel's panic handler
 pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, return_address: ?usize) noreturn {
     _ = error_return_trace;
 

src/mem.zig

@@ -1,3 +1,5 @@
+//! Platform-independent memory management functions.
+
 const std = @import("std");
 
 pub const vmm = @import("mem/vmm.zig");
@@ -5,36 +7,56 @@ pub const phys = @import("mem/phys.zig");
 
 pub const TranslationLevel = vmm.TranslationLevel;
 
+/// A representation for physical address.
 pub const PhysicalAddress = packed struct(u64) {
     raw: u64,
 
+    /// NULL/zero physical address constant.
     pub const NULL: @This() = .{ .raw = 0 };
 
-    pub var gVirtualizeBase: usize = 0;
-    pub var gVirtualizeSize: usize = 0;
+    /// Base address to add to a given `PhysicalAddress` in order to "virtualize" it.
+    pub var g_virtualize_base: usize = 0;
+    /// Maximum `PhysicalAddress` that can be represented as a virtual address.
+    pub var g_virtualize_size: usize = 0;
 
+    /// Adds an `offset` to this `PhysicalAddress`
     pub fn add(self: @This(), offset: usize) @This() {
         return .{ .raw = self.raw + @as(u64, @intCast(offset)) };
     }
 
+    /// "Virtualizes" a `PhysicalAddress` by turning it into a virtual address representation of
+    /// the memory pointed to by this physical address.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the physical address points to a memory that cannot be represented by a virtual
+    /// address.
     pub fn virtualize(self: @This()) usize {
-        if (self.raw > gVirtualizeSize) {
+        if (self.raw > g_virtualize_size) {
             @panic("Physical address out of virtualize bounds");
         }
 
-        return self.raw + gVirtualizeBase;
+        return self.raw + g_virtualize_base;
     }
 
+    /// "De-virtualizes" a previously "virtualized" physical address by mapping it back into its
+    /// physical form.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the virtual address provided is outside of virtualizable memory range.
     pub fn from_virtualized(virt: usize) @This() {
-        if ((virt < gVirtualizeBase) || (virt - gVirtualizeBase > gVirtualizeSize)) {
+        if (virt < g_virtualize_base or virt - g_virtualize_base > g_virtualize_size) {
             @panic("Invalid virtualized physical address");
         }
 
-        return .{ .raw = virt - gVirtualizeBase };
+        return .{ .raw = virt - g_virtualize_base };
     }
 };
 
-pub fn formatSize(buffer: []u8, size: u64) []const u8 {
+/// Helper function to format a byte quantity into a human-readable size.
+/// Writes its result to the `buffer` provided and returns a pointer to it.
+pub fn format_size(buffer: []u8, size: u64) []const u8 {
     const KIBI: u64 = 1024;
     const MIBI: u64 = KIBI * 1024;
     const GIBI: u64 = MIBI * 1024;
@@ -51,23 +73,20 @@ pub fn formatSize(buffer: []u8, size: u64) []const u8 {
     const integer = size / div;
     const dot = size >= 1024;
 
-    const iLen = std.fmt.formatIntBuf(buffer, integer, 10, .lower, .{});
-    var len = iLen;
-    var fLen: usize = 0;
+    const ilen = std.fmt.formatIntBuf(buffer, integer, 10, .lower, .{});
+    var len = ilen;
+    var flen: usize = 0;
 
     if (dot and integer < 100) {
         const fractional = (((size * 1000) / div) % 1000) / 10;
 
-        if (iLen < buffer.len + 1) {
-            buffer[iLen] = '.';
-            fLen = 1 + std.fmt.formatIntBuf(buffer[iLen + 1..], fractional, 10, .lower, .{
-                .fill = '0',
-                .width = 2
-            });
+        if (ilen < buffer.len + 1) {
+            buffer[ilen] = '.';
+            flen = 1 + std.fmt.formatIntBuf(buffer[ilen + 1 ..], fractional, 10, .lower, .{ .fill = '0', .width = 2 });
         }
     }
 
-    len += fLen;
+    len += flen;
 
     if (len + suffix.len < buffer.len) {
         std.mem.copyForwards(u8, buffer[len..], suffix);

src/mem/phys.zig

@@ -1,3 +1,5 @@
+//! Physical memory management implementation.
+
 const std = @import("std");
 
 const kernel = @import("../kernel.zig");
@@ -7,76 +9,104 @@ const vmm = @import("vmm.zig");
 const sync = @import("../sync.zig");
 
 const Range = @import("../util/range.zig").Range;
-const Spinlock = sync.IrqSafeSpinlock;
+const Spinlock = sync.Spinlock;
 
-pub const MemoryRegion = struct { name: []const u8, range: Range(u64) };
+/// Represents a single region of physical memory (reserved or available).
+pub const MemoryRegion = struct {
+    /// Name string, used to represent where the memory comes from.
+    name: []const u8,
+    /// Page frame number range of the region.
+    range: Range(u64),
+};
 
-const Page = extern struct {
-    refcount: u32 = 0,
-    unused: [3]u32 = undefined,
+const Bitmap = struct {
+    data: []u64,
 
-    fn isUsed(self: *const @This()) bool {
-        return self.refcount != 0;
+    const Self = @This();
+
+    pub const empty: Self = .{ .data = &.{} };
+
+    fn get_bit(self: *Self, index: usize) u1 {
+        const word_index = index / 64;
+        const bit_index = index % 64;
+        const masked = self.data[word_index] & (@as(u64, 1) << @intCast(bit_index));
+        return if (masked == 0) 0 else 1;
     }
 
-    fn makeAvailable(self: *@This()) void {
-        self.refcount = 0;
+    fn set_bit(self: *Self, index: usize) void {
+        const word_index = index / 64;
+        const bit_index = index % 64;
+        self.data[word_index] |= (@as(u64, 1) << @intCast(bit_index));
     }
 
-    fn makeReserved(self: *@This()) void {
-        self.refcount = std.math.maxInt(u32);
+    fn clear_bit(self: *Self, index: usize) void {
+        const word_index = index / 64;
+        const bit_index = index % 64;
+        self.data[word_index] &= ~(@as(u64, 1) << @intCast(bit_index));
     }
 };
 
 const PhysicalMemoryManager = struct {
-    pageArray: []Page,
-    offset: u64 = 0,
-    lastFree: usize = 0,
+    memory_start: u64,
+    last_free: usize,
+    len: usize,
 
-    const RECORDS_PER_PAGE: usize = vmm.PAGE_SIZE / @sizeOf(Page);
+    /// Each bit represents a page, there can be more u64s than needed
+    usage_bitmap: Bitmap,
+    page_refcounters: []u32,
+
+    const empty: @This() = .{
+        .memory_start = 0,
+        .last_free = 0,
+        .len = 0,
+        .usage_bitmap = .empty,
+        .page_refcounters = &.{},
+    };
 
     fn alloc_page(self: *@This()) ?mem.PhysicalAddress {
-        for (self.lastFree..self.pageArray.len) |i| {
-            if (self.pageArray[i].refcount == 0) {
-                self.pageArray[i].refcount += 1;
-                self.lastFree = (i + 1) % self.pageArray.len;
-                return .{ .raw = self.offset + i * vmm.PAGE_SIZE };
+        for (self.last_free..self.len) |i| {
+            if (!self.is_page_used(i)) {
+                self.page_refcounters[i] += 1;
+                self.set_page_used(i);
+                self.last_free = (i + 1) % self.len;
+                return .{ .raw = self.memory_start + i * vmm.PAGE_SIZE };
             }
         }
-        for (0..self.lastFree) |i| {
-            if (self.pageArray[i].refcount == 0) {
-                self.pageArray[i].refcount += 1;
-                self.lastFree = (i + 1) % self.pageArray.len;
-                return .{ .raw = self.offset + i * vmm.PAGE_SIZE };
+        for (0..self.last_free) |i| {
+            if (!self.is_page_used(i)) {
+                self.page_refcounters[i] += 1;
+                self.set_page_used(i);
+                self.last_free = (i + 1) % self.len;
+                return .{ .raw = self.memory_start + i * vmm.PAGE_SIZE };
             }
         }
         return null;
     }
 
     fn alloc_pages(self: *@This(), count: usize) ?mem.PhysicalAddress {
-        if (self.lastFree + count < self.pageArray.len) {
-            if (self.alloc_from(self.lastFree, self.pageArray.len, count)) |p| {
+        if (self.last_free + count < self.len) {
+            if (self.alloc_from(self.last_free, self.len, count)) |p| {
                 return p;
             }
         }
-        return self.alloc_from(0, self.lastFree, count);
+        return self.alloc_from(0, self.last_free, count);
     }
 
     fn alloc_from(self: *@This(), start: usize, end: usize, count: usize) ?mem.PhysicalAddress {
         for (start..end) |i| {
-            var taken = false;
-            for (0..count) |j| {
-                if (self.pageArray[i + j].isUsed()) {
-                    taken = true;
-                    break;
-                }
-            }
+            const taken = taken: {
+                for (0..count) |j|
+                    if (self.is_page_used(i + j))
+                        break :taken true;
+                break :taken false;
+            };
 
             if (!taken) {
                 for (0..count) |j| {
-                    self.pageArray[i + j].refcount = 1;
+                    self.page_refcounters[i + j] = 1;
+                    self.set_page_used(i + j);
                 }
-                return .{ .raw = self.offset + i * vmm.PAGE_SIZE };
+                return .{ .raw = self.memory_start + i * vmm.PAGE_SIZE };
             }
         }
 
@@ -84,11 +114,11 @@ const PhysicalMemoryManager = struct {
     }
 
     fn valid_index(self: *@This(), page: mem.PhysicalAddress) usize {
-        if (page.raw < self.offset) {
+        if (page.raw < self.memory_start) {
             log.panic("free_page: invalid page 0x{x}: outside of the allocation range", .{page.raw});
         }
-        const index = (page.raw - self.offset) / vmm.PAGE_SIZE;
-        if (index >= self.pageArray.len) {
+        const index = (page.raw - self.memory_start) / vmm.PAGE_SIZE;
+        if (index >= self.len) {
             log.panic("free_page: invalid page 0x{x}: outside of the allocation range", .{page.raw});
         }
         return index;
@@ -96,52 +126,87 @@ const PhysicalMemoryManager = struct {
 
     fn free_page(self: *@This(), page: mem.PhysicalAddress) void {
         const index = self.valid_index(page);
-        if (self.pageArray[index].refcount == 0) {
+        if (!self.is_page_used(index)) {
             log.panic("free_page: double free of page 0x{x} detected", .{page.raw});
         }
-        self.pageArray[index].refcount -= 1;
-        if (self.pageArray[index].refcount == 0) {
-            self.lastFree = index;
+        self.page_refcounters[index] -= 1;
+        if (self.page_refcounters[index] == 0) {
+            self.clear_page_used(index);
+            self.last_free = index;
         }
     }
 
-    fn get_page(self: *@This(), page: mem.PhysicalAddress) *Page {
-        const index = self.valid_index(page);
-        return &self.pageArray[index];
+    fn is_page_used(self: *@This(), index: usize) bool {
+        return self.usage_bitmap.get_bit(index) == 1;
+    }
+
+    fn set_page_used(self: *@This(), index: usize) void {
+        self.usage_bitmap.set_bit(index);
+    }
+
+    fn clear_page_used(self: *@This(), index: usize) void {
+        self.usage_bitmap.clear_bit(index);
     }
 };
 
-var gMemoryRegions: std.BoundedArray(MemoryRegion, 16) = .{};
-var gReservedRegions: std.BoundedArray(MemoryRegion, 16) = .{};
-var gPhysicalMemoryLock = Spinlock{};
-var gPhysicalMemory = PhysicalMemoryManager{ .pageArray = undefined };
+var g_memory_regions: std.BoundedArray(MemoryRegion, 16) = .{};
+var g_reserved_regions: std.BoundedArray(MemoryRegion, 16) = .{};
+var g_physical_memory_lock = Spinlock{};
+var g_physical_memory = PhysicalMemoryManager.empty;
 
-pub fn addMemoryRegion(name: []const u8, base: u64, size: u64) void {
+/// Adds an available memory region to the list.
+///
+/// `base` and `size` are in bytes. Regions are page-aligned "inwards", meaning the function will
+/// only add the range of full pages of the specified region. If a combination is provided that
+/// does not yield any full 4KiB pages (e.g. `base=0x1234, size=0x123`), it is ignored.
+///
+/// # Note
+///
+/// Only meaningful to call before calling `init()`.
+pub fn add_memory_region(name: []const u8, base: u64, size: u64) void {
     log.info("Memory: '{s}', base 0x{x}, size 0x{x}", .{ name, base, size });
-    gMemoryRegions.append(.{ .name = name, .range = .{ .start = base, .len = size } }) catch @panic("memory regions overflow");
+    const start = vmm.L3.align_up(base) / vmm.L3.SIZE;
+    const len = vmm.L3.align_down(base + size) / vmm.L3.SIZE - start;
+    if (len > 0) {
+        g_memory_regions.append(.{ .name = name, .range = .{ .start = start, .len = len } }) //
+        catch @panic("memory regions overflow");
+    }
 }
 
-pub fn addReservedRegion(name: []const u8, base: u64, size: u64) void {
+/// Adds an reserved memory region to the list.
+///
+/// `base` and `size` are in bytes. Regions are page-aligned "outwards", meaning that the
+/// reservation extends to any pages affected by the specified region.
+///
+/// # Note
+///
+/// Only meaningful to call before calling `init()`.
+pub fn add_reserved_region(name: []const u8, base: u64, size: u64) void {
     log.info("Reserved: '{s}', base 0x{x}, size 0x{x}", .{ name, base, size });
-    gReservedRegions.append(.{ .name = name, .range = .{ .start = base, .len = size } }) catch @panic("reserved regions overflow");
+    const start = base / vmm.L3.SIZE;
+    const len = vmm.L3.align_up(base + size) / vmm.L3.SIZE - start;
+    if (len > 0) {
+        g_reserved_regions.append(.{ .name = name, .range = .{ .start = start, .len = len } }) //
+        catch @panic("reserved regions overflow");
+    }
 }
 
-fn isReservedIn(page: u64) ?*const MemoryRegion {
-    for (0..gReservedRegions.len) |i| {
-        const region = &gReservedRegions.buffer[i];
-        if (page >= region.range.start and page < region.range.end()) {
+fn is_reserved_in(page_index: u64) ?*const MemoryRegion {
+    for (0..g_reserved_regions.len) |i| {
+        const region = &g_reserved_regions.buffer[i];
+        if (page_index >= region.range.start and page_index < region.range.end()) {
             return region;
         }
     }
     return null;
 }
 
-fn allocFromRegion(region: *const MemoryRegion, reason: []const u8, pageCount: usize) ?u64 {
+fn alloc_from_region(region: *const MemoryRegion, reason: []const u8, page_count: usize) ?u64 {
     var offset = @as(u64, 0);
     while (offset < region.range.len) {
         var taken: ?*const MemoryRegion = null;
-        for (0..pageCount) |i| {
-            if (isReservedIn(region.range.start + offset + i * vmm.PAGE_SIZE)) |resv| {
+        for (0..page_count) |i| {
+            if (is_reserved_in(region.range.start + offset + i)) |resv| {
                 taken = resv;
                 break;
             }
@@ -152,72 +217,110 @@ fn allocFromRegion(region: *const MemoryRegion, reason: []const u8, pageCount: u
             continue;
         }
 
-        const base = region.range.start + offset;
-        addReservedRegion(reason, base, pageCount * vmm.PAGE_SIZE);
+        const base = (region.range.start + offset) * vmm.L3.SIZE;
+        add_reserved_region(reason, base, page_count * vmm.PAGE_SIZE);
         return base;
     }
     return null;
 }
 
-fn allocPageArray(pageCount: usize) []Page {
-    for (gMemoryRegions.constSlice()) |region| {
-        if (allocFromRegion(&region, "page-array", pageCount)) |physAddress| {
+/// Allocates a slice of type `T` that spans the `page_count` pages.
+fn alloc_slice_pages(comptime T: type, reason: []const u8, page_count: usize) []T {
+    for (g_memory_regions.constSlice()) |region| {
+        if (alloc_from_region(&region, reason, page_count)) |physAddress| {
             const vaddr = (mem.PhysicalAddress{ .raw = physAddress }).virtualize();
-            const len = pageCount * PhysicalMemoryManager.RECORDS_PER_PAGE;
-            const ptr: [*]Page = @ptrFromInt(vaddr);
-            const slice: []Page = ptr[0..len];
-            for (0..len) |i| {
-                slice[i].refcount = std.math.maxInt(u32);
-            }
+            const len = (page_count * vmm.PAGE_SIZE) / @sizeOf(T);
+            const ptr: [*]T = @ptrFromInt(vaddr);
+            const slice: []T = ptr[0..len];
             return slice;
         }
     }
-    @panic("TODO");
+    @panic("Failed to allocate a slice");
 }
 
+/// Allocates a slice of type `T` that has at least `min_len` items, allocates in 4KiB pages.
+/// The items are zeroed out.
+fn alloc_slice(comptime T: type, reason: []const u8, min_len: usize) []T {
+    const min_alloc_bytes = min_len * @sizeOf(T);
+    // Round up to make sure we have enough space for the data
+    const needed_pages = vmm.L3.page_count(min_alloc_bytes);
+    const slice = alloc_slice_pages(T, reason, needed_pages);
+    const slice_as_bytes = std.mem.sliceAsBytes(slice);
+    @memset(slice_as_bytes, 0);
+    return slice;
+}
+
+/// Allocates a bitmap that has at least `bits_required` total bits.
+/// It can have more since we allocate 4KiB pages and the backing type is []u64
+fn alloc_bitmap(bits_required: usize) Bitmap {
+    // Round up to the upper u64 that has at least `pages` bits
+    const bitmap_entries = (bits_required + 63) / 64;
+    const bitmap = alloc_slice(u64, "bitmap", bitmap_entries);
+    return .{ .data = bitmap };
+}
+fn alloc_refcounters(count: usize) []u32 {
+    const refcounters = alloc_slice(u32, "refcounters", count);
+    @memset(refcounters, std.math.maxInt(u32));
+    return refcounters;
+}
+
+/// Initializes the physical memory management.
+///
+/// # Notes
+///
+/// Calls to `add***Region()` functions have no meaning past this point, so all the memory
+/// present in the system, along with memory reservations, should be added **prior** to this point.
 pub fn init() void {
-    var memoryStart: u64 = std.math.maxInt(u64);
-    var memoryEnd: u64 = std.math.minInt(u64);
+    var memory_start: u64 = std.math.maxInt(u64);
+    var memory_end: u64 = std.math.minInt(u64);
 
-    for (gMemoryRegions.constSlice()) |region| {
-        if (region.range.start < memoryStart) {
-            memoryStart = region.range.start;
+    for (g_memory_regions.constSlice()) |region| {
+        log.info("Region: {}..{}", .{ region.range.start, region.range.end() });
+        if (region.range.start < memory_start) {
+            memory_start = region.range.start;
         }
-        if (region.range.end() > memoryEnd) {
-            memoryEnd = region.range.end();
+        if (region.range.end() > memory_end) {
+            memory_end = region.range.end();
         }
     }
 
-    const memoryPages = (memoryEnd - memoryStart) / vmm.PAGE_SIZE; // == bitmap bits required
-    const pageArrayPages = (memoryPages + PhysicalMemoryManager.RECORDS_PER_PAGE - 1) / PhysicalMemoryManager.RECORDS_PER_PAGE;
+    const memory_pages = memory_end - memory_start; // == bitmap bits required
+    var bitmap = alloc_bitmap(memory_pages);
+    const refcounters = alloc_refcounters(memory_pages);
 
-    const pageArray = allocPageArray(pageArrayPages);
-    var availablePages: usize = 0;
+    var available_pages: usize = 0;
 
-    for (gMemoryRegions.constSlice()) |region| {
-        const offset = (region.range.start - memoryStart) / vmm.PAGE_SIZE;
-        for (0..region.range.len / vmm.PAGE_SIZE) |i| {
-            pageArray[offset + i].makeAvailable();
-            availablePages += 1;
+    for (g_memory_regions.constSlice()) |region| {
+        const offset = region.range.start - memory_start;
+        for (0..region.range.len) |i| {
+            refcounters[offset + i] = 0;
+            available_pages += 1;
         }
     }
-    for (gReservedRegions.constSlice()) |region| {
-        const offset = (region.range.start - memoryStart) / vmm.PAGE_SIZE;
-        for (0..region.range.len / vmm.PAGE_SIZE) |i| {
-            if (offset + i >= pageArray.len) {
+    for (g_reserved_regions.constSlice()) |region| {
+        const offset = region.range.start - memory_start;
+        for (0..region.range.len) |i| {
+            if (offset + i >= memory_pages) {
                 break;
             }
-            pageArray[offset + i].makeReserved();
-            availablePages -= 1;
+            refcounters[offset + i] = std.math.maxInt(u32);
+            bitmap.set_bit(offset + i);
+
+            available_pages -= 1;
         }
     }
 
-    var sizeFmt: [64]u8 = undefined;
-    const sizeFmtStr = mem.formatSize(&sizeFmt, availablePages * vmm.PAGE_SIZE);
-    log.info("Available memory: {s}, page array {*}", .{ sizeFmtStr, pageArray });
+    var size_fmt: [64]u8 = undefined;
+    const size_fmt_str = mem.format_size(&size_fmt, available_pages * vmm.PAGE_SIZE);
+    log.info(
+        "Available memory: {s}, bitmap {*}, refcounts {*}",
+        .{ size_fmt_str, bitmap.data, refcounters },
+    );
 
-    gPhysicalMemory.pageArray = pageArray;
-    gPhysicalMemory.offset = memoryStart;
+    g_physical_memory.len = memory_pages;
+    g_physical_memory.memory_start = memory_start * vmm.L3.SIZE;
+    g_physical_memory.usage_bitmap = bitmap;
+    g_physical_memory.page_refcounters = refcounters;
 }
 
 fn trace_allocation(count: usize, page: ?mem.PhysicalAddress) void {
@@ -232,36 +335,40 @@ fn trace_free(page: mem.PhysicalAddress) void {
     log.debug("free 0x{x}", .{page.raw});
 }
 
+/// Allocates a single 4KiB physical memory page.
 pub fn alloc_page() ?mem.PhysicalAddress {
-    gPhysicalMemoryLock.lock();
-    defer gPhysicalMemoryLock.release();
-    const page = gPhysicalMemory.alloc_page();
+    const guard = g_physical_memory_lock.lock_irqsave();
+    defer guard.release();
+    const page = g_physical_memory.alloc_page();
     if (comptime kernel.TRACE_PHYSICAL_ALLOCATOR) {
         trace_allocation(1, page);
     }
     return page;
 }
 
+/// Allocates a set of `count` contiguous 4KiB pages.
 pub fn alloc_pages(count: usize) ?mem.PhysicalAddress {
-    gPhysicalMemoryLock.lock();
-    defer gPhysicalMemoryLock.release();
-    const pages = gPhysicalMemory.alloc_pages(count);
+    const guard = g_physical_memory_lock.lock_irqsave();
+    defer guard.release();
+    const pages = g_physical_memory.alloc_pages(count);
     if (comptime kernel.TRACE_PHYSICAL_ALLOCATOR) {
         trace_allocation(count, pages);
     }
     return pages;
 }
 
+/// Deallocates a single page of physical memory.
+///
+/// # Panics
+///
+/// * If the `page` does not represent a valid managed page.
+/// * If the `page` was not previously allocated.
+/// * If a double free is detected.
 pub fn free_page(page: mem.PhysicalAddress) void {
     if (comptime kernel.TRACE_PHYSICAL_ALLOCATOR) {
         trace_free(page);
     }
-    gPhysicalMemoryLock.lock();
-    defer gPhysicalMemoryLock.release();
-    gPhysicalMemory.free_page(page);
-}
-
-// NOTE: Physical memory lock must be held
-pub fn get_page(page: mem.PhysicalAddress) *Page {
-    return gPhysicalMemory.get_page(page);
+    const guard = g_physical_memory_lock.lock_irqsave();
+    defer guard.release();
+    g_physical_memory.free_page(page);
 }

src/mem/tls.zig

@@ -0,0 +1,69 @@
+//! Thread-local storage implementation.
+
+const builtin = @import("builtin");
+
+const vmm = @import("vmm.zig");
+const phys_memory = @import("phys.zig");
+const kernel = @import("../kernel.zig");
+
+const PAGE_SIZE = vmm.PAGE_SIZE;
+const log = kernel.debug.log;
+
+/// Thread-local storage layout variant used by this target platform.
+pub const TLS_VARIANT: enum {
+    /// Variant I:
+    ///
+    /// [ TCB ] [ pad to p_align ] [ MODULE 0 ] [ MODULE 1 ] ...
+    /// |                          |            |
+    /// |                          |            |
+    /// tp                        off1          off2
+    variant1,
+    /// Variant II:
+    ///
+    /// ... [ MODULE 1 ] [ MODULE 0 ] [ TCB ]
+    ///     |            |            |
+    ///     |            |            |
+    ///    off2         off1          tp
+    variant2,
+} = switch (builtin.cpu.arch) {
+    .riscv64, .aarch64 => .variant1,
+    // x86-64 uses variant 2
+    else => @panic("Unsupported CPU architecture"),
+};
+
+extern var __tdata_start: u8;
+extern var __tdata_end: u8;
+extern var __tbss_start: u8;
+extern var __tbss_end: u8;
+
+/// Allocates a storage for one per-CPU TLS block, clones the TLS image
+/// (as described by .tbss/.tdata sections) and returns the result.
+pub fn load_kernel_tls_image() []u8 {
+    // Assume .tbss follows .tdata
+    const tdata_start = @intFromPtr(&__tdata_start);
+    const tdata_end = @intFromPtr(&__tdata_end);
+    const tdata_size = tdata_end - tdata_start;
+    const tbss_start = @intFromPtr(&__tbss_start);
+    const tbss_end = @intFromPtr(&__tbss_end);
+    const tbss_size = tbss_end - tbss_start;
+
+    const tdata_data = @as([*]u8, @ptrFromInt(tdata_start))[0..tdata_size];
+
+    switch (comptime TLS_VARIANT) {
+        .variant1 => {
+            const tls_size = tdata_size + tbss_size;
+            const tls_page_count = (tls_size + PAGE_SIZE - 1) / PAGE_SIZE;
+            // Variant I: TLS block 0 follows TP after a certain displacement
+            const tls_address = phys_memory.alloc_pages(tls_page_count).?.virtualize();
+            const tls_data = @as([*]u8, @ptrFromInt(tls_address))[0..tls_size];
+
+            log.info("Allocated TLS @ {*}", .{tls_data});
+
+            @memcpy(tls_data[0..tdata_size], tdata_data);
+            @memset(tls_data[tdata_size..], 0);
+
+            return tls_data;
+        },
+        .variant2 => @panic("TODO: TLS variant II"),
+    }
+}

src/mem/vmalloc.zig

@@ -0,0 +1,443 @@
+const std = @import("std");
+
+const Arena = @import("../arena.zig").Arena;
+const Range = @import("../util/range.zig").Range;
+
+/// Describes a single virtual memory range.
+///
+/// Used by `VirtualMemoryAllocator` to track allocated/used regions.
+pub const VirtualMemoryRange = struct {
+    range: Range(u64),
+
+    prev: ?*VirtualMemoryRange = null,
+    next: ?*VirtualMemoryRange = null,
+};
+
+/// Virtual memory allocator implementation.
+pub const VirtualMemoryAllocator = struct {
+    arena: *Arena,
+    head: ?*VirtualMemoryRange = null,
+    outer_range: Range(u64),
+
+    /// One of errors returned by the allocation logic + underlying allocator error.
+    pub const Error = error{ already_exists, invalid_region, cannot_fit };
+
+    pub const DrainIterator = struct {
+        vma: *VirtualMemoryAllocator,
+
+        pub fn next(self: *@This()) ?Range(u64) {
+            while (self.vma.head) |head| {
+                self.vma.head = head.next;
+                const range = head.range;
+                // TODO free the range
+                return range;
+            }
+            return null;
+        }
+    };
+
+    /// An iterator over VM regions being freed.
+    pub const FreeIterator = struct {
+        range: Range(u64),
+        vma: *VirtualMemoryAllocator,
+        current: ?*VirtualMemoryRange,
+
+        pub fn next(self: *@This()) Error!?Range(u64) {
+            while (self.current) |n| {
+                if (n.range.intersect(&self.range)) |xs| {
+                    if (xs.start == n.range.start) {
+                        if (xs.end() == n.range.end()) {
+                            // Whole range encompassed by requested range
+                            // Unlink the node
+                            if (n.next) |nn| {
+                                nn.prev = n.prev;
+                            }
+                            if (n.prev) |np| {
+                                np.next = n.next;
+                            } else {
+                                self.vma.head = n.next;
+                            }
+                            // Free it
+                            self.current = n.next;
+                            // self.vma.arena.destroy(n);
+
+                            return xs;
+                        }
+
+                        // Remove space from the start
+                        n.range.start += xs.len;
+                        n.range.len -= xs.len;
+                        // Does not touch the end, so can be sure this is the last node
+                        self.current = null;
+                        return xs;
+                    } else if (xs.end() == n.range.end()) {
+                        n.range.len -= xs.len;
+                        // Continue, there might be a following node affected
+                        self.current = n.next;
+                        return xs;
+                    } else {
+                        // Insert a new node after the current one
+                        const new_node = self.vma.arena.create(VirtualMemoryRange);
+                        new_node.* = VirtualMemoryRange{
+                            .range = .{ .start = xs.end(), .len = n.range.end() - xs.end() },
+                            .prev = n,
+                            .next = n.next,
+                        };
+                        n.range.len = xs.start - n.range.start;
+                        if (n.next) |nn| {
+                            nn.prev = new_node;
+                        }
+                        n.next = new_node;
+                        // Requested region is fully encompassed by this one, so no intersections
+                        // will follow
+                        self.current = null;
+                        return xs;
+                    }
+                } else {
+                    // No intersect
+                    self.current = n.next;
+                }
+            }
+            return null;
+        }
+    };
+
+    /// Creates a new instance of a virtual memory allocator.
+    pub fn init(arena: *Arena, outer_range: Range(u64)) @This() {
+        return .{
+            .outer_range = outer_range,
+            .arena = arena,
+        };
+    }
+
+    /// Allocates a free region of virtual memory of requested (`pfn_count`) size.
+    ///
+    /// # Errors
+    ///
+    /// * `cannot_fit` - if no free space found to fit the requested allocation.
+    /// * Underlying allocator error - if allocation of a new node fails.
+    pub fn allocate(self: *@This(), pfn_count: u64) Error!u64 {
+        // Try to fit before first entry
+        const gap_before_first = if (self.head) |n| (n.range.start - self.outer_range.start) else self.outer_range.len;
+
+        if (gap_before_first >= pfn_count) {
+            var new_node = self.arena.create(VirtualMemoryRange);
+
+            new_node.range = .{ .start = self.outer_range.start, .len = pfn_count };
+            new_node.next = self.head;
+            new_node.prev = null;
+
+            if (self.head) |n| {
+                n.prev = new_node;
+            }
+
+            self.head = new_node;
+
+            return self.outer_range.start;
+        }
+
+        // If cannot fit before first entry, find an entry to fit after
+        var node = self.head;
+        while (node) |n| {
+            const gap =
+                if (n.next) |nn|
+                    // Gap between this and next
+                    (nn.range.start - n.range.end())
+                else
+                    // Gap between this and the end
+                    (self.outer_range.end() - n.range.end());
+
+            if (gap >= pfn_count) {
+                // Insert after this
+                const result = n.range.end();
+                var new_node = self.arena.create(VirtualMemoryRange);
+                new_node.prev = n;
+                new_node.next = n.next;
+                new_node.range = .{ .start = result, .len = pfn_count };
+                if (n.next) |nn| {
+                    nn.prev = new_node;
+                }
+                n.next = new_node;
+                return result;
+            }
+
+            node = n.next;
+        }
+
+        return error.cannot_fit;
+    }
+
+    /// Inserts a reservation into the VM allocator.
+    ///
+    /// # Errors
+    ///
+    /// * `already_exists` - if the requested range intersects existing ranges.
+    /// * Underlying allocator error - if allocation of a new node fails.
+    pub fn insert(self: *@This(), region: Range(u64)) Error!void {
+        // Validate that the range does not escape the outer range
+        if (region.start < self.outer_range.start or region.end() > self.outer_range.end()) {
+            return error.invalid_region;
+        }
+
+        // Find the last node which is before the region supposed to be inserted
+        var node = self.head;
+        var insert_after: ?*VirtualMemoryRange = null;
+        while (node) |n| {
+            if (n.range.intersect(&region) != null) {
+                return error.already_exists;
+            }
+
+            if (n.range.end() <= region.start) {
+                insert_after = n;
+            }
+
+            node = n.next;
+        }
+
+        var new_node = self.arena.create(VirtualMemoryRange);
+
+        new_node.range = region;
+
+        if (insert_after) |ia| {
+            new_node.prev = ia;
+            new_node.next = ia.next;
+
+            if (ia.next) |ian| {
+                ian.prev = new_node;
+            }
+            ia.next = new_node;
+        } else {
+            new_node.next = null;
+            new_node.prev = null;
+
+            self.head = new_node;
+        }
+    }
+
+    /// Deallocates (shrinks/truncates) regions intersecting the requested range.
+    pub fn free(self: *@This(), start_pfn: u64, pfn_count: u64) FreeIterator {
+        const range = Range(u64){ .start = start_pfn, .len = pfn_count };
+        return FreeIterator{
+            .current = self.head,
+            .vma = self,
+            .range = range,
+        };
+    }
+
+    pub fn drain(self: *@This()) DrainIterator {
+        return DrainIterator{ .vma = self };
+    }
+};
+
+test "Inserted entries in vmalloc are properly ordered" {
+    var vma = VirtualMemoryAllocator.init(std.testing.allocator, .{ .start = 0x1000, .len = 0x2000 });
+    defer {
+        while (vma.head) |n| {
+            vma.head = n.next;
+            std.testing.allocator.destroy(n);
+        }
+    }
+    try vma.insert(.{ .start = 0x1200, .len = 0x200 });
+    {
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1200, n0.range.start);
+        try std.testing.expectEqual(0x200, n0.range.len);
+        try std.testing.expectEqual(null, n0.next);
+        try std.testing.expectEqual(null, n0.prev);
+    }
+    try vma.insert(.{ .start = 0x2000, .len = 0x200 });
+    {
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1200, n0.range.start);
+        try std.testing.expectEqual(0x200, n0.range.len);
+        try std.testing.expectEqual(null, n0.prev);
+        const n1 = n0.next.?;
+        try std.testing.expectEqual(0x2000, n1.range.start);
+        try std.testing.expectEqual(0x200, n1.range.len);
+        try std.testing.expectEqual(n0, n1.prev);
+        try std.testing.expectEqual(null, n1.next);
+    }
+    try vma.insert(.{ .start = 0x1400, .len = 0x200 });
+    {
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1200, n0.range.start);
+        try std.testing.expectEqual(0x200, n0.range.len);
+        try std.testing.expectEqual(null, n0.prev);
+        const n1 = n0.next.?;
+        try std.testing.expectEqual(0x1400, n1.range.start);
+        try std.testing.expectEqual(0x200, n1.range.len);
+        try std.testing.expectEqual(n0, n1.prev);
+        const n2 = n1.next.?;
+        try std.testing.expectEqual(0x2000, n2.range.start);
+        try std.testing.expectEqual(0x200, n2.range.len);
+        try std.testing.expectEqual(n1, n2.prev);
+        try std.testing.expectEqual(null, n2.next);
+    }
+}
+
+test "Overlapping insertions are denied" {
+    var vma = VirtualMemoryAllocator.init(std.testing.allocator, .{ .start = 0x1000, .len = 0x1000 });
+    defer {
+        while (vma.head) |n| {
+            vma.head = n.next;
+            std.testing.allocator.destroy(n);
+        }
+    }
+    try vma.insert(.{ .start = 0x1200, .len = 0x200 });
+    try std.testing.expectError(error.already_exists, vma.insert(.{ .start = 0x1100, .len = 0x200 }));
+    try std.testing.expectError(error.already_exists, vma.insert(.{ .start = 0x1300, .len = 0x200 }));
+    try std.testing.expectError(error.already_exists, vma.insert(.{ .start = 0x1100, .len = 0x400 }));
+}
+
+test "Insertions outside of bounds are denied" {
+    var vma = VirtualMemoryAllocator.init(std.testing.allocator, .{ .start = 0x1000, .len = 0x1000 });
+    // As above...
+    try std.testing.expectError(error.invalid_region, vma.insert(.{ .start = 0x2200, .len = 0x200 }));
+    // ... so below
+    try std.testing.expectError(error.invalid_region, vma.insert(.{ .start = 0x200, .len = 0x200 }));
+    // Crosses from below
+    try std.testing.expectError(error.invalid_region, vma.insert(.{ .start = 0x200, .len = 0x1000 }));
+    // Crosses into above
+    try std.testing.expectError(error.invalid_region, vma.insert(.{ .start = 0x1200, .len = 0x1000 }));
+    // Encompasses whole
+    try std.testing.expectError(error.invalid_region, vma.insert(.{ .start = 0x200, .len = 0x2000 }));
+}
+
+test "Allocations from vmalloc" {
+    var vma = VirtualMemoryAllocator.init(std.testing.allocator, .{ .start = 0x1000, .len = 0x1000 });
+    defer {
+        while (vma.head) |n| {
+            vma.head = n.next;
+            std.testing.allocator.destroy(n);
+        }
+    }
+    try vma.insert(.{ .start = 0x1200, .len = 0x200 });
+    try std.testing.expectEqual(0x1000, try vma.allocate(0x100));
+    try std.testing.expectEqual(0x1400, try vma.allocate(0x400));
+    try std.testing.expectEqual(0x1100, try vma.allocate(0x100));
+}
+
+test "vmalloc free" {
+    var vma = VirtualMemoryAllocator.init(std.testing.allocator, .{ .start = 0x1000, .len = 0x1000 });
+
+    try vma.insert(.{ .start = 0x1200, .len = 0x800 });
+    try vma.insert(.{ .start = 0x1A00, .len = 0x400 });
+
+    // Remove nothing
+    {
+        var free_it = vma.free(0x1000, 0x200);
+        try std.testing.expectEqual(null, free_it.next());
+    }
+
+    // Remove a chunk in the middle of a node
+    {
+        var free_it = vma.free(0x1400, 0x400);
+        const r0 = (try free_it.next()).?;
+        try std.testing.expectEqual(0x1400, r0.start);
+        try std.testing.expectEqual(0x400, r0.len);
+        try std.testing.expectEqual(null, free_it.next());
+
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1200, n0.range.start);
+        try std.testing.expectEqual(0x200, n0.range.len);
+        const n1 = n0.next.?;
+        try std.testing.expectEqual(0x1800, n1.range.start);
+        try std.testing.expectEqual(0x200, n1.range.len);
+        try std.testing.expectEqual(n0, n1.prev);
+        const n2 = n1.next.?;
+        try std.testing.expectEqual(0x1A00, n2.range.start);
+        try std.testing.expectEqual(0x400, n2.range.len);
+        try std.testing.expectEqual(n1, n2.prev);
+        try std.testing.expectEqual(null, n2.next);
+    }
+
+    // Remove from the start
+    {
+        var free_it = vma.free(0x1200, 0x100);
+        const r0 = (try free_it.next()).?;
+        try std.testing.expectEqual(0x1200, r0.start);
+        try std.testing.expectEqual(0x100, r0.len);
+        try std.testing.expectEqual(null, free_it.next());
+
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1300, n0.range.start);
+        try std.testing.expectEqual(0x100, n0.range.len);
+        const n1 = n0.next.?;
+        try std.testing.expectEqual(0x1800, n1.range.start);
+        try std.testing.expectEqual(0x200, n1.range.len);
+        try std.testing.expectEqual(n0, n1.prev);
+        const n2 = n1.next.?;
+        try std.testing.expectEqual(0x1A00, n2.range.start);
+        try std.testing.expectEqual(0x400, n2.range.len);
+        try std.testing.expectEqual(n1, n2.prev);
+        try std.testing.expectEqual(null, n2.next);
+    }
+
+    // Remove from the end
+    {
+        var free_it = vma.free(0x1900, 0x100);
+        const r0 = (try free_it.next()).?;
+        try std.testing.expectEqual(0x1900, r0.start);
+        try std.testing.expectEqual(0x100, r0.len);
+        try std.testing.expectEqual(null, free_it.next());
+
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1300, n0.range.start);
+        try std.testing.expectEqual(0x100, n0.range.len);
+        const n1 = n0.next.?;
+        try std.testing.expectEqual(0x1800, n1.range.start);
+        try std.testing.expectEqual(0x100, n1.range.len);
+        try std.testing.expectEqual(n0, n1.prev);
+        const n2 = n1.next.?;
+        try std.testing.expectEqual(0x1A00, n2.range.start);
+        try std.testing.expectEqual(0x400, n2.range.len);
+        try std.testing.expectEqual(n1, n2.prev);
+        try std.testing.expectEqual(null, n2.next);
+    }
+
+    // Remove single full
+    {
+        var free_it = vma.free(0x1000, 0x600);
+        const r0 = (try free_it.next()).?;
+        try std.testing.expectEqual(0x1300, r0.start);
+        try std.testing.expectEqual(0x100, r0.len);
+        try std.testing.expectEqual(null, free_it.next());
+
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1800, n0.range.start);
+        try std.testing.expectEqual(0x100, n0.range.len);
+        const n1 = n0.next.?;
+        try std.testing.expectEqual(0x1A00, n1.range.start);
+        try std.testing.expectEqual(0x400, n1.range.len);
+        try std.testing.expectEqual(n0, n1.prev);
+        try std.testing.expectEqual(null, n1.next);
+    }
+
+    // Remove one full + one partial
+    {
+        var free_it = vma.free(0x1600, 0x600);
+        const r0 = (try free_it.next()).?;
+        try std.testing.expectEqual(0x1800, r0.start);
+        try std.testing.expectEqual(0x100, r0.len);
+        const r1 = (try free_it.next()).?;
+        try std.testing.expectEqual(0x1A00, r1.start);
+        try std.testing.expectEqual(0x200, r1.len);
+        try std.testing.expectEqual(null, free_it.next());
+
+        const n0 = vma.head.?;
+        try std.testing.expectEqual(0x1C00, n0.range.start);
+        try std.testing.expectEqual(0x200, n0.range.len);
+        try std.testing.expectEqual(null, n0.next);
+    }
+
+    // Remove whatever remains
+    {
+        var free_it = vma.free(0, 0x20000);
+        const r0 = (try free_it.next()).?;
+        try std.testing.expectEqual(0x1C00, r0.start);
+        try std.testing.expectEqual(0x200, r0.len);
+        try std.testing.expectEqual(null, free_it.next());
+
+        try std.testing.expectEqual(null, vma.head);
+    }
+}

src/mem/vmm.zig

@@ -1,9 +1,31 @@
-pub const PAGE_SIZE: usize = 0x1000;
+//! Platform-independent virtual memory management definitions.
 
-pub fn TranslationLevel(comptime shift: usize) type {
+const mem = @import("../mem.zig");
+const arena = @import("../arena.zig");
+const vmalloc = @import("vmalloc.zig");
+const kernel = @import("../kernel.zig");
+const sync = @import("../sync.zig");
+
+const arch = kernel.arch;
+const log = kernel.log;
+const Arena = arena.Arena;
+
+/// Last virtual memory translation level. Always 4KiB on all platforms.
+pub const L3 = mem.TranslationLevel(12, null);
+
+/// Page size is 4KiB on all platforms.
+pub const PAGE_SIZE: usize = L3.SIZE;
+
+pub const AddressSpaceError = error{
+    out_of_pages,
+} || vmalloc.VirtualMemoryAllocator.Error;
+
+/// Helper function to construct a "Translation Level" struct type from a bit shift.
+pub fn TranslationLevel(comptime shift: usize, comptime Next: ?type) type {
     return struct {
         pub const SHIFT: usize = shift;
         pub const SIZE: usize = 1 << shift;
+        pub const NextLevel = Next;
 
         pub inline fn index(addr: usize) usize {
             return (addr >> shift) & 511;
@@ -24,5 +46,62 @@ pub fn TranslationLevel(comptime shift: usize) type {
         pub inline fn align_up(addr: usize) usize {
             return (addr + SIZE - 1) & ~(SIZE - 1);
         }
+
+        pub inline fn page_number(addr: usize) usize {
+            return addr >> shift;
+        }
+
+        pub inline fn page_count(size: usize) usize {
+            return (size + SIZE - 1) / SIZE;
+        }
     };
 }
+
+pub const ProcessAddressSpace = struct {
+    inner: arch.vmm.ProcessAddressSpace,
+    allocator: vmalloc.VirtualMemoryAllocator,
+    lock: sync.Spinlock,
+
+    pub fn init(a: *Arena) AddressSpaceError!ProcessAddressSpace {
+        // 0x200000..0x600000
+        const inner = try arch.vmm.ProcessAddressSpace.init();
+        const allocator = vmalloc.VirtualMemoryAllocator.init(a, .{ .start = 512, .len = 1024 });
+
+        return .{ .inner = inner, .allocator = allocator, .lock = .{} };
+    }
+
+    pub fn clear(self: *@This()) void {
+        var drain = self.allocator.drain();
+        while (drain.next()) |range| {
+            log.info("Free range: 0x{x}..0x{x}", .{ range.start * L3.SIZE, range.end() * L3.SIZE });
+            // TODO unmap/free pages
+        }
+    }
+
+    pub fn map_single_page(
+        self: *@This(),
+        virtual: usize,
+        physical: mem.PhysicalAddress,
+    ) AddressSpaceError!void {
+        self.lock.lock();
+        defer self.lock.release();
+
+        try self.allocator.insert(.{ .start = L3.page_number(virtual), .len = 1 });
+        errdefer {
+            var it = self.allocator.free(L3.page_number(virtual), 1);
+            while (it.next() catch unreachable) |n| {
+                // TODO: inner.unmap_page()
+                _ = n;
+            }
+        }
+        try self.inner.map_page(virtual, physical);
+    }
+
+    pub fn physical_address(self: *const @This()) mem.PhysicalAddress {
+        return self.inner.physical_address();
+    }
+
+    pub fn asid(self: *const @This()) u64 {
+        return self.inner.asid;
+    }
+};

src/object.zig

@@ -0,0 +1,103 @@
+const mem = @import("mem.zig");
+const kernel = @import("kernel.zig");
+const thread = @import("thread.zig");
+const abi = @import("abi");
+
+const ProcessAddressSpace = mem.vmm.ProcessAddressSpace;
+const log = kernel.log;
+const Thread = thread.Thread;
+
+pub const PhysicalMemoryObject = struct {
+    fn send(self: *const PhysicalMemoryObject, caller: *Thread, message: *const [5]usize) usize {
+        _ = self;
+        _ = caller;
+        _ = message;
+        @panic("TODO: physical memory object messaging");
+    }
+
+    fn sendrecv(self: *const PhysicalMemoryObject, caller: *Thread, message: *[5]usize) usize {
+        switch (@as(abi.PhysicalMemoryObjectAction, @enumFromInt(message[0]))) {
+            .ZO_physical_memory_allocate => {
+                // TODO somehow track ownership of the memory by the process
+                const pages = mem.phys.alloc_pages(message[1]) orelse {
+                    return 1;
+                };
+                log.info("{*}: allocated {} pages: 0x{x}", .{caller, message[1], pages.raw});
+                message[0] = pages.raw;
+                return 0;
+            },
+            .ZO_physical_memory_free => {
+                @panic("TODO: ZO_physical_memory_free");
+            },
+            else => {
+                @panic("TODO: invalid message to Physical Memory Object");
+            }
+        }
+        _ = self;
+    }
+};
+
+pub const ProcessObject = struct {
+    inner: *Thread,
+
+    fn send(self: *const ProcessObject, caller: *Thread, message: *const [5]usize) usize {
+        _ = self;
+        // TODO define this in IDL/ABI
+        switch (@as(abi.ProcessObjectAction, @enumFromInt(message[0]))) {
+            .ZO_process_exit => {
+                log.info("{*} exited with code 0x{x} ({})", .{caller, message[1], message[1]});
+                Thread.exit_current();
+            },
+            else => {
+                @panic("TODO: invalid message to Process Object");
+            }
+        }
+    }
+
+    fn sendrecv(self: *const ProcessObject, caller: *Thread, message: *[5]usize) usize {
+        _ = self;
+        _ = caller;
+        _ = message;
+        @panic("TODO: ProcessObject sendrecv()");
+    }
+};
+
+pub const DebugObject = struct {
+    fn send(self: *const DebugObject, caller: *Thread, message: *const [5]usize) usize {
+        _ = self;
+        // Debug has only one message
+        const text = @as([*]u8, @ptrFromInt(message[0]))[0..message[1]];
+        log.info("{*}: {s}", .{ caller, text });
+        return 0;
+    }
+
+    fn sendrecv(self: *const DebugObject, caller: *Thread, message: *[5]usize) usize {
+        _ = self;
+        _ = caller;
+        _ = message;
+        @panic("TODO: DebugObject sendrecv()");
+    }
+};
+
+pub const Object = union(enum) {
+    physical_memory: PhysicalMemoryObject,
+    process: ProcessObject,
+    debug: DebugObject,
+    // TODO userspace "object" can be placed here
+
+    pub fn send(self: Object, caller: *Thread, message: *const [5]usize) usize {
+        return switch (self) {
+            .physical_memory => |physical_memory| physical_memory.send(caller, message),
+            .process => |process| process.send(caller, message),
+            .debug => |debug| debug.send(caller, message),
+        };
+    }
+
+    pub fn sendrecv(self: Object, caller: *Thread, message: *[5]usize) usize {
+        return switch (self) {
+            .physical_memory => |physical_memory| physical_memory.sendrecv(caller, message),
+            .process => |process| process.sendrecv(caller, message),
+            .debug => |debug| debug.sendrecv(caller, message),
+        };
+    }
+};

src/sync.zig

@@ -1,19 +1,44 @@
+//! Kernel synchronization primitives.
+
 const std = @import("std");
 const arch = @import("kernel.zig").arch;
 
-pub const IrqSafeSpinlock = struct {
+/// Basic spinlock implementation
+pub const Spinlock = struct {
     state: std.atomic.Value(bool) = .{ .raw = false },
 
-    pub fn tryLock(self: *@This()) bool {
+    const Guard = struct {
+        lock: *Spinlock,
+        irq_mask: bool,
+
+        /// Releases the `Guard`, restoring the previous IRQ state and releasing the lock used
+        /// to acquire it.
+        pub fn release(self: @This()) void {
+            self.lock.release();
+            _ = arch.set_interrupt_mask(self.irq_mask);
+        }
+    };
+
+    /// Acquires a lock over `self`. Returns `false` if the lock is already held by someone else.
+    pub fn try_lock(self: *@This()) bool {
         return self.state.cmpxchgStrong(false, true, .acquire, .monotonic) orelse false;
     }
 
+    /// Acquires a lock over `self`. Will block until a lock can be acquired.
     pub fn lock(self: *@This()) void {
-        while (!self.tryLock()) {
-            arch.spinHint();
+        while (!self.try_lock()) {
+            arch.spin_hint();
         }
     }
 
+    /// Same as `lock()`, but additionally saves current IRQ state and masks IRQs.
+    pub fn lock_irqsave(self: *@This()) Guard {
+        const irq_mask = arch.set_interrupt_mask(true);
+        self.lock();
+        return .{ .irq_mask = irq_mask, .lock = self };
+    }
+
+    /// Releases a lock over `self`.
     pub fn release(self: *@This()) void {
         self.state.store(false, .release);
     }

src/syscall.zig

@@ -0,0 +1,55 @@
+const kernel = @import("kernel.zig");
+const abi = @import("abi");
+
+const thread = kernel.thread;
+const Thread = thread.Thread;
+const log = kernel.log;
+
+pub const SyscallFn = *const fn (*Thread, *[6]usize) void;
+
+pub const syscall_table: [abi.MAX_SYSCALL]?SyscallFn = make_syscall_table();
+
+fn make_syscall_table() [abi.MAX_SYSCALL]?SyscallFn {
+    const SC = abi.SyscallNumber;
+    var array = [_]?SyscallFn{undefined} ** abi.MAX_SYSCALL;
+    array[@intFromEnum(SC.SYS_send)] = sys_send;
+    array[@intFromEnum(SC.SYS_recv)] = sys_recv;
+    array[@intFromEnum(SC.SYS_sendrecv)] = sys_sendrecv;
+    return array;
+}
+
+fn sys_send(cthread: *Thread, frame: *[6]usize) void {
+    const handle: abi.Handle = @enumFromInt(frame[0]);
+    const object = cthread.handle(handle) orelse {
+        @panic("TODO: userspace invoked non-existent handle");
+    };
+    frame[0] = object.send(cthread, frame[1..6]);
+}
+
+fn sys_recv(cthread: *Thread, frame: *[6]usize) void {
+    _ = cthread;
+    _ = frame;
+    @panic("TODO: SYS_recv()");
+}
+
+fn sys_sendrecv(cthread: *Thread, frame: *[6]usize) void {
+    const handle: abi.Handle = @enumFromInt(frame[0]);
+    const object = cthread.handle(handle) orelse {
+        @panic("TODO: userspace invoked non-existent handle");
+    };
+    frame[0] = object.sendrecv(cthread, frame[1..6]);
+}
+
+fn sys_undefined_syscall(cthread: *Thread, frame: *[6]usize) void {
+    _ = frame;
+    log.warn("{*} invoked an undefined syscall", .{cthread});
+    Thread.exit_current();
+}
+
+pub fn syscall_handler(func: usize, frame: *[6]usize) void {
+    const cthread = thread.Thread.current();
+    const handler = if (func >= abi.MAX_SYSCALL) sys_undefined_syscall //
+        else syscall_table[func] //
+        orelse sys_undefined_syscall;
+    handler(cthread, frame);
+}

src/thread.zig

@@ -1,24 +1,166 @@
+//! Thread management utilities and data structures.
+
 const std = @import("std");
 
 const arena = @import("arena.zig");
 const arch = @import("kernel.zig").arch;
 const log = @import("debug.zig").log;
 const mem = @import("mem.zig");
+const sync = @import("sync.zig");
+const object = @import("object.zig");
+const abi = @import("abi");
 
+const ProcessAddressSpace = mem.vmm.ProcessAddressSpace;
+const Object = object.Object;
+const Handle = abi.Handle;
+
+// TODO: are kernel threads needed at all if we're doing a microkernel?
+
+/// Signature for kernel thread entry
+pub const KernelThreadFn = fn (usize) callconv(.C) void;
+
+pub fn kernel_return() callconv(.C) noreturn {
+    Thread.exit_current();
+}
+
+/// Task to run when there are no real threads in the queue
+pub fn idle_function(arg: usize) callconv(.C) noreturn {
+    _ = arg;
+    while (true) {
+        arch.wait_for_interrupt();
+    }
+}
+
+/// Represents a single execution thread.
+pub const Thread = struct {
+    const MAX_HANDLES: usize = 64;
+
+    /// Arena.
+    allocator: *arena.Arena,
+    /// Architecture-specific task context.
+    arch_context: arch.Context,
+
+    /// Queue to which this thread belongs
+    queue: ?*Queue = null,
+    /// Next thread in the queue.
+    next: ?*Thread = null,
+    /// Previous thread in the queue.
+    prev: ?*Thread = null,
+
+    // TODO move to process
+    address_space: ?ProcessAddressSpace = null,
+    // TODO move to process
+    handle_table: [MAX_HANDLES]?Object = [_]?Object{null} ** MAX_HANDLES,
+
+    pub const Error = error{out_of_memory} || mem.vmm.AddressSpaceError;
+
+    /// Creates a new (kernel) thread with given `function` and `arg`ument.
+    pub fn create_kernel(a: *arena.Arena, function: *const KernelThreadFn, arg: usize) *Thread {
+        const thread = a.create(Thread);
+        thread.* = .{
+            .allocator = a,
+            .arch_context = arch.Context.kernel(function, arg),
+        };
+        return thread;
+    }
+
+    pub fn create_user(
+        a: *arena.Arena,
+        address_space: ProcessAddressSpace,
+        pc: usize,
+        sp: usize,
+        arg: usize,
+    ) *Thread {
+        const thread = a.create(Thread);
+        thread.* = .{
+            .allocator = a,
+            .address_space = address_space,
+            .arch_context = arch.Context.user(&address_space, pc, sp, arg),
+        };
+        // "self" process object is granted to all processes
+        const process_object = Object{ .process = object.ProcessObject {
+            .inner = thread,
+        } };
+        _ = thread.grant(process_object);
+        return thread;
+    }
+
+    /// Enters the thread, does not return.
+    pub fn enter(self: *@This()) noreturn {
+        self.arch_context.enter();
+    }
+
+    /// Switches from `from` to `self` thread.
+    pub fn switch_from(self: *@This(), from: *@This()) void {
+        self.arch_context.switch_from(&from.arch_context);
+    }
+
+    pub fn grant(self: *@This(), obj: Object) Handle {
+        for (0..MAX_HANDLES) |i| {
+            if (self.handle_table[i] == null) {
+                self.handle_table[i] = obj;
+                return @enumFromInt(i + 1);
+            }
+        }
+        @panic("TODO: ran out of objects");
+    }
+
+    pub fn handle(self: *@This(), h: Handle) ?Object {
+        const index = @as(usize, @intFromEnum(h));
+        if (index > MAX_HANDLES or index == 0) {
+            return null;
+        }
+        return self.handle_table[index - 1];
+    }
+
+    pub fn dequeue(self: *@This()) void {
+        // TODO queueing information should be put under a lock for SMP to work properly
+        if (self.queue) |q| {
+            q.dequeue(self);
+        }
+    }
+
+    pub fn current() *@This() {
+        return Queue.t_this_cpu.?.current.?;
+    }
+
+    pub fn exit_current() noreturn {
+        // Mask IRQs so they don't break current thread's state
+        const mask = arch.IrqGuard.acquire();
+        defer mask.release();
+
+        const curr = Thread.current();
+        curr.dequeue();
+
+        yield();
+
+        @panic("This code should not be reachable");
+    }
+};
+
+/// Per-CPU thread queue structure.
 pub const Queue = struct {
+    /// Idle task context. Used when there are no other tasks running.
     idle: arch.Context,
+    /// Current thread pointer.
     current: ?*Thread = null,
+    /// Thread queue head pointer.
     head: ?*Thread = null,
+    /// Queue's lock
+    lock: sync.Spinlock = .{},
 
-    pub threadlocal var thisCpu: ?*Queue = null;
+    /// Pointer to this CPU's thread queue.
+    pub threadlocal var t_this_cpu: ?*Queue = null;
 
-    pub fn initThisCpu(a: *arena.Arena) void {
+    /// Sets up a thread queue for the current CPU.
+    pub fn init_this_cpu(a: *arena.Arena) void {
         const idle = arch.Context.idle();
         const q = a.create(Queue);
         q.* = .{ .idle = idle };
-        thisCpu = q;
+        t_this_cpu = q;
     }
 
+    /// Enters a task on this CPU.
     pub fn enter(self: *@This()) noreturn {
         if (self.head) |gt| {
             self.current = gt;
@@ -29,33 +171,47 @@ pub const Queue = struct {
         }
     }
 
+    /// Yields CPU to the next available task.
     pub fn yield(self: *@This()) void {
+        // TODO locking here
         if (self.current) |curr| {
             // Switching from thread
             if (curr.next) |next| {
                 // ... to thread
                 if (next != curr) {
                     self.current = next;
-                    next.switchFrom(curr);
+                    next.switch_from(curr);
+                }
+            } else if (self.head) |h| {
+                // ... to thread (head)
+                if (h != curr) {
+                    self.current = h;
+                    h.switch_from(curr);
                 }
             } else {
                 // ... to idle
                 self.current = null;
-                self.idle.switchFrom(&curr.archContext);
+                self.idle.switch_from(&curr.arch_context);
             }
         } else {
             // Switching from idle
             if (self.head) |gt| {
                 // ... to thread
                 self.current = gt;
-                gt.archContext.switchFrom(&self.idle);
+                gt.arch_context.switch_from(&self.idle);
                 return;
             }
             // ... back to idle
         }
     }
 
+    /// Adds an available task to this queue.
     pub fn enqueue(self: *@This(), t: *Thread) void {
+        var guard = self.lock.lock_irqsave();
+        defer guard.release();
+
+        t.queue = self;
+
         if (self.head) |gt| {
             t.next = gt;
             t.prev = gt.prev;
@@ -67,48 +223,79 @@ pub const Queue = struct {
             t.prev = t;
         }
     }
-};
 
-pub const Thread = struct {
-    allocator: *arena.Arena,
-    archContext: arch.Context,
+    /// # Invariants
+    ///
+    /// `t` must be a thread within the `self` queue.
+    fn dequeue(self: *@This(), t: *Thread) void {
+        var guard = self.lock.lock_irqsave();
+        defer guard.release();
 
-    next: ?*Thread = null,
-    prev: ?*Thread = null,
+        t.queue = null;
 
-    pub fn create(a: *arena.Arena, pc: usize, arg: usize) *Thread {
-        const thread = a.create(Thread);
-        thread.* = .{
-            .allocator = a,
-            .archContext = arch.Context.kernel(pc, arg),
-        };
-        return thread;
-    }
+        if (t == self.head) {
+            if (t.next == t) {
+                self.head = null;
+                t.next = null;
+                t.prev = null;
+                return;
+            }
 
-    pub fn enter(self: *@This()) noreturn {
-        self.archContext.enter();
-    }
+            if (t.next) |tn| {
+                tn.prev = t.prev;
+            }
+            if (t.prev) |tp| {
+                tp.next = t.next;
+            }
 
-    pub fn switchFrom(self: *@This(), from: *@This()) void {
-        self.archContext.switchFrom(&from.archContext);
+            self.head = t.next;
+        } else {
+            if (t.next) |tn| {
+                tn.prev = t.prev;
+            }
+            if (t.prev) |tp| {
+                tp.next = t.next;
+            }
+        }
+
+        t.next = null;
+        t.prev = null;
     }
 };
 
+/// Helper data structure to represent kernel stacks in task contexts.
 pub fn KStack(comptime SIZE: usize) type {
     return extern struct {
         // Has to be at exactly offset 0x00, used in assembly
+        /// Stack pointer. Aliases `data`.
         sp: *usize,
 
+        /// Stack data represented as a slice of `SIZE` machine-sized words.
         data: *[SIZE]usize,
-        physicalBase: mem.PhysicalAddress,
+        /// Physical base address at which the stack is allocated.
+        physical_base: mem.PhysicalAddress,
 
+        /// Allocates a new kernel stack.
+        ///
+        /// # Panics
+        ///
+        /// Panics on Out-of-Memory condition. TODO Fix this.
         pub fn create() @This() {
-            const physicalBase = mem.phys.alloc_pages(SIZE * @sizeOf(usize) / 0x1000) orelse @panic("OOM");
-            const ptr = @as(*[SIZE]usize, @ptrFromInt(physicalBase.virtualize()));
+            const physical_base = mem.phys.alloc_pages(SIZE * @sizeOf(usize) / 0x1000) orelse @panic("OOM");
+            const ptr = @as(*[SIZE]usize, @ptrFromInt(physical_base.virtualize()));
 
-            return .{ .data = ptr, .physicalBase = physicalBase, .sp = @ptrFromInt(@intFromPtr(&ptr[0]) + SIZE * @sizeOf(usize)) };
+            return .{
+                .data = ptr,
+                .physical_base = physical_base,
+                .sp = @ptrFromInt(@intFromPtr(&ptr[0]) + SIZE * @sizeOf(usize)),
+            };
         }
 
+        /// Pushes a machine-sized word onto the stack.
+        ///
+        /// # Panics
+        ///
+        /// Panics if a push would overflow the stack.
         pub fn push(self: *@This(), value: usize) void {
             if (self.sp == &self.data[0]) {
                 @panic("KStack overflow");
@@ -119,14 +306,63 @@ pub fn KStack(comptime SIZE: usize) type {
     };
 }
 
+/// Adds a thread to some CPU queue for execution.
 pub fn enqueue(t: *Thread) void {
-    Queue.thisCpu.?.enqueue(t);
+    Queue.t_this_cpu.?.enqueue(t);
 }
 
+/// Enters thread execution on the current CPU.
 pub fn enter() noreturn {
-    Queue.thisCpu.?.enter();
+    Queue.t_this_cpu.?.enter();
 }
 
+/// Yields this CPU's execution to a next thread.
 pub fn yield() void {
-    Queue.thisCpu.?.yield();
+    Queue.t_this_cpu.?.yield();
+}
+
+pub fn test_create_user_from_code(a: *arena.Arena, code: []const u8) Thread.Error!*Thread {
+    const L3 = mem.vmm.L3;
+    const CODE_BASE: usize = 0x200000;
+
+    var address_space = try ProcessAddressSpace.init(a);
+    errdefer {
+        address_space.clear();
+    }
+
+    // @ 0x200000
+    const code_page_count = L3.page_count(code.len);
+    log.info("Code is {} pages", .{code_page_count});
+    var offset: usize = 0;
+    var address: usize = CODE_BASE;
+    while (offset < code.len) {
+        const page_offset = address % L3.SIZE;
+        const amount = @min(L3.SIZE - page_offset, code.len - offset);
+
+        const page = mem.phys.alloc_page() orelse return error.out_of_memory;
+        try address_space.map_single_page(address, page);
+
+        const page_data = @as([*]u8, @ptrFromInt(page.virtualize()))[0..amount];
+        @memcpy(page_data, code[offset .. offset + amount]);
+
+        address += amount;
+        offset += amount;
+    }
+
+    // @ 0x400000
+    const sp_base = 0x400000;
+    for (0..8) |i| {
+        const page = mem.phys.alloc_page() orelse return error.out_of_memory;
+        try address_space.map_single_page(sp_base + i * L3.SIZE, page);
+    }
+    const sp = sp_base + 8 * L3.SIZE;
+
+    log.info("Enter with sp = 0x{x}", .{sp});
+
+    const thread = Thread.create_user(a, address_space, CODE_BASE, sp, 1234);
+
+    _ = thread.grant(Object { .physical_memory = object.PhysicalMemoryObject {} });
+    _ = thread.grant(Object { .debug = object.DebugObject {} });
+
+    return thread;
 }

src/util.zig

@@ -0,0 +1,4 @@
+pub const dtb = @import("util/dtb.zig");
+pub const range = @import("util/range.zig");
+pub const btree = @import("util/btree.zig");
+pub const rangemap = @import("util/rangemap.zig");

src/util/btree.zig

@@ -0,0 +1,368 @@
+const std = @import("std");
+
+const Allocator = std.mem.Allocator;
+pub const Order = std.math.Order;
+
+pub fn CompareFn(comptime N: type) type {
+    return fn (*const N, *const N) Order;
+}
+
+pub fn SearchFn(comptime N: type, comptime C: type) type {
+    return fn (*const N, C) Order;
+}
+
+pub fn BTree(comptime N: type, comptime compare_fn: CompareFn(N), comptime deinit_fn: ?fn (*N) void) type {
+    return struct {
+        gpa: Allocator,
+        root: ?*Node = null,
+
+        pub const Error = error{ already_exists, does_not_exist } || Allocator.Error;
+
+        pub fn WalkFn(comptime C: type) type {
+            return fn (*const Node, C) void;
+        }
+
+        pub const Iterator = struct {
+            current: ?*Node,
+
+            pub fn next(self: *Iterator) ?*const Node {
+                while (self.current) |n| {
+                    const v = n;
+
+                    if (n.right) |r| {
+                        // Emit
+                        self.current = Node.leftmost(r);
+                    } else {
+                        var nn = n;
+                        while (nn.parent) |p| {
+                            if (nn == p.right) {
+                                nn = p;
+                            } else {
+                                break;
+                            }
+                        }
+                        self.current = nn.parent;
+                    }
+
+                    return v;
+                }
+
+                return null;
+            }
+        };
+
+        pub const Node = struct {
+            key: N,
+            parent: ?*Node = null,
+            left: ?*Node = null,
+            right: ?*Node = null,
+
+            fn init(a: Allocator, key: N) Error!*Node {
+                const node = try a.create(Node);
+                node.* = .{
+                    .key = key,
+                };
+                return node;
+            }
+
+            fn deinit(node: ?*Node, a: Allocator) void {
+                if (node) |n| {
+                    if (comptime deinit_fn) |f| {
+                        f(&n.key);
+                    }
+
+                    Node.deinit(n.left, a);
+                    Node.deinit(n.right, a);
+
+                    // Free node itself
+                    a.destroy(n);
+                }
+            }
+
+            fn insert(node: ?*Node, a: Allocator, key: N) Error!struct { *Node, *Node } {
+                if (node) |n| {
+                    const ord = compare_fn(&n.key, &key);
+                    var inserted: *Node = undefined;
+                    switch (ord) {
+                        .lt => {
+                            const child, inserted = try Node.insert(n.right, a, key);
+                            child.parent = n;
+                            n.right = child;
+                        },
+                        .gt => {
+                            const child, inserted = try Node.insert(n.left, a, key);
+                            child.parent = n;
+                            n.left = child;
+                        },
+                        .eq => return error.already_exists,
+                    }
+                    return .{ n, inserted };
+                } else {
+                    const n = try Node.init(a, key);
+                    return .{ n, n };
+                }
+            }
+
+            fn remove_node(node: *Node, a: Allocator, destroy: bool) ?*Node {
+                if (node.left == null) {
+                    // Only right/none
+                    const tmp = node.right;
+                    if (tmp) |t| {
+                        t.parent = node.parent;
+                    }
+                    // Destroy the node
+                    if (comptime deinit_fn) |f| {
+                        if (destroy) {
+                            f(&node.key);
+                        }
+                    }
+                    a.destroy(node);
+                    return tmp;
+                }
+                if (node.right == null) {
+                    // Only left/none
+                    const tmp = node.left;
+                    if (tmp) |t| {
+                        t.parent = node.parent;
+                    }
+                    // Destroy the node
+                    if (comptime deinit_fn) |f| {
+                        if (destroy) {
+                            f(&node.key);
+                        }
+                    }
+                    a.destroy(node);
+                    return tmp;
+                }
+
+                // Both
+                var successor = node.right;
+                while (successor) |succ| {
+                    if (succ.left) |l| {
+                        successor = l;
+                    } else {
+                        break;
+                    }
+                }
+                if (successor) |succ| {
+                    node.key = succ.key;
+                    node.right = Node.remove(node.right, a, succ.key) catch unreachable;
+                }
+                return node;
+            }
+
+            fn remove(node: ?*Node, a: Allocator, key: N) Error!?*Node {
+                if (node) |n| {
+                    const ord = compare_fn(&n.key, &key);
+                    switch (ord) {
+                        .lt => n.right = try Node.remove(n.right, a, key),
+                        .gt => n.left = try Node.remove(n.left, a, key),
+                        .eq => return Node.remove_node(n, a, true),
+                    }
+                    return node;
+                } else {
+                    return error.does_not_exist;
+                }
+            }
+
+            fn walk(node: ?*Node, ctx: anytype, walk_fn: WalkFn(@TypeOf(ctx))) void {
+                if (node) |n| {
+                    Node.walk(n.left, ctx, walk_fn);
+                    walk_fn(n, ctx);
+                    Node.walk(n.right, ctx, walk_fn);
+                }
+            }
+
+            fn leftmost(node: ?*Node) ?*Node {
+                var n = node;
+                while (n) |nn| {
+                    if (nn.left == null) {
+                        break;
+                    }
+                    n = nn.left;
+                }
+                return n;
+            }
+        };
+
+        pub fn init(a: std.mem.Allocator) @This() {
+            return .{ .gpa = a };
+        }
+
+        pub fn deinit(self: *@This()) void {
+            Node.deinit(self.root, self.gpa);
+        }
+
+        pub fn iterator(self: *@This()) Iterator {
+            return .{ .current = Node.leftmost(self.root) };
+        }
+
+        pub fn insert(self: *@This(), key: N) Error!*Node {
+            self.root, const inserted = try Node.insert(self.root, self.gpa, key);
+            return inserted;
+        }
+
+        pub fn remove(self: *@This(), key: N) Error!void {
+            self.root = try Node.remove(self.root, self.gpa, key);
+        }
+
+        pub fn remove_node(self: *@This(), node: *Node, destroy: bool) Error!void {
+            if (node.parent) |p| {
+                // Non-root node
+                const np = Node.remove_node(node, self.gpa, destroy);
+                if (np) |npp| {
+                    npp.parent = p;
+                }
+                if (node == p.right) {
+                    p.right = np;
+                } else {
+                    p.left = np;
+                }
+            } else {
+                // Root node
+                const np = Node.remove_node(node, self.gpa, destroy);
+                if (np) |npp| {
+                    npp.parent = null;
+                }
+                self.root = np;
+            }
+        }
+
+        pub fn lookup(self: *const @This(), key: N) ?*Node {
+            const search_fn = struct {
+                fn call(n: *const N, cx: N) Order {
+                    return compare_fn(n, &cx);
+                }
+            }.call;
+
+            return self.search(key, search_fn);
+        }
+
+        pub fn search(
+            self: *const @This(),
+            ctx: anytype,
+            search_fn: SearchFn(N, @TypeOf(ctx)),
+        ) ?*Node {
+            var node = self.root;
+            while (node) |n| {
+                const ord = search_fn(&n.key, ctx);
+                switch (ord) {
+                    .gt => node = n.left,
+                    .eq => return n,
+                    .lt => node = n.right,
+                }
+            }
+            return null;
+        }
+
+        pub fn walk(self: *@This(), ctx: anytype, walk_fn: WalkFn(@TypeOf(ctx))) void {
+            Node.walk(self.root, ctx, walk_fn);
+        }
+    };
+}
+
+test "BTree insertion/removal" {
+    const int_compare_fn = struct {
+        fn call(a: *const u32, b: *const u32) Order {
+            if (a.* > b.*) {
+                return .gt;
+            } else if (a.* == b.*) {
+                return .eq;
+            } else {
+                return .lt;
+            }
+        }
+    }.call;
+    const Tree = BTree(u32, int_compare_fn, null);
+    var tree = Tree.init(std.testing.allocator);
+    defer tree.deinit();
+
+    for (50..100) |i| {
+        _ = try tree.insert(@truncate(i));
+    }
+    for (1..50) |i| {
+        _ = try tree.insert(@truncate(i));
+    }
+
+    for (1..100) |i| {
+        const k = @as(u32, @truncate(i));
+        try std.testing.expectEqual(k, tree.lookup(k).?.key);
+    }
+
+    for (1..100) |i| {
+        const k = 100 - @as(u32, @truncate(i));
+        if (i % 2 == 0) {
+            try tree.remove(k);
+        }
+    }
+
+    for (1..100) |i| {
+        const k = @as(u32, @truncate(i));
+        if (i % 2 == 0) {
+            try std.testing.expectEqual(null, tree.lookup(k));
+        } else {
+            try std.testing.expectEqual(k, tree.lookup(k).?.key);
+        }
+    }
+}
+
+test "BTree removal by node" {
+    const int_compare_fn = struct {
+        fn call(a: *const u32, b: *const u32) Order {
+            if (a.* > b.*) {
+                return .gt;
+            } else if (a.* == b.*) {
+                return .eq;
+            } else {
+                return .lt;
+            }
+        }
+    }.call;
+    const Tree = BTree(u32, int_compare_fn, null);
+    var tree = Tree.init(std.testing.allocator);
+    defer tree.deinit();
+
+    _ = try tree.insert(10);
+    _ = try tree.insert(11);
+    _ = try tree.insert(12);
+
+    {
+        const n = tree.lookup(10).?;
+        try tree.remove_node(n, true);
+    }
+
+    try std.testing.expectEqual(null, tree.lookup(10));
+    try std.testing.expectEqual(12, tree.lookup(12).?.key);
+    try std.testing.expectEqual(11, tree.lookup(11).?.key);
+}
+
+test "BTree iterator" {
+    const int_compare_fn = struct {
+        fn call(a: *const u32, b: *const u32) Order {
+            if (a.* > b.*) {
+                return .gt;
+            } else if (a.* == b.*) {
+                return .eq;
+            } else {
+                return .lt;
+            }
+        }
+    }.call;
+    const Tree = BTree(u32, int_compare_fn, null);
+    var tree = Tree.init(std.testing.allocator);
+    defer tree.deinit();
+
+    for (50..100) |i| {
+        _ = try tree.insert(@truncate(i));
+    }
+    for (1..50) |i| {
+        _ = try tree.insert(@truncate(i));
+    }
+
+    var it = tree.iterator();
+    for (1..100) |i| {
+        const n = it.next().?;
+        try std.testing.expectEqual(i, n.key);
+    }
+    try std.testing.expectEqual(null, it.next());
+}

src/util/dtb.zig

@@ -1,7 +1,11 @@
+//! Flattened Device Tree manipulation utilities
+
 const mem = @import("../mem.zig");
 const log = @import("../debug.zig").log;
 const std = @import("std");
 
+const phys_memory = mem.phys;
+
 const fdt_header = extern struct {
     magic: u32,
     totalsize: u32,
@@ -15,7 +19,14 @@ const fdt_header = extern struct {
     size_dt_struct: u32,
 };
 
-const fdt_op = enum(u32) { FDT_BEGIN_NODE = 1, FDT_END_NODE = 2, FDT_PROP = 3, FDT_NOP = 4, FDT_END = 9, _ };
+const fdt_op = enum(u32) {
+    FDT_BEGIN_NODE = 1,
+    FDT_END_NODE = 2,
+    FDT_PROP = 3,
+    FDT_NOP = 4,
+    FDT_END = 9,
+    _,
+};
 
 const fdt_prop = extern struct {
     len: u32,
@@ -36,26 +47,41 @@ const FdtTag = union(enum) {
     end,
 };
 
+/// Represents a memory region described by some device tree node.
+/// Can either mean a range of reserved memory or a range of available memory depending on the
+/// function used to obtain it.
 pub const FdtMemoryRegion = struct {
     name: []const u8,
     base: u64,
     size: u64,
 };
 
+/// Represents a single node within a device tree.
 pub const FdtNode = struct {
+    /// Parent device tree pointer.
     fdt: *const Fdt,
+    /// Offset within the device tree raw data blob.
     off: usize,
+    /// Name string of the node.
     name: []const u8,
+    /// Depth of the node within the tree.
     depth: usize,
 
-    pub fn propIterator(self: *const @This()) FdtNodePropIterator {
-        return .{ .node = self, .tagIter = self.fdt.tagIteratorAt(self.off) };
+    /// Returns an iterator over the node's properties.
+    pub fn prop_iterator(self: *const @This()) FdtNodePropIterator {
+        return .{ .node = self, .tag_iter = self.fdt.tag_iterator_at(self.off) };
     }
 
+    /// Returns an iterator over the node's children.
     pub fn children(self: *const @This()) FdtNodeIterator {
-        return .{ .tagIter = self.fdt.tagIteratorAt(self.off), .depth = self.depth + 1, .depthLower = self.depth };
+        return .{
+            .tag_iter = self.fdt.tag_iterator_at(self.off),
+            .depth = self.depth + 1,
+            .depth_lower = self.depth,
+        };
     }
 
+    /// Looks up a child with given `name` within the node.
     pub fn child(self: *const @This(), name: []const u8) ?FdtNode {
         var iter = self.children();
         while (iter.next()) |c| {
@@ -66,9 +92,10 @@ pub const FdtNode = struct {
         return null;
     }
 
+    /// Looks up a property with given `name` within the node.
     pub fn property(self: *const @This(), name: []const u8) ?FdtNodeProp {
-        var propIter = self.propIterator();
-        while (propIter.next()) |prop| {
+        var prop_iter = self.prop_iterator();
+        while (prop_iter.next()) |prop| {
             if (std.mem.eql(u8, name, prop.name)) {
                 return prop;
             }
@@ -77,33 +104,56 @@ pub const FdtNode = struct {
     }
 };
 
+/// Represents a property of some node within a device tree.
 pub const FdtNodeProp = struct {
+    /// Node to which this property belongs.
     node: *const FdtNode,
+    /// Name string.
     name: []const u8,
+    /// Value of the property represented as a raw byte slice.
     value: []const u8,
 
-    pub inline fn getStringArray(self: *const @This()) FdtStringArrayIterator {
+    /// Interprets the property's value as a list of strings.
+    pub inline fn get_string_array(self: *const @This()) FdtStringArrayIterator {
         return .{ .prop = self };
     }
 
-    pub inline fn lenU32(self: *const @This()) usize {
+    /// Interprets the property's value as a single string.
+    pub inline fn get_string(self: *const @This()) []const u8 {
+        var sa = self.get_string_array();
+        return sa.next() orelse "";
+    }
+
+    /// Returns the length of the property in full 32-bit cells.
+    pub inline fn len_cells(self: *const @This()) usize {
         return self.value.len / @sizeOf(u32);
     }
 
-    pub fn getU32(self: *const @This(), index: usize) ?u32 {
-        if (index >= self.lenU32()) {
+    /// Interprets the property's value as an array of 32-bit cells and returns a cell at a given
+    /// index.
+    pub fn get_cell(self: *const @This(), index: usize) ?u32 {
+        if (index >= self.len_cells()) {
             return null;
         }
-        return self.getU32Unchecked(index);
+        return self.get_cell_unchecked(index);
     }
 
-    fn getU32Unchecked(self: *const @This(), index: usize) u32 {
+    fn get_cell_unchecked(self: *const @This(), index: usize) u32 {
         return std.mem.bigToNative(u32, @as(*const u32, @ptrCast(@alignCast(&self.value[index * 4]))).*);
     }
 
-    pub fn readCells(self: *const @This(), index: usize, output: []u64, sizes: []const usize) bool {
+    /// Interprets the property's value as an array of tuples of cells with sizes described by
+    /// `sizes` and reads a single such tuple into the `output`.
+    ///
+    /// Returns `true` if a whole tuple can be read at given `index`.
+    ///
+    /// # Notes
+    ///
+    /// * `index` parameter means a 32-bit cell index, not a tuple index.
+    /// * Tuple length is assumed to be `@min(output.len, sizes.len)`.
+    pub fn read_cells(self: *const @This(), index: usize, output: []u64, sizes: []const usize) bool {
         const count = @min(output.len, sizes.len);
-        const len = self.lenU32();
+        const len = self.len_cells();
         var total: usize = 0;
         for (sizes[0..count]) |s| {
             total += s;
@@ -113,11 +163,11 @@ pub const FdtNodeProp = struct {
             for (0..count) |i| {
                 switch (sizes[i]) {
                     1 => {
-                        output[i] = self.getU32Unchecked(offset);
+                        output[i] = self.get_cell_unchecked(offset);
                     },
                     2 => {
-                        output[i] = self.getU32Unchecked(offset + 1);
-                        output[i] |= @as(u64, self.getU32Unchecked(offset)) << 32;
+                        output[i] = self.get_cell_unchecked(offset + 1);
+                        output[i] |= @as(u64, self.get_cell_unchecked(offset)) << 32;
                     },
                     else => @panic("Invalid cell size"),
                 }
@@ -131,6 +181,7 @@ pub const FdtNodeProp = struct {
     }
 };
 
+/// An iterator over a string list property value.
 pub const FdtStringArrayIterator = struct {
     prop: *const FdtNodeProp,
     off: usize = 0,
@@ -147,17 +198,18 @@ pub const FdtStringArrayIterator = struct {
     }
 };
 
+/// An iterator over available memory regions described by a device tree.
 pub const FdtMemoryRegionIterator = struct {
-    nodeIter: FdtNodeIterator,
-    cellSizes: [2]usize,
+    node_iter: FdtNodeIterator,
+    cell_sizes: [2]usize,
 
     pub fn next(self: *FdtMemoryRegionIterator) ?FdtMemoryRegion {
-        while (self.nodeIter.next()) |node| {
+        while (self.node_iter.next()) |node| {
             if (std.mem.startsWith(u8, node.name, "memory@")) {
                 const reg = node.property("reg") orelse continue;
                 var cells: [2]u64 = undefined;
 
-                if (reg.readCells(0, &cells, &self.cellSizes)) {
+                if (reg.read_cells(0, &cells, &self.cell_sizes)) {
                     return .{
                         .name = node.name,
                         .base = cells[0],
@@ -171,13 +223,14 @@ pub const FdtMemoryRegionIterator = struct {
     }
 };
 
+/// An iterator over a device tree's node properties.
 pub const FdtNodePropIterator = struct {
     node: *const FdtNode,
-    tagIter: FdtTagIterator,
+    tag_iter: FdtTagIterator,
     depth: usize = 0,
 
     fn next(self: *FdtNodePropIterator) ?FdtNodeProp {
-        while (self.tagIter.next()) |tag| {
+        while (self.tag_iter.next()) |tag| {
             switch (tag) {
                 .begin_node => |_| {
                     self.depth += 1;
@@ -185,7 +238,7 @@ pub const FdtNodePropIterator = struct {
                 .nop => {},
                 .prop => |prop| {
                     if (self.depth == 0) {
-                        const name = self.node.fdt.stringAt(prop.nameoff);
+                        const name = self.node.fdt.string_at(prop.nameoff);
                         return .{ .node = self.node, .value = prop.data, .name = name };
                     }
                 },
@@ -206,19 +259,20 @@ pub const FdtNodePropIterator = struct {
     }
 };
 
+/// An iterator over a device tree's nodes.
 pub const FdtNodeIterator = struct {
-    tagIter: FdtTagIterator,
+    tag_iter: FdtTagIterator,
     depth: usize = 0,
-    depthLower: ?usize = null,
+    depth_lower: ?usize = null,
 
     pub fn next(self: *FdtNodeIterator) ?FdtNode {
-        while (self.tagIter.next()) |tag| {
+        while (self.tag_iter.next()) |tag| {
             switch (tag) {
                 .begin_node => |name| {
                     self.depth += 1;
                     return .{
-                        .fdt = self.tagIter.fdt,
-                        .off = self.tagIter.off,
+                        .fdt = self.tag_iter.fdt,
+                        .off = self.tag_iter.off,
                         .name = name,
                         .depth = self.depth - 1,
                     };
@@ -226,7 +280,7 @@ pub const FdtNodeIterator = struct {
                 .end_node => {
                     self.depth -= 1;
 
-                    if (self.depthLower) |lower| {
+                    if (self.depth_lower) |lower| {
                         if (self.depth == lower) {
                             return null;
                         }
@@ -239,6 +293,7 @@ pub const FdtNodeIterator = struct {
     }
 };
 
+/// An iterator over a device tree's raw tags.
 pub const FdtTagIterator = struct {
     fdt: *const Fdt,
     raw: []const u8,
@@ -255,10 +310,10 @@ pub const FdtTagIterator = struct {
 
         switch (tag) {
             .FDT_BEGIN_NODE => {
-                const nameCStr: [*c]const u8 = @ptrCast(self.raw[self.off..]);
-                const nameLength = std.mem.len(nameCStr);
-                const name = self.raw[self.off .. self.off + nameLength];
-                self.off += (nameLength + 4) & ~@as(usize, 3);
+                const name_cstr: [*c]const u8 = @ptrCast(self.raw[self.off..]);
+                const name_length = std.mem.len(name_cstr);
+                const name = self.raw[self.off .. self.off + name_length];
+                self.off += (name_length + 4) & ~@as(usize, 3);
                 return .{ .begin_node = name };
             },
             .FDT_PROP => {
@@ -288,14 +343,23 @@ pub const FdtTagIterator = struct {
     }
 };
 
+/// An error returned by the FDT manipulation functions.
 pub const FdtError = error{invalid_magic};
 
+/// Magic number describing a valid device tree (Big-Endian).
 pub const FDT_MAGIC: u32 = 0xD00DFEED;
 
+/// Represents a Flattened Device Tree.
 pub const Fdt = struct {
+    /// Raw device tree bytes, including the header.
     bytes: []const u8,
 
-    pub fn fromPhysicalAddress(phys: mem.PhysicalAddress) FdtError!@This() {
+    /// Constructs a FDT struct from a physical address.
+    ///
+    /// # Errors
+    ///
+    /// * `invalid_magic` if the address provided does not have a valid magic number in its header.
+    pub fn from_physical_address(phys: mem.PhysicalAddress) FdtError!@This() {
         const virt = phys.virtualize();
         const hdr = @as(*const fdt_header, @ptrFromInt(virt));
         if (std.mem.bigToNative(u32, hdr.magic) != FDT_MAGIC) {
@@ -306,6 +370,7 @@ pub const Fdt = struct {
         return .{ .bytes = x[0..totalsize] };
     }
 
+    /// Returns the header pointer of this device tree.
     pub fn header(self: *const @This()) *const fdt_header {
         return @ptrCast(@alignCast(&self.bytes[0]));
     }
@@ -315,21 +380,30 @@ pub const Fdt = struct {
         return self.bytes[off..];
     }
 
-    pub fn tagIterator(self: *const @This()) FdtTagIterator {
-        return self.tagIteratorAt(0);
+    /// Returns an iterator over the device tree's raw tags.
+    pub fn tag_iterator(self: *const @This()) FdtTagIterator {
+        return self.tag_iterator_at(0);
     }
 
-    pub fn tagIteratorAt(self: *const @This(), off: usize) FdtTagIterator {
+    /// Returns an iterator over the device tree's raw tags at specific byte offset.
+    pub fn tag_iterator_at(self: *const @This(), off: usize) FdtTagIterator {
         return .{ .raw = self.data(), .fdt = self, .off = off };
     }
 
-    pub fn nodeIterator(self: *const @This()) FdtNodeIterator {
-        return .{ .tagIter = self.tagIterator() };
+    /// Returns an iterator over the device tree's nodes.
+    pub fn node_iterator(self: *const @This()) FdtNodeIterator {
+        return .{ .tag_iter = self.tag_iterator() };
     }
 
-    pub fn rootNode(self: *const @This()) FdtNode {
-        var nodeIter = self.nodeIterator();
-        while (nodeIter.next()) |node| {
+    /// Returns the root node of this device tree.
+    ///
+    /// # Panics
+    ///
+    /// Panics if the device tree does not have a root node (which means the device tree blob is
+    /// malformed and the OS shouldn't be running anyway).
+    pub fn root_node(self: *const @This()) FdtNode {
+        var node_iter = self.node_iterator();
+        while (node_iter.next()) |node| {
             if (node.depth == 0 and node.name.len == 0) {
                 return node;
             }
@@ -337,72 +411,161 @@ pub const Fdt = struct {
         @panic("Unreachable code");
     }
 
-    pub fn memoryRegionIterator(self: *const @This()) FdtMemoryRegionIterator {
-        const r = self.rootNode();
-        const addressCells = if (r.property("#address-cells")) |o| (if (o.getU32(0)) |p| p else 1) else 1;
-        const sizeCells = if (r.property("#size-cells")) |o| (if (o.getU32(0)) |p| p else 1) else 1;
+    /// Returns an iterator over available memory regions described by the device tree.
+    pub fn memory_region_iterator(self: *const @This()) FdtMemoryRegionIterator {
+        const r = self.root_node();
+        const address_cells = if (r.property("#address-cells")) |o| (if (o.get_cell(0)) |p| p else 1) else 1;
+        const size_cells = if (r.property("#size-cells")) |o| (if (o.get_cell(0)) |p| p else 1) else 1;
 
-        return .{ .nodeIter = self.nodeIterator(), .cellSizes = .{ addressCells, sizeCells } };
+        return .{ .node_iter = self.node_iterator(), .cell_sizes = .{ address_cells, size_cells } };
     }
 
-    fn stringData(self: *const @This()) [*c]const u8 {
-        const offStrings = std.mem.bigToNative(u32, self.header().off_dt_strings);
-        const sizeStrings = std.mem.bigToNative(u32, self.header().off_dt_strings);
-        const off = @min(offStrings, self.bytes.len);
-        const len = @min(sizeStrings, self.bytes.len - off);
+    fn string_data(self: *const @This()) [*c]const u8 {
+        const off_strings = std.mem.bigToNative(u32, self.header().off_dt_strings);
+        const size_strings = std.mem.bigToNative(u32, self.header().off_dt_strings);
+        const off = @min(off_strings, self.bytes.len);
+        const len = @min(size_strings, self.bytes.len - off);
         return @ptrCast(self.bytes[off .. off + len]);
     }
 
-    pub fn stringAt(self: *const @This(), off: usize) []const u8 {
-        const raw = self.stringData()[off..];
+    /// Returns a string slice at given byte offset into the device tree's strings section.
+    pub fn string_at(self: *const @This(), off: usize) []const u8 {
+        const raw = self.string_data()[off..];
         const len = std.mem.len(raw);
         return @ptrCast(raw[0..len]);
     }
 
-    pub fn dump(self: *const @This()) void {
-        const root = self.rootNode();
+    /// Adds information about the available and reserved memory regions described in this device
+    /// tree into the physical memory management structures.
+    pub fn add_physical_memory_to_system(self: *const @This()) void {
+        var memory_regions = self.memory_region_iterator();
         var cells: [2]u64 = undefined;
-        if (root.child("reserved-memory")) |resv| {
-            var regions = resv.children();
-            while (regions.next()) |region| {
+
+        while (memory_regions.next()) |region| {
+            phys_memory.add_memory_region(region.name, region.base, region.size);
+        }
+        const reserved_regions = self.root_node().child("reserved-memory");
+        if (reserved_regions) |resv| {
+            var children = resv.children();
+            while (children.next()) |region| {
                 if (region.property("reg")) |reg| {
-                    if (reg.readCells(0, &cells, &.{ 2, 2 })) {
-                        log.info("Reserved memory region {s}: base=0x{x}, size=0x{x}", .{ region.name, cells[0], cells[1] });
+                    // TODO #address-cells, #size-cells
+                    if (reg.read_cells(0, &cells, &.{ 2, 2 })) {
+                        phys_memory.add_reserved_region(region.name, cells[0], cells[1]);
                     }
                 }
             }
         }
-        // var memIter = self.memoryRegionIterator();
-        // while (memIter.next()) |region| {
-        //     log.info("base=0x{x}, size=0x{x}", .{ region.base, region.size });
-        // }
+    }
 
-        // var nodeIter = self.nodeIterator();
-        // while (nodeIter.next()) |node| {
-        //     for (0..node.depth) |_| {
-        //         log.writeRaw("  ");
-        //     }
-        //     if (node.name.len == 0) {
-        //         log.info("Root node, depth = {}", .{ node.depth });
-        //     } else {
-        //         log.info("Node {s}, depth {}", .{ node.name, node.depth });
-        //     }
-        //     var nodePropIter = node.propIterator();
-        //     while (nodePropIter.next()) |prop| {
-        //         for (0..node.depth + 1) |_| {
-        //             log.writeRaw("  ");
-        //         }
-        //         log.info("Prop {s}", .{ prop.name });
-        //         if (std.mem.eql(u8, prop.name, "compatible")) {
-        //             var strings = prop.getStringArray();
-        //             while (strings.next()) |s| {
-        //                 for (0..node.depth + 2) |_| {
-        //                     log.writeRaw("  ");
-        //                 }
-        //                 log.info("= {s}", .{ s });
-        //             }
-        //         }
-        //     }
-        // }
+    /// Looks up a `/slash/separated/path` inside the device tree.
+    pub fn find(self: *const @This(), path: []const u8) ?FdtNode {
+        const trimmed_path = std.mem.trimLeft(u8, path, "/");
+        var path_elements = std.mem.splitScalar(u8, trimmed_path, '/');
+        var current_node = self.root_node();
+        if (trimmed_path.len == 0) {
+            return current_node;
+        }
+        while (path_elements.next()) |element| {
+            var found: ?FdtNode = null;
+            var children = current_node.children();
+            while (children.next()) |child| {
+                if (std.mem.eql(u8, child.name, element)) {
+                    found = child;
+                    break;
+                }
+            }
+            if (found) |f| {
+                log.info("{s}", .{element});
+                current_node = f;
+            } else {
+                return null;
+            }
+        }
+        return current_node;
+    }
+
+    fn dump_property(property: *const FdtNodeProp, depth: usize, all_strings: bool) void {
+        for (0..depth) |_| {
+            log.write_raw("  ");
+        }
+        log.write("{s}", .{property.name});
+
+        if (std.mem.eql(u8, property.name, "#size-cells") //
+        or std.mem.eql(u8, property.name, "#address-cells") //
+        or std.mem.eql(u8, property.name, "#interrupt-cells") //
+        or std.mem.eql(u8, property.name, "phandle") //
+        or std.mem.eql(u8, property.name, "interrupt-parent") //
+        ) {
+            // Dump as a single cell
+            const v = property.get_cell(0) orelse 0;
+            log.write(" = {}", .{v});
+        } else if (all_strings //
+        or std.mem.eql(u8, property.name, "compatible") //
+        or std.mem.eql(u8, property.name, "model") //
+        or std.mem.endsWith(u8, property.name, "-names") //
+        or std.mem.endsWith(u8, property.name, "stdout-path") //
+        ) {
+            var v = property.get_string_array();
+            var f = true;
+            while (v.next()) |s| {
+                if (f) {
+                    log.write_raw(" = ");
+                } else {
+                    log.write_raw(", ");
+                }
+                f = false;
+                log.write("\"{s}\"", .{s});
+            }
+        } else {
+            // Dump the rest as a cell array
+            const len = property.len_cells();
+            log.write_raw(" = <");
+            for (0..len) |i| {
+                if (i != 0) {
+                    log.write_raw(", ");
+                }
+                log.write("0x{x}", .{property.get_cell_unchecked(i)});
+            }
+            log.write_raw(">");
+        }
+
+        log.write_raw(";\r\n");
+    }
+
+    fn dump_node(node: *const FdtNode, depth: usize) void {
+        var any_properties = false;
+        var first_child = true;
+
+        for (0..depth) |_| {
+            log.write_raw("  ");
+        }
+        if (node.name.len != 0) {
+            log.write("{s} ", .{node.name});
+        }
+        log.write_raw("{\r\n");
+        var properties = node.prop_iterator();
+        const all_strings = std.mem.eql(u8, node.name, "aliases");
+        while (properties.next()) |property| {
+            dump_property(&property, depth + 1, all_strings);
+            any_properties = true;
+        }
+        var children = node.children();
+        while (children.next()) |child| {
+            if (any_properties and first_child) {
+                log.write_raw("\r\n");
+            }
+            first_child = false;
+            dump_node(&child, depth + 1);
+        }
+        for (0..depth) |_| {
+            log.write_raw("  ");
+        }
+        log.write("}},\r\n", .{});
+    }
+
+    /// Dumps the structured device tree into the log output.
+    pub fn dump(self: *const @This()) void {
+        dump_node(&self.root_node(), 0);
     }
 };

src/util/range.zig

@@ -1,12 +1,21 @@
+//! Utilities for manipulating ranges.
+
+const std = @import("std");
+
+/// Non-inclusive range type over `T`.
 pub fn Range(comptime T: type) type {
     return struct {
+        /// Range start.
         start: T,
+        /// Range length.
         len: T,
 
+        /// Returns `start + len` of the range.
         pub fn end(self: *const @This()) T {
             return self.start + self.len;
         }
 
+        /// Returns a range representing an intersections of `self` with an`other` range.
         pub fn intersect(self: *const @This(), other: *const @This()) ?Range(T) {
             if (self.start < other.start) {
                 const p = other.start - self.start;
@@ -22,5 +31,19 @@ pub fn Range(comptime T: type) type {
 
             return null;
         }
+
+        pub fn contains(self: *const @This(), scalar: T) bool {
+            return scalar >= self.start and scalar - self.start < self.len;
+        }
+
+        pub fn compare_disjoint(a: *const @This(), b: *const @This()) std.math.Order {
+            if (a.start >= b.end()) {
+                return .gt;
+            } else if (b.start >= a.end()) {
+                return .lt;
+            } else {
+                return .eq;
+            }
+        }
     };
 }

src/util/rangemap.zig

@@ -0,0 +1,450 @@
+const std = @import("std");
+
+const btree = @import("btree.zig");
+
+const Range = @import("range.zig").Range;
+const Allocator = std.mem.Allocator;
+const BTree = btree.BTree;
+pub const Order = btree.Order;
+
+pub fn RangeMap(
+    comptime K: type,
+    comptime V: type,
+    comptime ops: struct {
+        deinit_fn: ?fn (*V) void = null,
+        merge_fn: ?fn (*const V, *const V) bool = null,
+    },
+) type {
+    return struct {
+        pub const Node = struct {
+            key: Range(K),
+            value: V,
+
+            pub fn len(self: *const @This()) K {
+                return self.key.len;
+            }
+        };
+
+        pub const WalkFn = fn (*const Node) void;
+
+        pub const Iterator = struct {
+            inner: Tree.Iterator,
+
+            pub fn next(self: *Iterator) ?*const Node {
+                if (self.inner.next()) |n| {
+                    return &n.key;
+                } else {
+                    return null;
+                }
+            }
+        };
+
+        pub const Tree = BTree(Node, compare_fn, deinit_node_fn);
+
+        pub const Error = error{
+            scalar_out_of_range,
+            range_out_of_bounds,
+        } || Tree.Error;
+
+        fn compare_fn(a: *const Node, b: *const Node) Order {
+            return Range(K).compare_disjoint(&a.key, &b.key);
+        }
+
+        fn deinit_node_fn(n: *Node) void {
+            if (comptime ops.deinit_fn) |f| {
+                f(&n.value);
+            }
+        }
+
+        btree: Tree,
+
+        pub fn init(gpa: Allocator) @This() {
+            return .{ .btree = Tree.init(gpa) };
+        }
+
+        pub fn deinit(self: *@This()) void {
+            self.btree.deinit();
+        }
+
+        /// Returns the value at a given scalar point, along with the full range it belongs to.
+        pub fn get_scalar(self: *const @This(), scalar: K) ?*Node {
+            return if (self.get_scalar_node(scalar)) |n| &n.key else null;
+        }
+
+        /// Same as `get_scalar()`, but returns the underlying BST node.
+        pub fn get_scalar_node(self: *const @This(), scalar: K) ?*Tree.Node {
+            return self.btree.search(scalar, struct {
+                fn call(n: *const Node, cx: K) Order {
+                    if (n.key.contains(cx)) {
+                        return .eq;
+                    } else if (cx < n.key.start) {
+                        return .gt;
+                    } else {
+                        return .lt;
+                    }
+                }
+            }.call);
+        }
+
+        /// Splits a given node at a scalar point inside its interval.
+        ///
+        /// The part of the interval before `at` is considered a "left" half, the remaining
+        /// part is considered a "right" half.
+        ///
+        /// # Note
+        ///
+        /// The "right" halve's value after the split is left uninitialized and it is up to the
+        /// caller to assign a proper value to it.
+        ///
+        /// # Errors
+        ///
+        /// * `scalar_out_of_range` if the given `at` value is not inside the node's interval.
+        pub fn split_node(
+            self: *@This(),
+            node: *Tree.Node,
+            at: K,
+        ) Error!?struct { *Tree.Node, *Tree.Node } {
+            if (!node.key.key.contains(at)) {
+                return error.scalar_out_of_range;
+            }
+
+            const start = node.key.key.start;
+            const end = node.key.key.end();
+
+            if (at == start or at == end - 1) {
+                // Nothing to split here
+                return null;
+            }
+
+            const value = node.key.value;
+
+            // Remove the node, don't drop the key
+            try self.btree.remove_node(node, false);
+
+            const lnode = try self.btree.insert(
+                .{ .key = .{ .start = start, .len = at - start }, .value = value },
+            );
+            const rnode = try self.btree.insert(
+                .{ .key = .{ .start = at, .len = end - at }, .value = undefined },
+            );
+
+            return .{ lnode, rnode };
+        }
+
+        /// Maps some range to a value. Returns an error if the requested range crosses another
+        /// mapped range.
+        pub fn insert(self: *@This(), start: K, len: K, value: V) Error!*Tree.Node {
+            try validate_range(start, len);
+
+            if (comptime ops.merge_fn) |merge_fn| {
+                const left: ?*Tree.Node = if (start > 0) self.get_scalar_node(start - 1) else null;
+                const right = self.get_scalar_node(start + len);
+
+                if (left) |l| {
+                    const l_start = l.key.key.start;
+
+                    if (merge_fn(&l.key.value, &value)) {
+                        if (right) |r| {
+                            if (merge_fn(&r.key.value, &value)) {
+                                l.key.key.len += len + r.key.key.len;
+                                try self.btree.remove_node(r, true);
+                                return self.get_scalar_node(l_start).?;
+                            }
+                        }
+
+                        l.key.key.len += len;
+                        return l;
+                    }
+                }
+
+                if (right) |r| {
+                    // Only right node to potentially merge with
+                    if (merge_fn(&r.key.value, &value)) {
+                        const r_len = r.key.key.len;
+                        try self.btree.remove_node(r, true);
+                        return self.btree.insert(.{
+                            .key = .{ .start = start, .len = len + r_len },
+                            .value = value,
+                        });
+                    }
+                }
+            }
+
+            return self.btree.insert(.{
+                .key = .{ .start = start, .len = len },
+                .value = value,
+            });
+        }
+
+        pub fn iterator(self: *@This()) Iterator {
+            return .{ .inner = self.btree.iterator() };
+        }
+
+        pub fn node_iterator(self: *@This()) Tree.Iterator {
+            return self.btree.iterator();
+        }
+
+        pub fn walk(self: *@This(), walk_fn: WalkFn) void {
+            self.btree.walk(walk_fn, struct {
+                fn call(n: *const Tree.Node, cx: WalkFn) void {
+                    cx(&n.key);
+                }
+            }.call);
+        }
+
+        fn validate_range(start: K, end: K) Error!void {
+            // Check for addition overflowing the K's bit size
+            if (std.math.add(K, start, end) == error.Overflow) {
+                return error.range_out_of_bounds;
+            }
+        }
+    };
+}
+
+test "Range map insertion" {
+    const Map = RangeMap(u32, []const u8, .{});
+    var map = Map.init(std.testing.allocator);
+    defer map.deinit();
+
+    _ = try map.insert(10, 10, "Range 2");
+    _ = try map.insert(0, 10, "Range 1");
+    _ = try map.insert(20, 10, "Range 3");
+
+    try std.testing.expectError(error.already_exists, map.insert(5, 10, "Invalid range"));
+
+    _ = try map.insert(1000, 10, "Range 4");
+}
+
+test "Range map merging insertion" {
+    const Map = RangeMap(u32, bool, .{
+        .merge_fn = struct {
+            fn call(lhs: *const bool, rhs: *const bool) bool {
+                return !lhs.* and !rhs.*;
+            }
+        }.call,
+    });
+    var map = Map.init(std.testing.allocator);
+    defer map.deinit();
+
+    // Should not merge
+    _ = try map.insert(10, 10, false);
+    _ = try map.insert(0, 10, true);
+
+    {
+        var it = map.iterator();
+        try std.testing.expectEqual(true, it.next().?.value);
+        try std.testing.expectEqual(false, it.next().?.value);
+        try std.testing.expectEqual(null, it.next());
+    }
+
+    // Merge left + inserted + right
+    _ = try map.insert(30, 10, false);
+    _ = try map.insert(20, 10, false);
+
+    {
+        var it = map.iterator();
+        const n0 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 0, .len = 10 }, n0.key);
+        try std.testing.expectEqual(true, n0.value);
+
+        const n1 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 10, .len = 30 }, n1.key);
+        try std.testing.expectEqual(false, n1.value);
+
+        try std.testing.expectEqual(null, it.next());
+    }
+
+    // Should not merge again
+    _ = try map.insert(40, 10, true);
+    _ = try map.insert(50, 10, false);
+
+    {
+        var it = map.iterator();
+        const n0 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 0, .len = 10 }, n0.key);
+        try std.testing.expectEqual(true, n0.value);
+
+        const n1 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 10, .len = 30 }, n1.key);
+        try std.testing.expectEqual(false, n1.value);
+
+        const n2 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 40, .len = 10 }, n2.key);
+        try std.testing.expectEqual(true, n2.value);
+
+        const n3 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 50, .len = 10 }, n3.key);
+        try std.testing.expectEqual(false, n3.value);
+
+        try std.testing.expectEqual(null, it.next());
+    }
+
+    // Should merge left + shouldn't merge right non-contiguous
+    _ = try map.insert(71, 9, false);
+    _ = try map.insert(60, 10, false);
+
+    {
+        var it = map.iterator();
+        const n0 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 0, .len = 10 }, n0.key);
+        try std.testing.expectEqual(true, n0.value);
+
+        const n1 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 10, .len = 30 }, n1.key);
+        try std.testing.expectEqual(false, n1.value);
+
+        const n2 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 40, .len = 10 }, n2.key);
+        try std.testing.expectEqual(true, n2.value);
+
+        const n3 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 50, .len = 20 }, n3.key);
+        try std.testing.expectEqual(false, n3.value);
+
+        const n4 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 71, .len = 9 }, n4.key);
+        try std.testing.expectEqual(false, n4.value);
+
+        try std.testing.expectEqual(null, it.next());
+    }
+
+    // Should merge left and right
+    _ = try map.insert(70, 1, false);
+
+    {
+        var it = map.iterator();
+        const n0 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 0, .len = 10 }, n0.key);
+        try std.testing.expectEqual(true, n0.value);
+
+        const n1 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 10, .len = 30 }, n1.key);
+        try std.testing.expectEqual(false, n1.value);
+
+        const n2 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 40, .len = 10 }, n2.key);
+        try std.testing.expectEqual(true, n2.value);
+
+        const n3 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 50, .len = 30 }, n3.key);
+        try std.testing.expectEqual(false, n3.value);
+
+        try std.testing.expectEqual(null, it.next());
+    }
+
+    // Should merge right
+    _ = try map.insert(110, 10, false);
+    _ = try map.insert(100, 10, false);
+
+    {
+        var it = map.iterator();
+        const n0 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 0, .len = 10 }, n0.key);
+        try std.testing.expectEqual(true, n0.value);
+
+        const n1 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 10, .len = 30 }, n1.key);
+        try std.testing.expectEqual(false, n1.value);
+
+        const n2 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 40, .len = 10 }, n2.key);
+        try std.testing.expectEqual(true, n2.value);
+
+        const n3 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 50, .len = 30 }, n3.key);
+        try std.testing.expectEqual(false, n3.value);
+
+        const n4 = it.next().?;
+        try std.testing.expectEqual(Range(u32){ .start = 100, .len = 20 }, n4.key);
+        try std.testing.expectEqual(false, n4.value);
+
+        try std.testing.expectEqual(null, it.next());
+    }
+}
+
+test "Range map get scalar" {
+    const Map = RangeMap(u32, []const u8, .{});
+    var map = Map.init(std.testing.allocator);
+    defer map.deinit();
+
+    _ = try map.insert(10, 10, "Range [10..20)");
+    _ = try map.insert(30, 10, "Range [30..40)");
+
+    {
+        const n = map.get_scalar(15).?;
+        try std.testing.expectEqual(10, n.key.start);
+        try std.testing.expectEqual(20, n.key.end());
+        try std.testing.expectEqualStrings("Range [10..20)", n.value);
+    }
+    {
+        const n = map.get_scalar(35).?;
+        try std.testing.expectEqual(30, n.key.start);
+        try std.testing.expectEqual(40, n.key.end());
+        try std.testing.expectEqualStrings("Range [30..40)", n.value);
+    }
+    {
+        const n = map.get_scalar(30).?;
+        try std.testing.expectEqual(30, n.key.start);
+        try std.testing.expectEqual(40, n.key.end());
+        try std.testing.expectEqualStrings("Range [30..40)", n.value);
+    }
+    try std.testing.expectEqual(null, map.get_scalar(20));
+    try std.testing.expectEqual(null, map.get_scalar(21));
+    try std.testing.expectEqual(null, map.get_scalar(9));
+    try std.testing.expectEqual(null, map.get_scalar(100));
+    try std.testing.expectEqual(null, map.get_scalar(40));
+    try std.testing.expectEqual(null, map.get_scalar(41));
+}
+
+test "Range map split" {
+    const Map = RangeMap(u32, []const u8, .{});
+    var map = Map.init(std.testing.allocator);
+    defer map.deinit();
+
+    _ = try map.insert(0x1000, 0x1000, "Range [0x1000..0x2000)");
+
+    const node = map.get_scalar_node(0x1000).?;
+    const lnode, const rnode = (try map.split_node(node, 0x1200)).?;
+
+    lnode.key.value = "Left";
+    rnode.key.value = "Right";
+
+    {
+        const n = map.get_scalar(0x1100).?;
+        try std.testing.expectEqual(0x1000, n.key.start);
+        try std.testing.expectEqual(0x200, n.key.len);
+        try std.testing.expectEqualStrings("Left", n.value);
+    }
+    {
+        const n = map.get_scalar(0x1300).?;
+        try std.testing.expectEqual(0x1200, n.key.start);
+        try std.testing.expectEqual(0xE00, n.key.len);
+        try std.testing.expectEqualStrings("Right", n.value);
+    }
+}
+
+test "Range map iterator" {
+    const Map = RangeMap(u32, []const u8, .{});
+    var map = Map.init(std.testing.allocator);
+    defer map.deinit();
+
+    _ = try map.insert(0x1000, 0x1000, "Range [0x1000..0x2000)");
+    _ = try map.insert(0x2000, 0x1000, "Range [0x2000..0x3000)");
+    _ = try map.insert(0x4000, 0x1000, "Range [0x4000..0x5000)");
+    _ = try map.insert(0x3000, 0x1000, "Range [0x3000..0x4000)");
+
+    var it = map.iterator();
+    try std.testing.expectEqualStrings("Range [0x1000..0x2000)", it.next().?.value);
+    try std.testing.expectEqualStrings("Range [0x2000..0x3000)", it.next().?.value);
+    try std.testing.expectEqualStrings("Range [0x3000..0x4000)", it.next().?.value);
+    try std.testing.expectEqualStrings("Range [0x4000..0x5000)", it.next().?.value);
+    try std.testing.expectEqual(null, it.next());
+}
+
+test "Range map should disallow overflowing ranges" {
+    const Map = RangeMap(u32, bool, .{});
+    var map = Map.init(std.testing.allocator);
+    defer map.deinit();
+
+    try std.testing.expectError(error.range_out_of_bounds, map.insert(0xF0000000, 0x20000000, false));
+}

user/aarch64.ld

@@ -0,0 +1,27 @@
+SECTIONS {
+    . = 0x200000;
+
+    .text : {
+        *(.text.entry*);
+        *(.text*)
+    }
+
+    .rodata : ALIGN(4K) {
+        *(.rodata*)
+    }
+
+    .eh_frame_hdr : {
+        *(.eh_frame_hdr*)
+    }
+    .eh_frame : {
+        *(.eh_frame*)
+    }
+
+    .data : ALIGN(4K) {
+        *(.data*)
+    }
+
+    .bss : ALIGN(4K) {
+        *(.bss*)
+    }
+}

user/arch.zig

@@ -0,0 +1,9 @@
+const builtin = @import("builtin");
+
+const impl = switch (builtin.cpu.arch) {
+    .riscv64 => @import("arch/riscv64.zig"),
+    .aarch64 => @import("arch/aarch64.zig"),
+    else => @compileError("Unsupported architecture"),
+};
+
+pub const syscall = impl.syscall;

user/arch/aarch64.zig

@@ -0,0 +1,55 @@
+const abi = @import("abi");
+
+pub const syscall = struct {
+    pub fn send(handle: u32, msg: *const [5]usize) usize {
+        return asm volatile ("svc #0"
+            : [result] "={x0}" (-> usize),
+            : [a0] "{x0}" (handle),
+              [a1] "{x1}" (msg[0]),
+              [a2] "{x2}" (msg[1]),
+              [a3] "{x3}" (msg[2]),
+              [a4] "{x4}" (msg[3]),
+              [a5] "{x5}" (msg[4]),
+              [func] "{x8}" (@intFromEnum(abi.SyscallNumber.SYS_send)),
+            : "memory"
+        );
+    }
+
+    pub fn sendrecv(handle: u32, msg: *const[5]usize, buffer: *[5]usize) usize {
+        return asm volatile (
+            \\ svc #0
+            \\ stp x1, x2, [%[buf], #16 * 0]
+            \\ stp x3, x4, [%[buf], #16 * 1]
+            \\ str x5,     [%[buf], #16 * 2]
+            : [result] "={x0}" (-> usize),
+            : [a0] "{x0}" (handle),
+              [a1] "{x1}" (msg[0]),
+              [a2] "{x2}" (msg[1]),
+              [a3] "{x3}" (msg[2]),
+              [a4] "{x4}" (msg[3]),
+              [a5] "{x5}" (msg[4]),
+              [func] "{x8}" (@intFromEnum(abi.SyscallNumber.SYS_sendrecv)),
+              [buf] "r" (buffer),
+            : "memory"
+        );
+    }
+
+    // pub fn syscall1(func: abi.SyscallNumber, arg0: usize) usize {
+    //     return asm volatile ("svc #0"
+    //         : [result] "={x0}" (-> usize),
+    //         : [arg0] "{x0}" (arg0),
+    //           [func] "{x8}" (@intFromEnum(func)),
+    //         : "memory"
+    //     );
+    // }
+
+    // pub fn syscall2(func: abi.SyscallNumber, arg0: usize, arg1: usize) usize {
+    //     return asm volatile ("svc #0"
+    //         : [result] "={x0}" (-> usize),
+    //         : [arg0] "{x0}" (arg0),
+    //           [arg1] "{x1}" (arg1),
+    //           [func] "{x8}" (@intFromEnum(func)),
+    //         : "memory"
+    //     );
+    // }
+};

user/arch/riscv64.zig

@@ -0,0 +1,22 @@
+const abi = @import("abi");
+
+pub const syscall = struct {
+    pub fn syscall1(func: abi.SyscallNumber, arg0: usize) usize {
+        return asm volatile ("ecall"
+            : [result] "={a0}" (-> usize),
+            : [func] "{a0}" (@intFromEnum(func)),
+              [arg0] "{a1}" (arg0),
+            : "memory"
+        );
+    }
+
+    pub fn syscall2(func: abi.SyscallNumber, arg0: usize, arg1: usize) usize {
+        return asm volatile ("ecall"
+            : [result] "={a0}" (-> usize),
+            : [func] "{a0}" (@intFromEnum(func)),
+              [arg0] "{a1}" (arg0),
+              [arg1] "{a2}" (arg1),
+            : "memory"
+        );
+    }
+};

user/log.zig

@@ -0,0 +1,28 @@
+const std = @import("std");
+const syscall = @import("syscall.zig");
+
+pub const BUFFER_SIZE: usize = 512;
+
+pub const Writer = struct {
+    buffer: [BUFFER_SIZE]u8 = undefined,
+    position: usize = 0,
+
+    fn write(self: *Writer, bytes: []const u8) error{}!usize {
+        const amount = @min(BUFFER_SIZE - self.position, bytes.len);
+        if (amount != 0) {
+            @memcpy(self.buffer[self.position .. self.position + amount], bytes[0..amount]);
+            self.position += amount;
+        }
+        return amount;
+    }
+};
+
+pub fn println(comptime format: []const u8, args: anytype) void {
+    const W = std.io.GenericWriter(*Writer, error{}, Writer.write);
+    var context = Writer{};
+    var w = W{ .context = &context };
+    w.print(format, args) catch return;
+    if (context.position != 0) {
+        syscall.debug_write(context.buffer[0..context.position]);
+    }
+}

user/main.zig

@@ -0,0 +1,58 @@
+pub const arch = @import("arch.zig");
+pub const syscall = @import("syscall.zig");
+pub const log = @import("log.zig");
+const abi = @import("abi");
+
+const Handle = abi.Handle;
+
+fn debug_write(handle: Handle, message: []const u8) void {
+    _ = syscall.send(handle, &.{
+        @intFromPtr(message.ptr),
+        message.len,
+        0,
+        0,
+        0,
+    });
+}
+
+fn exit_process(handle: Handle, code: u32) noreturn {
+    _ = syscall.send(handle, &.{
+        @intFromEnum(abi.ProcessObjectAction.ZO_process_exit),
+        code,
+        0,
+        0,
+        0,
+    });
+    unreachable;
+}
+
+fn allocate_memory(handle: Handle, count: usize) u64 {
+    var output: [5]usize = undefined;
+    // TODO error code
+    _ = syscall.sendrecv(
+        handle,
+        &.{
+            @intFromEnum(abi.PhysicalMemoryObjectAction.ZO_physical_memory_allocate),
+            count,
+            0,
+            0,
+            0,
+        },
+        &output,
+    );
+    return output[0];
+}
+
+export fn _start(arg: usize) linksection(".text.entry") callconv(.C) noreturn {
+    // TODO make the kernel provide those dynamically somehow
+    const SELF_PROCESS_HANDLE: Handle = @enumFromInt(1);
+    const PHYSICAL_MEMORY_HANDLE: Handle = @enumFromInt(2);
+    const DEBUG_HANDLE: Handle = @enumFromInt(3);
+
+    _ = arg;
+    const mem = allocate_memory(PHYSICAL_MEMORY_HANDLE, 8);
+    _ = mem;
+
+    debug_write(DEBUG_HANDLE, "Hello!!!");
+    exit_process(SELF_PROCESS_HANDLE, 4321);
+}

user/riscv64.ld

@@ -0,0 +1,20 @@
+SECTIONS {
+    . = 0x200000;
+
+    .text : {
+        *(.text.entry*);
+        *(.text*)
+    }
+
+    .rodata : ALIGN(4K) {
+        *(.rodata*)
+    }
+
+    .data : ALIGN(4K) {
+        *(.data*)
+    }
+
+    .bss : ALIGN(4K) {
+        *(.bss*)
+    }
+}

user/syscall.zig

@@ -0,0 +1,25 @@
+const arch = @import("arch.zig");
+const abi = @import("abi");
+
+const sc = arch.syscall;
+
+pub inline fn send(object: abi.Handle, msg: *const [5]usize) usize {
+    return sc.send(@intFromEnum(object), msg);
+}
+
+pub inline fn recv(object: abi.Handle, buffer: *[5]usize) usize {
+    return sc.recv(@intFromEnum(object), buffer);
+}
+
+pub inline fn sendrecv(object: abi.Handle, msg: *const [5]usize, buffer: *[5]usize) usize {
+    return sc.sendrecv(@intFromEnum(object), msg, buffer);
+}
+
+// pub fn exit(code: u32) noreturn {
+//     _ = sc.syscall1(SC.SYS_exit, code);
+//     unreachable;
+// }
+//
+// pub fn debug_write(text: []const u8) void {
+//     _ = sc.syscall2(SC.SYS_debug_write, @intFromPtr(text.ptr), text.len);
+// }