WIP: Add a userspace entry to riscv64

src/arch/riscv64/boot.zig

@@ -37,7 +37,7 @@ fn bsp_upper_entry(real_address: usize, unused: usize) callconv(.C) noreturn {
     exception.init();
 
     debug.log.set_write_fn(&sbi.debug_print_byte);
-    kernel.mem.PhysicalAddress.g_virtualize_base = 0;
+    kernel.mem.PhysicalAddress.g_virtualize_base = vmm.VIRTUALIZE_BASE;
     kernel.mem.PhysicalAddress.g_virtualize_size = vmm.virtualize_range();
 
     // Setup physical memory management

src/arch/riscv64/context.S

@@ -4,6 +4,7 @@
 
 .global __rv64_enter_task
 .global __rv64_switch_task
+.global __rv64_task_enter_user
 .global __rv64_task_enter_kernel
 
 .macro LOAD_TASK_STATE
@@ -44,6 +45,29 @@
     sd s0,  13 * 8(sp)
 .endm
 
+.set SSTATUS_SPP, (1 << 8)
+.set SSTATUS_SPIE, (1 << 5)
+
+__rv64_task_enter_user:
+    // TODO setup user thread pointer
+
+    ld a0, (sp)     // argument
+    ld ra, 16(sp)   // entry
+    ld sp, 8(sp)    // stack
+
+    // Clear SPP to zero to indicate a return to U-mode
+    li t1, SSTATUS_SPP
+    not t1, t1
+
+    csrr t0, sstatus
+    // TODO enable interrupts via SPIE
+    // ori t0, t0, SSTATUS_SPIE
+    and t0, t0, t1
+    csrw sstatus, t0
+    csrw sepc, ra
+
+    sret
+
 __rv64_task_enter_kernel:
     ld a0, (sp)     // argument
     ld ra, 8(sp)    // entry

src/arch/riscv64/context.zig

@@ -1,4 +1,11 @@
 const thread = @import("../../thread.zig");
+const mem = @import("../../mem.zig");
+const kernel = @import("../../kernel.zig");
+const regs = @import("regs.zig");
+const vmm = @import("vmm.zig");
+
+const ProcessAddressSpace = mem.vmm.ProcessAddressSpace;
+const log = kernel.log;
 
 fn idle_function() callconv(.naked) noreturn {
     asm volatile ("j .");
@@ -7,6 +14,7 @@ fn idle_function() callconv(.naked) noreturn {
 extern fn __rv64_enter_task(cx: *Context) callconv(.C) noreturn;
 extern fn __rv64_switch_task(dcx: *Context, scx: *Context) callconv(.C) void;
 extern fn __rv64_task_enter_kernel() callconv(.C) noreturn;
+extern fn __rv64_task_enter_user() callconv(.C) noreturn;
 
 pub const Context = extern struct {
     const STACK_SIZE: usize = 8192;
@@ -14,12 +22,52 @@ pub const Context = extern struct {
     // Has to be exactly at offset 0x00, used in assembly.
     kstack: thread.KStack(STACK_SIZE),
 
+    satp: u64 = 0,
+
     /// Constructs an idle context struct.
     pub fn idle() @This() {
         const entry = @intFromPtr(&idle_function);
         return Context.kernel(entry, 0);
     }
 
+    pub fn user(address_space: *const ProcessAddressSpace, pc: usize, sp: usize, arg: usize) @This() {
+        const space_physical = address_space.physical_address();
+        const space_asid = address_space.asid();
+
+        const satp = regs.SATP.Bits {
+            .PPN = @truncate(space_physical.raw >> 12),
+            .ASID = @truncate(space_asid),
+            .MODE = .sv39
+        };
+
+        var ks = thread.KStack(STACK_SIZE).create();
+        const entry = @intFromPtr(&__rv64_task_enter_user);
+
+        ks.push(pc);
+        ks.push(sp);
+        ks.push(arg);
+
+        ks.push(0); // x8/s0/fp
+        ks.push(0); // x9/s1
+        ks.push(0); // x18/s2
+        ks.push(0); // x19/s3
+        ks.push(0); // x20/s4
+        ks.push(0); // x21/s5
+        ks.push(0); // x22/s6
+        ks.push(0); // x23/s7
+        ks.push(0); // x24/s8
+        ks.push(0); // x25/s9
+        ks.push(0); // x26/s10
+        ks.push(0); // x27/s11
+        ks.push(0); // x4/gp
+        ks.push(entry); // x1/ra return address
+
+        return .{
+            .kstack = ks,
+            .satp = @bitCast(satp)
+        };
+    }
+
     /// Constructs a kernel task context with entry point in `pc` and an `arg`ument.
     pub fn kernel(pc: usize, arg: usize) @This() {
         var ks = thread.KStack(STACK_SIZE).create();
@@ -48,13 +96,29 @@ pub const Context = extern struct {
 
     /// Low-level task context entry function.
     pub fn enter(self: *@This()) noreturn {
+        self.load_state();
         __rv64_enter_task(self);
     }
 
     /// Low-level task context switch function.
     pub fn switch_from(self: *@This(), from: *@This()) void {
+        from.store_state();
+        self.load_state();
         __rv64_switch_task(self, from);
     }
+
+    fn load_state(self: *@This()) void {
+        if (self.satp != 0) {
+            log.info("Load SATP = 0x{x}", .{self.satp});
+            regs.SATP.set(self.satp);
+        } else {
+            vmm.load_kernel_table();
+        }
+    }
+
+    fn store_state(self: *@This()) void {
+        _ = self;
+    }
 };
 
 comptime {

src/arch/riscv64/regs.zig

@@ -4,6 +4,8 @@ fn Register(comptime name: []const u8, comptime bits: type) type {
         else => bits,
     };
     return enum(repr) {
+        pub const Bits = bits;
+
         pub fn set(value: repr) void {
             asm volatile ("csrw " ++ name ++ ", %[value]"
                 :

src/arch/riscv64/vmm.zig

@@ -1,17 +1,23 @@
+const std = @import("std");
 const sync = @import("../../sync.zig");
 const regs = @import("regs.zig");
 const mem = @import("../../mem.zig");
-const arch = @import("../../kernel.zig").arch;
+const kernel = @import("../../kernel.zig");
 
+const log = kernel.log;
+const arch = kernel.arch;
 const PhysicalAddress = mem.PhysicalAddress;
+const AtomicU8 = std.atomic.Value(u8);
 
 pub const KERNEL_VIRTUAL_BASE: usize = 0xFFFFFFF000000000;
 pub const KERNEL_VIRTUAL_L1I: usize = (KERNEL_VIRTUAL_BASE >> L1.SHIFT) & 511;
+pub const VIRTUALIZE_BASE: usize = KERNEL_VIRTUAL_BASE + L1.SIZE;
+pub const VIRTUALIZE_BASE_L1I: usize = L1.index(VIRTUALIZE_BASE);
 // 16 GiB
 const EARLY_MAPPING_SIZE: usize = 16;
 
-pub const L1 = mem.TranslationLevel(30);
-pub const L2 = mem.TranslationLevel(21);
+pub const L1 = mem.TranslationLevel(30, L2);
+pub const L2 = mem.TranslationLevel(21, L3);
 pub const L3 = mem.vmm.L3;
 
 pub const RawEntry = packed struct(u64) {
@@ -45,7 +51,7 @@ pub const RawEntry = packed struct(u64) {
     }
 
     pub fn clear(self: *@This(), mask: @This()) void {
-        const lhs = @as(*u64, @bitCast(self));
+        const lhs = @as(*u64, @ptrCast(self));
         const rhs = @as(u64, @bitCast(mask));
         lhs.* &= ~rhs;
     }
@@ -83,12 +89,14 @@ pub fn TableEntry(comptime Level: type) type {
         }
 
         pub fn table(addr: PhysicalAddress, flags: RawEntry) @This() {
-            flags.clear(.{ .r = true, .w = true, .x = true });
-            return .{ .raw = flags.make_union(.{
+            var f = flags;
+            f.clear(.{ .r = true, .w = true, .x = true });
+            return .{ .raw = f.make_union(.{
                 .address = @as(u39, @intCast(addr.raw >> 12)),
                 .v = true,
             }) };
         }
+
     };
 }
 
@@ -98,16 +106,108 @@ pub fn Table(comptime Level: type) type {
 
         entries: [512]Entry align(4096),
 
+        pub const Error = mem.vmm.AddressSpaceError;
+
         pub fn empty() @This() {
             return .{ .entries = [_]Entry{.INVALID} ** 512 };
         }
 
+        pub fn allocate_empty() Error!*@This() {
+            const page = mem.phys.alloc_page() orelse return error.out_of_pages;
+            const table = @as(*@This(), @ptrFromInt(page.virtualize()));
+            for (0..512) |i| {
+                table.entry(i).* = .INVALID;
+            }
+            return table;
+        }
+
+        pub fn physical_address(self: *const @This()) PhysicalAddress {
+            return PhysicalAddress.from_virtualized(@intFromPtr(self));
+        }
+
         pub inline fn entry(self: *@This(), index: usize) *Entry {
             return &self.entries[index];
         }
+
+        pub usingnamespace if (Level.NextLevel) |NextLevel| struct {
+            pub fn get_next_level(self: *Table(Level), index: usize) ?*Table(NextLevel) {
+                _ = self;
+                _ = index;
+                @panic("TODO");
+            }
+
+            pub fn get_or_create_next_level(self: *Table(Level), index: usize) Error!*Table(NextLevel) {
+                const ent = self.entry(index);
+
+                if (ent.raw.v) {
+                    // TODO handle mixed hugepages + tables
+                    if (ent.raw.r or ent.raw.w or ent.raw.x) {
+                        @panic("TODO: handle mixed hugepages and tables");
+                    }
+                    // It is a table
+                    @panic("OOO");
+                } else {
+                    // Allocate a new entry
+                    const table = try Table(NextLevel).allocate_empty();
+                    const physical = table.physical_address();
+                    ent.* = TableEntry(Level).table(physical, .{});
+                    return table;
+                }
+            }
+        } else struct {};
     };
 }
 
+pub const ProcessAddressSpace = struct {
+    l1: *Table(L1),
+    asid: u8,
+
+    pub const Error = mem.vmm.AddressSpaceError;
+
+    var g_asid: AtomicU8 = .{ .raw = 1 };
+
+    pub fn init() Error!ProcessAddressSpace {
+        const table = try Table(L1).allocate_empty();
+        // Copy kernel's mappings
+        for (KERNEL_VIRTUAL_L1I..512) |i| {
+            table.entry(i).* = g_fixed.entry(i).*;
+        }
+        const asid = g_asid.fetchAdd(1, .seq_cst);
+        return .{ .l1 = table, .asid = asid };
+    }
+
+    pub fn physical_address(self: *const @This()) PhysicalAddress {
+        return self.l1.physical_address();
+    }
+
+    pub fn map_page(self: *@This(), virtual: usize, physical: PhysicalAddress) Error!void {
+        // TODO align check on both virtual and physical
+
+        const l1i = L1.index(virtual);
+        const l2i = L2.index(virtual);
+        const l3i = L3.index(virtual);
+
+        const l2 = try self.l1.get_or_create_next_level(l1i);
+        const l3 = try l2.get_or_create_next_level(l2i);
+
+        const entry = l3.entry(l3i);
+
+        if (entry.raw.v) {
+            @panic("TODO: handle already present");
+        }
+
+        entry.* = TableEntry(L3).page(physical, .{
+            .r = true,
+            .w = true,
+            .x = true,
+            .u = true,
+        });
+        flush_vma_asid(virtual, self.asid);
+
+        log.debug("Map 0x{x} -> page 0x{x}", .{ virtual, physical.raw });
+    }
+};
+
 var g_fixed = Table(L1).empty();
 var g_fixed_lock: sync.Spinlock = .{};
 
@@ -120,13 +220,15 @@ pub fn unmap_early() void {
     const guard = g_fixed_lock.lock_irqsave();
     defer guard.release();
     for (0..EARLY_MAPPING_SIZE) |i| {
-        g_fixed.entry(i).* = .page(
-            .{ .raw = L1.address(i) },
-            .{ .r = true, .w = true },
-        );
+        g_fixed.entry(i).* = .INVALID;
     }
 }
 
+pub fn load_kernel_table() void {
+    const address = @as(usize, @intFromPtr(&g_fixed));
+    regs.SATP.write(.{ .PPN = @intCast(address >> 12), .MODE = .sv39 });
+}
+
 pub fn map_early(real_address: usize) void {
     const real_l1 = L1.index(real_address);
 
@@ -138,14 +240,20 @@ pub fn map_early(real_address: usize) void {
         );
     }
 
+    for (0..EARLY_MAPPING_SIZE) |i| {
+        g_fixed.entry(i + VIRTUALIZE_BASE_L1I).* = .page(
+            .{ .raw = L1.address(i) },
+            .{ .r = true, .w = true },
+        );
+    }
+
     // Map 1GiB at KERNEL_VIRTUAL_BASE -> physical 1GiB where the kernel is loaded
     g_fixed.entry(KERNEL_VIRTUAL_L1I).* = .page(
         .{ .raw = L1.address(real_l1) },
         .{ .r = true, .w = true, .x = true },
     );
 
-    const address = @as(usize, @intFromPtr(&g_fixed));
-    regs.SATP.write(.{ .PPN = @intCast(address >> 12), .MODE = .sv39 });
+    load_kernel_table();
 }
 
 pub inline fn flush_vma(page: usize) void {

src/kernel.zig

@@ -38,15 +38,21 @@ noinline fn f1(arg: usize, c: usize) void {
 /// * Physical memory must be initialized.
 /// * (optional) Logging should be set up.
 pub export fn kernel_main() callconv(.C) noreturn {
-    log.write("\x1B[2J", .{});
     var a = arena.Arena.init(256 * 0x1000) orelse @panic("Could not setup kernel arena");
     thread.Queue.init_this_cpu(&a);
 
-    const pc = @intFromPtr(&f0);
-    for (0..4) |i| {
-        const t = thread.Thread.create(&a, pc, i);
-        thread.enqueue(t);
-    }
+    const t = thread.test_create_user_from_code(&a, &[_]u8 {
+        0x6F, 0x00, 0x00, 0x00
+    });
+    thread.enqueue(t);
+
+    log.info("Test", .{});
+    // log.write("\x1B[2J", .{});
+    // const pc = @intFromPtr(&f0);
+    // for (0..4) |i| {
+    //     const t = thread.Thread.create_kernel(&a, pc, i);
+    //     thread.enqueue(t);
+    // }
 
     thread.enter();
 }

src/mem.zig

@@ -46,7 +46,7 @@ pub const PhysicalAddress = packed struct(u64) {
     ///
     /// Panics if the virtual address provided is outside of virtualizable memory range.
     pub fn from_virtualized(virt: usize) @This() {
-        if ((virt < g_virtualize_base) || (virt - g_virtualize_base > g_virtualize_size)) {
+        if (virt < g_virtualize_base or virt - g_virtualize_base > g_virtualize_size) {
             @panic("Invalid virtualized physical address");
         }
 

src/mem/vmalloc.zig

@@ -1,9 +1,8 @@
 const std = @import("std");
 
+const Arena = @import("../arena.zig").Arena;
 const Range = @import("../util/range.zig").Range;
 
-const Allocator = std.mem.Allocator;
-
 /// Describes a single virtual memory range.
 ///
 /// Used by `VirtualMemoryAllocator` to track allocated/used regions.
@@ -16,12 +15,12 @@ pub const VirtualMemoryRange = struct {
 
 /// Virtual memory allocator implementation.
 pub const VirtualMemoryAllocator = struct {
-    gpa: Allocator,
+    arena: *Arena,
     head: ?*VirtualMemoryRange = null,
     outer_range: Range(u64),
 
     /// One of errors returned by the allocation logic + underlying allocator error.
-    pub const Error = error{ already_exists, invalid_region, cannot_fit } || Allocator.Error;
+    pub const Error = error{ already_exists, invalid_region, cannot_fit };
 
     /// An iterator over VM regions being freed.
     pub const FreeIterator = struct {
@@ -46,7 +45,7 @@ pub const VirtualMemoryAllocator = struct {
                             }
                             // Free it
                             self.current = n.next;
-                            self.vma.gpa.destroy(n);
+                            // self.vma.arena.destroy(n);
 
                             return xs;
                         }
@@ -64,7 +63,7 @@ pub const VirtualMemoryAllocator = struct {
                         return xs;
                     } else {
                         // Insert a new node after the current one
-                        const new_node = try self.vma.gpa.create(VirtualMemoryRange);
+                        const new_node = self.vma.arena.create(VirtualMemoryRange);
                         new_node.* = VirtualMemoryRange {
                             .range = .{ .start = xs.end(), .len = n.range.end() - xs.end() },
                             .prev = n,
@@ -90,10 +89,10 @@ pub const VirtualMemoryAllocator = struct {
     };
 
     /// Creates a new instance of a virtual memory allocator.
-    pub fn init(gpa: Allocator, outer_range: Range(u64)) @This() {
+    pub fn init(arena: *Arena, outer_range: Range(u64)) @This() {
         return .{
             .outer_range = outer_range,
-            .gpa = gpa,
+            .arena = arena,
         };
     }
 
@@ -108,7 +107,7 @@ pub const VirtualMemoryAllocator = struct {
         const gap_before_first = if (self.head) |n| (n.range.start - self.outer_range.start) else self.outer_range.len;
 
         if (gap_before_first >= pfn_count) {
-            var new_node = try self.gpa.create(VirtualMemoryRange);
+            var new_node = self.arena.create(VirtualMemoryRange);
 
             new_node.range = .{ .start = self.outer_range.start, .len = pfn_count };
             new_node.next = self.head;
@@ -137,7 +136,7 @@ pub const VirtualMemoryAllocator = struct {
             if (gap >= pfn_count) {
                 // Insert after this
                 const result = n.range.end();
-                var new_node = try self.gpa.create(VirtualMemoryRange);
+                var new_node = self.arena.create(VirtualMemoryRange);
                 new_node.prev = n;
                 new_node.next = n.next;
                 new_node.range = .{ .start = result, .len = pfn_count };
@@ -181,7 +180,7 @@ pub const VirtualMemoryAllocator = struct {
             node = n.next;
         }
 
-        var new_node = try self.gpa.create(VirtualMemoryRange);
+        var new_node = self.arena.create(VirtualMemoryRange);
 
         new_node.range = region;
 

src/mem/vmm.zig

@@ -1,18 +1,30 @@
 //! Platform-independent virtual memory management definitions.
 
 const mem = @import("../mem.zig");
+const arena = @import("../arena.zig");
+const vmalloc = @import("vmalloc.zig");
+const kernel = @import("../kernel.zig");
+const sync = @import("../sync.zig");
+
+const arch = kernel.arch;
+const Arena = arena.Arena;
 
 /// Last virtual memory translation level. Always 4KiB on all platforms.
-pub const L3 = mem.TranslationLevel(12);
+pub const L3 = mem.TranslationLevel(12, null);
 
 /// Page size is 4KiB on all platforms.
 pub const PAGE_SIZE: usize = L3.SIZE;
 
+pub const AddressSpaceError = error{
+    out_of_pages,
+};
+
 /// Helper function to construct a "Translation Level" struct type from a bit shift.
-pub fn TranslationLevel(comptime shift: usize) type {
+pub fn TranslationLevel(comptime shift: usize, comptime Next: ?type) type {
     return struct {
         pub const SHIFT: usize = shift;
         pub const SIZE: usize = 1 << shift;
+        pub const NextLevel = Next;
 
         pub inline fn index(addr: usize) usize {
             return (addr >> shift) & 511;
@@ -43,3 +55,38 @@ pub fn TranslationLevel(comptime shift: usize) type {
         }
     };
 }
+
+pub const ProcessAddressSpace = struct {
+    inner: arch.vmm.ProcessAddressSpace,
+    allocator: vmalloc.VirtualMemoryAllocator,
+    lock: sync.Spinlock,
+
+    pub fn init(a: *Arena) AddressSpaceError!ProcessAddressSpace {
+        // 0x200000..0x600000
+        const inner = try arch.vmm.ProcessAddressSpace.init();
+        const allocator = vmalloc.VirtualMemoryAllocator.init(a, .{ .start = 512, .len = 1024 });
+
+        return .{ .inner = inner, .allocator = allocator, .lock = .{} };
+    }
+
+    pub fn map_single_page(
+        self: *@This(),
+        virtual: usize,
+        physical: mem.PhysicalAddress,
+    ) AddressSpaceError!void {
+        self.lock.lock();
+        defer self.lock.release();
+
+        // TODO If allocation succeeds, but mapping fails, rollback
+        self.allocator.insert(.{ .start = L3.page_number(virtual), .len = 1 }) catch @panic("TODO error");
+        try self.inner.map_page(virtual, physical);
+    }
+
+    pub fn physical_address(self: *const @This()) mem.PhysicalAddress {
+        return self.inner.physical_address();
+    }
+
+    pub fn asid(self: *const @This()) u64 {
+        return self.inner.asid;
+    }
+};

src/thread.zig

@@ -7,6 +7,8 @@ const arch = @import("kernel.zig").arch;
 const log = @import("debug.zig").log;
 const mem = @import("mem.zig");
 
+const ProcessAddressSpace = mem.vmm.ProcessAddressSpace;
+
 /// Per-CPU thread queue structure.
 pub const Queue = struct {
     /// Idle task context. Used when there are no other tasks running.
@@ -92,8 +94,11 @@ pub const Thread = struct {
     /// Previous thread in the queue.
     prev: ?*Thread = null,
 
+    // TODO move to process
+    address_space: ?ProcessAddressSpace = null,
+
     /// Creates a new (kernel) thread with given `pc` (entry point) and `arg`ument.
-    pub fn create(a: *arena.Arena, pc: usize, arg: usize) *Thread {
+    pub fn create_kernel(a: *arena.Arena, pc: usize, arg: usize) *Thread {
         const thread = a.create(Thread);
         thread.* = .{
             .allocator = a,
@@ -102,6 +107,22 @@ pub const Thread = struct {
         return thread;
     }
 
+    pub fn create_user(
+        a: *arena.Arena,
+        address_space: ProcessAddressSpace,
+        pc: usize,
+        sp: usize,
+        arg: usize,
+    ) *Thread {
+        const thread = a.create(Thread);
+        thread.* = .{
+            .allocator = a,
+            .address_space = address_space,
+            .arch_context = arch.Context.user(&address_space, pc, sp, arg),
+        };
+        return thread;
+    }
+
     /// Enters the thread, does not return.
     pub fn enter(self: *@This()) noreturn {
         self.arch_context.enter();
@@ -170,3 +191,18 @@ pub fn enter() noreturn {
 pub fn yield() void {
     Queue.t_this_cpu.?.yield();
 }
+
+pub fn test_create_user_from_code(a: *arena.Arena, code: []const u8) *Thread {
+    var address_space = ProcessAddressSpace.init(a) catch @panic("TODO");
+
+    // Map 0x200000
+    const page = mem.phys.alloc_page() orelse @panic("TODO error");
+    address_space.map_single_page(0x200000, page) catch @panic("TODO error map");
+
+    const page_data = @as([*]u8, @ptrFromInt(page.virtualize()))[0..code.len];
+    @memcpy(page_data, code);
+
+    const thread = Thread.create_user(a, address_space, 0x200000, 0, 0);
+
+    return thread;
+}