From 307d87d6d66ed20be5f3df5cca540e8e866349ef Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Tue, 25 Mar 2025 17:01:11 +0200 Subject: [PATCH] WIP: Add a userspace entry to riscv64 --- src/arch/riscv64/boot.zig | 2 +- src/arch/riscv64/context.S | 24 +++++++ src/arch/riscv64/context.zig | 64 +++++++++++++++++ src/arch/riscv64/regs.zig | 2 + src/arch/riscv64/vmm.zig | 132 +++++++++++++++++++++++++++++++---- src/kernel.zig | 18 +++-- src/mem.zig | 2 +- src/mem/vmalloc.zig | 21 +++--- src/mem/vmm.zig | 51 +++++++++++++- src/thread.zig | 38 +++++++++- 10 files changed, 320 insertions(+), 34 deletions(-) diff --git a/src/arch/riscv64/boot.zig b/src/arch/riscv64/boot.zig index 81d3ed1..e466c19 100644 --- a/src/arch/riscv64/boot.zig +++ b/src/arch/riscv64/boot.zig @@ -37,7 +37,7 @@ fn bsp_upper_entry(real_address: usize, unused: usize) callconv(.C) noreturn { exception.init(); debug.log.set_write_fn(&sbi.debug_print_byte); - kernel.mem.PhysicalAddress.g_virtualize_base = 0; + kernel.mem.PhysicalAddress.g_virtualize_base = vmm.VIRTUALIZE_BASE; kernel.mem.PhysicalAddress.g_virtualize_size = vmm.virtualize_range(); // Setup physical memory management diff --git a/src/arch/riscv64/context.S b/src/arch/riscv64/context.S index 2596b12..08d954b 100644 --- a/src/arch/riscv64/context.S +++ b/src/arch/riscv64/context.S @@ -4,6 +4,7 @@ .global __rv64_enter_task .global __rv64_switch_task +.global __rv64_task_enter_user .global __rv64_task_enter_kernel .macro LOAD_TASK_STATE @@ -44,6 +45,29 @@ sd s0, 13 * 8(sp) .endm +.set SSTATUS_SPP, (1 << 8) +.set SSTATUS_SPIE, (1 << 5) + +__rv64_task_enter_user: + // TODO setup user thread pointer + + ld a0, (sp) // argument + ld ra, 16(sp) // entry + ld sp, 8(sp) // stack + + // Clear SPP to zero to indicate a return to U-mode + li t1, SSTATUS_SPP + not t1, t1 + + csrr t0, sstatus + // TODO enable interrupts via SPIE + // ori t0, t0, SSTATUS_SPIE + and t0, t0, t1 + csrw sstatus, t0 + csrw sepc, ra + + sret + __rv64_task_enter_kernel: ld a0, (sp) // argument ld ra, 8(sp) // entry diff --git a/src/arch/riscv64/context.zig b/src/arch/riscv64/context.zig index 34f90f7..df33329 100644 --- a/src/arch/riscv64/context.zig +++ b/src/arch/riscv64/context.zig @@ -1,4 +1,11 @@ const thread = @import("../../thread.zig"); +const mem = @import("../../mem.zig"); +const kernel = @import("../../kernel.zig"); +const regs = @import("regs.zig"); +const vmm = @import("vmm.zig"); + +const ProcessAddressSpace = mem.vmm.ProcessAddressSpace; +const log = kernel.log; fn idle_function() callconv(.naked) noreturn { asm volatile ("j ."); @@ -7,6 +14,7 @@ fn idle_function() callconv(.naked) noreturn { extern fn __rv64_enter_task(cx: *Context) callconv(.C) noreturn; extern fn __rv64_switch_task(dcx: *Context, scx: *Context) callconv(.C) void; extern fn __rv64_task_enter_kernel() callconv(.C) noreturn; +extern fn __rv64_task_enter_user() callconv(.C) noreturn; pub const Context = extern struct { const STACK_SIZE: usize = 8192; @@ -14,12 +22,52 @@ pub const Context = extern struct { // Has to be exactly at offset 0x00, used in assembly. kstack: thread.KStack(STACK_SIZE), + satp: u64 = 0, + /// Constructs an idle context struct. pub fn idle() @This() { const entry = @intFromPtr(&idle_function); return Context.kernel(entry, 0); } + pub fn user(address_space: *const ProcessAddressSpace, pc: usize, sp: usize, arg: usize) @This() { + const space_physical = address_space.physical_address(); + const space_asid = address_space.asid(); + + const satp = regs.SATP.Bits { + .PPN = @truncate(space_physical.raw >> 12), + .ASID = @truncate(space_asid), + .MODE = .sv39 + }; + + var ks = thread.KStack(STACK_SIZE).create(); + const entry = @intFromPtr(&__rv64_task_enter_user); + + ks.push(pc); + ks.push(sp); + ks.push(arg); + + ks.push(0); // x8/s0/fp + ks.push(0); // x9/s1 + ks.push(0); // x18/s2 + ks.push(0); // x19/s3 + ks.push(0); // x20/s4 + ks.push(0); // x21/s5 + ks.push(0); // x22/s6 + ks.push(0); // x23/s7 + ks.push(0); // x24/s8 + ks.push(0); // x25/s9 + ks.push(0); // x26/s10 + ks.push(0); // x27/s11 + ks.push(0); // x4/gp + ks.push(entry); // x1/ra return address + + return .{ + .kstack = ks, + .satp = @bitCast(satp) + }; + } + /// Constructs a kernel task context with entry point in `pc` and an `arg`ument. pub fn kernel(pc: usize, arg: usize) @This() { var ks = thread.KStack(STACK_SIZE).create(); @@ -48,13 +96,29 @@ pub const Context = extern struct { /// Low-level task context entry function. pub fn enter(self: *@This()) noreturn { + self.load_state(); __rv64_enter_task(self); } /// Low-level task context switch function. pub fn switch_from(self: *@This(), from: *@This()) void { + from.store_state(); + self.load_state(); __rv64_switch_task(self, from); } + + fn load_state(self: *@This()) void { + if (self.satp != 0) { + log.info("Load SATP = 0x{x}", .{self.satp}); + regs.SATP.set(self.satp); + } else { + vmm.load_kernel_table(); + } + } + + fn store_state(self: *@This()) void { + _ = self; + } }; comptime { diff --git a/src/arch/riscv64/regs.zig b/src/arch/riscv64/regs.zig index 4af215e..48749bd 100644 --- a/src/arch/riscv64/regs.zig +++ b/src/arch/riscv64/regs.zig @@ -4,6 +4,8 @@ fn Register(comptime name: []const u8, comptime bits: type) type { else => bits, }; return enum(repr) { + pub const Bits = bits; + pub fn set(value: repr) void { asm volatile ("csrw " ++ name ++ ", %[value]" : diff --git a/src/arch/riscv64/vmm.zig b/src/arch/riscv64/vmm.zig index 43f4ed8..4efc59c 100644 --- a/src/arch/riscv64/vmm.zig +++ b/src/arch/riscv64/vmm.zig @@ -1,17 +1,23 @@ +const std = @import("std"); const sync = @import("../../sync.zig"); const regs = @import("regs.zig"); const mem = @import("../../mem.zig"); -const arch = @import("../../kernel.zig").arch; +const kernel = @import("../../kernel.zig"); +const log = kernel.log; +const arch = kernel.arch; const PhysicalAddress = mem.PhysicalAddress; +const AtomicU8 = std.atomic.Value(u8); pub const KERNEL_VIRTUAL_BASE: usize = 0xFFFFFFF000000000; pub const KERNEL_VIRTUAL_L1I: usize = (KERNEL_VIRTUAL_BASE >> L1.SHIFT) & 511; +pub const VIRTUALIZE_BASE: usize = KERNEL_VIRTUAL_BASE + L1.SIZE; +pub const VIRTUALIZE_BASE_L1I: usize = L1.index(VIRTUALIZE_BASE); // 16 GiB const EARLY_MAPPING_SIZE: usize = 16; -pub const L1 = mem.TranslationLevel(30); -pub const L2 = mem.TranslationLevel(21); +pub const L1 = mem.TranslationLevel(30, L2); +pub const L2 = mem.TranslationLevel(21, L3); pub const L3 = mem.vmm.L3; pub const RawEntry = packed struct(u64) { @@ -45,7 +51,7 @@ pub const RawEntry = packed struct(u64) { } pub fn clear(self: *@This(), mask: @This()) void { - const lhs = @as(*u64, @bitCast(self)); + const lhs = @as(*u64, @ptrCast(self)); const rhs = @as(u64, @bitCast(mask)); lhs.* &= ~rhs; } @@ -83,12 +89,14 @@ pub fn TableEntry(comptime Level: type) type { } pub fn table(addr: PhysicalAddress, flags: RawEntry) @This() { - flags.clear(.{ .r = true, .w = true, .x = true }); - return .{ .raw = flags.make_union(.{ + var f = flags; + f.clear(.{ .r = true, .w = true, .x = true }); + return .{ .raw = f.make_union(.{ .address = @as(u39, @intCast(addr.raw >> 12)), .v = true, }) }; } + }; } @@ -98,16 +106,108 @@ pub fn Table(comptime Level: type) type { entries: [512]Entry align(4096), + pub const Error = mem.vmm.AddressSpaceError; + pub fn empty() @This() { return .{ .entries = [_]Entry{.INVALID} ** 512 }; } + pub fn allocate_empty() Error!*@This() { + const page = mem.phys.alloc_page() orelse return error.out_of_pages; + const table = @as(*@This(), @ptrFromInt(page.virtualize())); + for (0..512) |i| { + table.entry(i).* = .INVALID; + } + return table; + } + + pub fn physical_address(self: *const @This()) PhysicalAddress { + return PhysicalAddress.from_virtualized(@intFromPtr(self)); + } + pub inline fn entry(self: *@This(), index: usize) *Entry { return &self.entries[index]; } + + pub usingnamespace if (Level.NextLevel) |NextLevel| struct { + pub fn get_next_level(self: *Table(Level), index: usize) ?*Table(NextLevel) { + _ = self; + _ = index; + @panic("TODO"); + } + + pub fn get_or_create_next_level(self: *Table(Level), index: usize) Error!*Table(NextLevel) { + const ent = self.entry(index); + + if (ent.raw.v) { + // TODO handle mixed hugepages + tables + if (ent.raw.r or ent.raw.w or ent.raw.x) { + @panic("TODO: handle mixed hugepages and tables"); + } + // It is a table + @panic("OOO"); + } else { + // Allocate a new entry + const table = try Table(NextLevel).allocate_empty(); + const physical = table.physical_address(); + ent.* = TableEntry(Level).table(physical, .{}); + return table; + } + } + } else struct {}; }; } +pub const ProcessAddressSpace = struct { + l1: *Table(L1), + asid: u8, + + pub const Error = mem.vmm.AddressSpaceError; + + var g_asid: AtomicU8 = .{ .raw = 1 }; + + pub fn init() Error!ProcessAddressSpace { + const table = try Table(L1).allocate_empty(); + // Copy kernel's mappings + for (KERNEL_VIRTUAL_L1I..512) |i| { + table.entry(i).* = g_fixed.entry(i).*; + } + const asid = g_asid.fetchAdd(1, .seq_cst); + return .{ .l1 = table, .asid = asid }; + } + + pub fn physical_address(self: *const @This()) PhysicalAddress { + return self.l1.physical_address(); + } + + pub fn map_page(self: *@This(), virtual: usize, physical: PhysicalAddress) Error!void { + // TODO align check on both virtual and physical + + const l1i = L1.index(virtual); + const l2i = L2.index(virtual); + const l3i = L3.index(virtual); + + const l2 = try self.l1.get_or_create_next_level(l1i); + const l3 = try l2.get_or_create_next_level(l2i); + + const entry = l3.entry(l3i); + + if (entry.raw.v) { + @panic("TODO: handle already present"); + } + + entry.* = TableEntry(L3).page(physical, .{ + .r = true, + .w = true, + .x = true, + .u = true, + }); + flush_vma_asid(virtual, self.asid); + + log.debug("Map 0x{x} -> page 0x{x}", .{ virtual, physical.raw }); + } +}; + var g_fixed = Table(L1).empty(); var g_fixed_lock: sync.Spinlock = .{}; @@ -120,13 +220,15 @@ pub fn unmap_early() void { const guard = g_fixed_lock.lock_irqsave(); defer guard.release(); for (0..EARLY_MAPPING_SIZE) |i| { - g_fixed.entry(i).* = .page( - .{ .raw = L1.address(i) }, - .{ .r = true, .w = true }, - ); + g_fixed.entry(i).* = .INVALID; } } +pub fn load_kernel_table() void { + const address = @as(usize, @intFromPtr(&g_fixed)); + regs.SATP.write(.{ .PPN = @intCast(address >> 12), .MODE = .sv39 }); +} + pub fn map_early(real_address: usize) void { const real_l1 = L1.index(real_address); @@ -138,14 +240,20 @@ pub fn map_early(real_address: usize) void { ); } + for (0..EARLY_MAPPING_SIZE) |i| { + g_fixed.entry(i + VIRTUALIZE_BASE_L1I).* = .page( + .{ .raw = L1.address(i) }, + .{ .r = true, .w = true }, + ); + } + // Map 1GiB at KERNEL_VIRTUAL_BASE -> physical 1GiB where the kernel is loaded g_fixed.entry(KERNEL_VIRTUAL_L1I).* = .page( .{ .raw = L1.address(real_l1) }, .{ .r = true, .w = true, .x = true }, ); - const address = @as(usize, @intFromPtr(&g_fixed)); - regs.SATP.write(.{ .PPN = @intCast(address >> 12), .MODE = .sv39 }); + load_kernel_table(); } pub inline fn flush_vma(page: usize) void { diff --git a/src/kernel.zig b/src/kernel.zig index 58e5375..ef9f42c 100644 --- a/src/kernel.zig +++ b/src/kernel.zig @@ -38,15 +38,21 @@ noinline fn f1(arg: usize, c: usize) void { /// * Physical memory must be initialized. /// * (optional) Logging should be set up. pub export fn kernel_main() callconv(.C) noreturn { - log.write("\x1B[2J", .{}); var a = arena.Arena.init(256 * 0x1000) orelse @panic("Could not setup kernel arena"); thread.Queue.init_this_cpu(&a); - const pc = @intFromPtr(&f0); - for (0..4) |i| { - const t = thread.Thread.create(&a, pc, i); - thread.enqueue(t); - } + const t = thread.test_create_user_from_code(&a, &[_]u8 { + 0x6F, 0x00, 0x00, 0x00 + }); + thread.enqueue(t); + + log.info("Test", .{}); + // log.write("\x1B[2J", .{}); + // const pc = @intFromPtr(&f0); + // for (0..4) |i| { + // const t = thread.Thread.create_kernel(&a, pc, i); + // thread.enqueue(t); + // } thread.enter(); } diff --git a/src/mem.zig b/src/mem.zig index 822ff23..7eb2de8 100644 --- a/src/mem.zig +++ b/src/mem.zig @@ -46,7 +46,7 @@ pub const PhysicalAddress = packed struct(u64) { /// /// Panics if the virtual address provided is outside of virtualizable memory range. pub fn from_virtualized(virt: usize) @This() { - if ((virt < g_virtualize_base) || (virt - g_virtualize_base > g_virtualize_size)) { + if (virt < g_virtualize_base or virt - g_virtualize_base > g_virtualize_size) { @panic("Invalid virtualized physical address"); } diff --git a/src/mem/vmalloc.zig b/src/mem/vmalloc.zig index ed04d20..8afc857 100644 --- a/src/mem/vmalloc.zig +++ b/src/mem/vmalloc.zig @@ -1,9 +1,8 @@ const std = @import("std"); +const Arena = @import("../arena.zig").Arena; const Range = @import("../util/range.zig").Range; -const Allocator = std.mem.Allocator; - /// Describes a single virtual memory range. /// /// Used by `VirtualMemoryAllocator` to track allocated/used regions. @@ -16,12 +15,12 @@ pub const VirtualMemoryRange = struct { /// Virtual memory allocator implementation. pub const VirtualMemoryAllocator = struct { - gpa: Allocator, + arena: *Arena, head: ?*VirtualMemoryRange = null, outer_range: Range(u64), /// One of errors returned by the allocation logic + underlying allocator error. - pub const Error = error{ already_exists, invalid_region, cannot_fit } || Allocator.Error; + pub const Error = error{ already_exists, invalid_region, cannot_fit }; /// An iterator over VM regions being freed. pub const FreeIterator = struct { @@ -46,7 +45,7 @@ pub const VirtualMemoryAllocator = struct { } // Free it self.current = n.next; - self.vma.gpa.destroy(n); + // self.vma.arena.destroy(n); return xs; } @@ -64,7 +63,7 @@ pub const VirtualMemoryAllocator = struct { return xs; } else { // Insert a new node after the current one - const new_node = try self.vma.gpa.create(VirtualMemoryRange); + const new_node = self.vma.arena.create(VirtualMemoryRange); new_node.* = VirtualMemoryRange { .range = .{ .start = xs.end(), .len = n.range.end() - xs.end() }, .prev = n, @@ -90,10 +89,10 @@ pub const VirtualMemoryAllocator = struct { }; /// Creates a new instance of a virtual memory allocator. - pub fn init(gpa: Allocator, outer_range: Range(u64)) @This() { + pub fn init(arena: *Arena, outer_range: Range(u64)) @This() { return .{ .outer_range = outer_range, - .gpa = gpa, + .arena = arena, }; } @@ -108,7 +107,7 @@ pub const VirtualMemoryAllocator = struct { const gap_before_first = if (self.head) |n| (n.range.start - self.outer_range.start) else self.outer_range.len; if (gap_before_first >= pfn_count) { - var new_node = try self.gpa.create(VirtualMemoryRange); + var new_node = self.arena.create(VirtualMemoryRange); new_node.range = .{ .start = self.outer_range.start, .len = pfn_count }; new_node.next = self.head; @@ -137,7 +136,7 @@ pub const VirtualMemoryAllocator = struct { if (gap >= pfn_count) { // Insert after this const result = n.range.end(); - var new_node = try self.gpa.create(VirtualMemoryRange); + var new_node = self.arena.create(VirtualMemoryRange); new_node.prev = n; new_node.next = n.next; new_node.range = .{ .start = result, .len = pfn_count }; @@ -181,7 +180,7 @@ pub const VirtualMemoryAllocator = struct { node = n.next; } - var new_node = try self.gpa.create(VirtualMemoryRange); + var new_node = self.arena.create(VirtualMemoryRange); new_node.range = region; diff --git a/src/mem/vmm.zig b/src/mem/vmm.zig index 1f7f483..a59e946 100644 --- a/src/mem/vmm.zig +++ b/src/mem/vmm.zig @@ -1,18 +1,30 @@ //! Platform-independent virtual memory management definitions. const mem = @import("../mem.zig"); +const arena = @import("../arena.zig"); +const vmalloc = @import("vmalloc.zig"); +const kernel = @import("../kernel.zig"); +const sync = @import("../sync.zig"); + +const arch = kernel.arch; +const Arena = arena.Arena; /// Last virtual memory translation level. Always 4KiB on all platforms. -pub const L3 = mem.TranslationLevel(12); +pub const L3 = mem.TranslationLevel(12, null); /// Page size is 4KiB on all platforms. pub const PAGE_SIZE: usize = L3.SIZE; +pub const AddressSpaceError = error{ + out_of_pages, +}; + /// Helper function to construct a "Translation Level" struct type from a bit shift. -pub fn TranslationLevel(comptime shift: usize) type { +pub fn TranslationLevel(comptime shift: usize, comptime Next: ?type) type { return struct { pub const SHIFT: usize = shift; pub const SIZE: usize = 1 << shift; + pub const NextLevel = Next; pub inline fn index(addr: usize) usize { return (addr >> shift) & 511; @@ -43,3 +55,38 @@ pub fn TranslationLevel(comptime shift: usize) type { } }; } + +pub const ProcessAddressSpace = struct { + inner: arch.vmm.ProcessAddressSpace, + allocator: vmalloc.VirtualMemoryAllocator, + lock: sync.Spinlock, + + pub fn init(a: *Arena) AddressSpaceError!ProcessAddressSpace { + // 0x200000..0x600000 + const inner = try arch.vmm.ProcessAddressSpace.init(); + const allocator = vmalloc.VirtualMemoryAllocator.init(a, .{ .start = 512, .len = 1024 }); + + return .{ .inner = inner, .allocator = allocator, .lock = .{} }; + } + + pub fn map_single_page( + self: *@This(), + virtual: usize, + physical: mem.PhysicalAddress, + ) AddressSpaceError!void { + self.lock.lock(); + defer self.lock.release(); + + // TODO If allocation succeeds, but mapping fails, rollback + self.allocator.insert(.{ .start = L3.page_number(virtual), .len = 1 }) catch @panic("TODO error"); + try self.inner.map_page(virtual, physical); + } + + pub fn physical_address(self: *const @This()) mem.PhysicalAddress { + return self.inner.physical_address(); + } + + pub fn asid(self: *const @This()) u64 { + return self.inner.asid; + } +}; diff --git a/src/thread.zig b/src/thread.zig index 20ad958..8a60d3b 100644 --- a/src/thread.zig +++ b/src/thread.zig @@ -7,6 +7,8 @@ const arch = @import("kernel.zig").arch; const log = @import("debug.zig").log; const mem = @import("mem.zig"); +const ProcessAddressSpace = mem.vmm.ProcessAddressSpace; + /// Per-CPU thread queue structure. pub const Queue = struct { /// Idle task context. Used when there are no other tasks running. @@ -92,8 +94,11 @@ pub const Thread = struct { /// Previous thread in the queue. prev: ?*Thread = null, + // TODO move to process + address_space: ?ProcessAddressSpace = null, + /// Creates a new (kernel) thread with given `pc` (entry point) and `arg`ument. - pub fn create(a: *arena.Arena, pc: usize, arg: usize) *Thread { + pub fn create_kernel(a: *arena.Arena, pc: usize, arg: usize) *Thread { const thread = a.create(Thread); thread.* = .{ .allocator = a, @@ -102,6 +107,22 @@ pub const Thread = struct { return thread; } + pub fn create_user( + a: *arena.Arena, + address_space: ProcessAddressSpace, + pc: usize, + sp: usize, + arg: usize, + ) *Thread { + const thread = a.create(Thread); + thread.* = .{ + .allocator = a, + .address_space = address_space, + .arch_context = arch.Context.user(&address_space, pc, sp, arg), + }; + return thread; + } + /// Enters the thread, does not return. pub fn enter(self: *@This()) noreturn { self.arch_context.enter(); @@ -170,3 +191,18 @@ pub fn enter() noreturn { pub fn yield() void { Queue.t_this_cpu.?.yield(); } + +pub fn test_create_user_from_code(a: *arena.Arena, code: []const u8) *Thread { + var address_space = ProcessAddressSpace.init(a) catch @panic("TODO"); + + // Map 0x200000 + const page = mem.phys.alloc_page() orelse @panic("TODO error"); + address_space.map_single_page(0x200000, page) catch @panic("TODO error map"); + + const page_data = @as([*]u8, @ptrFromInt(page.virtualize()))[0..code.len]; + @memcpy(page_data, code); + + const thread = Thread.create_user(a, address_space, 0x200000, 0, 0); + + return thread; +}