diff --git a/etc/riscv64-unknown-none.ld b/etc/riscv64-unknown-none.ld index b31e5a4..8a1e884 100644 --- a/etc/riscv64-unknown-none.ld +++ b/etc/riscv64-unknown-none.ld @@ -12,6 +12,7 @@ SECTIONS { .rodata : ALIGN(4K) { *(.rodata*) + *(.srodata*) *(.got*) *(.plt*) } @@ -28,6 +29,7 @@ SECTIONS { .data : ALIGN(4K) { *(.data*) + *(.sdata*) } .bss : { @@ -35,6 +37,7 @@ SECTIONS { PROVIDE(__bss_start = .); *(COMMON) *(.bss*) + *(.sbss*) . = ALIGN(4K); PROVIDE(__bss_end = .); } diff --git a/src/arch/riscv64.zig b/src/arch/riscv64.zig index 87c0080..a613e89 100644 --- a/src/arch/riscv64.zig +++ b/src/arch/riscv64.zig @@ -20,8 +20,8 @@ pub fn arch() type { // Has to be exactly at offset 0x00, used in assembly kstack: thread.KStack(STACK_SIZE), - pub fn kernel(a: *Arena, pc: usize, arg: usize) @This() { - var ks = thread.KStack(STACK_SIZE).create(a); + pub fn kernel(pc: usize, arg: usize) @This() { + var ks = thread.KStack(STACK_SIZE).create(); const entry = @intFromPtr(&__rv64_task_enter_kernel); ks.push(pc); diff --git a/src/arch/riscv64/boot.zig b/src/arch/riscv64/boot.zig index 0c96909..46abf8c 100644 --- a/src/arch/riscv64/boot.zig +++ b/src/arch/riscv64/boot.zig @@ -65,6 +65,8 @@ fn setupMemoryFromFdt(realAddress: usize) void { physMemory.addReservedRegion("kernel", kernelStart - (vmm.KERNEL_VIRTUAL_BASE + vmm.L1.offset(realAddress)) + realAddress, kernelEnd - kernelStart); physMemory.addReservedRegion("fdt", gDtbAddress, vmm.L3.align_up(fdt.bytes.len)); + + physMemory.init(); } fn bspUpperEntry(realAddress: usize, unused: usize) callconv(.C) noreturn { diff --git a/src/arena.zig b/src/arena.zig index cbd66d9..eec022e 100644 --- a/src/arena.zig +++ b/src/arena.zig @@ -8,11 +8,11 @@ pub const Arena = struct { len: usize, pub fn setup(cap: usize) ?Arena { - const base = physMemory.allocateChunk(cap / 0x1000) orelse return null; + const physBase = physMemory.alloc_pages(cap / mem.vmm.PAGE_SIZE) orelse return null; return .{ - .physBase = .{ .raw = base }, + .physBase = physBase, .capacity = cap, - .len = 0, + .len = 0 }; } diff --git a/src/kernel.zig b/src/kernel.zig index 4456806..ba0ee4d 100644 --- a/src/kernel.zig +++ b/src/kernel.zig @@ -8,15 +8,15 @@ pub const thread = @import("thread.zig"); pub const log = debug.log; pub const vmm = mem.vmm; +pub const TRACE_PHYSICAL_ALLOCATOR: bool = false; + const std = @import("std"); fn f0(arg: usize) callconv(.C) noreturn { - log.write("\x1B[2J", .{}); var c: usize = 0; while (true) { f1(arg, c); c += 1; - thread.yield(); } } @@ -28,7 +28,8 @@ pub export fn kernel_main() callconv(.C) noreturn { var a = arena.Arena.setup(256 * 0x1000) orelse @panic("Could not setup kernel arena"); const pc = @intFromPtr(&f0); - for (0..8) |i| { + // log.write("\x1B[2J", .{}); + for (0..32) |i| { const t = thread.Thread.create(&a, pc, i); thread.addThread(t); } @@ -44,8 +45,8 @@ pub fn panic(msg: []const u8, error_return_trace: ?*std.builtin.StackTrace, retu const ra = return_address orelse @returnAddress(); log.err("!!! Kernel panic !!!", .{}); - log.err(" Reason: {s}", .{ msg }); - log.err(" At: 0x{x}", .{ ra }); + log.err(" Reason: {s}", .{msg}); + log.err(" At: 0x{x}", .{ra}); arch.halt(); } diff --git a/src/mem.zig b/src/mem.zig index b397cdd..70d2878 100644 --- a/src/mem.zig +++ b/src/mem.zig @@ -1,4 +1,7 @@ +const std = @import("std"); + pub const vmm = @import("mem/vmm.zig"); +pub const phys = @import("mem/phys.zig"); pub const translationLevel = vmm.translationLevel; @@ -30,3 +33,46 @@ pub const PhysicalAddress = packed struct(u64) { return .{ .raw = virt - gVirtualizeBase }; } }; + +pub fn formatSize(buffer: []u8, size: u64) []const u8 { + const KIBI: u64 = 1024; + const MIBI: u64 = KIBI * 1024; + const GIBI: u64 = MIBI * 1024; + + const log2: u64 = std.math.log2_int(u64, size); + const opts: struct { u64, []const u8 } = switch (log2) { + 0...9 => .{ 1, "B" }, + 10...19 => .{ KIBI, " KiB" }, + 20...29 => .{ MIBI, " MiB" }, + else => .{ GIBI, " GiB" }, + }; + const div: u64 = opts[0]; + const suffix: []const u8 = opts[1]; + const integer = size / div; + const dot = size >= 1024; + + const iLen = std.fmt.formatIntBuf(buffer, integer, 10, .lower, .{}); + var len = iLen; + var fLen: usize = 0; + + if (dot and integer < 100) { + const fractional = (((size * 1000) / div) % 1000) / 10; + + if (iLen < buffer.len + 1) { + buffer[iLen] = '.'; + fLen = 1 + std.fmt.formatIntBuf(buffer[iLen + 1..], fractional, 10, .lower, .{ + .fill = '0', + .width = 2 + }); + } + } + + len += fLen; + + if (len + suffix.len < buffer.len) { + std.mem.copyForwards(u8, buffer[len..], suffix); + len += suffix.len; + } + + return buffer[0..len]; +} diff --git a/src/mem/phys.zig b/src/mem/phys.zig index 1ee5b95..4dd34f8 100644 --- a/src/mem/phys.zig +++ b/src/mem/phys.zig @@ -1,66 +1,268 @@ +const std = @import("std"); + +const kernel = @import("../kernel.zig"); const vec = @import("../util/vec.zig"); const log = @import("../debug.zig").log; +const mem = @import("../mem.zig"); +const vmm = @import("vmm.zig"); +const sync = @import("../sync.zig"); -pub const MemoryRegion = struct { - name: []const u8, - base: u64, - size: u64, +const Range = @import("../util/range.zig").Range; +const Spinlock = sync.IrqSafeSpinlock; - pub fn contains(self: *const @This(), page: u64) bool { - return (page >= self.base) and (page - self.base < self.size); +pub const MemoryRegion = struct { name: []const u8, range: Range(u64) }; + +const Page = extern struct { + refcount: u32 = 0, + unused: [3]u32 = undefined, + + fn isUsed(self: *const @This()) bool { + return self.refcount != 0; + } + + fn makeAvailable(self: *@This()) void { + self.refcount = 0; + } + + fn makeReserved(self: *@This()) void { + self.refcount = std.math.maxInt(u32); + } +}; + +const PhysicalMemoryManager = struct { + pageArray: []Page, + offset: u64 = 0, + lastFree: usize = 0, + + const RECORDS_PER_PAGE: usize = vmm.PAGE_SIZE / @sizeOf(Page); + + fn alloc_page(self: *@This()) ?mem.PhysicalAddress { + for (self.lastFree..self.pageArray.len) |i| { + if (self.pageArray[i].refcount == 0) { + self.pageArray[i].refcount += 1; + self.lastFree = (i + 1) % self.pageArray.len; + return .{ .raw = self.offset + i * vmm.PAGE_SIZE }; + } + } + for (0..self.lastFree) |i| { + if (self.pageArray[i].refcount == 0) { + self.pageArray[i].refcount += 1; + self.lastFree = (i + 1) % self.pageArray.len; + return .{ .raw = self.offset + i * vmm.PAGE_SIZE }; + } + } + return null; + } + + fn alloc_pages(self: *@This(), count: usize) ?mem.PhysicalAddress { + if (self.lastFree + count < self.pageArray.len) { + if (self.alloc_from(self.lastFree, self.pageArray.len, count)) |p| { + return p; + } + } + return self.alloc_from(0, self.lastFree, count); + } + + fn alloc_from(self: *@This(), start: usize, end: usize, count: usize) ?mem.PhysicalAddress { + for (start..end) |i| { + var taken = false; + for (0..count) |j| { + if (self.pageArray[i + j].isUsed()) { + taken = true; + break; + } + } + + if (!taken) { + for (0..count) |j| { + self.pageArray[i + j].refcount = 1; + } + return .{ .raw = self.offset + i * vmm.PAGE_SIZE }; + } + } + + return null; + } + + fn valid_index(self: *@This(), page: mem.PhysicalAddress) usize { + if (page.raw < self.offset) { + log.panic("free_page: invalid page 0x{x}: outside of the allocation range", .{page.raw}); + } + const index = (page.raw - self.offset) / vmm.PAGE_SIZE; + if (index >= self.pageArray.len) { + log.panic("free_page: invalid page 0x{x}: outside of the allocation range", .{page.raw}); + } + return index; + } + + fn free_page(self: *@This(), page: mem.PhysicalAddress) void { + const index = self.valid_index(page); + if (self.pageArray[index].refcount == 0) { + log.panic("free_page: double free of page 0x{x} detected", .{page.raw}); + } + self.pageArray[index].refcount -= 1; + if (self.pageArray[index].refcount == 0) { + self.lastFree = index; + } + } + + fn get_page(self: *@This(), page: mem.PhysicalAddress) *Page { + const index = self.valid_index(page); + return &self.pageArray[index]; } }; var gMemoryRegions: vec.FixedVec(MemoryRegion, 16) = .{}; var gReservedRegions: vec.FixedVec(MemoryRegion, 16) = .{}; - -fn isReserved(page: u64) bool { - for (gReservedRegions.asConstSlice()) |region| { - if (region.contains(page)) { - return true; - } - } - return false; -} +var gPhysicalMemoryLock = Spinlock{}; +var gPhysicalMemory = PhysicalMemoryManager{ .pageArray = undefined }; pub fn addMemoryRegion(name: []const u8, base: u64, size: u64) void { log.info("Memory: '{s}', base 0x{x}, size 0x{x}", .{ name, base, size }); - gMemoryRegions.push(.{ - .name = name, - .base = base, - .size = size - }); + gMemoryRegions.push(.{ .name = name, .range = .{ .start = base, .len = size } }); } pub fn addReservedRegion(name: []const u8, base: u64, size: u64) void { log.info("Reserved: '{s}', base 0x{x}, size 0x{x}", .{ name, base, size }); - gReservedRegions.push(.{ - .name = name, - .base = base, - .size = size - }); + gReservedRegions.push(.{ .name = name, .range = .{ .start = base, .len = size } }); } -pub fn allocateChunk(pageCount: usize) ?u64 { - for (gMemoryRegions.asConstSlice()) |region| { - var i: usize = 0; - while (i < region.size) { - var res = false; - for (0..pageCount) |j| { - if (isReserved(region.base + i + j * 0x1000)) { - res = true; - i += (j + 1) * 0x1000; - break; - } - } - if (res) { - continue; - } - - const addr = region.base + i; - addReservedRegion("alloc", addr, pageCount * 0x1000); - return addr; +fn isReservedIn(page: u64) ?*const MemoryRegion { + for (0..gReservedRegions.len) |i| { + const region = &gReservedRegions.data[i]; + if (page >= region.range.start and page < region.range.end()) { + return region; } } return null; } + +fn allocFromRegion(region: *const MemoryRegion, reason: []const u8, pageCount: usize) ?u64 { + var offset = @as(u64, 0); + while (offset < region.range.len) { + var taken: ?*const MemoryRegion = null; + for (0..pageCount) |i| { + if (isReservedIn(region.range.start + offset + i * vmm.PAGE_SIZE)) |resv| { + taken = resv; + break; + } + } + + if (taken) |resv| { + offset = (resv.range.start + resv.range.len) - region.range.start; + continue; + } + + const base = region.range.start + offset; + addReservedRegion(reason, base, pageCount * vmm.PAGE_SIZE); + return base; + } + return null; +} + +fn allocPageArray(pageCount: usize) []Page { + for (gMemoryRegions.asConstSlice()) |region| { + if (allocFromRegion(®ion, "page-array", pageCount)) |physAddress| { + const vaddr = (mem.PhysicalAddress{ .raw = physAddress }).virtualize(); + const len = pageCount * PhysicalMemoryManager.RECORDS_PER_PAGE; + const ptr: [*]Page = @ptrFromInt(vaddr); + const slice: []Page = ptr[0..len]; + for (0..len) |i| { + slice[i].refcount = std.math.maxInt(u32); + } + return slice; + } + } + @panic("TODO"); +} + +pub fn init() void { + var memoryStart: u64 = std.math.maxInt(u64); + var memoryEnd: u64 = std.math.minInt(u64); + + for (gMemoryRegions.asConstSlice()) |region| { + if (region.range.start < memoryStart) { + memoryStart = region.range.start; + } + if (region.range.end() > memoryEnd) { + memoryEnd = region.range.end(); + } + } + + const memoryPages = (memoryEnd - memoryStart) / vmm.PAGE_SIZE; // == bitmap bits required + const pageArrayPages = (memoryPages + PhysicalMemoryManager.RECORDS_PER_PAGE - 1) / PhysicalMemoryManager.RECORDS_PER_PAGE; + + const pageArray = allocPageArray(pageArrayPages); + var availablePages: usize = 0; + + for (gMemoryRegions.asConstSlice()) |region| { + const offset = (region.range.start - memoryStart) / vmm.PAGE_SIZE; + for (0..region.range.len / vmm.PAGE_SIZE) |i| { + pageArray[offset + i].makeAvailable(); + availablePages += 1; + } + } + for (gReservedRegions.asConstSlice()) |region| { + const offset = (region.range.start - memoryStart) / vmm.PAGE_SIZE; + for (0..region.range.len / vmm.PAGE_SIZE) |i| { + if (offset + i >= pageArray.len) { + break; + } + pageArray[offset + i].makeReserved(); + availablePages -= 1; + } + } + + var sizeFmt: [64]u8 = undefined; + const sizeFmtStr = mem.formatSize(&sizeFmt, availablePages * vmm.PAGE_SIZE); + log.info("Available memory: {s}, page array {*}", .{ sizeFmtStr, pageArray }); + + gPhysicalMemory.pageArray = pageArray; + gPhysicalMemory.offset = memoryStart; +} + +fn trace_allocation(count: usize, page: ?mem.PhysicalAddress) void { + if (page) |p| { + log.debug("alloc {} = 0x{x}", .{ count, p.raw }); + } else { + log.debug("alloc {} = FAIL", .{count}); + } +} + +fn trace_free(page: mem.PhysicalAddress) void { + log.debug("free 0x{x}", .{page.raw}); +} + +pub fn alloc_page() ?mem.PhysicalAddress { + gPhysicalMemoryLock.lock(); + defer gPhysicalMemoryLock.release(); + const page = gPhysicalMemory.alloc_page(); + if (comptime kernel.TRACE_PHYSICAL_ALLOCATOR) { + trace_allocation(1, page); + } + return page; +} + +pub fn alloc_pages(count: usize) ?mem.PhysicalAddress { + gPhysicalMemoryLock.lock(); + defer gPhysicalMemoryLock.release(); + const pages = gPhysicalMemory.alloc_pages(count); + if (comptime kernel.TRACE_PHYSICAL_ALLOCATOR) { + trace_allocation(count, pages); + } + return pages; +} + +pub fn free_page(page: mem.PhysicalAddress) void { + if (comptime kernel.TRACE_PHYSICAL_ALLOCATOR) { + trace_free(page); + } + gPhysicalMemoryLock.lock(); + defer gPhysicalMemoryLock.release(); + gPhysicalMemory.free_page(page); +} + +// NOTE: Physical memory lock must be held +pub fn get_page(page: mem.PhysicalAddress) *Page { + return gPhysicalMemory.get_page(page); +} diff --git a/src/mem/vmm.zig b/src/mem/vmm.zig index 7437cf6..0d06e20 100644 --- a/src/mem/vmm.zig +++ b/src/mem/vmm.zig @@ -1,3 +1,5 @@ +pub const PAGE_SIZE: usize = 0x1000; + pub fn translationLevel(comptime shift: usize) type { return struct { pub const SHIFT: usize = shift; diff --git a/src/thread.zig b/src/thread.zig index f542c78..c03c508 100644 --- a/src/thread.zig +++ b/src/thread.zig @@ -1,6 +1,7 @@ const arena = @import("arena.zig"); const arch = @import("kernel.zig").arch; const log = @import("debug.zig").log; +const mem = @import("mem.zig"); pub const Thread = struct { allocator: *arena.Arena, @@ -13,7 +14,7 @@ pub const Thread = struct { const thread = a.create(Thread); thread.* = .{ .allocator = a, - .archContext = arch.Context.kernel(a, pc, sp), + .archContext = arch.Context.kernel(pc, sp), }; return thread; } @@ -33,11 +34,15 @@ pub fn KStack(comptime SIZE: usize) type { sp: *usize, data: *[SIZE]usize, + physicalBase: mem.PhysicalAddress, + + pub fn create() @This() { + const physicalBase = mem.phys.alloc_pages(SIZE * @sizeOf(usize) / 0x1000) orelse @panic("OOM"); + const ptr = @as(*[SIZE]usize, @ptrFromInt(physicalBase.virtualize())); - pub fn create(a: *arena.Arena) @This() { - const ptr = a.create([SIZE]usize); return .{ .data = ptr, + .physicalBase = physicalBase, .sp = @ptrFromInt(@intFromPtr(&ptr[0]) + SIZE * @sizeOf(usize)) }; } diff --git a/src/util/range.zig b/src/util/range.zig new file mode 100644 index 0000000..d0002c1 --- /dev/null +++ b/src/util/range.zig @@ -0,0 +1,26 @@ +pub fn Range(comptime T: type) type { + return struct { + start: T, + len: T, + + pub fn end(self: *const @This()) T { + return self.start + self.len; + } + + pub fn intersect(self: *const @This(), other: *const @This()) ?Range(T) { + if (self.start < other.start) { + const p = other.start - self.start; + if (p < self.len) { + return .{ .start = other.start, .len = @min(self.len - p, other.len) }; + } + } else { + const p = self.start - other.start; + if (p < other.len) { + return .{ .start = self.start, .len = @min(other.len - p, self.len) }; + } + } + + return null; + } + }; +}