From 8ba37c97624a53565f9f9418ac140c4f250142ea Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 20 Jan 2025 00:54:26 +0200 Subject: [PATCH] rv64: boot into usermode --- kernel/arch/interface/src/task.rs | 1 + kernel/arch/riscv64/src/context.S | 22 +- kernel/arch/riscv64/src/context.rs | 70 +++++- kernel/arch/riscv64/src/lib.rs | 20 +- kernel/arch/riscv64/src/mem/mod.rs | 25 +- kernel/arch/riscv64/src/mem/process.rs | 181 ++++++++++++-- kernel/arch/riscv64/src/mem/table.rs | 63 ++++- kernel/arch/riscv64/src/registers.rs | 2 + kernel/arch/riscv64/src/sbi.rs | 2 + kernel/libk/libk-mm/interface/src/process.rs | 4 +- kernel/libk/libk-mm/src/process.rs | 10 +- kernel/libk/src/task/binary/elf.rs | 2 +- kernel/libk/src/task/binary/mod.rs | 12 +- kernel/libk/src/task/process.rs | 5 +- kernel/src/arch/riscv64/boot/entry.S | 2 +- kernel/src/arch/riscv64/exception.rs | 226 ++++++++++++++--- kernel/src/arch/riscv64/mod.rs | 11 +- kernel/src/arch/riscv64/vectors.S | 230 +++++++++++------- kernel/src/init.rs | 2 +- kernel/src/main.rs | 1 + kernel/src/syscall/imp/sys_process.rs | 3 + kernel/tools/gentables/src/riscv64.rs | 2 +- lib/qemu/src/riscv64.rs | 14 +- lib/runtime/src/process/thread_local/mod.rs | 61 +++-- .../src/process/thread_local/riscv64.rs | 13 +- .../src/process/thread_local/variant1.rs | 17 +- .../src/process/thread_local/variant2.rs | 11 +- lib/runtime/src/sys/riscv64.rs | 80 +++--- userspace/arch/riscv64/inittab | 4 + userspace/dyn-loader/src/main.rs | 14 +- userspace/dyn-loader/src/relocation/mod.rs | 2 + .../dyn-loader/src/relocation/riscv64.rs | 33 +++ userspace/dyn-loader/src/thread_local/mod.rs | 2 +- xtask/src/build/mod.rs | 7 +- xtask/src/env.rs | 9 +- xtask/src/qemu.rs | 11 +- 36 files changed, 907 insertions(+), 267 deletions(-) create mode 100644 userspace/arch/riscv64/inittab create mode 100644 userspace/dyn-loader/src/relocation/riscv64.rs diff --git a/kernel/arch/interface/src/task.rs b/kernel/arch/interface/src/task.rs index de147b19..54112edb 100644 --- a/kernel/arch/interface/src/task.rs +++ b/kernel/arch/interface/src/task.rs @@ -83,6 +83,7 @@ pub struct UserContextInfo { pub stack_pointer: usize, pub thread_pointer: usize, pub address_space: u64, + pub asid: u64, pub single_step: bool, } diff --git a/kernel/arch/riscv64/src/context.S b/kernel/arch/riscv64/src/context.S index 515e77ec..a40e921c 100644 --- a/kernel/arch/riscv64/src/context.S +++ b/kernel/arch/riscv64/src/context.S @@ -97,14 +97,32 @@ __rv64_task_enter_kernel: csrw sstatus, t0 csrw sepc, ra + csrw sscratch, zero sret .size __rv64_task_enter_kernel, . - __rv64_task_enter_kernel .type __rv64_task_enter_user, @function __rv64_task_enter_user: - // TODO - j . + csrw sscratch, tp + + ld a0, 0 * 8(sp) // argument + ld ra, 1 * 8(sp) // entry + ld tp, 2 * 8(sp) // thread pointer + ld sp, 3 * 8(sp) // user stack + + // Set SPIE to enable interrupts + // Set SPP = 0 to indicate a return to U-mode + li t1, (1 << 8) + not t1, t1 + + csrr t0, sstatus + ori t0, t0, (1 << 5) + and t0, t0, t1 + csrw sstatus, t0 + csrw sepc, ra + + sret .size __rv64_task_enter_user, . - __rv64_task_enter_user .option pop diff --git a/kernel/arch/riscv64/src/context.rs b/kernel/arch/riscv64/src/context.rs index 8b842b37..02457c43 100644 --- a/kernel/arch/riscv64/src/context.rs +++ b/kernel/arch/riscv64/src/context.rs @@ -3,16 +3,29 @@ use core::{arch::global_asm, cell::UnsafeCell, marker::PhantomData}; use kernel_arch_interface::{ mem::{KernelTableManager, PhysicalMemoryAllocator}, task::{StackBuilder, TaskContext, UserContextInfo}, + Architecture, }; use libk_mm_interface::address::PhysicalAddress; +use tock_registers::{ + interfaces::{Readable, Writeable}, + registers::InMemoryRegister, +}; use yggdrasil_abi::error::Error; +use crate::{ + mem::{self, KERNEL_VIRT_OFFSET}, + registers::SATP, + ArchitectureImpl, PerCpuData, +}; + pub const CONTEXT_SIZE: usize = 14 * size_of::(); #[repr(C, align(0x10))] struct TaskContextInner { // 0x00 sp: usize, + + satp: InMemoryRegister, } pub struct TaskContextImpl< @@ -22,6 +35,7 @@ pub struct TaskContextImpl< inner: UnsafeCell, // fp_context: UnsafeCell, stack_base_phys: PhysicalAddress, + stack_top: usize, stack_size: usize, _pd: PhantomData<(K, PA)>, @@ -32,6 +46,15 @@ impl() }; + + // Copy new SATP + let satp = inner.satp.get(); + if satp != SATP.get() { + SATP.set(satp); + } + cpu.smode_sp = self.stack_top; } unsafe fn store_state(&self) {} @@ -40,12 +63,46 @@ impl> TaskContext for TaskContextImpl { - const USER_STACK_EXTRA_ALIGN: usize = 0; + const USER_STACK_EXTRA_ALIGN: usize = 8; const SIGNAL_STACK_EXTRA_ALIGN: usize = 0; fn user(context: UserContextInfo) -> Result { - let _ = context; - todo!() + const USER_TASK_PAGES: usize = 16; + let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?; + let stack_base = stack_base_phys.raw_virtualize::(); + + let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + + log::info!( + "Set up user task: pc={:#x}, sp={:#x}, tp={:#x}", + context.entry, + context.stack_pointer, + context.thread_pointer + ); + stack.push(context.stack_pointer); + stack.push(context.thread_pointer); + stack.push(context.entry); + stack.push(context.argument); + + setup_common_context(&mut stack, __rv64_task_enter_user as _); + + let sp = stack.build(); + let satp = InMemoryRegister::new(0); + satp.write( + SATP::MODE::Sv39 + + SATP::ASID.val(context.asid) + + SATP::PPN.val(context.address_space >> 12), + ); + + Ok(Self { + inner: UnsafeCell::new(TaskContextInner { sp, satp }), + // fp_context: UnsafeCell::new(FpContext::new()), + stack_base_phys, + stack_top: stack_base + USER_TASK_PAGES * 0x1000, + stack_size: USER_TASK_PAGES * 0x1000, + + _pd: PhantomData, + }) } fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result { @@ -65,11 +122,16 @@ impl> 12)); Ok(Self { - inner: UnsafeCell::new(TaskContextInner { sp }), + inner: UnsafeCell::new(TaskContextInner { sp, satp }), // fp_context: UnsafeCell::new(FpContext::new()), stack_base_phys, + stack_top: 0, stack_size: KERNEL_TASK_PAGES * 0x1000, _pd: PhantomData, diff --git a/kernel/arch/riscv64/src/lib.rs b/kernel/arch/riscv64/src/lib.rs index e2695b4e..63eded75 100644 --- a/kernel/arch/riscv64/src/lib.rs +++ b/kernel/arch/riscv64/src/lib.rs @@ -10,9 +10,9 @@ use kernel_arch_interface::{ task::Scheduler, Architecture, }; -use tock_registers::interfaces::{ReadWriteable, Readable, Writeable}; +use tock_registers::interfaces::{ReadWriteable, Readable}; -use registers::{SSCRATCH, SSTATUS}; +use registers::SSTATUS; pub mod mem; pub use mem::{process::ProcessAddressSpaceImpl, KernelTableManagerImpl}; @@ -28,7 +28,8 @@ pub struct ArchitectureImpl; pub struct PerCpuData { // Used in assembly pub tmp_t0: usize, // 0x00 - pub smode_sp: usize, // 0x08 + pub umode_sp: usize, // 0x08 + pub smode_sp: usize, // 0x10 // Used elsewhere pub bootstrap: bool, @@ -69,7 +70,6 @@ impl Architecture for ArchitectureImpl { } unsafe fn set_local_cpu(cpu: *mut ()) { - SSCRATCH.set(cpu.addr() as u64); unsafe { core::arch::asm!("mv tp, {0}", in(reg) cpu) }; } @@ -94,7 +94,7 @@ impl Architecture for ArchitectureImpl { fn ipi_queue(cpu_id: u32) -> Option<&'static IpiQueue> { let _ = cpu_id; - loop {} + todo!() } #[inline] @@ -125,25 +125,25 @@ impl Architecture for ArchitectureImpl { } fn cpu_index() -> u32 { - loop {} + CpuImpl::::local().id() } fn cpu_enabled_features(cpu: &CpuImpl) -> Option<&Self::CpuFeatures> { let _ = cpu; - loop {} + todo!() } fn cpu_available_features(cpu: &CpuImpl) -> Option<&Self::CpuFeatures> { let _ = cpu; - loop {} + todo!() } fn local_interrupt_controller() -> Option<&'static dyn LocalInterruptController> { - loop {} + todo!() } fn message_interrupt_controller() -> Option<&'static dyn MessageInterruptController> { - loop {} + todo!() } fn idle_task() -> extern "C" fn(usize) -> ! { diff --git a/kernel/arch/riscv64/src/mem/mod.rs b/kernel/arch/riscv64/src/mem/mod.rs index c0fc0dc5..33dffe9b 100644 --- a/kernel/arch/riscv64/src/mem/mod.rs +++ b/kernel/arch/riscv64/src/mem/mod.rs @@ -229,13 +229,23 @@ pub fn auto_address(x: *const T) -> usize { } } +/// Enables the memory translation. +/// +/// # Safety +/// +/// Only meant to be called once per each HART during their early init. pub unsafe fn enable_mmu() { let l1_phys = auto_address(&raw const KERNEL_TABLES) as u64; SATP.write(SATP::PPN.val(l1_phys >> 12) + SATP::MODE::Sv39); } -// Also unmaps the lower half +/// Sets up run-time kernel translation tables and removed the lower-half mapping. +/// +/// # Safety +/// +/// The caller must ensure MMU is already enabled and that lower-half addresses will no +/// longer be referred to. pub unsafe fn setup_fixed_tables() { let kernel_l1i_lower = page_index::(KERNEL_PHYS_BASE); let mut tables = KERNEL_TABLES.lock(); @@ -275,3 +285,16 @@ pub fn tlb_flush_va(va: usize) { core::arch::asm!("sfence.vma zero, {0}", in(reg) va); } } + +pub fn tlb_flush_va_asid(va: usize, asid: usize) { + unsafe { + core::arch::asm!("sfence.vma {0}, {1}", in(reg) asid, in(reg) va); + } +} + +pub fn clone_kernel_tables(dst: &mut PageTable) { + let tables = KERNEL_TABLES.lock(); + for l1i in page_index::(USER_BOUNDARY)..512 { + dst[l1i] = unsafe { PageEntry::from_raw(tables.l1.data[l1i]) }; + } +} diff --git a/kernel/arch/riscv64/src/mem/process.rs b/kernel/arch/riscv64/src/mem/process.rs index 132239b0..463a8c90 100644 --- a/kernel/arch/riscv64/src/mem/process.rs +++ b/kernel/arch/riscv64/src/mem/process.rs @@ -1,22 +1,56 @@ -use core::marker::PhantomData; +use core::{ + marker::PhantomData, + sync::atomic::{AtomicU16, Ordering}, +}; use libk_mm_interface::{ - address::PhysicalAddress, + address::{AsPhysicalAddress, PhysicalAddress}, + pointer::PhysicalRefMut, process::ProcessAddressSpaceManager, - table::{MapAttributes, TableAllocator}, + table::{EntryLevel, EntryLevelExt, MapAttributes, NextPageTable, TableAllocator}, }; +use memtables::riscv64::PageAttributes; use yggdrasil_abi::error::Error; +use crate::mem::{clone_kernel_tables, table::PageEntry}; + +use super::{ + table::{PageTable, L1, L2, L3}, + tlb_flush_va_asid, KernelTableManagerImpl, USER_BOUNDARY, +}; + pub struct ProcessAddressSpaceImpl { + l1: PhysicalRefMut<'static, PageTable, KernelTableManagerImpl>, + asid: u16, _pd: PhantomData, } impl ProcessAddressSpaceManager for ProcessAddressSpaceImpl { - const LOWER_LIMIT_PFN: usize = 0; - const UPPER_LIMIT_PFN: usize = 0; + const LOWER_LIMIT_PFN: usize = 8; + const UPPER_LIMIT_PFN: usize = (16 << 30) / L3::SIZE; fn new() -> Result { - todo!() + static LAST_ASID: AtomicU16 = AtomicU16::new(1); + + let mut l1 = unsafe { + PhysicalRefMut::<'static, PageTable, KernelTableManagerImpl>::map( + TA::allocate_page_table()?, + ) + }; + + for i in 0..512 { + l1[i] = PageEntry::INVALID; + } + // Copy the kernel mappings + clone_kernel_tables(&mut l1); + + let asid = LAST_ASID.fetch_add(1, Ordering::AcqRel); + + Ok(Self { + l1, + asid, + _pd: PhantomData, + }) } unsafe fn map_page( @@ -25,27 +59,138 @@ impl ProcessAddressSpaceManager for ProcessAddressSpaceI physical: PhysicalAddress, flags: MapAttributes, ) -> Result<(), Error> { - let _ = address; - let _ = physical; - let _ = flags; - todo!() + self.write_l3_entry( + address, + PageEntry::page(physical, to_page_attributes(flags)), + false, + ) + .unwrap(); + Ok(()) } unsafe fn unmap_page(&mut self, address: usize) -> Result { - let _ = address; - todo!() + self.pop_l3_entry(address) } fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error> { - let _ = address; - todo!() + self.read_l3_entry(address).ok_or(Error::DoesNotExist) } - fn as_address_with_asid(&self) -> u64 { - todo!() + fn as_address_with_asid(&self) -> (u64, u64) { + let physical = unsafe { self.l1.as_physical_address() }.into_u64(); + (physical, self.asid as u64) } - unsafe fn clear(&mut self) { - todo!() + unsafe fn clear(&mut self) {} +} + +impl ProcessAddressSpaceImpl { + // Write a single 4KiB entry + fn write_l3_entry( + &mut self, + virt: usize, + entry: PageEntry, + overwrite: bool, + ) -> Result<(), Error> { + if virt >= USER_BOUNDARY { + log::warn!("Tried to map a userspace page to a non-userspace virtual region"); + return Err(Error::InvalidArgument); + } + + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); + + let mut l2 = self.l1.get_mut_or_alloc::(l1i)?; + let mut l3 = l2.get_mut_or_alloc::(l2i)?; + + if l3[l3i].is_present() && !overwrite { + todo!(); + } + + l3[l3i] = entry; + tlb_flush_va_asid(virt, self.asid as usize); + // dc_cvac((&raw const l3[l3i]).addr()); + // tlb_flush_vaae1(virt); + + Ok(()) + } + + fn pop_l3_entry(&mut self, virt: usize) -> Result { + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); + + // TODO somehow drop tables if they're known to be empty? + let mut l2 = self.l1.get_mut(l1i).ok_or(Error::DoesNotExist)?; + let mut l3 = l2.get_mut(l2i).ok_or(Error::DoesNotExist)?; + + let page = l3[l3i].as_page().ok_or(Error::DoesNotExist)?; + + l3[l3i] = PageEntry::INVALID; + tlb_flush_va_asid(virt, self.asid as usize); + // ic_iallu(); + // dc_cvac((&raw const l3[l3i]).addr()); + // tlb_flush_vaae1(virt); + + Ok(page) + } + + fn read_l3_entry(&self, virt: usize) -> Option<(PhysicalAddress, MapAttributes)> { + if virt >= USER_BOUNDARY { + log::warn!("Tried read an userspace page to a non-userspace virtual region"); + return None; + } + + let l1i = virt.page_index::(); + let l2i = virt.page_index::(); + let l3i = virt.page_index::(); + + let l2 = self.l1.get(l1i)?; + let l3 = l2.get(l2i)?; + + let page = l3[l3i].as_page()?; + + Some(( + page.add(virt & 0xFFF), + to_map_attributes(l3[l3i].attributes()), + )) } } + +impl Drop for ProcessAddressSpaceImpl { + fn drop(&mut self) { + // // SAFETY: with safe usage of the ProcessAddressSpaceImpl, clearing and dropping + // // is safe, no one refers to the memory + // unsafe { + // self.clear(); + // let l1_phys = self.l1.as_physical_address(); + // TA::free_page_table(l1_phys); + // } + } +} + +fn to_page_attributes(src: MapAttributes) -> PageAttributes { + let mut result = PageAttributes::R | PageAttributes::X; + if src.contains(MapAttributes::USER_WRITE) { + result |= PageAttributes::W; + } + if src.intersects(MapAttributes::USER_READ | MapAttributes::USER_WRITE) { + result |= PageAttributes::U; + } + result +} + +fn to_map_attributes(src: PageAttributes) -> MapAttributes { + let mut result = MapAttributes::NON_GLOBAL; + + if src.contains(PageAttributes::U) { + result |= MapAttributes::USER_READ; + + if src.contains(PageAttributes::W) { + result |= MapAttributes::USER_WRITE; + } + } + + result +} diff --git a/kernel/arch/riscv64/src/mem/table.rs b/kernel/arch/riscv64/src/mem/table.rs index 5e1e3d20..8a807441 100644 --- a/kernel/arch/riscv64/src/mem/table.rs +++ b/kernel/arch/riscv64/src/mem/table.rs @@ -4,7 +4,7 @@ use core::{ }; use libk_mm_interface::{ - address::PhysicalAddress, + address::{AsPhysicalAddress, PhysicalAddress}, pointer::{PhysicalRef, PhysicalRefMut}, table::{EntryLevel, NextPageTable, NonTerminalEntryLevel, TableAllocator}, }; @@ -42,7 +42,7 @@ pub struct PageTable { } #[derive(Clone, Copy, Debug, PartialEq)] -pub struct PageEntry(u64, PhantomData); +pub struct PageEntry(pub u64, PhantomData); impl NonTerminalEntryLevel for L1 { type NextLevel = L2; @@ -57,11 +57,33 @@ impl PageTable { entries: [PageEntry::INVALID; 512], } } + + pub fn new_zeroed<'a, TA: TableAllocator>( + ) -> Result, KernelTableManagerImpl>, Error> { + let physical = TA::allocate_page_table()?; + let mut table = + unsafe { PhysicalRefMut::<'a, Self, KernelTableManagerImpl>::map(physical) }; + + for i in 0..512 { + table[i] = PageEntry::INVALID; + } + + Ok(table) + } } impl PageEntry { pub const INVALID: Self = Self(0, PhantomData); + /// Constructs a [PageEntry] from its raw representation. + /// + /// # Safety + /// + /// The caller must ensure `value` is actually a "valid" PTE. + pub const unsafe fn from_raw(value: u64) -> Self { + Self(value, PhantomData) + } + pub const fn is_present(&self) -> bool { self.0 & PageAttributes::V.bits() != 0 } @@ -76,19 +98,31 @@ impl NextPageTable for PageTable { type TableRef = PhysicalRef<'static, PageTable, KernelTableManagerImpl>; type TableRefMut = PhysicalRefMut<'static, PageTable, KernelTableManagerImpl>; - fn get(&self, _index: usize) -> Option { - loop {} + fn get(&self, index: usize) -> Option { + let table = self[index].as_table()?; + Some(unsafe { PhysicalRef::map(table) }) } - fn get_mut(&mut self, _index: usize) -> Option { - loop {} + fn get_mut(&mut self, index: usize) -> Option { + let table = self[index].as_table()?; + Some(unsafe { PhysicalRefMut::map(table) }) } fn get_mut_or_alloc( &mut self, - _index: usize, + index: usize, ) -> Result { - loop {} + if let Some(table) = self[index].as_table() { + Ok(unsafe { PhysicalRefMut::map(table) }) + } else { + let table = PageTable::new_zeroed::()?; + self[index] = PageEntry::::table( + unsafe { table.as_physical_address() }, + PageAttributes::empty(), + ); + // dc_cvac((&raw const self[index]).addr()); + Ok(table) + } } } @@ -108,6 +142,15 @@ impl PageEntry { PhantomData, ) } + + pub fn as_table(&self) -> Option { + (self.0 + & (PageAttributes::R | PageAttributes::W | PageAttributes::X | PageAttributes::V) + .bits() + == PageAttributes::V.bits()) + .then_some((self.0 << 2) & !0xFFF) + .map(PhysicalAddress::from_u64) + } } impl PageEntry { @@ -119,7 +162,9 @@ impl PageEntry { } pub fn as_page(&self) -> Option { - loop {} + (self.0 & PageAttributes::V.bits() != 0) + .then_some((self.0 << 2) & !0xFFF) + .map(PhysicalAddress::from_u64) } } diff --git a/kernel/arch/riscv64/src/registers.rs b/kernel/arch/riscv64/src/registers.rs index 75f7229d..aa6c0fad 100644 --- a/kernel/arch/riscv64/src/registers.rs +++ b/kernel/arch/riscv64/src/registers.rs @@ -448,6 +448,8 @@ pub mod sstatus { register_bitfields!( u64, pub SSTATUS [ + SUM OFFSET(18) NUMBITS(1) [], + SPP OFFSET(8) NUMBITS(1) [], SIE OFFSET(1) NUMBITS(1) [], ] ); diff --git a/kernel/arch/riscv64/src/sbi.rs b/kernel/arch/riscv64/src/sbi.rs index e2f98451..6722f7a2 100644 --- a/kernel/arch/riscv64/src/sbi.rs +++ b/kernel/arch/riscv64/src/sbi.rs @@ -1,3 +1,5 @@ +#[allow(clippy::too_many_arguments)] +#[inline(always)] unsafe fn sbi_do_call( extension: u64, function: u64, diff --git a/kernel/libk/libk-mm/interface/src/process.rs b/kernel/libk/libk-mm/interface/src/process.rs index 8cdbe3dc..e2fa6b72 100644 --- a/kernel/libk/libk-mm/interface/src/process.rs +++ b/kernel/libk/libk-mm/interface/src/process.rs @@ -39,8 +39,8 @@ pub trait ProcessAddressSpaceManager: Sized { /// if one is mapped fn translate(&self, address: usize) -> Result<(PhysicalAddress, MapAttributes), Error>; - /// Returns the implementation specific physical address of this space, with ASID applied - fn as_address_with_asid(&self) -> u64; + /// Returns the physical address of the translation table along with its ASID + fn as_address_with_asid(&self) -> (u64, u64); /// Clears the address space by dropping and non-global tables. /// diff --git a/kernel/libk/libk-mm/src/process.rs b/kernel/libk/libk-mm/src/process.rs index 730965a9..0ecdcc03 100644 --- a/kernel/libk/libk-mm/src/process.rs +++ b/kernel/libk/libk-mm/src/process.rs @@ -335,7 +335,8 @@ impl ProcessAddressSpace { ProcessAddressSpaceImpl::::LOWER_LIMIT_PFN, ProcessAddressSpaceImpl::::UPPER_LIMIT_PFN, ); - log::debug!("New AddressSpace {:#x}", table.as_address_with_asid()); + let (physical, asid) = table.as_address_with_asid(); + log::debug!("New AddressSpace {:#x}, asid {:#x}", physical, asid); Ok(Self { inner: IrqSafeSpinlock::new(Inner { table, allocator }), }) @@ -451,8 +452,8 @@ impl ProcessAddressSpace { lock.unmap_range(address, size / L3_PAGE_SIZE) } - /// Returns the physical address of this table, with ASID applied - pub fn as_address_with_asid(&self) -> u64 { + /// Returns the physical address of the translation table along with its ASID + pub fn as_address_with_asid(&self) -> (u64, u64) { self.inner.lock().table.as_address_with_asid() } @@ -465,7 +466,8 @@ impl ProcessAddressSpace { impl Drop for ProcessAddressSpace { fn drop(&mut self) { - log::debug!("Drop AddressSpace {:#x}", self.as_address_with_asid()); + let (physical, asid) = self.as_address_with_asid(); + log::debug!("Drop AddressSpace {:#x}, asid {:#x}", physical, asid); self.clear().ok(); } } diff --git a/kernel/libk/src/task/binary/elf.rs b/kernel/libk/src/task/binary/elf.rs index 8507039e..ea94f6c3 100644 --- a/kernel/libk/src/task/binary/elf.rs +++ b/kernel/libk/src/task/binary/elf.rs @@ -488,7 +488,7 @@ fn write_rela(rela: &Rela, space: &ProcessAddressSpace, b: usize) -> Result<(), let rel_field = rela.r_offset as usize + b; let (value, width) = match rela.r_type { - elf::abi::R_X86_64_RELATIVE | elf::abi::R_AARCH64_RELATIVE => { + elf::abi::R_X86_64_RELATIVE | elf::abi::R_AARCH64_RELATIVE | elf::abi::R_RISCV_RELATIVE => { // B + A // Width: qword (b as i64 + a, 8) diff --git a/kernel/libk/src/task/binary/mod.rs b/kernel/libk/src/task/binary/mod.rs index 4caaf390..99dd1d58 100644 --- a/kernel/libk/src/task/binary/mod.rs +++ b/kernel/libk/src/task/binary/mod.rs @@ -217,13 +217,21 @@ where // let tls_address = elf::clone_tls(space, image)?; - log::debug!("argument = {:#x}", argument); + log::debug!( + "argument = {:#x}, user_sp = {:#x}, stack: {:#x}..{:#x}", + argument, + user_sp, + virt_stack_base, + virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContextImpl::USER_STACK_EXTRA_ALIGN + ); + let (address_space, asid) = space.as_address_with_asid(); TaskContext::user(UserContextInfo { entry: image.entry, argument, stack_pointer: ptr.addr(), thread_pointer: 0, - address_space: space.as_address_with_asid(), + address_space, + asid, single_step: options.single_step, }) } diff --git a/kernel/libk/src/task/process.rs b/kernel/libk/src/task/process.rs index 9904a784..f187e0e0 100644 --- a/kernel/libk/src/task/process.rs +++ b/kernel/libk/src/task/process.rs @@ -194,10 +194,13 @@ impl Process { let sp = TaskContextImpl::align_stack_for_entry(options.stack_top) as *mut usize; let sp = unsafe { Thread::setup_stack_header(&space, sp, options.argument)? }; + let (address_space, asid) = space.as_address_with_asid(); + let info = UserContextInfo { entry: options.entry as _, argument: options.argument, - address_space: space.as_address_with_asid(), + address_space, + asid, stack_pointer: sp.addr(), single_step: false, thread_pointer: 0, diff --git a/kernel/src/arch/riscv64/boot/entry.S b/kernel/src/arch/riscv64/boot/entry.S index c17cb7ed..25142ecb 100644 --- a/kernel/src/arch/riscv64/boot/entry.S +++ b/kernel/src/arch/riscv64/boot/entry.S @@ -14,7 +14,7 @@ __rv64_entry: // a0 - bootstrap HART ID // a1 - device tree blob // mhartid == a0 - // satp == 0 + csrw satp, zero // Zero the .bss LOAD_PCREL .L00, t0, __bss_start_phys diff --git a/kernel/src/arch/riscv64/exception.rs b/kernel/src/arch/riscv64/exception.rs index d864c6a3..da9f7f3b 100644 --- a/kernel/src/arch/riscv64/exception.rs +++ b/kernel/src/arch/riscv64/exception.rs @@ -1,24 +1,56 @@ use core::arch::global_asm; -use kernel_arch::{task::Scheduler, Architecture}; -use libk::arch::Cpu; -use tock_registers::interfaces::{ReadWriteable, Readable}; +use abi::{arch::SavedFrame, primitive_enum, process::Signal, SyscallFunction}; +use kernel_arch::{ + task::{Scheduler, TaskFrame}, + Architecture, +}; +use libk::{arch::Cpu, task::thread::Thread}; +use tock_registers::interfaces::ReadWriteable; use kernel_arch_riscv64::{ intrinsics, - registers::{SCAUSE, SEPC, STVAL, STVEC}, + registers::{SSTATUS, STVEC}, sbi, ArchitectureImpl, }; +use crate::syscall; + +primitive_enum! { + pub enum Cause: u64 { + MisalignedInstruction = 0, + InstructionAccessFault = 1, + IllegalInstruction = 2, + Breakpoint = 3, + LoadAddressMisaligned = 4, + LoadAccessFault = 5, + StoreAddressMisaligned = 6, + StoreAccessFault = 7, + EcallUmode = 8, + EcallSmode = 9, + EcallMmode = 11, + InstructionPageFault = 12, + LoadPageFault = 13, + StorePageFault = 15, + } +} + #[derive(Debug)] #[repr(C)] pub struct TrapFrame { + // General-purpose pub ra: u64, pub gp: u64, - pub t0_2: [u64; 3], - pub a0_7: [u64; 8], - pub t3_6: [u64; 4], - pub s0: u64, + pub tn: [u64; 7], + pub sn: [u64; 12], + pub an: [usize; 8], + // Special + pub sp: u64, + pub sstatus: u64, + pub sepc: u64, + pub stval: u64, + pub scause: u64, + pub tp: u64, } pub fn init_smode_exceptions() { @@ -31,42 +63,180 @@ pub fn init_smode_exceptions() { STVEC.modify(STVEC::MODE::Vectored); } -unsafe fn smode_exception_handler(frame: *mut TrapFrame) { - let _ = frame; - let cause = SCAUSE.read(SCAUSE::CODE); - let tval = STVAL.get(); - let epc = SEPC.get(); +unsafe fn umode_exception_handler(frame: &mut TrapFrame) { + let thread = Thread::current(); - log::error!("S-mode exception cause={cause}, tval={tval:#x}, epc={epc:#x}"); + let cause = Cause::try_from(frame.scause).ok(); + + let dump = match cause { + Some(Cause::LoadPageFault) + | Some(Cause::StorePageFault) + | Some(Cause::LoadAccessFault) + | Some(Cause::StoreAccessFault) + | Some(Cause::InstructionPageFault) + | Some(Cause::InstructionAccessFault) => { + let translation = if let Some(space) = thread.try_get_process().map(|p| p.space()) { + space.translate(frame.stval as usize).ok() + } else { + None + }; + + thread.raise_signal(Signal::MemoryAccessViolation); + + if let Some(physical) = translation { + log::warn!(" * tval translates to {physical:#x}"); + } else { + log::warn!(" * tval does not translate"); + } + + true + } + Some(Cause::EcallUmode) => { + // TODO more granular control over how U-mode pages are accessed from S-mode + SSTATUS.modify(SSTATUS::SUM::SET); + + let func = frame.an[0]; + if func == usize::from(SyscallFunction::ExitSignal) { + todo!() + } + + let args = &frame.an[1..]; + let result = syscall::raw_syscall_handler(func, args) as _; + frame.an[0] = result; + frame.sepc += 4; + false + } + _ => { + thread.raise_signal(Signal::MemoryAccessViolation); + true + } + }; + + if dump { + log::warn!( + "U-mode exception cause={:?} ({}), epc={:#x}, sp={:#x}, tval={:#x}", + cause, + frame.scause, + frame.sepc, + frame.sp, + frame.stval + ); + } +} + +unsafe fn smode_exception_handler(frame: &mut TrapFrame) { + let cause = Cause::try_from(frame.scause).expect("Invalid exception cause"); + log::error!( + "S-mode exception cause={:?} ({}), tval={:#x}, epc={:#x}, sp={:#x}", + cause, + frame.scause, + frame.stval, + frame.sepc, + frame.sp + ); + match cause { + Cause::LoadPageFault + | Cause::StorePageFault + | Cause::LoadAccessFault + | Cause::StoreAccessFault + | Cause::InstructionPageFault + | Cause::InstructionAccessFault => { + let translation = if let Some(space) = Thread::get_current() + .and_then(|t| t.try_get_process()) + .map(|p| p.space()) + { + space.translate(frame.stval as usize).ok() + } else { + None + }; + + if let Some(physical) = translation { + log::warn!(" * tval translates to {physical:#x}"); + } else { + log::warn!(" * tval does not translate"); + } + } + _ => (), + } ArchitectureImpl::halt(); } unsafe extern "C" fn smode_interrupt_handler(frame: *mut TrapFrame) { - let _ = frame; - let cause = SCAUSE.read(SCAUSE::CODE); - - match cause { + let frame = &*frame; + match frame.scause & !(1 << 63) { // S-mode timer interrupt 5 => { - sbi::sbi_set_timer(intrinsics::rdtime() + 1_000_000); + sbi::sbi_set_timer(intrinsics::rdtime() + 100_000); // TODO runtime tick, time accounting Cpu::local().scheduler().yield_cpu(); } - _ => { - log::warn!("Unknown/unhandled S-mode interrupt {cause}"); - ArchitectureImpl::halt(); - } + n => todo!("Unhandled interrupt #{n}"), } } unsafe extern "C" fn smode_general_trap_handler(frame: *mut TrapFrame) { - let interrupt = SCAUSE.matches_all(SCAUSE::INTERRUPT::SET); + let frame = &mut *frame; - if interrupt { - smode_interrupt_handler(frame); - } else { - smode_exception_handler(frame); + let interrupt = frame.scause & (1 << 63) != 0; + let smode = frame.sstatus & (1 << 8) != 0; + + match (interrupt, smode) { + (true, _) => smode_interrupt_handler(frame), + (false, true) => smode_exception_handler(frame), + (false, false) => umode_exception_handler(frame), + } + + if !smode && let Some(thread) = Thread::get_current() { + thread.handle_pending_signals(frame); + } +} + +impl TaskFrame for TrapFrame { + fn store(&self) -> SavedFrame { + todo!() + } + + fn restore(&mut self, saved: &SavedFrame) { + let _ = saved; + todo!() + } + + fn user_sp(&self) -> usize { + todo!() + } + + fn user_ip(&self) -> usize { + todo!() + } + + fn argument(&self) -> u64 { + todo!() + } + + fn set_user_sp(&mut self, value: usize) { + let _ = value; + todo!() + } + + fn set_user_ip(&mut self, value: usize) { + let _ = value; + todo!() + } + + fn set_argument(&mut self, value: u64) { + let _ = value; + todo!() + } + + fn set_single_step(&mut self, step: bool) { + let _ = step; + todo!() + } + + fn set_return_value(&mut self, value: u64) { + let _ = value; + todo!() } } diff --git a/kernel/src/arch/riscv64/mod.rs b/kernel/src/arch/riscv64/mod.rs index 2927079a..44795861 100644 --- a/kernel/src/arch/riscv64/mod.rs +++ b/kernel/src/arch/riscv64/mod.rs @@ -8,11 +8,12 @@ use device_api::{ ResetDevice, }; use device_tree::{driver::unflatten_device_tree, DeviceTree, DeviceTreeNodeExt}; +use kernel_arch::Architecture; use kernel_arch_riscv64::{ intrinsics, mem::{self, KERNEL_VIRT_OFFSET}, registers::SIE, - sbi, PerCpuData, + sbi, ArchitectureImpl, PerCpuData, }; use libk::{arch::Cpu, config}; use libk_mm::{ @@ -59,7 +60,7 @@ impl Platform for Riscv64 { type L3 = L3; unsafe fn reset(&self) -> ! { - loop {} + ArchitectureImpl::halt(); } unsafe fn send_ipi(&self, target: IpiDeliveryTarget, msg: IpiMessage) -> Result { @@ -76,7 +77,7 @@ impl Platform for Riscv64 { fn register_reset_device(&self, reset: Arc) -> Result<(), Error> { let _ = reset; - loop {} + Ok(()) } } @@ -119,6 +120,7 @@ impl Riscv64 { let aligned_end = end.page_align_up::(); let size = aligned_end - aligned_start; + log::info!("Reserve initrd @ {:#x}..{:#x}", aligned_start, aligned_end); reserve_region( "initrd", PhysicalMemoryRegion { @@ -159,6 +161,7 @@ impl Riscv64 { let per_cpu = PerCpuData { tmp_t0: 0, + umode_sp: 0, smode_sp: 0, bootstrap: is_bsp, @@ -220,7 +223,7 @@ impl Riscv64 { // Setup the timer SIE.modify(SIE::STIE::SET); - sbi::sbi_set_timer(intrinsics::rdtime() + 1_000_000); + sbi::sbi_set_timer(intrinsics::rdtime() + 100_000); // Test call into M-mode // core::arch::asm!("ecall", in("a0") MModeFunction::WriteTimerComparator as u64, in("a1") 0x4321); diff --git a/kernel/src/arch/riscv64/vectors.S b/kernel/src/arch/riscv64/vectors.S index 7d5ca744..da122a84 100644 --- a/kernel/src/arch/riscv64/vectors.S +++ b/kernel/src/arch/riscv64/vectors.S @@ -2,112 +2,158 @@ .section .text -.set SMODE_TRAP_STATE_SIZE, (8 * 18) +// ra+gp, 7 tN, 12 sN, 8 aN +.set GP_REGS_SIZE, (2 + 7 + 12 + 8) * 8 +// U-mode sp, sstatus, sepc, stval, scause, sscratch +.set CTL_REGS_SIZE, 6 * 8 +.set TRAP_CONTEXT_SIZE, (GP_REGS_SIZE) + (CTL_REGS_SIZE) -.macro SAVE_TRAP_CONTEXT - addi sp, sp, -SMODE_TRAP_STATE_SIZE - sd ra, 8 * 0(sp) - sd gp, 8 * 1(sp) - sd t0, 8 * 2(sp) - sd t1, 8 * 3(sp) - sd t2, 8 * 4(sp) - sd a0, 8 * 5(sp) - sd a1, 8 * 6(sp) - sd a2, 8 * 7(sp) - sd a3, 8 * 8(sp) - sd a4, 8 * 9(sp) - sd a5, 8 * 10(sp) - sd a6, 8 * 11(sp) - sd a7, 8 * 12(sp) - sd t3, 8 * 13(sp) - sd t4, 8 * 14(sp) - sd t5, 8 * 15(sp) - sd t6, 8 * 16(sp) - sd s0, 8 * 17(sp) +.set REG_UMODE_SP, (GP_REGS_SIZE + 0 * 8) +.set REG_SSTATUS, (GP_REGS_SIZE + 1 * 8) +.set REG_SEPC, (GP_REGS_SIZE + 2 * 8) +.set REG_STVAL, (GP_REGS_SIZE + 3 * 8) +.set REG_SCAUSE, (GP_REGS_SIZE + 4 * 8) +.set REG_SSCRATCH, (GP_REGS_SIZE + 5 * 8) + +.macro SAVE_GP_REGS + // Save all general-purpose registers, except: + // * sp (saved elsewhere) + // * tp (saved elsewhere) + sd ra, 0 * 8(sp) + sd gp, 1 * 8(sp) + + sd t0, 2 * 8(sp) + sd t1, 3 * 8(sp) + sd t2, 4 * 8(sp) + sd t3, 5 * 8(sp) + sd t4, 6 * 8(sp) + sd t5, 7 * 8(sp) + sd t6, 8 * 8(sp) + + sd s0, 9 * 8(sp) + sd s1, 10 * 8(sp) + sd s2, 11 * 8(sp) + sd s3, 12 * 8(sp) + sd s4, 13 * 8(sp) + sd s5, 14 * 8(sp) + sd s6, 15 * 8(sp) + sd s7, 16 * 8(sp) + sd s8, 17 * 8(sp) + sd s9, 18 * 8(sp) + sd s10, 19 * 8(sp) + sd s11, 20 * 8(sp) + + sd a0, 21 * 8(sp) + sd a1, 22 * 8(sp) + sd a2, 23 * 8(sp) + sd a3, 24 * 8(sp) + sd a4, 25 * 8(sp) + sd a5, 26 * 8(sp) + sd a6, 27 * 8(sp) + sd a7, 28 * 8(sp) .endm -.macro LOAD_TRAP_CONTEXT - ld ra, 8 * 0(sp) - ld gp, 8 * 1(sp) - ld t0, 8 * 2(sp) - ld t1, 8 * 3(sp) - ld t2, 8 * 4(sp) - ld a0, 8 * 5(sp) - ld a1, 8 * 6(sp) - ld a2, 8 * 7(sp) - ld a3, 8 * 8(sp) - ld a4, 8 * 9(sp) - ld a5, 8 * 10(sp) - ld a6, 8 * 11(sp) - ld a7, 8 * 12(sp) - ld t3, 8 * 13(sp) - ld t4, 8 * 14(sp) - ld t5, 8 * 15(sp) - ld t6, 8 * 16(sp) - ld s0, 8 * 17(sp) - addi sp, sp, SMODE_TRAP_STATE_SIZE -.endm +.macro LOAD_GP_REGS + ld ra, 0 * 8(sp) + ld gp, 1 * 8(sp) -// * Switch stack to kernel if needed -// * Store pre-trap register state on the stack -// * Make a0 point to the frame -// * Make s0 = original_tp -.macro SMODE_TRAP_ENTER - // Stack may be either U-mode or S-mode stack depending on sstatus.SPP - // Original tp -> sscratch - // Per-CPU struct -> tp - csrrw tp, sscratch, tp + ld t0, 2 * 8(sp) + ld t1, 3 * 8(sp) + ld t2, 4 * 8(sp) + ld t3, 5 * 8(sp) + ld t4, 6 * 8(sp) + ld t5, 7 * 8(sp) + ld t6, 8 * 8(sp) - // Store t0 in per-CPU scratch space - sd t0, 0(tp) + ld s0, 9 * 8(sp) + ld s1, 10 * 8(sp) + ld s2, 11 * 8(sp) + ld s3, 12 * 8(sp) + ld s4, 13 * 8(sp) + ld s5, 14 * 8(sp) + ld s6, 15 * 8(sp) + ld s7, 16 * 8(sp) + ld s8, 17 * 8(sp) + ld s9, 18 * 8(sp) + ld s10, 19 * 8(sp) + ld s11, 20 * 8(sp) - // Determine where the interrupt came from (SPP is bit 8) - csrr t0, sstatus - andi t0, t0, (1 << 8) - bnez t0, 1f - - // Trap came from U-mode - // TODO - j . -1: - // Trap came from S-mode -2: - // Either stack was adjusted or the trap came from S-mode - - // Load t0 back - ld t0, 0(tp) - SAVE_TRAP_CONTEXT - - mv a0, sp - csrr s0, sscratch -.endm - -// * Set sscratch to pre-trap tp -// * Restore the pre-trap register state -// * Return -.macro SMODE_TRAP_LEAVE_TO_SMODE - csrw sscratch, s0 - - // Restore the state - LOAD_TRAP_CONTEXT - - // Swap the tp<->scratch back - csrrw tp, sscratch, tp - sret + ld a0, 21 * 8(sp) + ld a1, 22 * 8(sp) + ld a2, 23 * 8(sp) + ld a3, 24 * 8(sp) + ld a4, 25 * 8(sp) + ld a5, 26 * 8(sp) + ld a6, 27 * 8(sp) + ld a7, 28 * 8(sp) .endm .macro SMODE_TRAP n, handler .type __rv64_smode_trap_\n, @function __rv64_smode_trap_\n: - SMODE_TRAP_ENTER + // If coming from userspace, sscratch = kernel-mode tp + // If coming from kernelspace, sscratch = 0 + csrrw tp, sscratch, tp + bnez tp, 1f - // TODO when coming through a non-zero vector, trap is always asyncrhonous, so - // the interrupt handler can be called directly instead of a more generic - // trap handler to avoid an extra indirection + // Coming from S-mode + // tp = 0, sscratch contains kernel tp + csrr tp, sscratch + sd sp, 16(tp) // Set proper S-mode sp +1: + sd sp, 8(tp) // Store U-mode sp + ld sp, 16(tp) // Load S-mode sp + + // Store pre-trap context + addi sp, sp, -(TRAP_CONTEXT_SIZE) + + SAVE_GP_REGS + // Save special registers + ld t0, 8(tp) + csrr t1, sstatus + csrr t2, sepc + csrr t3, stval + csrr t4, scause + csrr t5, sscratch + + sd t0, REG_UMODE_SP (sp) + sd t1, REG_SSTATUS (sp) + sd t2, REG_SEPC (sp) + sd t3, REG_STVAL (sp) + sd t4, REG_SCAUSE (sp) + sd t5, REG_SSCRATCH (sp) + + // Reset sscratch to zero to make sure a S-mode -> S-mode nested exception + // happens properly + csrw sscratch, zero + + mv a0, sp call \handler - // TODO U-mode trap return - SMODE_TRAP_LEAVE_TO_SMODE + // Return from exception + ld t0, REG_SSTATUS (sp) + andi t0, t0, (1 << 8) + bnez t0, 2f + + // Return to U-mode + // Restore SSCRATCH to a proper value + csrw sscratch, tp +2: + // Return to S-mode + + // Restore special registers + ld t0, REG_SSTATUS (sp) + ld t1, REG_SEPC (sp) + csrw sstatus, t0 + csrw sepc, t1 + + // Restore general-purpose registers + LOAD_GP_REGS + + ld tp, REG_SSCRATCH (sp) + ld sp, REG_UMODE_SP (sp) + + sret .size __rv64_smode_trap_\n, . - __rv64_smode_trap_\n .endm diff --git a/kernel/src/init.rs b/kernel/src/init.rs index f4e0b2a3..eeaf2208 100644 --- a/kernel/src/init.rs +++ b/kernel/src/init.rs @@ -58,7 +58,7 @@ pub fn kinit() -> Result<(), Error> { // TODO move this to userspace so it doesn't block the init process, maybe lazy-load on first // attempt to load a module? - #[cfg(not(target_arch = "aarch64"))] + #[cfg(all(not(target_arch = "aarch64"), not(target_arch = "riscv64")))] { use libk::module::load_kernel_symbol_table; diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 0fc18cc0..282f1f66 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -29,6 +29,7 @@ clippy::match_ref_pats, clippy::match_single_binding, clippy::missing_transmute_annotations, + clippy::modulo_one, async_fn_in_trait )] #![deny(missing_docs)] diff --git a/kernel/src/syscall/imp/sys_process.rs b/kernel/src/syscall/imp/sys_process.rs index f440c23b..5c2f1384 100644 --- a/kernel/src/syscall/imp/sys_process.rs +++ b/kernel/src/syscall/imp/sys_process.rs @@ -53,6 +53,9 @@ pub(crate) fn map_memory( space.allocate(None, len, backing, attrs) }) + .inspect_err(|error| { + log::warn!("map_memory({len}) failed: {error:?}"); + }) } pub(crate) fn unmap_memory(address: usize, len: usize) -> Result<(), Error> { diff --git a/kernel/tools/gentables/src/riscv64.rs b/kernel/tools/gentables/src/riscv64.rs index 0a84fcc0..a6fa185f 100644 --- a/kernel/tools/gentables/src/riscv64.rs +++ b/kernel/tools/gentables/src/riscv64.rs @@ -151,7 +151,7 @@ impl Riscv64Builder { let entry = shift_pfn(paddr) | (PageAttributes::V | flags).bits(); - let l2i = (vaddr >> L2_SHIFT) as usize & 0x1FF - start_l2i; + let l2i = ((vaddr >> L2_SHIFT) as usize & 0x1FF) - start_l2i; let l3i = (vaddr >> L3_SHIFT) as usize & 0x1FF; let l3 = &mut tables.kernel_l3s[l2i]; diff --git a/lib/qemu/src/riscv64.rs b/lib/qemu/src/riscv64.rs index 8ffadc1d..7f48de03 100644 --- a/lib/qemu/src/riscv64.rs +++ b/lib/qemu/src/riscv64.rs @@ -17,7 +17,11 @@ pub struct QemuRiscv64; #[derive(Debug)] pub enum Image { - OpenSBI { kernel: PathBuf, bios: PathBuf }, + OpenSBI { + bios: PathBuf, + kernel: PathBuf, + initrd: PathBuf, + }, } impl IntoArgs for Machine { @@ -41,9 +45,15 @@ impl IntoArgs for Cpu { impl IntoArgs for Image { fn add_args(&self, command: &mut Command) { match self { - Self::OpenSBI { kernel, bios } => { + Self::OpenSBI { + bios, + kernel, + initrd, + } => { command.arg("-kernel"); command.arg(kernel); + command.arg("-initrd"); + command.arg(initrd); command.arg("-bios"); command.arg(bios); } diff --git a/lib/runtime/src/process/thread_local/mod.rs b/lib/runtime/src/process/thread_local/mod.rs index 56808929..d4431956 100644 --- a/lib/runtime/src/process/thread_local/mod.rs +++ b/lib/runtime/src/process/thread_local/mod.rs @@ -63,6 +63,13 @@ pub struct Dtv { specific: Vec<*mut c_void>, } +#[allow(missing_docs)] +pub struct TlsInfo { + base: usize, + tp: usize, + module0_offset: Option, +} + struct TcbHeader { #[allow(unused)] self_pointer: usize, @@ -170,8 +177,10 @@ impl Dtv { /// Will panic if key == 0. /// Will panic if key is longer than the DTV itself. pub fn set_specific(&mut self, key: usize, value: *mut c_void, grow: bool) { - self.try_set_specific(key, value, grow) - .expect("Dtv::set_specific(): invalid key") + if self.try_set_specific(key, value, grow).is_err() { + crate::debug_trace!("Dtv::set_specific(): invalid key {key}"); + panic!("Dtv::set_specific(): invalid key {key})") + } } /// Sets a DTV entry for a thread-specific key. @@ -203,7 +212,13 @@ impl Dtv { /// Will panic if key == 0. /// Will panic if key is larger than the DTV itself. pub fn get(&self, key: usize) -> *mut c_void { - Self::get_key(&self.entries, key).expect("Out-of-bounds DTV key") + match Self::get_key(&self.entries, key) { + Some(value) => value, + None => { + crate::debug_trace!("Dtv::get(): out-of-bounds DTV key: {key}"); + panic!("Dtv::get(): out-of-bounds DTV key: {key}"); + } + } } /// Sets a DTV entry, growing the DTV allocation if necessary @@ -216,7 +231,8 @@ impl Dtv { self.entries.resize(key, null_mut()); } if !Self::set_key(&mut self.entries, key, value) { - panic!("Dtv::set(): invalid key"); + crate::debug_trace!("Dtv::set(): invalid key {key}"); + panic!("Dtv::set(): invalid key {key}"); } } } @@ -231,9 +247,9 @@ pub fn init_tls_from_auxv<'a, I: Iterator>( }; if force || !tls_image.already_initialized { - let (base, tp) = clone_tls(&tls_image)?; - unsafe { set_thread_pointer(tp) }?; - setup_dtv(&tls_image, base)?; + let tls = clone_tls(&tls_image)?; + unsafe { set_thread_pointer(tls.tp) }?; + setup_dtv(&tls_image, &tls)?; } Ok(Some(tls_image)) @@ -248,9 +264,9 @@ pub fn init_tls(image: Option<&TlsImage>, force: bool) -> Result<(), Error> { }; if force || !image.already_initialized { - let (base, tp) = clone_tls(image)?; - unsafe { set_thread_pointer(tp) }?; - setup_dtv(image, base)?; + let tls = clone_tls(image)?; + unsafe { set_thread_pointer(tls.tp) }?; + setup_dtv(image, &tls)?; } Ok(()) @@ -265,16 +281,31 @@ fn get_tcb_mut() -> &'static mut TcbHeader { unsafe { &mut *get_tcb_raw() } } -fn setup_dtv(image: &TlsImage, tls_base: usize) -> Result<(), Error> { +fn setup_dtv(image: &TlsImage, tls_info: &TlsInfo) -> Result<(), Error> { + #[cfg(any(target_arch = "riscv64", rust_analyzer))] + const DTV_OFFSET: usize = 0x800; + #[cfg(any(not(target_arch = "riscv64"), rust_analyzer))] + const DTV_OFFSET: usize = 0; + + let dtv = get_dtv(); + // Executable itself + // NOTE if module 1 is specified again by the dynamic loader, it will be overriden with + // what dynamic loader says + if let Some(module0_offset) = tls_info.module0_offset { + dtv.set( + 1, + core::ptr::without_provenance_mut(tls_info.base + module0_offset + DTV_OFFSET), + ); + } + if image.module_offsets.is_empty() { return Ok(()); } - let dtv = get_dtv(); for &(module_id, module_offset) in image.module_offsets.iter() { assert!(module_offset < image.full_size); dtv.set( module_id, - core::ptr::with_exposed_provenance_mut(tls_base + module_offset), + core::ptr::with_exposed_provenance_mut(tls_info.base + module_offset + DTV_OFFSET), ); } Ok(()) @@ -292,14 +323,12 @@ pub fn get_dtv() -> &'static mut Dtv { } #[cfg(all( - any(target_arch = "x86", target_arch = "x86_64"), + any(target_arch = "x86", target_arch = "x86_64", target_arch = "riscv64"), any(feature = "__tls_get_addr", rust_analyzer) ))] #[no_mangle] unsafe extern "C" fn __tls_get_addr(index: *mut usize) -> *mut c_void { let module_id = index.read(); let offset = index.add(1).read(); - assert!(module_id > 0); - get_dtv().get(module_id).add(offset) } diff --git a/lib/runtime/src/process/thread_local/riscv64.rs b/lib/runtime/src/process/thread_local/riscv64.rs index 4ebaefe4..53188a80 100644 --- a/lib/runtime/src/process/thread_local/riscv64.rs +++ b/lib/runtime/src/process/thread_local/riscv64.rs @@ -3,10 +3,17 @@ use abi::error::Error; pub fn get_thread_pointer() -> usize { - todo!() + let output: usize; + unsafe { core::arch::asm!("mv {0}, tp", out(reg) output) }; + output } +/// Writes `value` into `tp` register. +/// +/// # Safety +/// +/// Usual pointer rules apply. pub unsafe fn set_thread_pointer(value: usize) -> Result<(), Error> { - let _ = value; - todo!() + core::arch::asm!("mv tp, {0}", in(reg) value); + Ok(()) } diff --git a/lib/runtime/src/process/thread_local/variant1.rs b/lib/runtime/src/process/thread_local/variant1.rs index 33a9f38d..e380d7a0 100644 --- a/lib/runtime/src/process/thread_local/variant1.rs +++ b/lib/runtime/src/process/thread_local/variant1.rs @@ -1,9 +1,10 @@ use abi::{ error::Error, mem::{MappingFlags, MappingSource}, + process::ExitCode, }; -use super::TlsImage; +use super::{TlsImage, TlsInfo}; // Variant I TLS layout: // @@ -16,14 +17,16 @@ use super::TlsImage; /// Creates a new TLS image in the process memory, copying data from the TLS master copy (if any). /// Returns the resulting thread pointer. -pub fn clone_tls(image: &TlsImage) -> Result<(usize, usize), Error> { +pub fn clone_tls(image: &TlsImage) -> Result { const TCB_SIZE: usize = size_of::() * 2; if !image.align.is_power_of_two() { - panic!("TLS layout not aligned to a power of two: {}", image.align) + crate::debug_trace!("TLS layout not aligned to a power of two: {}", image.align); + unsafe { crate::sys::exit_process(ExitCode::Exited(1)) }; } if image.align > 0x1000 { - panic!("TODO: TLS alignment larger than a page size is not supported"); + crate::debug_trace!("TODO: TLS alignment larger than a page size is not supported"); + unsafe { crate::sys::exit_process(ExitCode::Exited(1)) }; } // TCB size, padded to align. Also the start of the first module @@ -72,7 +75,11 @@ pub fn clone_tls(image: &TlsImage) -> Result<(usize, usize), Error> { crate::debug_trace!("TLS: base={:#x}, tp={:#x}", base, tp); - Ok((base, tp)) + Ok(TlsInfo { + base, + tp, + module0_offset: Some(tcb_aligned_size), + }) } pub(super) fn get_tcb_raw(tp: usize) -> *mut u8 { diff --git a/lib/runtime/src/process/thread_local/variant2.rs b/lib/runtime/src/process/thread_local/variant2.rs index 90d0fb9c..4f5463de 100644 --- a/lib/runtime/src/process/thread_local/variant2.rs +++ b/lib/runtime/src/process/thread_local/variant2.rs @@ -3,7 +3,7 @@ use abi::{ mem::{MappingFlags, MappingSource}, }; -use super::TlsImage; +use super::{TlsImage, TlsInfo}; // Variant II TLS layout: // @@ -14,7 +14,7 @@ use super::TlsImage; /// Creates a new TLS image in the process memory, copying data from the TLS master copy (if any). /// Returns the resulting thread pointer. -pub fn clone_tls(image: &TlsImage) -> Result<(usize, usize), Error> { +pub fn clone_tls(image: &TlsImage) -> Result { // Basically, the layout is: // * align(image.full_size) below the TP // * tcb_size starting with the TP @@ -73,7 +73,12 @@ pub fn clone_tls(image: &TlsImage) -> Result<(usize, usize), Error> { crate::debug_trace!("TLS: base={:#x}, tp={:#x}", base, tp); - Ok((base, tp)) + Ok(TlsInfo { + base, + tp, + module0_offset: None, + }) + // Ok((base, tp)) } // In Variant II, the TP points directly at the TCB start diff --git a/lib/runtime/src/sys/riscv64.rs b/lib/runtime/src/sys/riscv64.rs index ad2c6618..2eed7ac4 100644 --- a/lib/runtime/src/sys/riscv64.rs +++ b/lib/runtime/src/sys/riscv64.rs @@ -2,52 +2,50 @@ #[macro_export] macro_rules! syscall { ($num:expr $(,)?) => {{ - let _ = $num; - todo!() + let mut a0 = usize::from($num); + core::arch::asm!("ecall", inlateout("a0") a0); + a0 }}; - ($num:expr, $a0:expr $(,)?) => {{ - let _ = $num; - let _ = $a0; - todo!() + ($num:expr, $a1:expr $(,)?) => {{ + let mut a0 = usize::from($num); + core::arch::asm!("ecall", inlateout("a0") a0, in("a1") $a1); + a0 }}; - ($num:expr, $a0:expr, $a1:expr $(,)?) => {{ - let _ = $num; - let _ = $a0; - let _ = $a1; - todo!() + ($num:expr, $a1:expr, $a2:expr $(,)?) => {{ + let mut a0 = usize::from($num); + core::arch::asm!("ecall", inlateout("a0") a0, in("a1") $a1, in("a2") $a2); + a0 }}; - ($num:expr, $a0:expr, $a1:expr, $a2:expr $(,)?) => {{ - let _ = $num; - let _ = $a0; - let _ = $a1; - let _ = $a2; - todo!() + ($num:expr, $a1:expr, $a2:expr, $a3:expr $(,)?) => {{ + let mut a0 = usize::from($num); + core::arch::asm!("ecall", inlateout("a0") a0, in("a1") $a1, in("a2") $a2, in("a3") $a3); + a0 }}; - ($num:expr, $a0:expr, $a1:expr, $a2:expr, $a3:expr $(,)?) => {{ - let _ = $num; - let _ = $a0; - let _ = $a1; - let _ = $a2; - let _ = $a3; - todo!() + ($num:expr, $a1:expr, $a2:expr, $a3:expr, $a4:expr $(,)?) => {{ + let mut a0 = usize::from($num); + core::arch::asm!( + "ecall", + inlateout("a0") a0, + in("a1") $a1, in("a2") $a2, in("a3") $a3, in("a4") $a4 + ); + a0 }}; - ($num:expr, $a0:expr, $a1:expr, $a2:expr, $a3:expr, $a4:expr $(,)?) => {{ - let _ = $num; - let _ = $a0; - let _ = $a1; - let _ = $a2; - let _ = $a3; - let _ = $a4; - todo!() + ($num:expr, $a1:expr, $a2:expr, $a3:expr, $a4:expr, $a5:expr $(,)?) => {{ + let mut a0 = usize::from($num); + core::arch::asm!( + "ecall", + inlateout("a0") a0, + in("a1") $a1, in("a2") $a2, in("a3") $a3, in("a4") $a4, in("a5") $a5 + ); + a0 }}; - ($num:expr, $a0:expr, $a1:expr, $a2:expr, $a3:expr, $a4:expr, $a5:expr $(,)?) => {{ - let _ = $num; - let _ = $a0; - let _ = $a1; - let _ = $a2; - let _ = $a3; - let _ = $a4; - let _ = $a5; - todo!() + ($num:expr, $a1:expr, $a2:expr, $a3:expr, $a4:expr, $a5:expr, $a6:expr $(,)?) => {{ + let mut a0 = usize::from($num); + core::arch::asm!( + "ecall", + inlateout("a0") a0, + in("a1") $a1, in("a2") $a2, in("a3") $a3, in("a4") $a4, in("a5") $a5, in("a6") $a6 + ); + a0 }}; } diff --git a/userspace/arch/riscv64/inittab b/userspace/arch/riscv64/inittab new file mode 100644 index 00000000..9c848816 --- /dev/null +++ b/userspace/arch/riscv64/inittab @@ -0,0 +1,4 @@ +init:1:wait:/sbin/rc default +logd:1:once:/sbin/logd + +user:1:once:/sbin/login /dev/ttyS0 diff --git a/userspace/dyn-loader/src/main.rs b/userspace/dyn-loader/src/main.rs index 1845d725..7397c514 100644 --- a/userspace/dyn-loader/src/main.rs +++ b/userspace/dyn-loader/src/main.rs @@ -71,17 +71,13 @@ fn run(binary: &str, args: &[String]) -> Result { }); for module in layout.segments.iter() { - if module.object_id == 0 { - continue; - } - auxv.push(AuxValue { tag: auxv::TLS_MODULE_ID, - val: module.object_id as _, + val: module.object_id as u64 + 1, }); auxv.push(AuxValue { tag: auxv::TLS_MODULE_OFFSET, - val: module.offset as _, + val: module.offset as u64, }); } } @@ -124,6 +120,12 @@ unsafe fn enter(entry: extern "C" fn(usize), argument: usize) -> ! { options(att_syntax, noreturn) ); } + #[cfg(any(target_arch = "riscv64", rust_analyzer))] + { + let _ = entry; + let _ = argument; + todo!() + } #[cfg(any(target_arch = "aarch64", target_arch = "x86_64", rust_analyzer))] { entry(argument); diff --git a/userspace/dyn-loader/src/relocation/mod.rs b/userspace/dyn-loader/src/relocation/mod.rs index 755aec6a..1b92af33 100644 --- a/userspace/dyn-loader/src/relocation/mod.rs +++ b/userspace/dyn-loader/src/relocation/mod.rs @@ -8,6 +8,8 @@ mod aarch64; mod x86_64; #[cfg(any(target_arch = "x86", rust_analyzer))] mod i686; +#[cfg(any(target_arch = "riscv64", rust_analyzer))] +mod riscv64; pub enum RelaValue { DQWord(i64, i64), diff --git a/userspace/dyn-loader/src/relocation/riscv64.rs b/userspace/dyn-loader/src/relocation/riscv64.rs new file mode 100644 index 00000000..4f004d1a --- /dev/null +++ b/userspace/dyn-loader/src/relocation/riscv64.rs @@ -0,0 +1,33 @@ +use elf::relocation::{Rel, Rela}; + +use crate::{error::Error, object::ResolvedSymbol, state::State}; + +use super::{RelValue, RelaValue, Relocation}; + +impl Relocation for Rel { + type Value = RelValue; + + fn resolve( + &self, + _state: &State, + _name: &str, + _symbol: &ResolvedSymbol, + _load_base: usize, + ) -> Result, Error> { + todo!() + } +} + +impl Relocation for Rela { + type Value = RelaValue; + + fn resolve( + &self, + _state: &State, + _name: &str, + _symbol: &ResolvedSymbol, + _load_base: usize, + ) -> Result, Error> { + todo!() + } +} diff --git a/userspace/dyn-loader/src/thread_local/mod.rs b/userspace/dyn-loader/src/thread_local/mod.rs index 1968c5c5..119d43f4 100644 --- a/userspace/dyn-loader/src/thread_local/mod.rs +++ b/userspace/dyn-loader/src/thread_local/mod.rs @@ -13,7 +13,7 @@ cfg_if! { } else if #[cfg(any(target_arch = "x86_64", target_arch = "x86"))] { mod variant2; pub use variant2::TlsLayoutImpl; - } else if #[cfg(target_arch = "aarch64")] { + } else if #[cfg(any(target_arch = "aarch64", target_arch = "riscv64"))] { mod variant1; pub use variant1::TlsLayoutImpl; } diff --git a/xtask/src/build/mod.rs b/xtask/src/build/mod.rs index 72451330..1ad8a4f0 100644 --- a/xtask/src/build/mod.rs +++ b/xtask/src/build/mod.rs @@ -42,7 +42,7 @@ pub struct KernelProcessed(pub KernelBuilt); pub struct InitrdGenerated(pub PathBuf); pub struct ImageBuilt(pub PathBuf); pub enum AllBuilt { - Riscv64(KernelProcessed), + Riscv64(KernelProcessed, InitrdGenerated), X86_64(ImageBuilt), AArch64(KernelProcessed, InitrdGenerated), I686(ImageBuilt), @@ -99,16 +99,13 @@ pub fn build_all(env: &BuildEnv) -> Result { // for module in modules { // install_extra.push((module.clone(), module.file_name().unwrap().into())); // } - if env.arch == Arch::riscv64 { - return Ok(AllBuilt::Riscv64(kernel)); - } // Userspace stuff let initrd = userspace::build_initrd(env, install_extra, check)?; // Build target-specific image let image = match env.arch { - Arch::riscv64 => AllBuilt::Riscv64(kernel), + Arch::riscv64 => AllBuilt::Riscv64(kernel, initrd), Arch::aarch64 => AllBuilt::AArch64(kernel, initrd), Arch::x86_64 => AllBuilt::X86_64(x86_64::build_image(env, kernel, initrd)?), Arch::i686 => AllBuilt::I686(i686::build_image(env, kernel, initrd)?), diff --git a/xtask/src/env.rs b/xtask/src/env.rs index d3b5d1fe..49831a92 100644 --- a/xtask/src/env.rs +++ b/xtask/src/env.rs @@ -22,6 +22,12 @@ pub struct AArch64TargetConfig { pub components: BuildComponents, } +#[derive(Debug, Default, serde::Deserialize, serde::Serialize)] +#[serde(default)] +pub struct Riscv64TargetConfig { + pub components: BuildComponents, +} + #[derive(Debug, Default, serde::Deserialize, serde::Serialize)] #[serde(default)] pub struct X86_64TargetConfig { @@ -38,6 +44,7 @@ pub struct I686TargetConfig { #[serde(default)] pub struct TargetConfig { pub aarch64: AArch64TargetConfig, + pub riscv64: Riscv64TargetConfig, pub x86_64: X86_64TargetConfig, pub i686: I686TargetConfig, } @@ -193,7 +200,7 @@ impl BuildEnv { impl XTaskConfig { pub fn components(&self, env: &BuildEnv) -> &BuildComponents { match env.arch { - Arch::riscv64 => todo!(), + Arch::riscv64 => &self.target.riscv64.components, Arch::aarch64 => &self.target.aarch64.components, Arch::x86_64 => &self.target.x86_64.components, Arch::i686 => &self.target.i686.components, diff --git a/xtask/src/qemu.rs b/xtask/src/qemu.rs index dae0727c..ce4fadcd 100644 --- a/xtask/src/qemu.rs +++ b/xtask/src/qemu.rs @@ -260,6 +260,7 @@ fn run_riscv64( qemu_bin: Option, devices: Vec, kernel: PathBuf, + initrd: PathBuf, ) -> Result { let _ = config; let _ = devices; @@ -272,7 +273,11 @@ fn run_riscv64( qemu.with_serial(QemuSerialTarget::MonStdio) .with_machine(riscv64::Machine::Virt) .with_cpu(riscv64::Cpu::Rv64) - .with_boot_image(riscv64::Image::OpenSBI { kernel, bios }); + .with_boot_image(riscv64::Image::OpenSBI { + kernel, + initrd, + bios, + }); Ok(qemu.into_command()) } @@ -349,8 +354,8 @@ pub fn run( add_devices_from_config(&mut devices, disk.as_ref(), &config)?; let mut command = match built { - AllBuilt::Riscv64(KernelProcessed(KernelBuilt(kernel))) => { - run_riscv64(&config, &env, qemu, devices, kernel)? + AllBuilt::Riscv64(KernelProcessed(KernelBuilt(kernel)), InitrdGenerated(initrd)) => { + run_riscv64(&config, &env, qemu, devices, kernel, initrd)? } AllBuilt::AArch64(KernelProcessed(KernelBuilt(kernel)), InitrdGenerated(initrd)) => { make_kernel_bin(kernel, &kernel_bin)?;