diff --git a/kernel/arch/aarch64/src/context.rs b/kernel/arch/aarch64/src/context.rs index 64b677ba..c5bddc6c 100644 --- a/kernel/arch/aarch64/src/context.rs +++ b/kernel/arch/aarch64/src/context.rs @@ -202,7 +202,7 @@ impl: Sized /// Only meant to be called from the scheduler code after the `thread` has terminated. unsafe fn switch_and_drop(&self, thread: *const ()); + /// Replaces the current thread pointer with the provided one. + fn set_thread_pointer(&self, tp: usize); + // XXX /// Constructs a safe wrapper process to execute a kernel-space closure fn kernel_closure ! + Send + 'static>(f: F) -> Result { diff --git a/kernel/arch/x86/src/registers.rs b/kernel/arch/x86/src/registers.rs index a04ee465..7bfa6a82 100644 --- a/kernel/arch/x86/src/registers.rs +++ b/kernel/arch/x86/src/registers.rs @@ -291,6 +291,18 @@ mod msr { pub const MSR_IA32_KERNEL_GS_BASE: Reg = Reg; } + pub mod ia32_fs_base { + const ADDR: u32 = 0xC0000100; + pub struct Reg; + + msr_impl_read!(Reg, ADDR); + msr_impl_write!(Reg, ADDR); + + /// IA32_FS_BASE model-specific register. Provides the base address for %fs-relative + /// loads/stores. + pub const MSR_IA32_FS_BASE: Reg = Reg; + } + pub mod ia32_apic_base { use tock_registers::{interfaces::Readable, register_bitfields}; @@ -420,6 +432,7 @@ pub use cr3::CR3; pub use cr4::CR4; pub use msr::ia32_apic_base::MSR_IA32_APIC_BASE; pub use msr::ia32_efer::MSR_IA32_EFER; +pub use msr::ia32_fs_base::MSR_IA32_FS_BASE; pub use msr::ia32_kernel_gs_base::MSR_IA32_KERNEL_GS_BASE; pub use msr::ia32_lstar::MSR_IA32_LSTAR; pub use msr::ia32_sfmask::MSR_IA32_SFMASK; diff --git a/kernel/arch/x86_64/src/context.S b/kernel/arch/x86_64/src/context.S index 84c64874..91dbd481 100644 --- a/kernel/arch/x86_64/src/context.S +++ b/kernel/arch/x86_64/src/context.S @@ -3,53 +3,21 @@ .set MSR_IA32_FS_BASE, 0xC0000100 .macro SAVE_TASK_STATE - sub ${context_size}, %rsp - - mov %rbx, 0(%rsp) - mov %r12, 8(%rsp) - mov %r13, 16(%rsp) - mov %r14, 24(%rsp) - mov %r15, 32(%rsp) - - // Store FS_BASE - mov $MSR_IA32_FS_BASE, %ecx - rdmsr - mov %edx, %ecx - shl $32, %rcx - or %rax, %rcx - - mov %rcx, 40(%rsp) - - // TODO save %fs - mov %rbp, 48(%rsp) - - mov %cr3, %rbx - mov %rbx, 56(%rsp) + push %rbp + push %r15 + push %r14 + push %r13 + push %r12 + push %rbx .endm .macro LOAD_TASK_STATE - mov 56(%rsp), %rbx - mov %rbx, %cr3 - - mov 0(%rsp), %rbx - mov 8(%rsp), %r12 - mov 16(%rsp), %r13 - mov 24(%rsp), %r14 - mov 32(%rsp), %r15 - - // Load FS_BASE - // edx:eax = fs_base - mov 40(%rsp), %rdx - mov %edx, %eax - shr $32, %rdx - - mov $MSR_IA32_FS_BASE, %ecx - wrmsr - - // mov 40(%rsp), %fs - mov 48(%rsp), %rbp - - add ${context_size}, %rsp + pop %rbx + pop %r12 + pop %r13 + pop %r14 + pop %r15 + pop %rbp .endm .global __x86_64_task_enter_user @@ -134,55 +102,27 @@ __x86_64_task_enter_kernel: // %rsi - from struct ptr, %rdi - to struct ptr __x86_64_switch_task: + // save state to source stack SAVE_TASK_STATE mov %rsp, 0(%rsi) - // TSS.RSP0 - mov 8(%rdi), %rax - // Kernel stack - mov 0(%rdi), %rdi - - mov %rdi, %rsp - - // Load TSS.RSP0 - mov %gs:(8), %rdi - mov %rax, 4(%rdi) - + // load destination stack + mov 0(%rdi), %rsp LOAD_TASK_STATE - ret __x86_64_switch_and_drop: - // TSS.RSP0 - mov 8(%rdi), %rax - // Kernel stack - mov 0(%rdi), %rdi - - mov %rdi, %rsp - - // Load TSS.RSP0 - mov %gs:(8), %rdi - mov %rax, 4(%rdi) + mov 0(%rdi), %rsp + // Call thread drop before loading the state mov %rsi, %rdi call __arch_drop_thread LOAD_TASK_STATE - ret // %rdi - to struct ptr __x86_64_enter_task: - // TSS.RSP0 - mov 8(%rdi), %rax - // Kernel stack - mov 0(%rdi), %rdi - - mov %rdi, %rsp - - // Load TSS.RSP0 - mov %gs:(8), %rdi - mov %rax, 4(%rdi) - + mov 0(%rdi), %rsp LOAD_TASK_STATE ret diff --git a/kernel/arch/x86_64/src/context.rs b/kernel/arch/x86_64/src/context.rs index ca87acca..658f6012 100644 --- a/kernel/arch/x86_64/src/context.rs +++ b/kernel/arch/x86_64/src/context.rs @@ -4,11 +4,12 @@ use kernel_arch_interface::{ mem::{KernelTableManager, PhysicalMemoryAllocator}, task::{ForkFrame, StackBuilder, TaskContext, TaskFrame, UserContextInfo}, }; -use kernel_arch_x86::registers::FpuContext; +use kernel_arch_x86::registers::{FpuContext, CR3, MSR_IA32_FS_BASE}; use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress}; +use tock_registers::interfaces::Writeable; use yggdrasil_abi::{arch::SavedFrame, error::Error}; -use crate::mem::KERNEL_TABLES; +use crate::{mem::KERNEL_TABLES, ArchitectureImpl}; /// Frame saved onto the stack when taking an IRQ #[derive(Debug)] @@ -93,8 +94,8 @@ pub struct SyscallFrame { struct Inner { // 0x00 sp: usize, - // 0x08 - tss_rsp0: usize, + + fs_base: usize, } /// x86-64 implementation of a task context @@ -107,14 +108,14 @@ pub struct TaskContextImpl< fpu_context: UnsafeCell, stack_base_phys: PhysicalAddress, stack_size: usize, + tss_rsp0: usize, + + cr3: usize, _alloc: PhantomData, _table_manager: PhantomData, } -// 8 registers + return address (which is not included) -const COMMON_CONTEXT_SIZE: usize = 8 * 8; - impl TaskFrame for IrqFrame { fn store(&self) -> SavedFrame { SavedFrame { @@ -348,52 +349,71 @@ impl { /// Constructs a new task context from a "forked" syscall frame - pub(super) unsafe fn from_syscall_frame(frame: &SyscallFrame, cr3: u64) -> Result { - const USER_TASK_PAGES: usize = 8; + pub(super) unsafe fn from_syscall_frame( + _frame: &SyscallFrame, + _cr3: u64, + ) -> Result { + todo!() + // const USER_TASK_PAGES: usize = 8; - let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?; - let stack_base = stack_base_phys.raw_virtualize::(); + // let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?; + // let stack_base = stack_base_phys.raw_virtualize::(); - let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); + // let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000); - // iretq frame - stack.push(0x1B); - stack.push(frame.user_sp as _); - stack.push(0x200); - stack.push(0x23); - stack.push(frame.user_ip as _); + // // iretq frame + // stack.push(0x1B); + // stack.push(frame.user_sp as _); + // stack.push(0x200); + // stack.push(0x23); + // stack.push(frame.user_ip as _); - stack.push(frame.args[5] as _); // r9 - stack.push(frame.args[4] as _); // r8 - stack.push(frame.args[3] as _); // r10 - stack.push(frame.args[2] as _); // rdx - stack.push(frame.args[1] as _); // rsi - stack.push(frame.args[0] as _); // rdi + // stack.push(frame.args[5] as _); // r9 + // stack.push(frame.args[4] as _); // r8 + // stack.push(frame.args[3] as _); // r10 + // stack.push(frame.args[2] as _); // rdx + // stack.push(frame.args[1] as _); // rsi + // stack.push(frame.args[0] as _); // rdi - // callee-saved registers - stack.push(__x86_64_task_enter_from_fork as _); + // // callee-saved registers + // stack.push(__x86_64_task_enter_from_fork as _); - stack.push(cr3 as _); + // stack.push(cr3 as _); - stack.push(frame.rbp as _); - stack.push(0x12345678); // XXX TODO: fs_base from SyscallFrame - stack.push(frame.r15 as _); - stack.push(frame.r14 as _); - stack.push(frame.r13 as _); - stack.push(frame.r12 as _); - stack.push(frame.rbx as _); + // stack.push(frame.rbp as _); + // stack.push(0x12345678); // XXX TODO: fs_base from SyscallFrame + // stack.push(frame.r15 as _); + // stack.push(frame.r14 as _); + // stack.push(frame.r13 as _); + // stack.push(frame.r12 as _); + // stack.push(frame.rbx as _); - let sp = stack.build(); - let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; + // let sp = stack.build(); + // let rsp0 = stack_base + USER_TASK_PAGES * 0x1000; - Ok(Self { - inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), - fpu_context: UnsafeCell::new(FpuContext::new(true)), - stack_base_phys, - stack_size: USER_TASK_PAGES * 0x1000, - _alloc: PhantomData, - _table_manager: PhantomData, - }) + // Ok(Self { + // inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }), + // fpu_context: UnsafeCell::new(FpuContext::new(true)), + // stack_base_phys, + // stack_size: USER_TASK_PAGES * 0x1000, + // _alloc: PhantomData, + // _table_manager: PhantomData, + // }) + } + + unsafe fn store_state(&self) { + FpuContext::store(self.fpu_context.get()); + // No need to save TSS/%cr3/%fs base back into the TCB, only the kernel + // can make changes to those + } + + unsafe fn load_state(&self) { + FpuContext::restore(self.fpu_context.get()); + // When the task is interrupted from Ring 3, make the CPU load + // the top of its kernel stack + ArchitectureImpl::set_local_tss_sp0(self.tss_rsp0); + MSR_IA32_FS_BASE.set((*self.inner.get()).fs_base as u64); + CR3.set_address(self.cr3); } } @@ -411,6 +431,8 @@ impl !, arg: usize) -> Result { const KERNEL_TASK_PAGES: usize = 32; + let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into(); + let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?; let stack_base = stack_base_phys.raw_virtualize::(); @@ -420,23 +442,21 @@ impl ! { - FpuContext::restore(self.fpu_context.get()); - + self.load_state(); __x86_64_enter_task(self.inner.get()) } unsafe fn switch(&self, from: &Self) { - let dst = self.inner.get(); - let src = from.inner.get(); - - if dst != src { - // Save the old context - FpuContext::store(from.fpu_context.get()); - // Load next context - FpuContext::restore(self.fpu_context.get()); - - __x86_64_switch_task(dst, src); + if core::ptr::addr_eq(self, from) { + return; } + + from.store_state(); + self.load_state(); + __x86_64_switch_task(self.inner.get(), from.inner.get()) } unsafe fn switch_and_drop(&self, thread: *const ()) { - let dst = self.inner.get(); + self.load_state(); + __x86_64_switch_and_drop(self.inner.get(), thread) + } - FpuContext::restore(self.fpu_context.get()); - - __x86_64_switch_and_drop(dst, thread) + fn set_thread_pointer(&self, tp: usize) { + unsafe { (*self.inner.get()).fs_base = tp }; + MSR_IA32_FS_BASE.set(tp as _); } } @@ -524,13 +542,10 @@ impl Option<&'static mut PerCpuData> { unsafe { (Self::local_cpu() as *mut PerCpuData).as_mut() } } + + fn set_local_tss_sp0(sp: usize) { + let local_cpu = Self::local_cpu_data().unwrap(); + unsafe { + (core::ptr::with_exposed_provenance_mut::(local_cpu.tss_address + 4)) + .write_unaligned(sp); + } + } } impl Architecture for ArchitectureImpl { diff --git a/kernel/libk/src/task/binary/elf.rs b/kernel/libk/src/task/binary/elf.rs index b4393027..4e57fa13 100644 --- a/kernel/libk/src/task/binary/elf.rs +++ b/kernel/libk/src/task/binary/elf.rs @@ -5,6 +5,7 @@ use alloc::sync::Arc; use cfg_if::cfg_if; use elf::{ endian::AnyEndian, + io_traits::InputStream, relocation::{Rel, Rela}, segment::ProgramHeader, ElfStream, ParseError, @@ -13,18 +14,14 @@ use libk_mm::{ pointer::PhysicalRefMut, process::{ProcessAddressSpace, VirtualRangeBacking}, table::MapAttributes, - PageBox, + PageBox, L3_PAGE_SIZE, }; use libk_util::io::{Read, Seek}; use yggdrasil_abi::{error::Error, io::SeekFrom, path::PathBuf}; use crate::{ random, - task::{ - mem::ForeignPointer, - process::ProcessImage, - types::{ProcessTlsInfo, ProcessTlsLayout}, - }, + task::{mem::ForeignPointer, process::ProcessImage, types::TlsImage}, }; cfg_if! { @@ -203,15 +200,13 @@ pub fn load_elf_from_file( ElfSegmentType::Load => { load_segment(space, &file, &segment, image_load_base, vaddr_min)?; } - // TODO not yet handled ElfSegmentType::Tls => (), - // TODO not yet handled, contains __tls_get_addr ElfSegmentType::Dynamic => (), } } - // Load TLS master copy - let tls = handle_tls(&elf, &file)?; + // Load TLS master copy somewhere into process memory + let tls_image = handle_tls(space, &elf, &file)?; // Fixup relocations handle_relocations(&mut elf, space, image_load_base)?; @@ -223,69 +218,11 @@ pub fn load_elf_from_file( Ok(ProcessImage { entry, ip_offset, - tls, load_base: image_load_base, + tls_image, }) } -/// Creates a new copy of the TLS from given master image -pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result { - let Some(tls) = image.tls.as_ref() else { - // No TLS - return Ok(0); - }; - - assert_ne!(tls.layout.full_size, 0); - let aligned_size = (tls.layout.full_size + 0xFFF) & !0xFFF; - - let address = space.allocate( - None, - aligned_size, - VirtualRangeBacking::anonymous(), - MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, - )?; - - // Copy master image data - if tls.layout.data_size > 0 { - let master_copy = tls.master_copy.as_ref().unwrap(); - load_bytes( - space, - address + tls.layout.data_offset, - tls.layout.data_size, - |off, mut data| { - let len = data.len(); - data.copy_from_slice(&master_copy[off..off + len]); - Ok(()) - }, - )?; - } - - // Zero zeroed data - if tls.layout.mem_size > tls.layout.data_size { - load_bytes( - space, - address + tls.layout.data_offset + tls.layout.data_size, - tls.layout.mem_size - tls.layout.data_size, - |_, mut data| { - data.fill(0); - Ok(()) - }, - )?; - } - - // Zero slots and setup self-pointer - unsafe { - let slots: *mut usize = core::ptr::without_provenance_mut(address + tls.layout.slot_offset); - slots.try_write_foreign_volatile(space, 0)?; - slots.add(1).try_write_foreign_volatile(space, 0)?; - let self_ptr: *mut usize = - core::ptr::without_provenance_mut(address + tls.layout.ptr_offset); - self_ptr.try_write_foreign_volatile(space, address + tls.layout.ptr_offset)?; - } - - Ok(address + tls.layout.ptr_offset) -} - fn is_dynamic( elf: &mut ElfStream>, ) -> Result { @@ -366,7 +303,6 @@ fn handle_relocations( space: &ProcessAddressSpace, image_load_base: usize, ) -> Result<(), Error> { - // TODO handle SHT_REL as well let rel_section = elf .section_headers() .iter() @@ -401,45 +337,73 @@ fn handle_relocations( } fn handle_tls( + space: &ProcessAddressSpace, elf: &ElfStream>, file: &FileReader, -) -> Result, Error> { - // TODO handle different TLS models - // TODO handle TLS segment attributes - // TODO check if it's possible to have more than one TLS segment - - // Locate TLS master copy information, if any - let tls_segment = elf.segments().iter().find(|s| s.p_type == elf::abi::PT_TLS); - - if let Some(tls_segment) = tls_segment { - // Can't yet handle higher align values - assert!(tls_segment.p_align.is_power_of_two()); - assert!(tls_segment.p_align < 0x1000); - - let layout = ProcessTlsLayout::new( - tls_segment.p_align as _, - tls_segment.p_filesz as _, - tls_segment.p_memsz as _, - ); - - assert!(layout.mem_size <= layout.full_size); - - let master_copy = if layout.data_size > 0 { - let mut data = PageBox::new_slice(0, layout.data_size)?; - file.file.seek(SeekFrom::Start(tls_segment.p_offset))?; - file.file.read_exact(&mut data)?; - Some(Arc::new(data)) - } else { - None - }; - - Ok(Some(ProcessTlsInfo { - master_copy, - layout, - })) - } else { - Ok(None) +) -> Result, Error> { + let mut tls_segments = elf + .segments() + .iter() + .filter(|s| s.p_type == elf::abi::PT_TLS); + let Some(tls_segment) = tls_segments.next() else { + return Ok(None); + }; + if tls_segments.next().is_some() { + todo!("Handle ELFs with more than one PT_TLS?"); } + + if !tls_segment.p_align.is_power_of_two() { + log::warn!("PT_TLS align not power of two: {}", tls_segment.p_align); + return Err(Error::UnrecognizedExecutable); + } + if tls_segment.p_align > 0x1000 { + todo!("PT_TLS align > 0x1000"); + } + + // Load the master copy somewhere. + // This will be the pointer passed to the program as an argument so that it can + // clone the TLS master copy when, for example, spawning a new thread. + // + // Only do the allocation if there's any actual data in the master copy, + // i.e., don't allocate a master copy if there's only a zeroed .tbss + let master_copy_base = if tls_segment.p_filesz > 0 { + // NOTE: don't need to load more than filesz bytes, rest will get zeroed anyway + let page_aligned_size = + (tls_segment.p_filesz as usize + L3_PAGE_SIZE - 1) & !(L3_PAGE_SIZE - 1); + log::debug!( + "Allocate TLS master copy for {:#x} bytes", + page_aligned_size + ); + // User doesn't need to write/execute a master copy, it's for cloning only + let master_copy_base = space.allocate( + None, + page_aligned_size, + VirtualRangeBacking::Anonymous, + MapAttributes::USER_READ, + )?; + + load_bytes( + space, + master_copy_base, + tls_segment.p_filesz as usize, + |off, mut dst| { + file.file + .seek(SeekFrom::Start(tls_segment.p_offset + off as u64))?; + file.file.read_exact(dst.deref_mut()) + }, + )?; + + master_copy_base + } else { + 0 + }; + + Ok(Some(TlsImage { + master_copy_base, + align: tls_segment.p_align as usize, + data_size: tls_segment.p_filesz as usize, + full_size: tls_segment.p_memsz as usize, + })) } fn load_segment( diff --git a/kernel/libk/src/task/binary/mod.rs b/kernel/libk/src/task/binary/mod.rs index c67f1000..72b1bde2 100644 --- a/kernel/libk/src/task/binary/mod.rs +++ b/kernel/libk/src/task/binary/mod.rs @@ -4,8 +4,10 @@ use alloc::{ borrow::ToOwned, string::String, sync::{Arc, Weak}, + vec, vec::Vec, }; +use bytemuck::Pod; use kernel_arch::task::{TaskContext, UserContextInfo}; use libk_mm::{ pointer::PhysicalRefMut, @@ -18,7 +20,7 @@ use yggdrasil_abi::{ io::SeekFrom, pass::{Place, Placer}, path::Path, - process::{ProcessGroupId, ProgramArgumentInner}, + process::{auxv, AuxValue, ProcessGroupId}, }; use crate::{ @@ -47,59 +49,126 @@ pub struct LoadOptions<'e, P: AsRef> { pub disable_aslr: bool, } -struct BufferPlacer<'a> { +// struct BufferPlacer<'a> { +// buffer: &'a mut [u8], +// virtual_offset: usize, +// offset: usize, +// } +// +// impl<'a> BufferPlacer<'a> { +// pub fn new(virtual_offset: usize, buffer: &'a mut [u8]) -> Self { +// Self { +// buffer, +// virtual_offset, +// offset: 0, +// } +// } +// +// unsafe fn alloc_layout( +// &mut self, +// layout: Layout, +// ) -> Result<(NonNull, NonNull), Error> { +// // TODO checks +// let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1); +// self.offset = aligned + layout.size(); +// Ok(( +// NonNull::new_unchecked(self.buffer.as_mut_ptr().add(aligned) as *mut T), +// NonNull::new_unchecked((self.virtual_offset + aligned) as *mut T), +// )) +// } +// } +// +// unsafe impl<'a> Placer for BufferPlacer<'a> { +// fn place_ref(&mut self, r: &T) -> Result, Error> { +// let layout = Layout::new::(); +// unsafe { +// let (kernel, user) = self.alloc_layout::(layout)?; +// kernel.as_ptr().write(r.place(self)?); +// Ok(user) +// } +// } +// +// fn place_slice(&mut self, r: &[T]) -> Result, Error> { +// let layout = Layout::array::(r.len()).unwrap(); +// unsafe { +// let (kernel, user) = self.alloc_layout::(layout)?; +// let kernel = NonNull::slice_from_raw_parts(kernel, r.len()); +// let user = NonNull::slice_from_raw_parts(user, r.len()); +// for (i, elem) in r.iter().enumerate() { +// kernel +// .get_unchecked_mut(i) +// .as_ptr() +// .write(elem.place(self)?); +// } +// Ok(user) +// } +// } +// } + +struct ArgPlacer<'a> { buffer: &'a mut [u8], - virtual_offset: usize, - offset: usize, + position: usize, } -impl<'a> BufferPlacer<'a> { - pub fn new(virtual_offset: usize, buffer: &'a mut [u8]) -> Self { +impl<'a> ArgPlacer<'a> { + fn new(buffer: &'a mut [u8]) -> Self { Self { buffer, - virtual_offset, - offset: 0, + position: 0, } } - unsafe fn alloc_layout( - &mut self, - layout: Layout, - ) -> Result<(NonNull, NonNull), Error> { - // TODO checks - let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1); - self.offset = aligned + layout.size(); - Ok(( - NonNull::new_unchecked(self.buffer.as_mut_ptr().add(aligned) as *mut T), - NonNull::new_unchecked((self.virtual_offset + aligned) as *mut T), - )) - } -} - -unsafe impl<'a> Placer for BufferPlacer<'a> { - fn place_ref(&mut self, r: &T) -> Result, Error> { - let layout = Layout::new::(); - unsafe { - let (kernel, user) = self.alloc_layout::(layout)?; - kernel.as_ptr().write(r.place(self)?); - Ok(user) + fn align(&mut self, align: usize) -> Result<(), Error> { + debug_assert!(align.is_power_of_two()); + let aligned = (self.position + align - 1) & !(align - 1); + if aligned > self.buffer.len() { + return Err(Error::InvalidArgument); } + self.position = aligned; + Ok(()) } - fn place_slice(&mut self, r: &[T]) -> Result, Error> { - let layout = Layout::array::(r.len()).unwrap(); - unsafe { - let (kernel, user) = self.alloc_layout::(layout)?; - let kernel = NonNull::slice_from_raw_parts(kernel, r.len()); - let user = NonNull::slice_from_raw_parts(user, r.len()); - for (i, elem) in r.iter().enumerate() { - kernel - .get_unchecked_mut(i) - .as_ptr() - .write(elem.place(self)?); - } - Ok(user) + fn put_str(&mut self, s: &str) -> Result { + if self.position + s.len() >= self.buffer.len() { + return Err(Error::InvalidArgument); } + let off = self.position; + self.buffer[off..off + s.len()].copy_from_slice(s.as_bytes()); + self.buffer[off + s.len()] = 0; + self.position += s.len() + 1; + Ok(off) + } + + fn put(&mut self, v: &T) -> Result { + if self.position + size_of::() > self.buffer.len() { + return Err(Error::InvalidArgument); + } + let off = self.position; + self.buffer[off..off + size_of::()].copy_from_slice(bytemuck::bytes_of(v)); + self.position += size_of::(); + Ok(off) + } + + fn put_aux_array(&mut self, s: &[AuxValue]) -> Result { + self.align(size_of::())?; + let off = self.position; + for item in s { + self.put(&item.tag)?; + self.put(&item.val)?; + } + self.put(&auxv::NULL)?; + self.put(&0u64)?; + Ok(off) + } + + fn put_ptr_array(&mut self, s: &[usize]) -> Result { + self.align(size_of::())?; + let off = self.position; + for item in s { + self.put(item)?; + } + self.put(&0usize)?; + Ok(off) } } @@ -110,6 +179,7 @@ fn setup_program_env( virt: usize, args: &Vec, envs: &Vec, + aux: &[AuxValue], ) -> Result { // TODO growing buffer let phys_page = space.map_single( @@ -118,18 +188,37 @@ fn setup_program_env( MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL, )?; let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) }; - let mut placer = BufferPlacer::new(virt, &mut buffer); + let mut placer = ArgPlacer::new(&mut buffer); + let mut arg_ptrs = vec![]; + let mut env_ptrs = vec![]; - let args = args.iter().map(String::as_ref).collect::>(); - let envs = envs.iter().map(String::as_ref).collect::>(); + log::debug!("place karg: {:#x}", virt); + for arg in args.iter() { + let ptr = placer.put_str(arg)? + virt; + log::debug!("put arg {:?} -> {:#x}", arg, ptr); + arg_ptrs.push(ptr); + } + for env in envs.iter() { + let ptr = placer.put_str(env)? + virt; + log::debug!("put env {:?} -> {:#x}", env, ptr); + env_ptrs.push(ptr); + } - let in_kernel = ProgramArgumentInner { - args: &args, - env: &envs, - }; - let in_user = in_kernel.place_ref(&mut placer)?; + let argv = placer.put_ptr_array(&arg_ptrs)? + virt; + log::debug!("put argv {} -> {:#x}", arg_ptrs.len(), argv); + let envp = placer.put_ptr_array(&env_ptrs)? + virt; + log::debug!("put envp {} -> {:#x}", env_ptrs.len(), envp); + let auxv = placer.put_aux_array(aux)? + virt; + log::debug!("put auxv {} -> {:#x}", aux.len(), auxv); - Ok(in_user as *const _ as usize) + // Put ProgramArgumentInner struct + let arg_address = placer.position + virt; + + placer.put(&argv)?; + placer.put(&envp)?; + placer.put(&auxv)?; + + Ok(arg_address) } fn setup_context

( @@ -156,7 +245,31 @@ where MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL, )?; - let arg = setup_program_env(space, virt_args_base, args, envs)?; + let mut auxv = vec![]; + if let Some(tls_image) = image.tls_image.as_ref() { + if tls_image.master_copy_base != 0 { + auxv.push(AuxValue { + tag: auxv::TLS_MASTER_COPY, + val: tls_image.master_copy_base as _, + }); + auxv.push(AuxValue { + tag: auxv::TLS_DATA_SIZE, + val: tls_image.data_size as _, + }); + } else { + assert_eq!(tls_image.data_size, 0); + } + auxv.push(AuxValue { + tag: auxv::TLS_ALIGN, + val: tls_image.align as _, + }); + auxv.push(AuxValue { + tag: auxv::TLS_FULL_SIZE, + val: tls_image.full_size as _, + }); + } + + let argument = setup_program_env(space, virt_args_base, args, envs, &auxv)?; let user_sp = virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContextImpl::USER_STACK_EXTRA_ALIGN; @@ -171,15 +284,16 @@ where } } - let ptr = unsafe { Thread::setup_stack_header(space, ptr, arg)? }; + let ptr = unsafe { Thread::setup_stack_header(space, ptr, argument)? }; - let tls_address = elf::clone_tls(space, image)?; + // let tls_address = elf::clone_tls(space, image)?; + log::debug!("argument = {:#x}", argument); TaskContext::user(UserContextInfo { entry: image.entry, - argument: arg, + argument, stack_pointer: ptr.addr(), - tls: tls_address, + thread_pointer: 0, address_space: space.as_address_with_asid(), single_step: options.single_step, }) diff --git a/kernel/libk/src/task/process.rs b/kernel/libk/src/task/process.rs index 839b3e19..dca22ddc 100644 --- a/kernel/libk/src/task/process.rs +++ b/kernel/libk/src/task/process.rs @@ -29,15 +29,14 @@ use yggdrasil_abi::{ use crate::{ task::{ - binary, - futex::UserspaceMutex, - thread::Thread, - types::{AllocateProcessId, ProcessTlsInfo}, - TaskContextImpl, ThreadId, + binary, futex::UserspaceMutex, thread::Thread, types::AllocateProcessId, TaskContextImpl, + ThreadId, }, vfs::{FileReadiness, FileSet, IoContext, NodeRef}, }; +use super::types::TlsImage; + pub trait ForkFrame = kernel_arch::task::ForkFrame; pub struct ProcessIo { @@ -57,8 +56,8 @@ pub struct ProcessImage { pub ip_offset: usize, /// Entry point address pub entry: usize, - /// Thread-local storage information - pub tls: Option, + /// TLS master copy, if any + pub tls_image: Option, } pub struct ProcessInner { @@ -133,16 +132,11 @@ impl Process { /// Spawns a new child thread within the process pub fn spawn_thread(self: &Arc, options: &ThreadSpawnOptions) -> Result { log::debug!("Spawn thread in {} with options: {:#x?}", self.id, options); + let mut inner = self.inner.write(); let space = inner.space.clone().unwrap(); - let tls_address = if let Some(image) = inner.image.as_ref() { - binary::elf::clone_tls(&space, image)? - } else { - 0 - }; - let sp = (options.stack_top - 8) as *mut usize; let sp = unsafe { Thread::setup_stack_header(&space, sp, options.argument)? }; @@ -152,7 +146,7 @@ impl Process { address_space: space.as_address_with_asid(), stack_pointer: sp.addr(), single_step: false, - tls: tls_address, + thread_pointer: 0, }; let context = TaskContextImpl::user(info)?; let thread = Thread::new_uthread(self.id, None, space.clone(), context); diff --git a/kernel/libk/src/task/thread.rs b/kernel/libk/src/task/thread.rs index ac4e6898..d77daed0 100644 --- a/kernel/libk/src/task/thread.rs +++ b/kernel/libk/src/task/thread.rs @@ -204,6 +204,10 @@ impl Thread { self.context.replace(context) } + pub fn set_thread_pointer(&self, tp: usize) { + unsafe { (*self.context.as_ptr()).set_thread_pointer(tp) }; + } + // Get/Set pub fn is_user(&self) -> bool { self.id.is_user() diff --git a/kernel/libk/src/task/types.rs b/kernel/libk/src/task/types.rs index 7fa2dd40..55eee8bb 100644 --- a/kernel/libk/src/task/types.rs +++ b/kernel/libk/src/task/types.rs @@ -40,39 +40,53 @@ pub enum ThreadId { User(u64), } -// TLS layout (x86-64): -// | mem_size | uthread_size | -// | Data .......| self, ??? | -// -// TLS layout (aarch64): -// | uthread_size (0x10?) | mem_size | -// | ??? | Data .....| -// -// TLS layout (i686): -// | self | ... | - -/// Describes Thread-Local Storage of a process -#[derive(Clone, Debug)] -pub struct ProcessTlsInfo { - /// Master copy of the TLS - pub master_copy: Option>>, - /// Layout of the TLS - pub layout: ProcessTlsLayout, -} - -/// Describes TLS layout for a program image -#[derive(Clone, Debug)] -pub struct ProcessTlsLayout { - /// Pointer offset from the TLS base. The pointer is passed to the userspace - pub ptr_offset: usize, - pub data_offset: usize, +/// Describes a TLS master copy +#[derive(Debug, Clone)] +pub struct TlsImage { + /// Where the kernel placed the segment in the process' memory. + /// NOTE: is zero if there's no actual data in the master copy. + pub master_copy_base: usize, + /// Segment alignment + pub align: usize, + /// Part of the segment occupied by actual data pub data_size: usize, - pub mem_size: usize, - pub slot_offset: usize, - /// Overall allocation size of the TLS data + /// Full size of the segment. `full_size - data_size` is zeroed on load. pub full_size: usize, } +// // TLS layout (x86-64): +// // | mem_size | uthread_size | +// // | Data .......| self, ??? | +// // +// // TLS layout (aarch64): +// // | uthread_size (0x10?) | mem_size | +// // | ??? | Data .....| +// // +// // TLS layout (i686): +// // | self | ... | +// +// /// Describes Thread-Local Storage of a process +// #[derive(Clone, Debug)] +// pub struct ProcessTlsInfo { +// /// Master copy of the TLS +// pub master_copy: Option>>, +// /// Layout of the TLS +// pub layout: ProcessTlsLayout, +// } +// +// /// Describes TLS layout for a program image +// #[derive(Clone, Debug)] +// pub struct ProcessTlsLayout { +// /// Pointer offset from the TLS base. The pointer is passed to the userspace +// pub ptr_offset: usize, +// pub data_offset: usize, +// pub data_size: usize, +// pub mem_size: usize, +// pub slot_offset: usize, +// /// Overall allocation size of the TLS data +// pub full_size: usize, +// } + impl ThreadAffinity { /// Mask value for a thread to be scheduled onto any CPU pub const ANY_CPU: u64 = u64::MAX; @@ -151,79 +165,79 @@ impl AllocateProcessId for ProcessId { } } -#[cfg(any(target_arch = "aarch64", rust_analyzer))] -impl ProcessTlsLayout { - /// Constructs a new thread-local storage layout info struct - pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { - use core::mem::size_of; - - debug_assert!(align.is_power_of_two()); - let tls_block0_offset = (size_of::() * 2 + align - 1) & !(align - 1); - // Allocate two words in the back for extra storage - let back_size = size_of::() * 2; - - let full_size = back_size + (tls_block0_offset + mem_size + align - 1) & !(align - 1); - - Self { - data_offset: tls_block0_offset + back_size, - ptr_offset: back_size, - slot_offset: 0, - - data_size, - mem_size, - full_size, - } - } -} - -#[cfg(any(target_arch = "x86_64", rust_analyzer))] -impl ProcessTlsLayout { - /// Constructs a new thread-local storage layout info struct - pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { - use core::mem::size_of; - - // The static TLS blocks are placed below TP - // TP points to the TCB - debug_assert!(align.is_power_of_two()); - let back_size = (mem_size + align - 1) & !(align - 1); - // 0: self-pointer - // 1: - - // 2: segment pointer - // 3: segment size - let forward_size = size_of::() * 4; - - let full_size = back_size + forward_size; - - Self { - data_offset: 0, - ptr_offset: back_size, - slot_offset: back_size + 2 * size_of::(), - - data_size, - mem_size, - full_size, - } - } -} - -#[cfg(any(target_arch = "x86", rust_analyzer))] -impl ProcessTlsLayout { - pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { - use core::mem::size_of; - - debug_assert!(align.is_power_of_two()); - let back_size = (mem_size + align - 1) & !(align - 1); - let forward_size = size_of::() * 4; - let full_size = back_size + forward_size; - - Self { - data_offset: 0, - ptr_offset: back_size, - slot_offset: back_size + 2 * size_of::(), - - data_size, - mem_size, - full_size, - } - } -} +// #[cfg(any(target_arch = "aarch64", rust_analyzer))] +// impl ProcessTlsLayout { +// /// Constructs a new thread-local storage layout info struct +// pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { +// use core::mem::size_of; +// +// debug_assert!(align.is_power_of_two()); +// let tls_block0_offset = (size_of::() * 2 + align - 1) & !(align - 1); +// // Allocate two words in the back for extra storage +// let back_size = size_of::() * 2; +// +// let full_size = back_size + (tls_block0_offset + mem_size + align - 1) & !(align - 1); +// +// Self { +// data_offset: tls_block0_offset + back_size, +// ptr_offset: back_size, +// slot_offset: 0, +// +// data_size, +// mem_size, +// full_size, +// } +// } +// } +// +// #[cfg(any(target_arch = "x86_64", rust_analyzer))] +// impl ProcessTlsLayout { +// /// Constructs a new thread-local storage layout info struct +// pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { +// use core::mem::size_of; +// +// // The static TLS blocks are placed below TP +// // TP points to the TCB +// debug_assert!(align.is_power_of_two()); +// let back_size = (mem_size + align - 1) & !(align - 1); +// // 0: self-pointer +// // 1: - +// // 2: segment pointer +// // 3: segment size +// let forward_size = size_of::() * 4; +// +// let full_size = back_size + forward_size; +// +// Self { +// data_offset: 0, +// ptr_offset: back_size, +// slot_offset: back_size + 2 * size_of::(), +// +// data_size, +// mem_size, +// full_size, +// } +// } +// } +// +// #[cfg(any(target_arch = "x86", rust_analyzer))] +// impl ProcessTlsLayout { +// pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self { +// use core::mem::size_of; +// +// debug_assert!(align.is_power_of_two()); +// let back_size = (mem_size + align - 1) & !(align - 1); +// let forward_size = size_of::() * 4; +// let full_size = back_size + forward_size; +// +// Self { +// data_offset: 0, +// ptr_offset: back_size, +// slot_offset: back_size + 2 * size_of::(), +// +// data_size, +// mem_size, +// full_size, +// } +// } +// } diff --git a/kernel/src/syscall/imp/mod.rs b/kernel/src/syscall/imp/mod.rs index 7a124bb0..6c671f47 100644 --- a/kernel/src/syscall/imp/mod.rs +++ b/kernel/src/syscall/imp/mod.rs @@ -6,7 +6,7 @@ pub(crate) use abi::{ }, mem::{MappingFlags, MappingSource}, net::SocketType, - process::{Signal, SignalEntryData, SpawnOptions}, + process::{Signal, SignalEntryData, SpawnOptions, ThreadOption}, system::SystemInfo, }; use abi::{ diff --git a/kernel/src/syscall/imp/sys_process.rs b/kernel/src/syscall/imp/sys_process.rs index 413c5b36..73c7ad20 100644 --- a/kernel/src/syscall/imp/sys_process.rs +++ b/kernel/src/syscall/imp/sys_process.rs @@ -6,7 +6,7 @@ use abi::{ mem::{MappingFlags, MappingSource}, process::{ ExitCode, MutexOperation, ProcessGroupId, ProcessId, Signal, SpawnFlags, SpawnOption, - SpawnOptions, ThreadSpawnOptions, + SpawnOptions, ThreadOption, ThreadSpawnOptions, }, }; use alloc::sync::Arc; @@ -276,3 +276,18 @@ pub(crate) fn wait_thread(id: u32) -> Result<(), Error> { Ok(()) } + +pub(crate) fn get_thread_option(option: &mut ThreadOption) -> Result<(), Error> { + todo!() +} + +pub(crate) fn set_thread_option(option: &ThreadOption) -> Result<(), Error> { + let thread = Thread::current(); + match option { + &ThreadOption::ThreadPointer(tp) => { + log::debug!("{:?}: set thread pointer: {:#x}", thread.id, tp); + thread.set_thread_pointer(tp); + Ok(()) + } + } +} diff --git a/lib/abi/def/yggdrasil.abi b/lib/abi/def/yggdrasil.abi index 31acd1ce..1fae1ab2 100644 --- a/lib/abi/def/yggdrasil.abi +++ b/lib/abi/def/yggdrasil.abi @@ -1,4 +1,4 @@ -// vi:syntax=yggdrasil_abi: +// vi:syntax=yggdrasil-abi: extern { type Duration = core::time::Duration; @@ -13,6 +13,7 @@ extern { type MutexOperation = yggdrasil_abi::process::MutexOperation; #[thin] type ExitCode = yggdrasil_abi::process::ExitCode; + type ThreadOption = yggdrasil_abi::process::ThreadOption; type SocketAddr = core::net::SocketAddr; type SocketOption = yggdrasil_abi::net::SocketOption; @@ -82,6 +83,8 @@ syscall get_pid() -> ProcessId; syscall spawn_thread(options: &ThreadSpawnOptions) -> Result; syscall exit_thread() -> !; syscall wait_thread(tid: u32) -> Result<()>; +syscall get_thread_option(option: &mut ThreadOption) -> Result<()>; +syscall set_thread_option(option: &ThreadOption) -> Result<()>; syscall nanosleep(dur: &Duration) -> Result<()>; diff --git a/lib/abi/src/process/mod.rs b/lib/abi/src/process/mod.rs index 9241aada..a6b13c28 100644 --- a/lib/abi/src/process/mod.rs +++ b/lib/abi/src/process/mod.rs @@ -1,19 +1,33 @@ //! Data structures for process management -use core::{fmt, time::Duration}; +use core::{ffi::CStr, fmt, marker::PhantomData, time::Duration}; -use crate::{ - impl_place_lifetime_struct, - io::{FileMode, RawFd}, -}; +use crate::io::{FileMode, RawFd}; mod exit; +pub mod thread; pub use crate::generated::{ ExecveOptions, ProcessGroupId, ProcessId, Signal, SignalEntryData, SpawnFlags, SpawnOptions, ThreadId, ThreadSpawnOptions, }; pub use exit::ExitCode; +pub use thread::ThreadOption; + +// TODO this is ugly +pub mod auxv { + pub const TLS_MASTER_COPY: u64 = 0x80; + pub const TLS_FULL_SIZE: u64 = 0x81; + pub const TLS_DATA_SIZE: u64 = 0x82; + pub const TLS_ALIGN: u64 = 0x83; + + pub const TLS_MODULE_ID: u64 = 0x84; + pub const TLS_MODULE_OFFSET: u64 = 0x85; + + pub const TLS_ALREADY_INITIALIZED: u64 = 0x86; + + pub const NULL: u64 = 0x00; +} /// Defines an optional argument for controlling process creation #[derive(Debug)] @@ -53,16 +67,30 @@ pub enum ProcessInfoElement { Umask(FileMode), } -// TODO not sure if I really need #[repr(C)] ABI here -impl_place_lifetime_struct! { - #[doc = "Argument struct passed from the kernel to a spawned process"] - #[derive(Debug)] - pub struct ProgramArgumentInner<'a> { - #[doc = "Argument list"] - pub args: &'a [&'a str], - #[doc = "List of KEY=VALUE environment variable pairs"] - pub env: &'a [&'a str], - } +#[derive(Debug, Clone, Copy)] +#[repr(C)] +pub struct AuxValue { + pub tag: u64, + pub val: u64, +} + +#[derive(Clone, Copy)] +pub struct StringArgIter<'a> { + ptr: *const *const u8, + _pd: PhantomData<&'a ProgramArgumentInner>, +} + +#[derive(Clone, Copy)] +pub struct AuxValueIter<'a> { + ptr: *const AuxValue, + _pd: PhantomData<&'a ProgramArgumentInner>, +} + +#[repr(C)] +pub struct ProgramArgumentInner { + pub args: *const *const u8, + pub envs: *const *const u8, + pub auxv: *const AuxValue, } impl ProcessId { @@ -80,3 +108,59 @@ impl fmt::Display for ProcessGroupId { write!(f, "", self.0) } } + +impl<'a> Iterator for StringArgIter<'a> { + type Item = &'a CStr; + + fn next(&mut self) -> Option { + if self.ptr.is_null() { + return None; + } + let entry = unsafe { self.ptr.read() }; + if entry.is_null() { + return None; + } + self.ptr = unsafe { self.ptr.add(1) }; + let entry = unsafe { CStr::from_ptr(entry.cast()) }; + Some(entry) + } +} + +impl<'a> Iterator for AuxValueIter<'a> { + type Item = &'a AuxValue; + + fn next(&mut self) -> Option { + if self.ptr.is_null() { + return None; + } + let entry = unsafe { &*self.ptr }; + if entry.tag == auxv::NULL { + return None; + } + self.ptr = unsafe { self.ptr.add(1) }; + Some(entry) + } +} + +impl ProgramArgumentInner { + pub fn args(&self) -> impl Iterator { + StringArgIter { + ptr: self.args, + _pd: PhantomData, + } + } + + pub fn envs(&self) -> impl Iterator { + StringArgIter { + ptr: self.envs, + _pd: PhantomData, + } + } + + pub fn auxv(&self) -> impl Iterator { + AuxValueIter { + ptr: self.auxv, + _pd: PhantomData, + } + } +} diff --git a/lib/abi/src/process/thread.rs b/lib/abi/src/process/thread.rs new file mode 100644 index 00000000..d2b2b233 --- /dev/null +++ b/lib/abi/src/process/thread.rs @@ -0,0 +1,4 @@ +#[derive(Debug)] +pub enum ThreadOption { + ThreadPointer(usize), +} diff --git a/lib/runtime/Cargo.toml b/lib/runtime/Cargo.toml index cd6445a0..47b16568 100644 --- a/lib/runtime/Cargo.toml +++ b/lib/runtime/Cargo.toml @@ -6,7 +6,7 @@ authors = ["Mark Poliakov "] build = "build.rs" [dependencies] -yggdrasil-abi = { path = "../abi" } +yggdrasil-abi = { path = "../abi", features = ["alloc"] } core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" } alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-alloc" } @@ -22,7 +22,7 @@ cc = "*" [features] default = [] -alloc = ["dep:alloc", "yggdrasil-abi/alloc"] +__tls_get_addr = [] rustc-dep-of-std = [ "core", "alloc", diff --git a/lib/runtime/src/lib.rs b/lib/runtime/src/lib.rs index db5ae90d..cbe3272c 100644 --- a/lib/runtime/src/lib.rs +++ b/lib/runtime/src/lib.rs @@ -9,6 +9,8 @@ extern crate compiler_builtins; extern crate yggdrasil_abi as abi; +extern crate alloc; + pub use abi::error::Error; pub use abi::path; pub mod debug; diff --git a/lib/runtime/src/process/aarch64.rs b/lib/runtime/src/process/aarch64.rs deleted file mode 100644 index dfacd2a9..00000000 --- a/lib/runtime/src/process/aarch64.rs +++ /dev/null @@ -1,27 +0,0 @@ -#![allow(missing_docs)] - -pub fn get_tls() -> usize { - let tp: usize; - unsafe { - core::arch::asm!("mrs {0}, tpidr_el0", out(reg) tp); - } - tp -} - -pub unsafe fn set_pthread_self(value: usize) { - todo!() -} - -pub fn pthread_self() -> usize { - todo!() -} - -pub fn thread_local_area_impl() -> &'static mut [*mut u8] { - let tp = get_tls(); - - // Kernel reserves two words below the TP - assert_eq!(tp % size_of::(), 0); - let tp = tp - size_of::() * 2; - - unsafe { super::thread_local_area_common(tp) } -} diff --git a/lib/runtime/src/process/i686.rs b/lib/runtime/src/process/i686.rs deleted file mode 100644 index e69d442b..00000000 --- a/lib/runtime/src/process/i686.rs +++ /dev/null @@ -1,27 +0,0 @@ -#![allow(missing_docs)] - -pub fn get_tls() -> usize { - let tp: usize; - unsafe { - core::arch::asm!("mov %gs:0, {0}", out(reg) tp, options(att_syntax)); - } - tp -} - -pub unsafe fn set_pthread_self(value: usize) { - todo!() -} - -pub fn pthread_self() -> usize { - todo!() -} - -pub fn thread_local_area_impl() -> &'static mut [*mut u8] { - let tp = get_tls(); - - // Skip self-pointer - assert_eq!(tp % size_of::(), 0); - let tp = tp + size_of::() * 2; - - unsafe { super::thread_local_area_common(tp) } -} diff --git a/lib/runtime/src/process/mod.rs b/lib/runtime/src/process/mod.rs index 6eaf1767..6dad4b98 100644 --- a/lib/runtime/src/process/mod.rs +++ b/lib/runtime/src/process/mod.rs @@ -1,65 +1,9 @@ //! Process management data types -use abi::mem::{MappingFlags, MappingSource}; pub use abi::process::{ - ExecveOptions, ExitCode, MutexOperation, ProcessGroupId, ProcessId, ProcessInfoElement, - ProgramArgumentInner, Signal, SignalEntryData, SpawnFlags, SpawnOption, SpawnOptions, ThreadId, - ThreadSpawnOptions, + auxv, AuxValue, AuxValueIter, ExecveOptions, ExitCode, MutexOperation, ProcessGroupId, + ProcessId, ProcessInfoElement, ProgramArgumentInner, Signal, SignalEntryData, SpawnFlags, + SpawnOption, SpawnOptions, StringArgIter, ThreadId, ThreadSpawnOptions, }; -use crate::sys; - -const THREAD_LOCAL_AREA_SIZE: usize = 4096 / size_of::(); - -#[cfg(any(target_arch = "aarch64", rust_analyzer))] -mod aarch64; -#[cfg(any(target_arch = "aarch64", rust_analyzer))] -use aarch64 as imp; - -#[cfg(any(target_arch = "x86_64", rust_analyzer))] -mod x86_64; -#[cfg(any(target_arch = "x86_64", rust_analyzer))] -use x86_64 as imp; - -#[cfg(any(target_arch = "x86", rust_analyzer))] -mod i686; -#[cfg(any(target_arch = "x86", rust_analyzer))] -use i686 as imp; - -pub use imp::{get_tls, pthread_self, set_pthread_self}; - -/// # Safety -/// -/// `tp` must be a proper and aligned pointer to a pair of [usize] words. -unsafe fn thread_local_area_common(tp: usize) -> &'static mut [*mut u8] { - // 0: segment pointer, 1: segment len - // seg ptr -> thread-local pointer list base - let ptr: *mut *mut *mut u8 = core::ptr::with_exposed_provenance_mut(tp); - let len: *mut usize = core::ptr::with_exposed_provenance_mut(tp + size_of::()); - - let (base, len) = if unsafe { (*ptr).is_null() } { - // If segment is NULL, allocate some memory and map it here - let base = - unsafe { sys::map_memory(None, 4096, MappingFlags::WRITE, &MappingSource::Anonymous) } - .expect("Could not allocate TLS storage"); - let base = core::ptr::with_exposed_provenance_mut::<*mut u8>(base); - - unsafe { - *ptr = base; - *len = THREAD_LOCAL_AREA_SIZE; - } - - (base, THREAD_LOCAL_AREA_SIZE) - } else { - // Otherwise return the existing values - unsafe { (*ptr, *len) } - }; - - unsafe { core::slice::from_raw_parts_mut(base, len) } -} - -/// Returns a reference to the thread-local area, which contains thread-local pointers -pub fn thread_local_area() -> &'static mut [*mut u8] { - // TODO grow thread local areas based on size requested? - imp::thread_local_area_impl() -} +pub mod thread_local; diff --git a/lib/runtime/src/process/thread_local/aarch64.rs b/lib/runtime/src/process/thread_local/aarch64.rs new file mode 100644 index 00000000..056c7db6 --- /dev/null +++ b/lib/runtime/src/process/thread_local/aarch64.rs @@ -0,0 +1,15 @@ +#![allow(missing_docs)] + +use abi::error::Error; + +pub fn get_thread_pointer() -> usize { + let tp: usize; + unsafe { + core::arch::asm!("mrs {0}, tpidr_el0", out(reg) tp); + } + tp +} + +pub unsafe fn set_thread_pointer(value: usize) -> Result<(), Error> { + todo!() +} diff --git a/lib/runtime/src/process/thread_local/i686.rs b/lib/runtime/src/process/thread_local/i686.rs new file mode 100644 index 00000000..35b2a8c1 --- /dev/null +++ b/lib/runtime/src/process/thread_local/i686.rs @@ -0,0 +1,15 @@ +#![allow(missing_docs)] + +use abi::error::Error; + +pub fn get_thread_pointer() -> usize { + let tp: usize; + unsafe { + core::arch::asm!("mov %gs:0, {0}", out(reg) tp, options(att_syntax)); + } + tp +} + +pub unsafe fn set_thread_pointer(value: usize) -> Result<(), Error> { + todo!() +} diff --git a/lib/runtime/src/process/thread_local/mod.rs b/lib/runtime/src/process/thread_local/mod.rs new file mode 100644 index 00000000..adc972d6 --- /dev/null +++ b/lib/runtime/src/process/thread_local/mod.rs @@ -0,0 +1,242 @@ +//! Yggdrasil Runtime functions for handling Thread-Local Storage +use core::{ + ffi::c_void, + ptr::{null_mut, NonNull}, +}; + +use alloc::{boxed::Box, vec::Vec}; + +use abi::{ + error::Error, + process::{auxv, AuxValue, ThreadOption}, +}; + +#[cfg(any(target_arch = "aarch64", rust_analyzer))] +mod aarch64; +#[cfg(any(target_arch = "aarch64", rust_analyzer))] +use aarch64 as imp; + +#[cfg(any(target_arch = "x86_64", rust_analyzer))] +mod x86_64; +#[cfg(any(target_arch = "x86_64", rust_analyzer))] +use x86_64 as imp; + +#[cfg(any(target_arch = "x86", rust_analyzer))] +mod i686; +#[cfg(any(target_arch = "x86", rust_analyzer))] +use i686 as imp; + +#[cfg(any(target_arch = "aarch64", rust_analyzer))] +mod variant1; +#[cfg(any(target_arch = "aarch64", rust_analyzer))] +use variant1 as layout; + +#[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))] +mod variant2; +#[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))] +use variant2 as layout; + +pub use imp::{get_thread_pointer, set_thread_pointer}; +pub use layout::clone_tls; + +/// Describes a TLS master copy of this executable +#[derive(Debug)] +pub struct TlsImage { + master_copy: Option>, + full_size: usize, + align: usize, + already_initialized: bool, + // Pairs of dynamic module_id:offset from allocation base + module_offsets: Vec<(usize, usize)>, +} + +/// DTV table, containing dynamic TLS module mappings +pub struct Dtv { + entries: Vec<*mut c_void>, +} + +struct TcbHeader { + self_pointer: usize, + dtv_pointer: *mut Dtv, +} + +impl TlsImage { + /// Extracts information about the TLS master copy from aux vector + pub fn from_auxv<'a, I: Iterator>(auxv: I) -> Option { + let mut master_copy = None; + let mut data_size = None; + let mut full_size = None; + let mut align = None; + let mut current_module = None; + let mut module_offsets = Vec::new(); + let mut already_initialized = false; + + for item in auxv { + match item.tag { + auxv::TLS_MASTER_COPY => master_copy = Some(item.val as usize), + auxv::TLS_DATA_SIZE => data_size = Some(item.val as usize), + auxv::TLS_FULL_SIZE => full_size = Some(item.val as usize), + auxv::TLS_ALIGN => align = Some(item.val as usize), + auxv::TLS_MODULE_ID => current_module = Some(item.val as usize), + auxv::TLS_MODULE_OFFSET => { + if let Some(module_id) = current_module.take() { + module_offsets.push((module_id, item.val as usize)); + } + } + auxv::TLS_ALREADY_INITIALIZED => { + already_initialized = true; + } + _ => (), + } + } + + let full_size = full_size?; + let align = align?; + + let master_copy = if let (Some(master_copy), Some(data_size)) = (master_copy, data_size) { + assert_ne!(master_copy, 0); + assert_ne!(data_size, 0); + + let base = unsafe { + NonNull::new_unchecked(core::ptr::with_exposed_provenance_mut::(master_copy)) + }; + + Some(NonNull::slice_from_raw_parts(base, data_size)) + } else { + None + }; + + Some(Self { + master_copy, + full_size, + align, + module_offsets, + already_initialized, + }) + } +} + +impl Dtv { + fn new() -> Self { + Self { + entries: Vec::new(), + } + } + + /// Returns a value associated with a given key. + /// + /// # Panics + /// + /// Will panic if key == 0. + /// Will panic if key is larger than the DTV itself. + pub fn get(&self, key: usize) -> *mut c_void { + if key == 0 { + panic!("Zero key used when calling DTV::get()"); + } + let key = key - 1; + if key >= self.entries.len() { + panic!("Key not in DTV: {}", key + 1); + } + self.entries[key] + } + + /// Sets a DTV entry, growing the DTV allocation if necessary + pub fn set(&mut self, key: usize, value: *mut c_void) -> Result<(), Error> { + if key == 0 { + return Err(Error::InvalidArgument); + } + let key = key - 1; + if key >= self.entries.len() { + // TODO return an error if resize fails + self.entries.resize(key + 1, null_mut()); + } + self.entries[key] = value; + Ok(()) + } +} + +/// Initializes thread-local storage for this thread from aux vector provided to the executable. +pub fn init_tls_from_auxv<'a, I: Iterator>( + auxv: I, + force: bool, + tcb_size: usize, +) -> Result, Error> { + let Some(tls_image) = TlsImage::from_auxv(auxv) else { + return Ok(None); + }; + + if force || !tls_image.already_initialized { + let (base, tp) = clone_tls(&tls_image, tcb_size)?; + unsafe { set_thread_pointer(tp) }?; + setup_dtv(&tls_image, base)?; + } + + Ok(Some(tls_image)) +} + +/// Initializes thread-local storage for this thread from a [TlsImage]. If no image is provided, +/// a dummy (platform-specific) TLS copy will be created. +pub fn init_tls(image: Option<&TlsImage>, force: bool, tcb_size: usize) -> Result<(), Error> { + let Some(image) = image else { + crate::debug_trace!("TODO: handle executables with no TLS"); + return Err(Error::NotImplemented); + }; + + if force || !image.already_initialized { + let (base, tp) = clone_tls(image, tcb_size)?; + unsafe { set_thread_pointer(tp) }?; + setup_dtv(&image, base)?; + } + + Ok(()) +} + +fn get_tcb_raw() -> *mut TcbHeader { + let tp = imp::get_thread_pointer(); + layout::get_tcb_raw(tp).cast() +} + +fn get_tcb_mut() -> &'static mut TcbHeader { + unsafe { &mut *get_tcb_raw() } +} + +fn setup_dtv(image: &TlsImage, tls_base: usize) -> Result<(), Error> { + if image.module_offsets.is_empty() { + return Ok(()); + } + let dtv = get_dtv(); + for &(module_id, module_offset) in image.module_offsets.iter() { + assert!(module_offset < image.full_size); + dtv.set( + module_id, + core::ptr::with_exposed_provenance_mut(tls_base + module_offset), + )?; + } + Ok(()) +} + +/// Returns the pointer to the TCB contents after the self-pointer and the DTV +pub fn get_tcb() -> *mut u8 { + unsafe { get_tcb_raw().add(1).cast() } +} + +/// Gets or allocates a DTV for current thread. +pub fn get_dtv() -> &'static mut Dtv { + let tcb = get_tcb_mut(); + if tcb.dtv_pointer.is_null() { + let dtv = Box::new(Dtv::new()); + let dtv = Box::into_raw(dtv); + tcb.dtv_pointer = dtv; + } + unsafe { &mut *tcb.dtv_pointer } +} + +#[cfg(any(feature = "__tls_get_addr", rust_analyzer))] +#[no_mangle] +unsafe extern "C" fn __tls_get_addr(index: *mut usize) -> *mut c_void { + let module_id = index.read(); + let offset = index.add(1).read(); + assert!(module_id > 0); + + get_dtv().get(module_id).add(offset) +} diff --git a/lib/runtime/src/process/thread_local/variant1.rs b/lib/runtime/src/process/thread_local/variant1.rs new file mode 100644 index 00000000..bb33203b --- /dev/null +++ b/lib/runtime/src/process/thread_local/variant1.rs @@ -0,0 +1,13 @@ +use abi::error::Error; + +use super::TlsImage; + +/// Creates a new TLS image in the process memory, copying data from the TLS master copy (if any). +/// Returns the resulting thread pointer. +pub fn clone_tls(image: &TlsImage, tcb_size: usize) -> Result<(usize, usize), Error> { + Err(Error::NotImplemented) +} + +pub(super) fn get_tcb_raw(_tp: usize) -> *mut u8 { + todo!() +} diff --git a/lib/runtime/src/process/thread_local/variant2.rs b/lib/runtime/src/process/thread_local/variant2.rs new file mode 100644 index 00000000..1a56c20e --- /dev/null +++ b/lib/runtime/src/process/thread_local/variant2.rs @@ -0,0 +1,85 @@ +use abi::{ + error::Error, + mem::{MappingFlags, MappingSource}, +}; + +use super::TlsImage; + +// Variant II TLS layout: +// +// ... [ MODULE ] [ MODULE ] [ MODULE ] [ TCB ] +// | | | | +// | | | | +// off3 off2 off1 tp + +/// Creates a new TLS image in the process memory, copying data from the TLS master copy (if any). +/// Returns the resulting thread pointer. +pub fn clone_tls(image: &TlsImage, tcb_size: usize) -> Result<(usize, usize), Error> { + // Basically, the layout is: + // * align(image.full_size) below the TP + // * tcb_size starting with the TP + + // If the caller asked for no TCB to be placed, just reserve two words, the first one + // will be used to put the self-pointer + let tcb_size = if tcb_size < size_of::() * 2 { + size_of::() * 2 + } else { + (tcb_size + size_of::() - 1) & !(size_of::() - 1) + }; + + if !image.align.is_power_of_two() { + panic!("TLS layout not aligned to a power of two: {}", image.align); + } + if image.align > 0x1000 { + todo!("Cannot yet handle TLS block alignmnent larger than page size") + } + let align = image.align; + + let aligned_size = (image.full_size + tcb_size + align - 1) & !(align - 1); + let page_aligned_size = (aligned_size + 0xFFf) & !0xFFF; + + let base = unsafe { + crate::sys::map_memory( + None, + page_aligned_size, + MappingFlags::WRITE, + &MappingSource::Anonymous, + ) + }?; + + let ptr = core::ptr::with_exposed_provenance_mut::(base); + + let module_data = unsafe { core::slice::from_raw_parts_mut(ptr, image.full_size) }; + let tcb = unsafe { core::slice::from_raw_parts_mut(ptr.add(image.full_size), tcb_size) }; + + // Clone the Master Copy, if any + let data_size = if let Some(master_copy) = image.master_copy { + let source = unsafe { master_copy.as_ref() }; + module_data[..source.len()].copy_from_slice(source); + source.len() + } else { + 0 + }; + + // Zero the rest + module_data[data_size..image.full_size].fill(0); + + // Place the self-pointer to point at the TP itself + let tp = base + image.full_size; + assert_eq!( + tp % size_of::(), + 0, + "TLS self-pointer should at least be word-aligned" + ); + tcb[..size_of::()].copy_from_slice(&tp.to_ne_bytes()); + + // Zero the TCB after the self-pointer + tcb[size_of::()..].fill(0); + + Ok((base, tp)) +} + +// In Variant II, the TP points directly at the TCB start +pub(super) fn get_tcb_raw(tp: usize) -> *mut u8 { + core::ptr::with_exposed_provenance_mut(tp) +} diff --git a/lib/runtime/src/process/thread_local/x86_64.rs b/lib/runtime/src/process/thread_local/x86_64.rs new file mode 100644 index 00000000..ce2b1001 --- /dev/null +++ b/lib/runtime/src/process/thread_local/x86_64.rs @@ -0,0 +1,15 @@ +#![allow(missing_docs)] + +use abi::{error::Error, process::ThreadOption}; + +pub fn get_thread_pointer() -> usize { + let tp: usize; + unsafe { + core::arch::asm!("mov %fs:0, {0}", out(reg) tp, options(att_syntax)); + } + tp +} + +pub unsafe fn set_thread_pointer(value: usize) -> Result<(), Error> { + crate::sys::set_thread_option(&ThreadOption::ThreadPointer(value)) +} diff --git a/lib/runtime/src/process/x86_64.rs b/lib/runtime/src/process/x86_64.rs deleted file mode 100644 index 45aa628b..00000000 --- a/lib/runtime/src/process/x86_64.rs +++ /dev/null @@ -1,35 +0,0 @@ -#![allow(missing_docs)] - -pub fn get_tls() -> usize { - let tp: usize; - unsafe { - core::arch::asm!("mov %fs:0, {0}", out(reg) tp, options(att_syntax)); - } - tp -} - -pub unsafe fn set_pthread_self(value: usize) { - let tls: *mut usize = core::ptr::with_exposed_provenance_mut(get_tls()); - if tls.is_null() { - panic!("TLS pointer is NULL"); - } - tls.add(1).write(value); -} - -pub fn pthread_self() -> usize { - let tls: *const usize = core::ptr::with_exposed_provenance(get_tls()); - if tls.is_null() { - panic!("TLS pointer is NULL"); - } - unsafe { tls.add(1).read() } -} - -pub fn thread_local_area_impl() -> &'static mut [*mut u8] { - let tp = get_tls(); - - // Skip self-pointer - assert_eq!(tp % size_of::(), 0); - let tp = tp + size_of::() * 2; - - unsafe { super::thread_local_area_common(tp) } -} diff --git a/test.c b/test.c index f3eb283c..76698e10 100644 --- a/test.c +++ b/test.c @@ -2,14 +2,22 @@ #include #include +_Thread_local int x = 123; + static void *function(void *arg) { printf("This is thread %u\n", pthread_self()); printf("argument: %s\n", (const char *) arg); + printf("[child] x = %d\n", x); + x += 1; + printf("[child] x = %d\n", x); return NULL; } int main(int argc, const char **argv) { pthread_t thread; + printf("[main] x = %d\n", x); + x = 321; + printf("[main] x = %d\n", x); assert(pthread_create(&thread, NULL, function, (void *) "thread1") == 0); diff --git a/test.cpp b/test.cpp index 29626e7a..230c9057 100644 --- a/test.cpp +++ b/test.cpp @@ -1,4 +1,6 @@ #include +#include +#include int main() { std::vector items; @@ -9,5 +11,11 @@ int main() { for (const auto &item: items) { std::cout << "Item: " << item << std::endl; } + + std::cout << "&errno = " << &errno << std::endl; + std::cout << "errno = " << errno << std::endl; + open("/xxxsaa", 0, 0); + std::cout << "errno = " << errno << std::endl; + return 0; } diff --git a/userspace/Cargo.lock b/userspace/Cargo.lock index 5a8249ed..be69e51f 100644 --- a/userspace/Cargo.lock +++ b/userspace/Cargo.lock @@ -375,6 +375,7 @@ dependencies = [ name = "dyn-loader" version = "0.1.0" dependencies = [ + "bytemuck", "elf", "thiserror", "yggdrasil-rt", diff --git a/userspace/dyn-loader/Cargo.toml b/userspace/dyn-loader/Cargo.toml index 800c08f5..9cf3f89e 100644 --- a/userspace/dyn-loader/Cargo.toml +++ b/userspace/dyn-loader/Cargo.toml @@ -8,6 +8,7 @@ yggdrasil-rt.workspace = true thiserror.workspace = true elf = "0.7.4" +bytemuck = "1.19.0" [lints.rust] unexpected_cfgs = { level = "allow", check-cfg = ['cfg(rust_analyzer)'] } diff --git a/userspace/dyn-loader/src/builtins.rs b/userspace/dyn-loader/src/builtins.rs new file mode 100644 index 00000000..e69de29b diff --git a/userspace/dyn-loader/src/env.rs b/userspace/dyn-loader/src/env.rs new file mode 100644 index 00000000..0719ebcd --- /dev/null +++ b/userspace/dyn-loader/src/env.rs @@ -0,0 +1,99 @@ +use bytemuck::Pod; +use yggdrasil_rt::{mem::MappingFlags, process::{auxv, AuxValue, ProgramArgumentInner}}; + +use crate::{error::Error, mapping::Mapping, thread_local::{TlsImage, TlsLayout}}; + +struct ArgPlacer<'a> { + buffer: &'a mut [u8], + position: usize, +} + +impl<'a> ArgPlacer<'a> { + fn new(buffer: &'a mut [u8]) -> Self { + Self { + buffer, + position: 0, + } + } + + fn align(&mut self, align: usize) -> Result<(), Error> { + debug_assert!(align.is_power_of_two()); + let aligned = (self.position + align - 1) & !(align - 1); + if aligned > self.buffer.len() { + todo!() + } + self.position = aligned; + Ok(()) + } + + fn put_str(&mut self, s: &str) -> Result { + if self.position + s.len() >= self.buffer.len() { + todo!() + } + let off = self.position; + self.buffer[off..off + s.len()].copy_from_slice(s.as_bytes()); + self.buffer[off + s.len()] = 0; + self.position += s.len() + 1; + Ok(off) + } + + fn put(&mut self, v: &T) -> Result { + if self.position + size_of::() > self.buffer.len() { + todo!() + } + let off = self.position; + self.buffer[off..off + size_of::()].copy_from_slice(bytemuck::bytes_of(v)); + self.position += size_of::(); + Ok(off) + } + + fn put_aux_array(&mut self, s: &[AuxValue]) -> Result { + self.align(size_of::())?; + let off = self.position; + for item in s { + self.put(&item.tag)?; + self.put(&item.val)?; + } + self.put(&auxv::NULL)?; + self.put(&0u64)?; + Ok(off) + } + + fn put_ptr_array(&mut self, s: &[usize]) -> Result { + self.align(size_of::())?; + let off = self.position; + for item in s { + self.put(item)?; + } + self.put(&0usize)?; + Ok(off) + } +} + + +pub fn build_argument(args: &[String], auxv: &[AuxValue]) -> Result { + let mut buffer = Mapping::new(0x1000, MappingFlags::WRITE)?; + let arg_base = buffer.as_ptr().addr(); + let mut placer = ArgPlacer::new(&mut buffer[..]); + + let mut argv = vec![]; + + for arg in args { + argv.push(placer.put_str(arg)? + arg_base); + } + + // TODO env + let argv = placer.put_ptr_array(&argv)? + arg_base; + let envp = placer.put_ptr_array(&[])? + arg_base; + let auxv = placer.put_aux_array(&auxv)? + arg_base; + + let argument = placer.position + arg_base; + + placer.put(&argv)?; + placer.put(&envp)?; + placer.put(&auxv)?; + + buffer.leak(); + + Ok(argument) +} diff --git a/userspace/dyn-loader/src/main.rs b/userspace/dyn-loader/src/main.rs index c5045308..8c73c081 100644 --- a/userspace/dyn-loader/src/main.rs +++ b/userspace/dyn-loader/src/main.rs @@ -1,43 +1,31 @@ -#![feature(yggdrasil_os, never_type, map_try_insert, slice_ptr_get)] +#![feature(yggdrasil_os, never_type, map_try_insert, slice_ptr_get, iter_chain)] -use std::{collections::HashMap, env, process::ExitCode}; +use std::process::ExitCode; use error::Error; use object::Object; -use state::State; -use yggdrasil_rt::process::ProgramArgumentInner; +use state::ObjectSet; +use yggdrasil_rt::process::{auxv, AuxValue}; +pub mod builtins; +pub mod env; pub mod error; -pub mod object; pub mod mapping; -pub mod state; +pub mod object; pub mod relocation; pub mod search; +pub mod state; +pub mod thread_local; fn run(binary: &str, args: &[String]) -> Result { - let mut state = State::new(); - let mut root = Object::open(binary)?; - let mut libraries = HashMap::new(); - for needed in root.needed() { - // TODO load needed of needed - let path = search::find_library(needed.as_str())?; - if libraries.contains_key(&path) { - continue; - } + // Open root and its dependencies + let root = Object::open(binary)?; + let mut objects = ObjectSet::new(root); + objects.open_root_dependencies()?; + let tls_image = objects.load_all()?; + objects.resolve_symbols()?; - let object = Object::open(&path)?; - libraries.insert(path, object); - } - root.load(&mut state)?; - for (_, lib) in libraries.iter_mut() { - lib.load(&mut state)?; - } - root.resolve_symbols(&mut state); - for (_, lib) in libraries.iter_mut() { - lib.resolve_symbols(&mut state); - } - - if let Err(undefined) = state.no_undefined_symbols() { + if let Err(undefined) = objects.state.no_undefined_symbols() { eprintln!("Undefined symbols:"); for (path, syms) in undefined { eprintln!(" in {path:?}"); @@ -48,42 +36,74 @@ fn run(binary: &str, args: &[String]) -> Result { return Err(Error::UnresolvedSymbols); } - root.relocate(&mut state)?; - for (_, lib) in libraries.iter_mut() { - lib.relocate(&mut state)?; + objects.relocate()?; + + let mut auxv = vec![]; + if let Some(image) = tls_image { + let layout = objects.state.tls_layout.as_ref().unwrap(); + + let size = image.size; + let base = image.data.leak(); + + auxv.push(AuxValue { + tag: auxv::TLS_MASTER_COPY, + val: base as _, + }); + auxv.push(AuxValue { + tag: auxv::TLS_ALIGN, + val: image.align as _, + }); + auxv.push(AuxValue { + tag: auxv::TLS_DATA_SIZE, + val: size as _, + }); + auxv.push(AuxValue { + tag: auxv::TLS_FULL_SIZE, + val: size as _, + }); + + for module in layout.segments.iter() { + if module.object_id == 0 { + continue; + } + + auxv.push(AuxValue { + tag: auxv::TLS_MODULE_ID, + val: module.object_id as _, + }); + auxv.push(AuxValue { + tag: auxv::TLS_MODULE_OFFSET, + val: module.offset as _, + }); + } } - // Call global constructors before handing off control to the program - // .preinit_array, .preinit, ... - for (_, lib) in libraries.iter_mut() { - unsafe { lib.call_early_constructors() }; - } - unsafe { root.call_early_constructors() }; - // .init_array, .init, ... - for (_, lib) in libraries.iter_mut() { - unsafe { lib.call_constructors() }; - } - unsafe { root.call_constructors() }; + // Set up TLS for the main thread. This needs to be done here, because + // we still need to call initializers before handing off control to the executable + // TODO use an ELF note to indicate the extra size required in the TCB + yggdrasil_rt::process::thread_local::init_tls_from_auxv(auxv.iter(), true, size_of::() * 4) + .expect("Could not initialize main thread TLS for the executable"); - let entry = root.entry().ok_or(Error::NoEntrypoint)?; + auxv.push(AuxValue { + tag: auxv::TLS_ALREADY_INITIALIZED, + val: 0, + }); + + // Call initializers + objects.initialize(); + + // Setup arguments and enter the main object + let entry = objects.root.entry().ok_or(Error::NoEntrypoint)?; debug_trace!("entry = {:p}", entry); - let args = args.iter().map(|s| s.as_str()).collect::>(); - // TODO - let envs = vec![]; + let argument = env::build_argument(args, &auxv)?; - let arg = Box::new(ProgramArgumentInner { - args: &args, - env: &envs - }); - let arg = Box::into_raw(arg).addr(); - - entry(arg); + entry(argument); unreachable!() } fn main() -> ExitCode { - let args: Vec = env::args().skip(1).collect(); + let args: Vec = std::env::args().skip(1).collect(); if args.is_empty() { // Dump help and exit diff --git a/userspace/dyn-loader/src/mapping.rs b/userspace/dyn-loader/src/mapping.rs index 247d0581..14f9bfcb 100644 --- a/userspace/dyn-loader/src/mapping.rs +++ b/userspace/dyn-loader/src/mapping.rs @@ -1,6 +1,5 @@ use std::{ - ops::{Deref, DerefMut, Range}, - ptr::NonNull, + mem, ops::{Deref, DerefMut, Range}, ptr::NonNull }; use yggdrasil_rt::mem::{MappingFlags, MappingSource}; @@ -23,6 +22,12 @@ impl Mapping { Ok(Self { data }) } + pub fn leak(self) -> usize { + let base = self.data.addr().into(); + mem::forget(self); + base + } + pub fn qword(&mut self, offset: u64) -> NonNull { unsafe { self.data.as_non_null_ptr().add(offset as usize).cast() } } diff --git a/userspace/dyn-loader/src/object.rs b/userspace/dyn-loader/src/object.rs index 097c0ff5..03e2976f 100644 --- a/userspace/dyn-loader/src/object.rs +++ b/userspace/dyn-loader/src/object.rs @@ -13,9 +13,10 @@ use elf::{ DF_1_PIE, DT_FINI, DT_FINI_ARRAY, DT_FINI_ARRAYSZ, DT_FLAGS_1, DT_INIT, DT_INIT_ARRAY, DT_INIT_ARRAYSZ, DT_NEEDED, DT_PREINIT_ARRAY, DT_PREINIT_ARRAYSZ, ET_DYN, ET_EXEC, PT_DYNAMIC, PT_GNU_EH_FRAME, PT_GNU_RELRO, PT_GNU_STACK, PT_INTERP, PT_LOAD, PT_NOTE, - PT_NULL, PT_PHDR, SHN_UNDEF, SHT_REL, SHT_RELA, STB_GLOBAL, STB_LOCAL, STB_WEAK, + PT_NULL, PT_PHDR, PT_TLS, SHN_UNDEF, SHT_REL, SHT_RELA, STB_GLOBAL, STB_LOCAL, STB_WEAK, }, endian::AnyEndian, + segment::ProgramHeader, symbol::Symbol, ElfStream, }; @@ -25,7 +26,7 @@ use crate::{ error::Error, mapping::Mapping, relocation::{Relocation, RelocationValue}, - state::{ExportedSymbol, State}, + state::{ExportedNormalSymbol, ExportedSymbol, ExportedTlsSymbol, State}, thread_local::TlsLayout, }; pub enum ElfType { @@ -36,7 +37,8 @@ pub enum ElfType { } pub enum ResolvedSymbol<'s> { - Global(&'s ExportedSymbol), + Global(&'s ExportedNormalSymbol), + Tls(&'s ExportedTlsSymbol), Local, Null, } @@ -57,6 +59,10 @@ pub struct Object { pub ty: ElfType, needed: Vec, + pub tls: Option, + pub tls_offset: Option, + pub id: u32, + mapping: Option, dynamic_symbol_array: Vec, @@ -69,6 +75,7 @@ impl ResolvedSymbol<'_> { match *self { Self::Local => todo!(), Self::Global(sym) => sym.value as i64, + Self::Tls(_) => 0, Self::Null => 0, } } @@ -137,6 +144,16 @@ impl Object { } } + let tls_segments = elf + .segments() + .iter() + .filter(|p| p.p_type == PT_TLS) + .collect::>(); + if tls_segments.len() > 1 { + todo!("Support objects with more than one TLS segment"); + } + let tls = tls_segments.get(0).map(|s| **s); + Ok(Self { path, file, @@ -147,6 +164,10 @@ impl Object { vma_end, needed, + tls, + tls_offset: None, + id: 0, + mapping: None, dynamic_symbol_array, @@ -155,6 +176,10 @@ impl Object { }) } + pub fn set_id(&mut self, id: u32) { + self.id = id; + } + pub fn load(&mut self, state: &mut State) -> Result<(), Error> { // Already loaded if self.mapping.is_some() { @@ -236,6 +261,8 @@ impl Object { segment_data[file_size..].fill(0); } + // Handled separately + PT_TLS => (), // TODO handle GNU_STACK PT_DYNAMIC | PT_GNU_RELRO | PT_GNU_STACK | PT_INTERP | PT_PHDR | PT_GNU_EH_FRAME | PT_NOTE => (), @@ -243,6 +270,24 @@ impl Object { } } + self.mapping = Some(mapping); + Ok(()) + } + + pub fn place_tls_copy(&mut self, dst: &mut [u8]) -> Result<(), Error> { + let tls = self.tls.as_ref().unwrap(); + if tls.p_filesz > 0 { + self.file.seek(SeekFrom::Start(tls.p_offset))?; + self.file.read_exact(&mut dst[..tls.p_filesz as usize])?; + } + // Zero the rest + dst[tls.p_filesz as usize..].fill(0); + Ok(()) + } + + pub fn export_symbols(&self, state: &mut State) -> Result<(), Error> { + let mapping = self.mapping.as_ref().ok_or(Error::NotLoaded)?; + // Export dynamic symbols into the linking table for dynsym in self.dynamic_symbol_array.iter() { // Don't export undefined symbols @@ -257,14 +302,13 @@ impl Object { // S: "image" load base let offset = mapping.base() as isize - self.vma_start as isize; match dynsym.raw.st_bind() { - STB_GLOBAL => state.export(&self.path, dynsym, offset, false), - STB_WEAK => state.export(&self.path, dynsym, offset, true), + STB_GLOBAL => state.export(&self.path, dynsym, offset, false, self.id), + STB_WEAK => state.export(&self.path, dynsym, offset, true, self.id), STB_LOCAL => todo!(), _ => todo!(), } } - self.mapping = Some(mapping); Ok(()) } @@ -350,7 +394,10 @@ impl Object { for rela in rela_section { let dynsym = &self.dynamic_symbol_array[rela.r_sym as usize]; let sym = match dynsym.raw.st_bind() { - STB_GLOBAL | STB_WEAK => ResolvedSymbol::Global(state.lookup(dynsym).unwrap()), + STB_GLOBAL | STB_WEAK => match state.lookup(dynsym).unwrap() { + ExportedSymbol::Normal(export) => ResolvedSymbol::Global(export), + ExportedSymbol::Tls(export) => ResolvedSymbol::Tls(export), + } STB_LOCAL => { if dynsym.name.is_empty() { ResolvedSymbol::Null @@ -361,7 +408,7 @@ impl Object { _ => todo!(), }; - if let Some(value) = rela.resolve(&sym, mapping.base())? { + if let Some(value) = rela.resolve(&state, &dynsym.name, &sym, mapping.base())? { value.write(mapping, rela.r_offset); } } diff --git a/userspace/dyn-loader/src/relocation/mod.rs b/userspace/dyn-loader/src/relocation/mod.rs index 6db633fb..96f912c4 100644 --- a/userspace/dyn-loader/src/relocation/mod.rs +++ b/userspace/dyn-loader/src/relocation/mod.rs @@ -1,4 +1,4 @@ -use crate::{error::Error, mapping::Mapping, object::ResolvedSymbol}; +use crate::{error::Error, mapping::Mapping, object::ResolvedSymbol, state::State}; #[cfg(any(target_arch = "x86_64", rust_analyzer))] mod x86_64; @@ -14,7 +14,13 @@ pub trait RelocationValue { pub trait Relocation { type Value: RelocationValue; - fn resolve(&self, symbol: &ResolvedSymbol, load_base: usize) -> Result, Error>; + fn resolve( + &self, + state: &State, + name: &str, + symbol: &ResolvedSymbol, + load_base: usize, + ) -> Result, Error>; } impl RelocationValue for RelaValue { diff --git a/userspace/dyn-loader/src/relocation/x86_64.rs b/userspace/dyn-loader/src/relocation/x86_64.rs index 8ccf2bc4..7b98c031 100644 --- a/userspace/dyn-loader/src/relocation/x86_64.rs +++ b/userspace/dyn-loader/src/relocation/x86_64.rs @@ -3,12 +3,16 @@ use std::path::Path; use elf::{ abi::{ R_X86_64_64, R_X86_64_DTPMOD64, R_X86_64_DTPOFF64, R_X86_64_GLOB_DAT, R_X86_64_JUMP_SLOT, - R_X86_64_RELATIVE, + R_X86_64_RELATIVE, R_X86_64_TPOFF64, }, relocation::Rela, }; -use crate::{error::Error, object::{DynamicSymbol, ResolvedSymbol}, state::State}; +use crate::{ + error::Error, + object::{DynamicSymbol, ResolvedSymbol}, + state::State, +}; use super::{RelaValue, Relocation}; @@ -17,24 +21,47 @@ impl Relocation for Rela { fn resolve( &self, + state: &State, + name: &str, symbol: &ResolvedSymbol, load_base: usize, ) -> Result, Error> { - let s = symbol.value() as i64; - if s == 0 && self.r_type != R_X86_64_RELATIVE { - todo!() - } - match self.r_type { - // S - R_X86_64_JUMP_SLOT | R_X86_64_GLOB_DAT => Ok(Some(RelaValue::QWord(s))), - // S + A - R_X86_64_64 => Ok(Some(RelaValue::QWord(s + self.r_addend))), - // B + A - R_X86_64_RELATIVE => Ok(Some(RelaValue::QWord(load_base as i64 + self.r_addend))), - // TLS - R_X86_64_DTPOFF64 => todo!(), - R_X86_64_DTPMOD64 => todo!(), - _ => todo!(), + match symbol { + ResolvedSymbol::Tls(tls) => { + match self.r_type { + // Object ID (index into DTV) of the object containing this symbol + R_X86_64_DTPMOD64 => Ok(Some(RelaValue::QWord(tls.module_id as _))), + R_X86_64_DTPOFF64 => Ok(Some(RelaValue::QWord(tls.offset as _))), + R_X86_64_TPOFF64 => { + // Need to extract fixed global offset + let tls_layout = state.tls_layout.as_ref().unwrap(); + // Offset from TLS start + let offset = tls_layout.offset(tls.module_id, tls.offset).unwrap(); + let offset_from_tp = -((tls_layout.tp_offset - offset) as i64); + debug_trace!("{}@tpoff -> {}", name, offset_from_tp); + + Ok(Some(RelaValue::QWord(offset_from_tp))) + }, + _ => todo!(), + } + } + _ => { + let s = symbol.value() as i64; + if s == 0 && self.r_type != R_X86_64_RELATIVE { + todo!() + } + match self.r_type { + // S + R_X86_64_JUMP_SLOT | R_X86_64_GLOB_DAT => Ok(Some(RelaValue::QWord(s))), + // S + A + R_X86_64_64 => Ok(Some(RelaValue::QWord(s + self.r_addend))), + // B + A + R_X86_64_RELATIVE => { + Ok(Some(RelaValue::QWord(load_base as i64 + self.r_addend))) + } + _ => todo!(), + } + } } } // fn resolve<'a, F: Fn(u32) -> Result<&'a DynamicSymbol, Error>>( diff --git a/userspace/dyn-loader/src/state.rs b/userspace/dyn-loader/src/state.rs index c0c9ca97..a551a74e 100644 --- a/userspace/dyn-loader/src/state.rs +++ b/userspace/dyn-loader/src/state.rs @@ -6,31 +6,72 @@ use std::{ use elf::abi::{STT_FUNC, STT_NOTYPE, STT_OBJECT, STT_TLS}; -use crate::object::DynamicSymbol; +use crate::{ + error::Error, + object::{DynamicSymbol, Object}, + search, + thread_local::{self, TlsImage, TlsLayout}, +}; -pub struct ExportedSymbol { +pub struct ExportedNormalSymbol { pub source: PathBuf, pub value: usize, - pub weak: bool + pub weak: bool, +} + +pub struct ExportedTlsSymbol { + pub source: PathBuf, + // Offset from TLS block start + pub offset: usize, + // TLS module ID + pub module_id: u32, + pub weak: bool, +} + +pub enum ExportedSymbol<'a> { + Normal(&'a ExportedNormalSymbol), + Tls(&'a ExportedTlsSymbol), } pub struct State { - symbol_table: HashMap, ExportedSymbol>, - undefined_references: HashMap>> + symbol_table: HashMap, ExportedNormalSymbol>, + tls_symbol_table: HashMap, ExportedTlsSymbol>, + undefined_references: HashMap>>, + pub tls_layout: Option, +} + +pub struct ObjectSet { + pub root: Object, + libraries: HashMap, + library_path_map: HashMap, + + // Linker state + pub state: State, + + last_object_id: u32, } impl State { pub fn new() -> Self { Self { symbol_table: HashMap::new(), + tls_symbol_table: HashMap::new(), undefined_references: HashMap::new(), + tls_layout: None, } } - pub fn export(&mut self, source: impl AsRef, sym: &DynamicSymbol, offset: isize, weak: bool) { + pub fn export( + &mut self, + source: impl AsRef, + sym: &DynamicSymbol, + offset: isize, + weak: bool, + object_id: u32, + ) { + let source = source.as_ref().to_owned(); match sym.raw.st_symtype() { STT_FUNC | STT_OBJECT | STT_NOTYPE => { - let source = source.as_ref().to_owned(); let value: usize = (isize::try_from(sym.raw.st_value).unwrap() + offset) .try_into() .unwrap(); @@ -38,16 +79,53 @@ impl State { match self.symbol_table.get_mut(&sym.name) { // Stronger binding exported Some(export) if export.weak && !weak => { - *export = ExportedSymbol { source, value, weak }; - }, + *export = ExportedNormalSymbol { + source, + value, + weak, + }; + } // Do nothing, already strong or already weak Some(_) => (), None => { - self.symbol_table.insert(sym.name.clone(), ExportedSymbol { source, value, weak }); + self.symbol_table.insert( + sym.name.clone(), + ExportedNormalSymbol { + source, + value, + weak, + }, + ); + } + } + } + STT_TLS => { + let offset = sym.raw.st_value as usize; + + match self.tls_symbol_table.get_mut(&sym.name) { + Some(export) if export.weak && !weak => { + *export = ExportedTlsSymbol { + source, + offset, + module_id: object_id, + weak, + }; + } + Some(_) => (), + None => { + debug_trace!("{:?}: TLS {:?} -> {}:{:#x}", source, sym.name, object_id, offset); + self.tls_symbol_table.insert( + sym.name.clone(), + ExportedTlsSymbol { + source, + offset, + module_id: object_id, + weak, + }, + ); } } } - STT_TLS => todo!(), _ => todo!(), } } @@ -58,18 +136,24 @@ impl State { if !self.symbol_table.contains_key(&sym.name) { self.undefined(source, &sym.name); } - }, - STT_TLS => todo!(), + } + STT_TLS => { + if !self.tls_symbol_table.contains_key(&sym.name) { + self.undefined(source, &sym.name); + } + } _ => todo!(), } } - pub fn lookup(&mut self, sym: &DynamicSymbol) -> Option<&ExportedSymbol> { + pub fn lookup(&self, sym: &DynamicSymbol) -> Option { match sym.raw.st_symtype() { STT_FUNC | STT_OBJECT | STT_NOTYPE => { - self.symbol_table.get(&sym.name) + self.symbol_table.get(&sym.name).map(ExportedSymbol::Normal) + } + STT_TLS => { + self.tls_symbol_table.get(&sym.name).map(ExportedSymbol::Tls) }, - STT_TLS => todo!(), _ => todo!(), } } @@ -93,3 +177,106 @@ impl State { } } } + +impl ObjectSet { + pub fn new(mut root: Object) -> Self { + root.set_id(0); + Self { + root, + libraries: HashMap::new(), + library_path_map: HashMap::new(), + state: State::new(), + last_object_id: 0, + } + } + + pub fn open_root_dependencies(&mut self) -> Result<(), Error> { + for needed in self.root.needed() { + // TODO load needed of needed + let path = search::find_library(needed.as_str())?; + if self.library_path_map.contains_key(&path) { + continue; + } + let mut object = Object::open(&path)?; + self.last_object_id += 1; + let id = self.last_object_id; + object.id = id; + self.libraries.insert(id, object); + self.library_path_map.insert(path, id); + } + Ok(()) + } + + pub fn load_all(&mut self) -> Result, Error> { + // Load all objects somewhere, order doesn't matter + self.root.load(&mut self.state)?; + for (_, lib) in self.libraries.iter_mut() { + lib.load(&mut self.state)?; + } + + // Build a TLS master copy somewhere + match thread_local::build_tls_image(&mut self.root, &mut self.libraries)? { + Some((tls_image, tls_layout)) => { + self.state.tls_layout = Some(tls_layout); + Ok(Some(tls_image)) + } + None => Ok(None), + } + } + + pub fn resolve_symbols(&mut self) -> Result<(), Error> { + // Export all symbols + self.root.export_symbols(&mut self.state)?; + for (_, lib) in self.libraries.iter_mut() { + lib.export_symbols(&mut self.state)?; + } + + // Resolve all symbols + self.root.resolve_symbols(&mut self.state); + for (_, lib) in self.libraries.iter_mut() { + lib.resolve_symbols(&mut self.state); + } + + Ok(()) + } + + pub fn relocate(&mut self) -> Result<(), Error> { + self.root.relocate(&mut self.state)?; + for (_, lib) in self.libraries.iter_mut() { + lib.relocate(&mut self.state)?; + } + Ok(()) + } + + pub fn initialize(&mut self) { + // Call global constructors before handing off control to the program + // .preinit_array, .preinit, ... + for (_, lib) in self.libraries.iter_mut() { + unsafe { lib.call_early_constructors() }; + } + unsafe { self.root.call_early_constructors() }; + // .init_array, .init, ... + for (_, lib) in self.libraries.iter_mut() { + unsafe { lib.call_constructors() }; + } + unsafe { self.root.call_constructors() }; + } + + pub fn is_library_loaded(&self, p: impl AsRef) -> bool { + self.library_path_map.contains_key(p.as_ref()) + } + + pub fn get(&self, id: u32) -> Option<&Object> { + match id { + 0 => Some(&self.root), + _ => self.libraries.get(&id), + } + } + + pub fn get_mut(&mut self, id: u32) -> Option<&mut Object> { + match id { + 0 => Some(&mut self.root), + _ => self.libraries.get_mut(&id), + } + } +} diff --git a/userspace/dyn-loader/src/thread_local/mod.rs b/userspace/dyn-loader/src/thread_local/mod.rs new file mode 100644 index 00000000..de4e5706 --- /dev/null +++ b/userspace/dyn-loader/src/thread_local/mod.rs @@ -0,0 +1,93 @@ +#[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))] +mod variant2; +use std::collections::HashMap; + +use elf::segment::ProgramHeader; +#[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))] +pub use variant2::TlsLayoutImpl; +use yggdrasil_rt::mem::MappingFlags; + +use crate::{error::Error, mapping::Mapping, object::Object}; + +pub trait TlsLayoutBuilder { + fn build(align: usize, root: &Object, libraries: &HashMap) -> TlsLayout; +} + +#[derive(Debug)] +pub struct TlsLayout { + pub segments: Vec, + pub total_size: usize, + pub tp_offset: usize, +} + +#[derive(Debug)] +pub struct TlsSegment { + pub offset: usize, + pub size: usize, + pub object_id: u32, +} + +pub struct TlsImage { + // TLS master image that will be passed to the program + pub data: Mapping, + pub size: usize, + pub align: usize, + pub tp_offset: usize, +} + +impl TlsLayout { + pub fn offset(&self, object_id: u32, offset: usize) -> Option { + let segment = self.segments.iter().find(|seg| seg.object_id == object_id)?; + Some(segment.offset + offset) + } +} + +pub fn build_tls_image( + root: &mut Object, + libraries: &mut HashMap, +) -> Result, Error> { + // Find max align + let mut align = usize::MIN; + if let Some(tls) = root.tls.as_ref() { + align = align.max(tls.p_align as usize); + } + for tls in libraries.values().filter_map(|lib| lib.tls.as_ref()) { + align = align.max(tls.p_align as usize); + } + + // No TLS segment in any of the objects + if align == usize::MIN { + return Ok(None); + } + let align = align.max(size_of::()); + + let layout = TlsLayoutImpl::build(align, root, libraries); + + // Create a master image based on this layout + let mut image_data = Mapping::new(layout.total_size, MappingFlags::empty())?; + let tp_offset = layout.tp_offset; + for (i, segment) in layout.segments.iter().enumerate() { + debug_trace!( + "Load TLS segment: tlsoffset_i={:#x?}, module_id={}", + segment.offset..segment.offset + segment.size, + segment.object_id + ); + + let object = match segment.object_id { + 0 => &mut *root, + _ => libraries.get_mut(&segment.object_id).unwrap(), + }; + object.tls_offset = Some(segment.offset); + object.place_tls_copy(&mut image_data[segment.offset..segment.offset + segment.size])?; + } + + Ok(Some(( + TlsImage { + data: image_data, + size: layout.total_size, + align, + tp_offset, + }, + layout, + ))) +} diff --git a/userspace/dyn-loader/src/thread_local/variant2.rs b/userspace/dyn-loader/src/thread_local/variant2.rs new file mode 100644 index 00000000..a6d36aae --- /dev/null +++ b/userspace/dyn-loader/src/thread_local/variant2.rs @@ -0,0 +1,65 @@ +use std::collections::HashMap; + +use elf::segment::ProgramHeader; + +use crate::object::Object; + +use super::{TlsLayout, TlsLayoutBuilder, TlsSegment}; + +pub struct TlsLayoutImpl; + +impl TlsLayoutBuilder for TlsLayoutImpl { + fn build(align: usize, root: &Object, libraries: &HashMap) -> TlsLayout { + let mut total_size = 0; + // Layout: + // + // [ Module N | padding ] ... [ Module 1 | padding ] [ padding | Module 0 ] [ TP ] + + let mut modules = vec![]; + + if let Some(tls) = root.tls.as_ref() { + let size = tls.p_memsz as usize; + let aligned_size = (size + align - 1) & !(align - 1); + let pad = aligned_size - size; + + // module_id, offset from tp, full size + modules.push((0, size, aligned_size)); + + total_size += aligned_size; + } + + for (&id, lib) in libraries.iter() { + let Some(tls) = lib.tls.as_ref() else { + continue; + }; + + let size = tls.p_memsz as usize; + let aligned_size = (size + align - 1) & !(align - 1); + + modules.push((id, total_size + aligned_size, aligned_size)); + total_size += aligned_size; + } + + let tp_offset = total_size; + + // All the offsets calculated before are (tp_offset - x) offsets, recalculate those + // before returning the module list + for (_, off, _) in modules.iter_mut() { + *off = tp_offset - *off; + } + + let segments = modules + .into_iter() + .map(|(object_id, offset, size)| TlsSegment { + offset, + size, + object_id, + }).collect(); + + TlsLayout { + segments, + total_size, + tp_offset + } + } +} diff --git a/userspace/lib/ygglibc/Cargo.toml b/userspace/lib/ygglibc/Cargo.toml index 14ff47a9..39547d24 100644 --- a/userspace/lib/ygglibc/Cargo.toml +++ b/userspace/lib/ygglibc/Cargo.toml @@ -7,7 +7,7 @@ edition = "2021" crate-type = ["cdylib", "staticlib"] [dependencies] -yggdrasil-rt = { path = "../../../lib/runtime" } +yggdrasil-rt = { path = "../../../lib/runtime", features = ["__tls_get_addr"] } yggdrasil-abi = { path = "../../../lib/abi", features = ["alloc", "bytemuck"] } libyalloc = { path = "../../../lib/libyalloc" } diff --git a/userspace/lib/ygglibc/etc/x86_64-unknown-none.json b/userspace/lib/ygglibc/etc/x86_64-unknown-none.json index 77cc972d..ee5b4fa7 100644 --- a/userspace/lib/ygglibc/etc/x86_64-unknown-none.json +++ b/userspace/lib/ygglibc/etc/x86_64-unknown-none.json @@ -17,7 +17,7 @@ "position-independent-executables": true, "crt-static-allows-dylibs": true, - "has-thread-local": false, + "has-thread-local": true, "linker": "rust-lld", "linker-flavor": "ld.lld", diff --git a/userspace/lib/ygglibc/include/bits/errno.h b/userspace/lib/ygglibc/include/bits/errno.h index 39622d33..f71b69d9 100644 --- a/userspace/lib/ygglibc/include/bits/errno.h +++ b/userspace/lib/ygglibc/include/bits/errno.h @@ -5,7 +5,7 @@ extern "C" { #endif -extern int errno; +extern _Thread_local int errno; #if defined(__cplusplus) } diff --git a/userspace/lib/ygglibc/src/env.rs b/userspace/lib/ygglibc/src/env.rs index 0507bad6..6efcc510 100644 --- a/userspace/lib/ygglibc/src/env.rs +++ b/userspace/lib/ygglibc/src/env.rs @@ -6,17 +6,15 @@ use core::{ slice::memchr, }; -use alloc::{ffi::CString, vec::Vec}; -use yggdrasil_rt::process::ProgramArgumentInner; +use alloc::{borrow::ToOwned, ffi::CString, vec::Vec}; +use yggdrasil_rt::process::{thread_local, ProgramArgumentInner}; +// use yggdrasil_rt::process::ProgramArgumentInner; use crate::{ - allocator::{c_alloc, c_free}, - error::EResult, - headers::{ + allocator::{c_alloc, c_free}, error::EResult, headers::{ errno, string::{mem::memcpy, str::strlen}, - }, - util::PointerExt, + }, thread::{self, tls}, util::PointerExt }; #[no_mangle] @@ -78,16 +76,6 @@ unsafe fn push_env(str: NonNull) -> EResult<()> { EResult::Ok(()) } -unsafe fn setup_env(envs: impl IntoIterator) { - for &env in envs { - // Make a malloc()ed, NULL-terminated string - let entry = entry_from_raw(env.as_bytes()).expect("Couldn't allocate env variable"); - shadow.push(entry.as_ptr()); - } - shadow.push(null_mut()); - environ = shadow.as_mut_ptr(); -} - pub unsafe fn get_env(name: &[u8]) -> EResult>> { if name.is_empty() || name.contains(&b'=') { return EResult::Err(errno::EINVAL); @@ -211,18 +199,31 @@ pub unsafe fn put_env(str: NonNull) -> EResult<()> { push_env(str) } +unsafe fn setup_env<'a>(envs: impl Iterator) { + for env in envs { + // Make a malloc()ed, NULL-terminated string + let entry = entry_from_raw(env.to_bytes()).expect("Couldn't allocate env variable"); + shadow.push(entry.as_ptr()); + } + shadow.push(null_mut()); + environ = shadow.as_mut_ptr(); +} + pub fn handle_kernel_argument(arg: usize) -> Vec { let arg_ptr: *const ProgramArgumentInner = core::ptr::with_exposed_provenance(arg); let arg = unsafe { arg_ptr.ensure() }; let mut args = Vec::new(); - for &arg in arg.args { - let c_arg = CString::new(arg).unwrap(); + for arg in arg.args() { + let c_arg = arg.to_owned(); args.push(c_arg); } - unsafe { setup_env(arg.env) }; + unsafe { setup_env(arg.envs()) }; + + // Setup TLS from argument + thread::init_main_thread(arg); args } diff --git a/userspace/lib/ygglibc/src/error.rs b/userspace/lib/ygglibc/src/error.rs index 9f154d5b..3d2ccccb 100644 --- a/userspace/lib/ygglibc/src/error.rs +++ b/userspace/lib/ygglibc/src/error.rs @@ -23,6 +23,7 @@ macro impl_from_residual($($ty:ty),+) { )+ } +#[thread_local] #[no_mangle] #[allow(non_upper_case_globals)] pub static mut errno: Errno = Errno(0); diff --git a/userspace/lib/ygglibc/src/headers/fcntl/mod.rs b/userspace/lib/ygglibc/src/headers/fcntl/mod.rs index 07271ba4..74e57b6d 100644 --- a/userspace/lib/ygglibc/src/headers/fcntl/mod.rs +++ b/userspace/lib/ygglibc/src/headers/fcntl/mod.rs @@ -164,6 +164,7 @@ unsafe extern "C" fn creat(pathname: *const c_char, mode: mode_t) -> CFdResult { #[no_mangle] unsafe extern "C" fn open(pathname: *const c_char, opts: c_int, mut args: ...) -> CFdResult { + yggdrasil_rt::debug_trace!("&errno = {:p}", &crate::error::errno); vopenat(AT_FDCWD, pathname, opts, args.as_va_list()) } diff --git a/userspace/lib/ygglibc/src/lib.rs b/userspace/lib/ygglibc/src/lib.rs index 7d0db451..bc2f8ca5 100644 --- a/userspace/lib/ygglibc/src/lib.rs +++ b/userspace/lib/ygglibc/src/lib.rs @@ -10,7 +10,8 @@ linkage, rustc_private, naked_functions, - non_null_from_ref + non_null_from_ref, + thread_local )] #![allow(internal_features)] #![cfg_attr(not(test), no_std)] diff --git a/userspace/lib/ygglibc/src/thread/mod.rs b/userspace/lib/ygglibc/src/thread/mod.rs index cbdee20a..3f27757a 100644 --- a/userspace/lib/ygglibc/src/thread/mod.rs +++ b/userspace/lib/ygglibc/src/thread/mod.rs @@ -9,7 +9,7 @@ use core::{ use alloc::{boxed::Box, collections::BTreeMap, sync::Arc}; use yggdrasil_rt::{ mem::{MappingFlags, MappingSource}, - process::{get_tls, pthread_self, set_pthread_self, ThreadId, ThreadSpawnOptions}, + process::{thread_local, ProgramArgumentInner, ThreadId, ThreadSpawnOptions}, }; use crate::{ @@ -18,7 +18,7 @@ use crate::{ sync::Mutex, }; -mod tls; +pub mod tls; static THREADS: Mutex>> = Mutex::new(BTreeMap::new()); @@ -114,26 +114,43 @@ impl Thread { } pub fn this() -> EResult> { - let pthread_self: *const Self = ptr::with_exposed_provenance(pthread_self()); - if !pthread_self.is_null() { + let tcb: *const *const Self = thread_local::get_tcb().cast(); + let raw = unsafe { tcb.read() }; + if !raw.is_null() { // Arc::from_raw creates an owned Arc, which will decrement refcount when dropped, // prevent this by incrementing the count once more - unsafe { Arc::increment_strong_count(pthread_self) }; - let arc = unsafe { Arc::from_raw(pthread_self) }; + unsafe { Arc::increment_strong_count(raw) }; + let arc = unsafe { Arc::from_raw(raw) }; EResult::Ok(arc) } else { - panic!("pthread_self is NULL") + panic!("pthread_self() is NULL") } } + unsafe fn set_this(thread: Arc) { + let tcb: *mut usize = thread_local::get_tcb().cast(); + let raw = Arc::into_raw(thread); + tcb.write(raw.addr()); + } + pub fn result(&self) -> *mut c_void { self.result.load(Ordering::Acquire) } extern "C" fn thread_entry(argument: usize) -> ! { + // Set up TLS as soon as possible. Note the `force = true` parameter, because the image + // contains "already initialized" tag, which only matters for the main thread. + if let Err(err) = + unsafe { thread_local::init_tls(tls::TLS_IMAGE.as_ref(), true, size_of::() * 4) } + { + yggdrasil_rt::debug_trace!("thread_entry failed: TLS init error: {err:?}"); + unsafe { yggdrasil_rt::sys::exit_thread() }; + } + { assert_ne!(argument, 0); + let argument: Box = unsafe { Box::from_raw(ptr::with_exposed_provenance_mut(argument)) }; yggdrasil_rt::debug_trace!( @@ -141,13 +158,14 @@ impl Thread { argument.entry, argument.argument ); - yggdrasil_rt::debug_trace!("tls pointer: {:#x}", get_tls()); // TODO better way to initialize the thread ID while argument.thread.id.load(Ordering::Acquire) == 0 { core::hint::spin_loop(); } - argument.thread.clone().setup(); + + unsafe { Self::set_this(argument.thread.clone()) }; + let result = (argument.entry)(argument.argument); argument.thread.result.store(result, Ordering::Release); @@ -156,15 +174,9 @@ impl Thread { unsafe { Arc::decrement_strong_count(Arc::as_ptr(&this)) }; } - // Exit the thread when everything's dropped + // TODO call thread-local destructors unsafe { yggdrasil_rt::sys::exit_thread() } } - - fn setup(self: Arc) { - // Setup self-pointer - let pthread_self = Arc::into_raw(self); - unsafe { set_pthread_self(pthread_self.addr()) }; - } } impl Drop for Thread { @@ -176,3 +188,17 @@ impl Drop for Thread { } } } + +pub fn init_main_thread(arg: &ProgramArgumentInner) { + // Will create a TLS for the main thread (if not done already). + // Usually, a dynamic loader will do this for us, but this still needs to be + // done for statically-linked programs if the kernel transfers control directly to + // the program. + let tls_image = thread_local::init_tls_from_auxv(arg.auxv(), false, size_of::() * 4) + .expect("Could not initialize TLS"); + + // Store the TLS image, it'll be needed when creating new threads + unsafe { + tls::TLS_IMAGE = tls_image; + } +} diff --git a/userspace/lib/ygglibc/src/thread/tls.rs b/userspace/lib/ygglibc/src/thread/tls.rs index 8b137891..b7983d55 100644 --- a/userspace/lib/ygglibc/src/thread/tls.rs +++ b/userspace/lib/ygglibc/src/thread/tls.rs @@ -1 +1,4 @@ +use yggdrasil_rt::process::thread_local::TlsImage; +// TODO OnceLock +pub(super) static mut TLS_IMAGE: Option = None;