alnyan/yggdrasil
1
0
Fork 0
Code Issues 33 Pull Requests Packages Projects Releases Wiki Activity

proc/WIP: better handling for TLS

Browse Source
This commit is contained in:
Mark Poliakov 2024-11-17 23:32:07 +02:00
parent 17eca4c0c0
commit e0e39d2f23
52 changed files with 1766 additions and 753 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
kernel/arch/aarch64/src/context.rs
View File

@ -202,7 +202,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
&mut stack,
__aarch64_task_enter_user as _,
context.address_space,
context.tls as _,
context.thread_pointer as _,
);
let sp = stack.build();

2
kernel/arch/i686/src/context.rs
View File

@ -130,7 +130,7 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
tss_esp0: esp0 as _,
cr3: context.address_space.try_into().unwrap(),
gs_base: context.tls,
gs_base: context.thread_pointer,
_pd: PhantomData,
})

5
kernel/arch/interface/src/task.rs
View File

@ -81,7 +81,7 @@ pub struct UserContextInfo {
pub entry: usize,
pub argument: usize,
pub stack_pointer: usize,
pub tls: usize,
pub thread_pointer: usize,
pub address_space: u64,
pub single_step: bool,
}
@ -121,6 +121,9 @@ pub trait TaskContext<K: KernelTableManager, PA: PhysicalMemoryAllocator>: Sized
/// Only meant to be called from the scheduler code after the `thread` has terminated.
unsafe fn switch_and_drop(&self, thread: *const ());
/// Replaces the current thread pointer with the provided one.
fn set_thread_pointer(&self, tp: usize);
// XXX
/// Constructs a safe wrapper process to execute a kernel-space closure
fn kernel_closure<F: FnOnce() -> ! + Send + 'static>(f: F) -> Result<Self, Error> {

13
kernel/arch/x86/src/registers.rs
View File

@ -291,6 +291,18 @@ mod msr {
pub const MSR_IA32_KERNEL_GS_BASE: Reg = Reg;
}
pub mod ia32_fs_base {
const ADDR: u32 = 0xC0000100;
pub struct Reg;
msr_impl_read!(Reg, ADDR);
msr_impl_write!(Reg, ADDR);
/// IA32_FS_BASE model-specific register. Provides the base address for %fs-relative
/// loads/stores.
pub const MSR_IA32_FS_BASE: Reg = Reg;
}
pub mod ia32_apic_base {
use tock_registers::{interfaces::Readable, register_bitfields};
@ -420,6 +432,7 @@ pub use cr3::CR3;
pub use cr4::CR4;
pub use msr::ia32_apic_base::MSR_IA32_APIC_BASE;
pub use msr::ia32_efer::MSR_IA32_EFER;
pub use msr::ia32_fs_base::MSR_IA32_FS_BASE;
pub use msr::ia32_kernel_gs_base::MSR_IA32_KERNEL_GS_BASE;
pub use msr::ia32_lstar::MSR_IA32_LSTAR;
pub use msr::ia32_sfmask::MSR_IA32_SFMASK;

96
kernel/arch/x86_64/src/context.S
View File

@ -3,53 +3,21 @@
.set MSR_IA32_FS_BASE, 0xC0000100
.macro SAVE_TASK_STATE
sub ${context_size}, %rsp
mov %rbx, 0(%rsp)
mov %r12, 8(%rsp)
mov %r13, 16(%rsp)
mov %r14, 24(%rsp)
mov %r15, 32(%rsp)
// Store FS_BASE
mov $MSR_IA32_FS_BASE, %ecx
rdmsr
mov %edx, %ecx
shl $32, %rcx
or %rax, %rcx
mov %rcx, 40(%rsp)
// TODO save %fs
mov %rbp, 48(%rsp)
mov %cr3, %rbx
mov %rbx, 56(%rsp)
push %rbp
push %r15
push %r14
push %r13
push %r12
push %rbx
.endm
.macro LOAD_TASK_STATE
mov 56(%rsp), %rbx
mov %rbx, %cr3
mov 0(%rsp), %rbx
mov 8(%rsp), %r12
mov 16(%rsp), %r13
mov 24(%rsp), %r14
mov 32(%rsp), %r15
// Load FS_BASE
// edx:eax = fs_base
mov 40(%rsp), %rdx
mov %edx, %eax
shr $32, %rdx
mov $MSR_IA32_FS_BASE, %ecx
wrmsr
// mov 40(%rsp), %fs
mov 48(%rsp), %rbp
add ${context_size}, %rsp
pop %rbx
pop %r12
pop %r13
pop %r14
pop %r15
pop %rbp
.endm
.global __x86_64_task_enter_user
@ -134,55 +102,27 @@ __x86_64_task_enter_kernel:
// %rsi - from struct ptr, %rdi - to struct ptr
__x86_64_switch_task:
// save state to source stack
SAVE_TASK_STATE
mov %rsp, 0(%rsi)
// TSS.RSP0
mov 8(%rdi), %rax
// Kernel stack
mov 0(%rdi), %rdi
mov %rdi, %rsp
// Load TSS.RSP0
mov %gs:(8), %rdi
mov %rax, 4(%rdi)
// load destination stack
mov 0(%rdi), %rsp
LOAD_TASK_STATE
ret
__x86_64_switch_and_drop:
// TSS.RSP0
mov 8(%rdi), %rax
// Kernel stack
mov 0(%rdi), %rdi
mov %rdi, %rsp
// Load TSS.RSP0
mov %gs:(8), %rdi
mov %rax, 4(%rdi)
mov 0(%rdi), %rsp
// Call thread drop before loading the state
mov %rsi, %rdi
call __arch_drop_thread
LOAD_TASK_STATE
ret
// %rdi - to struct ptr
__x86_64_enter_task:
// TSS.RSP0
mov 8(%rdi), %rax
// Kernel stack
mov 0(%rdi), %rdi
mov %rdi, %rsp
// Load TSS.RSP0
mov %gs:(8), %rdi
mov %rax, 4(%rdi)
mov 0(%rdi), %rsp
LOAD_TASK_STATE
ret

177
kernel/arch/x86_64/src/context.rs
View File

@ -4,11 +4,12 @@ use kernel_arch_interface::{
mem::{KernelTableManager, PhysicalMemoryAllocator},
task::{ForkFrame, StackBuilder, TaskContext, TaskFrame, UserContextInfo},
};
use kernel_arch_x86::registers::FpuContext;
use kernel_arch_x86::registers::{FpuContext, CR3, MSR_IA32_FS_BASE};
use libk_mm_interface::address::{AsPhysicalAddress, PhysicalAddress};
use tock_registers::interfaces::Writeable;
use yggdrasil_abi::{arch::SavedFrame, error::Error};
use crate::mem::KERNEL_TABLES;
use crate::{mem::KERNEL_TABLES, ArchitectureImpl};
/// Frame saved onto the stack when taking an IRQ
#[derive(Debug)]
@ -93,8 +94,8 @@ pub struct SyscallFrame {
struct Inner {
// 0x00
sp: usize,
// 0x08
tss_rsp0: usize,
fs_base: usize,
}
/// x86-64 implementation of a task context
@ -107,14 +108,14 @@ pub struct TaskContextImpl<
fpu_context: UnsafeCell<FpuContext>,
stack_base_phys: PhysicalAddress,
stack_size: usize,
tss_rsp0: usize,
cr3: usize,
_alloc: PhantomData<PA>,
_table_manager: PhantomData<K>,
}
// 8 registers + return address (which is not included)
const COMMON_CONTEXT_SIZE: usize = 8 * 8;
impl TaskFrame for IrqFrame {
fn store(&self) -> SavedFrame {
SavedFrame {
@ -348,52 +349,71 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
TaskContextImpl<K, PA>
{
/// Constructs a new task context from a "forked" syscall frame
pub(super) unsafe fn from_syscall_frame(frame: &SyscallFrame, cr3: u64) -> Result<Self, Error> {
const USER_TASK_PAGES: usize = 8;
pub(super) unsafe fn from_syscall_frame(
_frame: &SyscallFrame,
_cr3: u64,
) -> Result<Self, Error> {
todo!()
// const USER_TASK_PAGES: usize = 8;
let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?;
let stack_base = stack_base_phys.raw_virtualize::<K>();
// let stack_base_phys = PA::allocate_contiguous_pages(USER_TASK_PAGES)?;
// let stack_base = stack_base_phys.raw_virtualize::<K>();
let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000);
// let mut stack = StackBuilder::new(stack_base, USER_TASK_PAGES * 0x1000);
// iretq frame
stack.push(0x1B);
stack.push(frame.user_sp as _);
stack.push(0x200);
stack.push(0x23);
stack.push(frame.user_ip as _);
// // iretq frame
// stack.push(0x1B);
// stack.push(frame.user_sp as _);
// stack.push(0x200);
// stack.push(0x23);
// stack.push(frame.user_ip as _);
stack.push(frame.args[5] as _); // r9
stack.push(frame.args[4] as _); // r8
stack.push(frame.args[3] as _); // r10
stack.push(frame.args[2] as _); // rdx
stack.push(frame.args[1] as _); // rsi
stack.push(frame.args[0] as _); // rdi
// stack.push(frame.args[5] as _); // r9
// stack.push(frame.args[4] as _); // r8
// stack.push(frame.args[3] as _); // r10
// stack.push(frame.args[2] as _); // rdx
// stack.push(frame.args[1] as _); // rsi
// stack.push(frame.args[0] as _); // rdi
// callee-saved registers
stack.push(__x86_64_task_enter_from_fork as _);
// // callee-saved registers
// stack.push(__x86_64_task_enter_from_fork as _);
stack.push(cr3 as _);
// stack.push(cr3 as _);
stack.push(frame.rbp as _);
stack.push(0x12345678); // XXX TODO: fs_base from SyscallFrame
stack.push(frame.r15 as _);
stack.push(frame.r14 as _);
stack.push(frame.r13 as _);
stack.push(frame.r12 as _);
stack.push(frame.rbx as _);
// stack.push(frame.rbp as _);
// stack.push(0x12345678); // XXX TODO: fs_base from SyscallFrame
// stack.push(frame.r15 as _);
// stack.push(frame.r14 as _);
// stack.push(frame.r13 as _);
// stack.push(frame.r12 as _);
// stack.push(frame.rbx as _);
let sp = stack.build();
let rsp0 = stack_base + USER_TASK_PAGES * 0x1000;
// let sp = stack.build();
// let rsp0 = stack_base + USER_TASK_PAGES * 0x1000;
Ok(Self {
inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }),
fpu_context: UnsafeCell::new(FpuContext::new(true)),
stack_base_phys,
stack_size: USER_TASK_PAGES * 0x1000,
_alloc: PhantomData,
_table_manager: PhantomData,
})
// Ok(Self {
// inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }),
// fpu_context: UnsafeCell::new(FpuContext::new(true)),
// stack_base_phys,
// stack_size: USER_TASK_PAGES * 0x1000,
// _alloc: PhantomData,
// _table_manager: PhantomData,
// })
}
unsafe fn store_state(&self) {
FpuContext::store(self.fpu_context.get());
// No need to save TSS/%cr3/%fs base back into the TCB, only the kernel
// can make changes to those
}
unsafe fn load_state(&self) {
FpuContext::restore(self.fpu_context.get());
// When the task is interrupted from Ring 3, make the CPU load
// the top of its kernel stack
ArchitectureImpl::set_local_tss_sp0(self.tss_rsp0);
MSR_IA32_FS_BASE.set((*self.inner.get()).fs_base as u64);
CR3.set_address(self.cr3);
}
}
@ -411,6 +431,8 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
fn kernel(entry: extern "C" fn(usize) -> !, arg: usize) -> Result<Self, Error> {
const KERNEL_TASK_PAGES: usize = 32;
let cr3: usize = unsafe { KERNEL_TABLES.lock().as_physical_address() }.into();
let stack_base_phys = PA::allocate_contiguous_pages(KERNEL_TASK_PAGES)?;
let stack_base = stack_base_phys.raw_virtualize::<K>();
@ -420,23 +442,21 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
stack.push(entry as _);
stack.push(arg);
setup_common_context(
&mut stack,
__x86_64_task_enter_kernel as _,
unsafe { KERNEL_TABLES.lock().as_physical_address() }.into(),
0,
);
setup_common_context(&mut stack, __x86_64_task_enter_kernel as _);
let sp = stack.build();
// TODO stack is leaked
Ok(Self {
inner: UnsafeCell::new(Inner { sp, tss_rsp0: 0 }),
inner: UnsafeCell::new(Inner { sp, fs_base: 0 }),
fpu_context: UnsafeCell::new(FpuContext::new(false)),
stack_base_phys,
stack_size: KERNEL_TASK_PAGES * 0x1000,
tss_rsp0: 0,
cr3,
_alloc: PhantomData,
_table_manager: PhantomData,
})
@ -460,53 +480,51 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
stack.push(context.argument);
stack.push(context.stack_pointer);
setup_common_context(
&mut stack,
__x86_64_task_enter_user as _,
context.address_space,
context.tls,
);
setup_common_context(&mut stack, __x86_64_task_enter_user as _);
let sp = stack.build();
let rsp0 = stack_base + USER_TASK_PAGES * 0x1000;
Ok(Self {
inner: UnsafeCell::new(Inner { sp, tss_rsp0: rsp0 }),
inner: UnsafeCell::new(Inner {
sp,
fs_base: context.thread_pointer,
}),
fpu_context: UnsafeCell::new(FpuContext::new(true)),
stack_base_phys,
stack_size: USER_TASK_PAGES * 0x1000,
tss_rsp0: rsp0,
cr3: context.address_space as usize,
_alloc: PhantomData,
_table_manager: PhantomData,
})
}
unsafe fn enter(&self) -> ! {
FpuContext::restore(self.fpu_context.get());
self.load_state();
__x86_64_enter_task(self.inner.get())
}
unsafe fn switch(&self, from: &Self) {
let dst = self.inner.get();
let src = from.inner.get();
if dst != src {
// Save the old context
FpuContext::store(from.fpu_context.get());
// Load next context
FpuContext::restore(self.fpu_context.get());
__x86_64_switch_task(dst, src);
if core::ptr::addr_eq(self, from) {
return;
}
from.store_state();
self.load_state();
__x86_64_switch_task(self.inner.get(), from.inner.get())
}
unsafe fn switch_and_drop(&self, thread: *const ()) {
let dst = self.inner.get();
self.load_state();
__x86_64_switch_and_drop(self.inner.get(), thread)
}
FpuContext::restore(self.fpu_context.get());
__x86_64_switch_and_drop(dst, thread)
fn set_thread_pointer(&self, tp: usize) {
unsafe { (*self.inner.get()).fs_base = tp };
MSR_IA32_FS_BASE.set(tp as _);
}
}
@ -524,13 +542,10 @@ impl<K: KernelTableManager, PA: PhysicalMemoryAllocator<Address = PhysicalAddres
}
}
fn setup_common_context(builder: &mut StackBuilder, entry: usize, cr3: u64, fs_base: usize) {
fn setup_common_context(builder: &mut StackBuilder, entry: usize) {
builder.push(entry);
builder.push(cr3 as _);
builder.push(0); // %rbp
builder.push(fs_base); // %fs_base
builder.push(0); // %r15
builder.push(0); // %r14
builder.push(0); // %r13
@ -547,8 +562,4 @@ extern "C" {
fn __x86_64_switch_and_drop(to: *mut Inner, from: *const ());
}
global_asm!(
include_str!("context.S"),
context_size = const COMMON_CONTEXT_SIZE,
options(att_syntax)
);
global_asm!(include_str!("context.S"), options(att_syntax));

8
kernel/arch/x86_64/src/lib.rs
View File

@ -84,6 +84,14 @@ impl ArchitectureImpl {
fn local_cpu_data() -> Option<&'static mut PerCpuData> {
unsafe { (Self::local_cpu() as *mut PerCpuData).as_mut() }
}
fn set_local_tss_sp0(sp: usize) {
let local_cpu = Self::local_cpu_data().unwrap();
unsafe {
(core::ptr::with_exposed_provenance_mut::<usize>(local_cpu.tss_address + 4))
.write_unaligned(sp);
}
}
}
impl Architecture for ArchitectureImpl {

160
kernel/libk/src/task/binary/elf.rs
View File

@ -5,6 +5,7 @@ use alloc::sync::Arc;
use cfg_if::cfg_if;
use elf::{
endian::AnyEndian,
io_traits::InputStream,
relocation::{Rel, Rela},
segment::ProgramHeader,
ElfStream, ParseError,
@ -13,18 +14,14 @@ use libk_mm::{
pointer::PhysicalRefMut,
process::{ProcessAddressSpace, VirtualRangeBacking},
table::MapAttributes,
PageBox,
PageBox, L3_PAGE_SIZE,
};
use libk_util::io::{Read, Seek};
use yggdrasil_abi::{error::Error, io::SeekFrom, path::PathBuf};
use crate::{
random,
task::{
mem::ForeignPointer,
process::ProcessImage,
types::{ProcessTlsInfo, ProcessTlsLayout},
},
task::{mem::ForeignPointer, process::ProcessImage, types::TlsImage},
};
cfg_if! {
@ -203,15 +200,13 @@ pub fn load_elf_from_file<F: Read + Seek>(
ElfSegmentType::Load => {
load_segment(space, &file, &segment, image_load_base, vaddr_min)?;
}
// TODO not yet handled
ElfSegmentType::Tls => (),
// TODO not yet handled, contains __tls_get_addr
ElfSegmentType::Dynamic => (),
}
}
// Load TLS master copy
let tls = handle_tls(&elf, &file)?;
// Load TLS master copy somewhere into process memory
let tls_image = handle_tls(space, &elf, &file)?;
// Fixup relocations
handle_relocations(&mut elf, space, image_load_base)?;
@ -223,69 +218,11 @@ pub fn load_elf_from_file<F: Read + Seek>(
Ok(ProcessImage {
entry,
ip_offset,
tls,
load_base: image_load_base,
tls_image,
})
}
/// Creates a new copy of the TLS from given master image
pub fn clone_tls(space: &ProcessAddressSpace, image: &ProcessImage) -> Result<usize, Error> {
let Some(tls) = image.tls.as_ref() else {
// No TLS
return Ok(0);
};
assert_ne!(tls.layout.full_size, 0);
let aligned_size = (tls.layout.full_size + 0xFFF) & !0xFFF;
let address = space.allocate(
None,
aligned_size,
VirtualRangeBacking::anonymous(),
MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL,
)?;
// Copy master image data
if tls.layout.data_size > 0 {
let master_copy = tls.master_copy.as_ref().unwrap();
load_bytes(
space,
address + tls.layout.data_offset,
tls.layout.data_size,
|off, mut data| {
let len = data.len();
data.copy_from_slice(&master_copy[off..off + len]);
Ok(())
},
)?;
}
// Zero zeroed data
if tls.layout.mem_size > tls.layout.data_size {
load_bytes(
space,
address + tls.layout.data_offset + tls.layout.data_size,
tls.layout.mem_size - tls.layout.data_size,
|_, mut data| {
data.fill(0);
Ok(())
},
)?;
}
// Zero slots and setup self-pointer
unsafe {
let slots: *mut usize = core::ptr::without_provenance_mut(address + tls.layout.slot_offset);
slots.try_write_foreign_volatile(space, 0)?;
slots.add(1).try_write_foreign_volatile(space, 0)?;
let self_ptr: *mut usize =
core::ptr::without_provenance_mut(address + tls.layout.ptr_offset);
self_ptr.try_write_foreign_volatile(space, address + tls.layout.ptr_offset)?;
}
Ok(address + tls.layout.ptr_offset)
}
fn is_dynamic<F: Read + Seek>(
elf: &mut ElfStream<AnyEndian, FileReader<F>>,
) -> Result<bool, Error> {
@ -366,7 +303,6 @@ fn handle_relocations<F: Read + Seek>(
space: &ProcessAddressSpace,
image_load_base: usize,
) -> Result<(), Error> {
// TODO handle SHT_REL as well
let rel_section = elf
.section_headers()
.iter()
@ -401,45 +337,73 @@ fn handle_relocations<F: Read + Seek>(
}
fn handle_tls<F: Read + Seek>(
space: &ProcessAddressSpace,
elf: &ElfStream<AnyEndian, FileReader<F>>,
file: &FileReader<F>,
) -> Result<Option<ProcessTlsInfo>, Error> {
// TODO handle different TLS models
// TODO handle TLS segment attributes
// TODO check if it's possible to have more than one TLS segment
) -> Result<Option<TlsImage>, Error> {
let mut tls_segments = elf
.segments()
.iter()
.filter(|s| s.p_type == elf::abi::PT_TLS);
let Some(tls_segment) = tls_segments.next() else {
return Ok(None);
};
if tls_segments.next().is_some() {
todo!("Handle ELFs with more than one PT_TLS?");
}
// Locate TLS master copy information, if any
let tls_segment = elf.segments().iter().find(|s| s.p_type == elf::abi::PT_TLS);
if !tls_segment.p_align.is_power_of_two() {
log::warn!("PT_TLS align not power of two: {}", tls_segment.p_align);
return Err(Error::UnrecognizedExecutable);
}
if tls_segment.p_align > 0x1000 {
todo!("PT_TLS align > 0x1000");
}
if let Some(tls_segment) = tls_segment {
// Can't yet handle higher align values
assert!(tls_segment.p_align.is_power_of_two());
assert!(tls_segment.p_align < 0x1000);
let layout = ProcessTlsLayout::new(
tls_segment.p_align as _,
tls_segment.p_filesz as _,
tls_segment.p_memsz as _,
// Load the master copy somewhere.
// This will be the pointer passed to the program as an argument so that it can
// clone the TLS master copy when, for example, spawning a new thread.
//
// Only do the allocation if there's any actual data in the master copy,
// i.e., don't allocate a master copy if there's only a zeroed .tbss
let master_copy_base = if tls_segment.p_filesz > 0 {
// NOTE: don't need to load more than filesz bytes, rest will get zeroed anyway
let page_aligned_size =
(tls_segment.p_filesz as usize + L3_PAGE_SIZE - 1) & !(L3_PAGE_SIZE - 1);
log::debug!(
"Allocate TLS master copy for {:#x} bytes",
page_aligned_size
);
// User doesn't need to write/execute a master copy, it's for cloning only
let master_copy_base = space.allocate(
None,
page_aligned_size,
VirtualRangeBacking::Anonymous,
MapAttributes::USER_READ,
)?;
assert!(layout.mem_size <= layout.full_size);
load_bytes(
space,
master_copy_base,
tls_segment.p_filesz as usize,
|off, mut dst| {
file.file
.seek(SeekFrom::Start(tls_segment.p_offset + off as u64))?;
file.file.read_exact(dst.deref_mut())
},
)?;
let master_copy = if layout.data_size > 0 {
let mut data = PageBox::new_slice(0, layout.data_size)?;
file.file.seek(SeekFrom::Start(tls_segment.p_offset))?;
file.file.read_exact(&mut data)?;
Some(Arc::new(data))
master_copy_base
} else {
None
0
};
Ok(Some(ProcessTlsInfo {
master_copy,
layout,
Ok(Some(TlsImage {
master_copy_base,
align: tls_segment.p_align as usize,
data_size: tls_segment.p_filesz as usize,
full_size: tls_segment.p_memsz as usize,
}))
} else {
Ok(None)
}
}
fn load_segment<F: Read + Seek>(

218
kernel/libk/src/task/binary/mod.rs
View File

@ -4,8 +4,10 @@ use alloc::{
borrow::ToOwned,
string::String,
sync::{Arc, Weak},
vec,
vec::Vec,
};
use bytemuck::Pod;
use kernel_arch::task::{TaskContext, UserContextInfo};
use libk_mm::{
pointer::PhysicalRefMut,
@ -18,7 +20,7 @@ use yggdrasil_abi::{
io::SeekFrom,
pass::{Place, Placer},
path::Path,
process::{ProcessGroupId, ProgramArgumentInner},
process::{auxv, AuxValue, ProcessGroupId},
};
use crate::{
@ -47,59 +49,126 @@ pub struct LoadOptions<'e, P: AsRef<Path>> {
pub disable_aslr: bool,
}
struct BufferPlacer<'a> {
// struct BufferPlacer<'a> {
// buffer: &'a mut [u8],
// virtual_offset: usize,
// offset: usize,
// }
//
// impl<'a> BufferPlacer<'a> {
// pub fn new(virtual_offset: usize, buffer: &'a mut [u8]) -> Self {
// Self {
// buffer,
// virtual_offset,
// offset: 0,
// }
// }
//
// unsafe fn alloc_layout<T>(
// &mut self,
// layout: Layout,
// ) -> Result<(NonNull<T>, NonNull<T>), Error> {
// // TODO checks
// let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1);
// self.offset = aligned + layout.size();
// Ok((
// NonNull::new_unchecked(self.buffer.as_mut_ptr().add(aligned) as *mut T),
// NonNull::new_unchecked((self.virtual_offset + aligned) as *mut T),
// ))
// }
// }
//
// unsafe impl<'a> Placer for BufferPlacer<'a> {
// fn place_ref<T: Place>(&mut self, r: &T) -> Result<NonNull<T::Output>, Error> {
// let layout = Layout::new::<T::Output>();
// unsafe {
// let (kernel, user) = self.alloc_layout::<T::Output>(layout)?;
// kernel.as_ptr().write(r.place(self)?);
// Ok(user)
// }
// }
//
// fn place_slice<T: Place>(&mut self, r: &[T]) -> Result<NonNull<[T::Output]>, Error> {
// let layout = Layout::array::<T>(r.len()).unwrap();
// unsafe {
// let (kernel, user) = self.alloc_layout::<T::Output>(layout)?;
// let kernel = NonNull::slice_from_raw_parts(kernel, r.len());
// let user = NonNull::slice_from_raw_parts(user, r.len());
// for (i, elem) in r.iter().enumerate() {
// kernel
// .get_unchecked_mut(i)
// .as_ptr()
// .write(elem.place(self)?);
// }
// Ok(user)
// }
// }
// }
struct ArgPlacer<'a> {
buffer: &'a mut [u8],
virtual_offset: usize,
offset: usize,
position: usize,
}
impl<'a> BufferPlacer<'a> {
pub fn new(virtual_offset: usize, buffer: &'a mut [u8]) -> Self {
impl<'a> ArgPlacer<'a> {
fn new(buffer: &'a mut [u8]) -> Self {
Self {
buffer,
virtual_offset,
offset: 0,
position: 0,
}
}
unsafe fn alloc_layout<T>(
&mut self,
layout: Layout,
) -> Result<(NonNull<T>, NonNull<T>), Error> {
// TODO checks
let aligned = (self.offset + layout.align() - 1) & !(layout.align() - 1);
self.offset = aligned + layout.size();
Ok((
NonNull::new_unchecked(self.buffer.as_mut_ptr().add(aligned) as *mut T),
NonNull::new_unchecked((self.virtual_offset + aligned) as *mut T),
))
fn align(&mut self, align: usize) -> Result<(), Error> {
debug_assert!(align.is_power_of_two());
let aligned = (self.position + align - 1) & !(align - 1);
if aligned > self.buffer.len() {
return Err(Error::InvalidArgument);
}
self.position = aligned;
Ok(())
}
unsafe impl<'a> Placer for BufferPlacer<'a> {
fn place_ref<T: Place>(&mut self, r: &T) -> Result<NonNull<T::Output>, Error> {
let layout = Layout::new::<T::Output>();
unsafe {
let (kernel, user) = self.alloc_layout::<T::Output>(layout)?;
kernel.as_ptr().write(r.place(self)?);
Ok(user)
fn put_str(&mut self, s: &str) -> Result<usize, Error> {
if self.position + s.len() >= self.buffer.len() {
return Err(Error::InvalidArgument);
}
let off = self.position;
self.buffer[off..off + s.len()].copy_from_slice(s.as_bytes());
self.buffer[off + s.len()] = 0;
self.position += s.len() + 1;
Ok(off)
}
fn place_slice<T: Place>(&mut self, r: &[T]) -> Result<NonNull<[T::Output]>, Error> {
let layout = Layout::array::<T>(r.len()).unwrap();
unsafe {
let (kernel, user) = self.alloc_layout::<T::Output>(layout)?;
let kernel = NonNull::slice_from_raw_parts(kernel, r.len());
let user = NonNull::slice_from_raw_parts(user, r.len());
for (i, elem) in r.iter().enumerate() {
kernel
.get_unchecked_mut(i)
.as_ptr()
.write(elem.place(self)?);
fn put<T: Pod>(&mut self, v: &T) -> Result<usize, Error> {
if self.position + size_of::<T>() > self.buffer.len() {
return Err(Error::InvalidArgument);
}
Ok(user)
let off = self.position;
self.buffer[off..off + size_of::<T>()].copy_from_slice(bytemuck::bytes_of(v));
self.position += size_of::<T>();
Ok(off)
}
fn put_aux_array(&mut self, s: &[AuxValue]) -> Result<usize, Error> {
self.align(size_of::<u64>())?;
let off = self.position;
for item in s {
self.put(&item.tag)?;
self.put(&item.val)?;
}
self.put(&auxv::NULL)?;
self.put(&0u64)?;
Ok(off)
}
fn put_ptr_array(&mut self, s: &[usize]) -> Result<usize, Error> {
self.align(size_of::<usize>())?;
let off = self.position;
for item in s {
self.put(item)?;
}
self.put(&0usize)?;
Ok(off)
}
}
@ -110,6 +179,7 @@ fn setup_program_env(
virt: usize,
args: &Vec<String>,
envs: &Vec<String>,
aux: &[AuxValue],
) -> Result<usize, Error> {
// TODO growing buffer
let phys_page = space.map_single(
@ -118,18 +188,37 @@ fn setup_program_env(
MapAttributes::USER_READ | MapAttributes::USER_WRITE | MapAttributes::NON_GLOBAL,
)?;
let mut buffer = unsafe { PhysicalRefMut::map_slice(phys_page, 4096) };
let mut placer = BufferPlacer::new(virt, &mut buffer);
let mut placer = ArgPlacer::new(&mut buffer);
let mut arg_ptrs = vec![];
let mut env_ptrs = vec![];
let args = args.iter().map(String::as_ref).collect::<Vec<_>>();
let envs = envs.iter().map(String::as_ref).collect::<Vec<_>>();
log::debug!("place karg: {:#x}", virt);
for arg in args.iter() {
let ptr = placer.put_str(arg)? + virt;
log::debug!("put arg {:?} -> {:#x}", arg, ptr);
arg_ptrs.push(ptr);
}
for env in envs.iter() {
let ptr = placer.put_str(env)? + virt;
log::debug!("put env {:?} -> {:#x}", env, ptr);
env_ptrs.push(ptr);
}
let in_kernel = ProgramArgumentInner {
args: &args,
env: &envs,
};
let in_user = in_kernel.place_ref(&mut placer)?;
let argv = placer.put_ptr_array(&arg_ptrs)? + virt;
log::debug!("put argv {} -> {:#x}", arg_ptrs.len(), argv);
let envp = placer.put_ptr_array(&env_ptrs)? + virt;
log::debug!("put envp {} -> {:#x}", env_ptrs.len(), envp);
let auxv = placer.put_aux_array(aux)? + virt;
log::debug!("put auxv {} -> {:#x}", aux.len(), auxv);
Ok(in_user as *const _ as usize)
// Put ProgramArgumentInner struct
let arg_address = placer.position + virt;
placer.put(&argv)?;
placer.put(&envp)?;
placer.put(&auxv)?;
Ok(arg_address)
}
fn setup_context<P>(
@ -156,7 +245,31 @@ where
MapAttributes::USER_WRITE | MapAttributes::USER_READ | MapAttributes::NON_GLOBAL,
)?;
let arg = setup_program_env(space, virt_args_base, args, envs)?;
let mut auxv = vec![];
if let Some(tls_image) = image.tls_image.as_ref() {
if tls_image.master_copy_base != 0 {
auxv.push(AuxValue {
tag: auxv::TLS_MASTER_COPY,
val: tls_image.master_copy_base as _,
});
auxv.push(AuxValue {
tag: auxv::TLS_DATA_SIZE,
val: tls_image.data_size as _,
});
} else {
assert_eq!(tls_image.data_size, 0);
}
auxv.push(AuxValue {
tag: auxv::TLS_ALIGN,
val: tls_image.align as _,
});
auxv.push(AuxValue {
tag: auxv::TLS_FULL_SIZE,
val: tls_image.full_size as _,
});
}
let argument = setup_program_env(space, virt_args_base, args, envs, &auxv)?;
let user_sp =
virt_stack_base + USER_STACK_PAGES * 0x1000 - TaskContextImpl::USER_STACK_EXTRA_ALIGN;
@ -171,15 +284,16 @@ where
}
}
let ptr = unsafe { Thread::setup_stack_header(space, ptr, arg)? };
let ptr = unsafe { Thread::setup_stack_header(space, ptr, argument)? };
let tls_address = elf::clone_tls(space, image)?;
// let tls_address = elf::clone_tls(space, image)?;
log::debug!("argument = {:#x}", argument);
TaskContext::user(UserContextInfo {
entry: image.entry,
argument: arg,
argument,
stack_pointer: ptr.addr(),
tls: tls_address,
thread_pointer: 0,
address_space: space.as_address_with_asid(),
single_step: options.single_step,
})

22
kernel/libk/src/task/process.rs
View File

@ -29,15 +29,14 @@ use yggdrasil_abi::{
use crate::{
task::{
binary,
futex::UserspaceMutex,
thread::Thread,
types::{AllocateProcessId, ProcessTlsInfo},
TaskContextImpl, ThreadId,
binary, futex::UserspaceMutex, thread::Thread, types::AllocateProcessId, TaskContextImpl,
ThreadId,
},
vfs::{FileReadiness, FileSet, IoContext, NodeRef},
};
use super::types::TlsImage;
pub trait ForkFrame = kernel_arch::task::ForkFrame<KernelTableManagerImpl, GlobalPhysicalAllocator>;
pub struct ProcessIo {
@ -57,8 +56,8 @@ pub struct ProcessImage {
pub ip_offset: usize,
/// Entry point address
pub entry: usize,
/// Thread-local storage information
pub tls: Option<ProcessTlsInfo>,
/// TLS master copy, if any
pub tls_image: Option<TlsImage>,
}
pub struct ProcessInner {
@ -133,16 +132,11 @@ impl Process {
/// Spawns a new child thread within the process
pub fn spawn_thread(self: &Arc<Self>, options: &ThreadSpawnOptions) -> Result<ThreadId, Error> {
log::debug!("Spawn thread in {} with options: {:#x?}", self.id, options);
let mut inner = self.inner.write();
let space = inner.space.clone().unwrap();
let tls_address = if let Some(image) = inner.image.as_ref() {
binary::elf::clone_tls(&space, image)?
} else {
0
};
let sp = (options.stack_top - 8) as *mut usize;
let sp = unsafe { Thread::setup_stack_header(&space, sp, options.argument)? };
@ -152,7 +146,7 @@ impl Process {
address_space: space.as_address_with_asid(),
stack_pointer: sp.addr(),
single_step: false,
tls: tls_address,
thread_pointer: 0,
};
let context = TaskContextImpl::user(info)?;
let thread = Thread::new_uthread(self.id, None, space.clone(), context);

4
kernel/libk/src/task/thread.rs
View File

@ -204,6 +204,10 @@ impl Thread {
self.context.replace(context)
}
pub fn set_thread_pointer(&self, tp: usize) {
unsafe { (*self.context.as_ptr()).set_thread_pointer(tp) };
}
// Get/Set
pub fn is_user(&self) -> bool {
self.id.is_user()

224
kernel/libk/src/task/types.rs
View File

@ -40,39 +40,53 @@ pub enum ThreadId {
User(u64),
}
// TLS layout (x86-64):
// | mem_size | uthread_size |
// | Data .......| self, ??? |
//
// TLS layout (aarch64):
// | uthread_size (0x10?) | mem_size |
// | ??? | Data .....|
//
// TLS layout (i686):
// | self | ... |
/// Describes Thread-Local Storage of a process
#[derive(Clone, Debug)]
pub struct ProcessTlsInfo {
/// Master copy of the TLS
pub master_copy: Option<Arc<PageBox<[u8]>>>,
/// Layout of the TLS
pub layout: ProcessTlsLayout,
}
/// Describes TLS layout for a program image
#[derive(Clone, Debug)]
pub struct ProcessTlsLayout {
/// Pointer offset from the TLS base. The pointer is passed to the userspace
pub ptr_offset: usize,
pub data_offset: usize,
/// Describes a TLS master copy
#[derive(Debug, Clone)]
pub struct TlsImage {
/// Where the kernel placed the segment in the process' memory.
/// NOTE: is zero if there's no actual data in the master copy.
pub master_copy_base: usize,
/// Segment alignment
pub align: usize,
/// Part of the segment occupied by actual data
pub data_size: usize,
pub mem_size: usize,
pub slot_offset: usize,
/// Overall allocation size of the TLS data
/// Full size of the segment. `full_size - data_size` is zeroed on load.
pub full_size: usize,
}
// // TLS layout (x86-64):
// // | mem_size | uthread_size |
// // | Data .......| self, ??? |
// //
// // TLS layout (aarch64):
// // | uthread_size (0x10?) | mem_size |
// // | ??? | Data .....|
// //
// // TLS layout (i686):
// // | self | ... |
//
// /// Describes Thread-Local Storage of a process
// #[derive(Clone, Debug)]
// pub struct ProcessTlsInfo {
// /// Master copy of the TLS
// pub master_copy: Option<Arc<PageBox<[u8]>>>,
// /// Layout of the TLS
// pub layout: ProcessTlsLayout,
// }
//
// /// Describes TLS layout for a program image
// #[derive(Clone, Debug)]
// pub struct ProcessTlsLayout {
// /// Pointer offset from the TLS base. The pointer is passed to the userspace
// pub ptr_offset: usize,
// pub data_offset: usize,
// pub data_size: usize,
// pub mem_size: usize,
// pub slot_offset: usize,
// /// Overall allocation size of the TLS data
// pub full_size: usize,
// }
impl ThreadAffinity {
/// Mask value for a thread to be scheduled onto any CPU
pub const ANY_CPU: u64 = u64::MAX;
@ -151,79 +165,79 @@ impl AllocateProcessId for ProcessId {
}
}
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
impl ProcessTlsLayout {
/// Constructs a new thread-local storage layout info struct
pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self {
use core::mem::size_of;
debug_assert!(align.is_power_of_two());
let tls_block0_offset = (size_of::<usize>() * 2 + align - 1) & !(align - 1);
// Allocate two words in the back for extra storage
let back_size = size_of::<usize>() * 2;
let full_size = back_size + (tls_block0_offset + mem_size + align - 1) & !(align - 1);
Self {
data_offset: tls_block0_offset + back_size,
ptr_offset: back_size,
slot_offset: 0,
data_size,
mem_size,
full_size,
}
}
}
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
impl ProcessTlsLayout {
/// Constructs a new thread-local storage layout info struct
pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self {
use core::mem::size_of;
// The static TLS blocks are placed below TP
// TP points to the TCB
debug_assert!(align.is_power_of_two());
let back_size = (mem_size + align - 1) & !(align - 1);
// 0: self-pointer
// 1: -
// 2: segment pointer
// 3: segment size
let forward_size = size_of::<usize>() * 4;
let full_size = back_size + forward_size;
Self {
data_offset: 0,
ptr_offset: back_size,
slot_offset: back_size + 2 * size_of::<usize>(),
data_size,
mem_size,
full_size,
}
}
}
#[cfg(any(target_arch = "x86", rust_analyzer))]
impl ProcessTlsLayout {
pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self {
use core::mem::size_of;
debug_assert!(align.is_power_of_two());
let back_size = (mem_size + align - 1) & !(align - 1);
let forward_size = size_of::<usize>() * 4;
let full_size = back_size + forward_size;
Self {
data_offset: 0,
ptr_offset: back_size,
slot_offset: back_size + 2 * size_of::<usize>(),
data_size,
mem_size,
full_size,
}
}
}
// #[cfg(any(target_arch = "aarch64", rust_analyzer))]
// impl ProcessTlsLayout {
// /// Constructs a new thread-local storage layout info struct
// pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self {
// use core::mem::size_of;
//
// debug_assert!(align.is_power_of_two());
// let tls_block0_offset = (size_of::<usize>() * 2 + align - 1) & !(align - 1);
// // Allocate two words in the back for extra storage
// let back_size = size_of::<usize>() * 2;
//
// let full_size = back_size + (tls_block0_offset + mem_size + align - 1) & !(align - 1);
//
// Self {
// data_offset: tls_block0_offset + back_size,
// ptr_offset: back_size,
// slot_offset: 0,
//
// data_size,
// mem_size,
// full_size,
// }
// }
// }
//
// #[cfg(any(target_arch = "x86_64", rust_analyzer))]
// impl ProcessTlsLayout {
// /// Constructs a new thread-local storage layout info struct
// pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self {
// use core::mem::size_of;
//
// // The static TLS blocks are placed below TP
// // TP points to the TCB
// debug_assert!(align.is_power_of_two());
// let back_size = (mem_size + align - 1) & !(align - 1);
// // 0: self-pointer
// // 1: -
// // 2: segment pointer
// // 3: segment size
// let forward_size = size_of::<usize>() * 4;
//
// let full_size = back_size + forward_size;
//
// Self {
// data_offset: 0,
// ptr_offset: back_size,
// slot_offset: back_size + 2 * size_of::<usize>(),
//
// data_size,
// mem_size,
// full_size,
// }
// }
// }
//
// #[cfg(any(target_arch = "x86", rust_analyzer))]
// impl ProcessTlsLayout {
// pub fn new(align: usize, data_size: usize, mem_size: usize) -> Self {
// use core::mem::size_of;
//
// debug_assert!(align.is_power_of_two());
// let back_size = (mem_size + align - 1) & !(align - 1);
// let forward_size = size_of::<usize>() * 4;
// let full_size = back_size + forward_size;
//
// Self {
// data_offset: 0,
// ptr_offset: back_size,
// slot_offset: back_size + 2 * size_of::<usize>(),
//
// data_size,
// mem_size,
// full_size,
// }
// }
// }

2
kernel/src/syscall/imp/mod.rs
View File

@ -6,7 +6,7 @@ pub(crate) use abi::{
},
mem::{MappingFlags, MappingSource},
net::SocketType,
process::{Signal, SignalEntryData, SpawnOptions},
process::{Signal, SignalEntryData, SpawnOptions, ThreadOption},
system::SystemInfo,
};
use abi::{

17
kernel/src/syscall/imp/sys_process.rs
View File

@ -6,7 +6,7 @@ use abi::{
mem::{MappingFlags, MappingSource},
process::{
ExitCode, MutexOperation, ProcessGroupId, ProcessId, Signal, SpawnFlags, SpawnOption,
SpawnOptions, ThreadSpawnOptions,
SpawnOptions, ThreadOption, ThreadSpawnOptions,
},
};
use alloc::sync::Arc;
@ -276,3 +276,18 @@ pub(crate) fn wait_thread(id: u32) -> Result<(), Error> {
Ok(())
}
pub(crate) fn get_thread_option(option: &mut ThreadOption) -> Result<(), Error> {
todo!()
}
pub(crate) fn set_thread_option(option: &ThreadOption) -> Result<(), Error> {
let thread = Thread::current();
match option {
&ThreadOption::ThreadPointer(tp) => {
log::debug!("{:?}: set thread pointer: {:#x}", thread.id, tp);
thread.set_thread_pointer(tp);
Ok(())
}
}
}

5
lib/abi/def/yggdrasil.abi
View File

@ -1,4 +1,4 @@
// vi:syntax=yggdrasil_abi:
// vi:syntax=yggdrasil-abi:
extern {
type Duration = core::time::Duration;
@ -13,6 +13,7 @@ extern {
type MutexOperation = yggdrasil_abi::process::MutexOperation;
#[thin]
type ExitCode = yggdrasil_abi::process::ExitCode;
type ThreadOption = yggdrasil_abi::process::ThreadOption;
type SocketAddr = core::net::SocketAddr;
type SocketOption = yggdrasil_abi::net::SocketOption;
@ -82,6 +83,8 @@ syscall get_pid() -> ProcessId;
syscall spawn_thread(options: &ThreadSpawnOptions) -> Result<u32>;
syscall exit_thread() -> !;
syscall wait_thread(tid: u32) -> Result<()>;
syscall get_thread_option(option: &mut ThreadOption) -> Result<()>;
syscall set_thread_option(option: &ThreadOption) -> Result<()>;
syscall nanosleep(dur: &Duration) -> Result<()>;

112
lib/abi/src/process/mod.rs
View File

@ -1,19 +1,33 @@
//! Data structures for process management
use core::{fmt, time::Duration};
use core::{ffi::CStr, fmt, marker::PhantomData, time::Duration};
use crate::{
impl_place_lifetime_struct,
io::{FileMode, RawFd},
};
use crate::io::{FileMode, RawFd};
mod exit;
pub mod thread;
pub use crate::generated::{
ExecveOptions, ProcessGroupId, ProcessId, Signal, SignalEntryData, SpawnFlags, SpawnOptions,
ThreadId, ThreadSpawnOptions,
};
pub use exit::ExitCode;
pub use thread::ThreadOption;
// TODO this is ugly
pub mod auxv {
pub const TLS_MASTER_COPY: u64 = 0x80;
pub const TLS_FULL_SIZE: u64 = 0x81;
pub const TLS_DATA_SIZE: u64 = 0x82;
pub const TLS_ALIGN: u64 = 0x83;
pub const TLS_MODULE_ID: u64 = 0x84;
pub const TLS_MODULE_OFFSET: u64 = 0x85;
pub const TLS_ALREADY_INITIALIZED: u64 = 0x86;
pub const NULL: u64 = 0x00;
}
/// Defines an optional argument for controlling process creation
#[derive(Debug)]
@ -53,16 +67,30 @@ pub enum ProcessInfoElement {
Umask(FileMode),
}
// TODO not sure if I really need #[repr(C)] ABI here
impl_place_lifetime_struct! {
#[doc = "Argument struct passed from the kernel to a spawned process"]
#[derive(Debug)]
pub struct ProgramArgumentInner<'a> {
#[doc = "Argument list"]
pub args: &'a [&'a str],
#[doc = "List of KEY=VALUE environment variable pairs"]
pub env: &'a [&'a str],
#[derive(Debug, Clone, Copy)]
#[repr(C)]
pub struct AuxValue {
pub tag: u64,
pub val: u64,
}
#[derive(Clone, Copy)]
pub struct StringArgIter<'a> {
ptr: *const *const u8,
_pd: PhantomData<&'a ProgramArgumentInner>,
}
#[derive(Clone, Copy)]
pub struct AuxValueIter<'a> {
ptr: *const AuxValue,
_pd: PhantomData<&'a ProgramArgumentInner>,
}
#[repr(C)]
pub struct ProgramArgumentInner {
pub args: *const *const u8,
pub envs: *const *const u8,
pub auxv: *const AuxValue,
}
impl ProcessId {
@ -80,3 +108,59 @@ impl fmt::Display for ProcessGroupId {
write!(f, "<Process group {}>", self.0)
}
}
impl<'a> Iterator for StringArgIter<'a> {
type Item = &'a CStr;
fn next(&mut self) -> Option<Self::Item> {
if self.ptr.is_null() {
return None;
}
let entry = unsafe { self.ptr.read() };
if entry.is_null() {
return None;
}
self.ptr = unsafe { self.ptr.add(1) };
let entry = unsafe { CStr::from_ptr(entry.cast()) };
Some(entry)
}
}
impl<'a> Iterator for AuxValueIter<'a> {
type Item = &'a AuxValue;
fn next(&mut self) -> Option<Self::Item> {
if self.ptr.is_null() {
return None;
}
let entry = unsafe { &*self.ptr };
if entry.tag == auxv::NULL {
return None;
}
self.ptr = unsafe { self.ptr.add(1) };
Some(entry)
}
}
impl ProgramArgumentInner {
pub fn args(&self) -> impl Iterator<Item = &CStr> {
StringArgIter {
ptr: self.args,
_pd: PhantomData,
}
}
pub fn envs(&self) -> impl Iterator<Item = &CStr> {
StringArgIter {
ptr: self.envs,
_pd: PhantomData,
}
}
pub fn auxv(&self) -> impl Iterator<Item = &AuxValue> {
AuxValueIter {
ptr: self.auxv,
_pd: PhantomData,
}
}
}

4
lib/abi/src/process/thread.rs Normal file
View File

@ -0,0 +1,4 @@
#[derive(Debug)]
pub enum ThreadOption {
ThreadPointer(usize),
}

4
lib/runtime/Cargo.toml
View File

@ -6,7 +6,7 @@ authors = ["Mark Poliakov <mark@alnyan.me>"]
build = "build.rs"
[dependencies]
yggdrasil-abi = { path = "../abi" }
yggdrasil-abi = { path = "../abi", features = ["alloc"] }
core = { version = "1.0.0", optional = true, package = "rustc-std-workspace-core" }
alloc = { version = "1.0.0", optional = true, package = "rustc-std-workspace-alloc" }
@ -22,7 +22,7 @@ cc = "*"
[features]
default = []
alloc = ["dep:alloc", "yggdrasil-abi/alloc"]
__tls_get_addr = []
rustc-dep-of-std = [
"core",
"alloc",

2
lib/runtime/src/lib.rs
View File

@ -9,6 +9,8 @@
extern crate compiler_builtins;
extern crate yggdrasil_abi as abi;
extern crate alloc;
pub use abi::error::Error;
pub use abi::path;
pub mod debug;

27
lib/runtime/src/process/aarch64.rs
View File

@ -1,27 +0,0 @@
#![allow(missing_docs)]
pub fn get_tls() -> usize {
let tp: usize;
unsafe {
core::arch::asm!("mrs {0}, tpidr_el0", out(reg) tp);
}
tp
}
pub unsafe fn set_pthread_self(value: usize) {
todo!()
}
pub fn pthread_self() -> usize {
todo!()
}
pub fn thread_local_area_impl() -> &'static mut [*mut u8] {
let tp = get_tls();
// Kernel reserves two words below the TP
assert_eq!(tp % size_of::<usize>(), 0);
let tp = tp - size_of::<usize>() * 2;
unsafe { super::thread_local_area_common(tp) }
}

27
lib/runtime/src/process/i686.rs
View File

@ -1,27 +0,0 @@
#![allow(missing_docs)]
pub fn get_tls() -> usize {
let tp: usize;
unsafe {
core::arch::asm!("mov %gs:0, {0}", out(reg) tp, options(att_syntax));
}
tp
}
pub unsafe fn set_pthread_self(value: usize) {
todo!()
}
pub fn pthread_self() -> usize {
todo!()
}
pub fn thread_local_area_impl() -> &'static mut [*mut u8] {
let tp = get_tls();
// Skip self-pointer
assert_eq!(tp % size_of::<usize>(), 0);
let tp = tp + size_of::<usize>() * 2;
unsafe { super::thread_local_area_common(tp) }
}

64
lib/runtime/src/process/mod.rs
View File

@ -1,65 +1,9 @@
//! Process management data types
use abi::mem::{MappingFlags, MappingSource};
pub use abi::process::{
ExecveOptions, ExitCode, MutexOperation, ProcessGroupId, ProcessId, ProcessInfoElement,
ProgramArgumentInner, Signal, SignalEntryData, SpawnFlags, SpawnOption, SpawnOptions, ThreadId,
ThreadSpawnOptions,
auxv, AuxValue, AuxValueIter, ExecveOptions, ExitCode, MutexOperation, ProcessGroupId,
ProcessId, ProcessInfoElement, ProgramArgumentInner, Signal, SignalEntryData, SpawnFlags,
SpawnOption, SpawnOptions, StringArgIter, ThreadId, ThreadSpawnOptions,
};
use crate::sys;
const THREAD_LOCAL_AREA_SIZE: usize = 4096 / size_of::<usize>();
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
mod aarch64;
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
use aarch64 as imp;
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
mod x86_64;
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
use x86_64 as imp;
#[cfg(any(target_arch = "x86", rust_analyzer))]
mod i686;
#[cfg(any(target_arch = "x86", rust_analyzer))]
use i686 as imp;
pub use imp::{get_tls, pthread_self, set_pthread_self};
/// # Safety
///
/// `tp` must be a proper and aligned pointer to a pair of [usize] words.
unsafe fn thread_local_area_common(tp: usize) -> &'static mut [*mut u8] {
// 0: segment pointer, 1: segment len
// seg ptr -> thread-local pointer list base
let ptr: *mut *mut *mut u8 = core::ptr::with_exposed_provenance_mut(tp);
let len: *mut usize = core::ptr::with_exposed_provenance_mut(tp + size_of::<usize>());
let (base, len) = if unsafe { (*ptr).is_null() } {
// If segment is NULL, allocate some memory and map it here
let base =
unsafe { sys::map_memory(None, 4096, MappingFlags::WRITE, &MappingSource::Anonymous) }
.expect("Could not allocate TLS storage");
let base = core::ptr::with_exposed_provenance_mut::<*mut u8>(base);
unsafe {
*ptr = base;
*len = THREAD_LOCAL_AREA_SIZE;
}
(base, THREAD_LOCAL_AREA_SIZE)
} else {
// Otherwise return the existing values
unsafe { (*ptr, *len) }
};
unsafe { core::slice::from_raw_parts_mut(base, len) }
}
/// Returns a reference to the thread-local area, which contains thread-local pointers
pub fn thread_local_area() -> &'static mut [*mut u8] {
// TODO grow thread local areas based on size requested?
imp::thread_local_area_impl()
}
pub mod thread_local;

15
lib/runtime/src/process/thread_local/aarch64.rs Normal file
View File

@ -0,0 +1,15 @@
#![allow(missing_docs)]
use abi::error::Error;
pub fn get_thread_pointer() -> usize {
let tp: usize;
unsafe {
core::arch::asm!("mrs {0}, tpidr_el0", out(reg) tp);
}
tp
}
pub unsafe fn set_thread_pointer(value: usize) -> Result<(), Error> {
todo!()
}

15
lib/runtime/src/process/thread_local/i686.rs Normal file
View File

@ -0,0 +1,15 @@
#![allow(missing_docs)]
use abi::error::Error;
pub fn get_thread_pointer() -> usize {
let tp: usize;
unsafe {
core::arch::asm!("mov %gs:0, {0}", out(reg) tp, options(att_syntax));
}
tp
}
pub unsafe fn set_thread_pointer(value: usize) -> Result<(), Error> {
todo!()
}

242
lib/runtime/src/process/thread_local/mod.rs Normal file
View File

@ -0,0 +1,242 @@
//! Yggdrasil Runtime functions for handling Thread-Local Storage
use core::{
ffi::c_void,
ptr::{null_mut, NonNull},
};
use alloc::{boxed::Box, vec::Vec};
use abi::{
error::Error,
process::{auxv, AuxValue, ThreadOption},
};
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
mod aarch64;
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
use aarch64 as imp;
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
mod x86_64;
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
use x86_64 as imp;
#[cfg(any(target_arch = "x86", rust_analyzer))]
mod i686;
#[cfg(any(target_arch = "x86", rust_analyzer))]
use i686 as imp;
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
mod variant1;
#[cfg(any(target_arch = "aarch64", rust_analyzer))]
use variant1 as layout;
#[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
mod variant2;
#[cfg(any(target_arch = "x86", target_arch = "x86_64", rust_analyzer))]
use variant2 as layout;
pub use imp::{get_thread_pointer, set_thread_pointer};
pub use layout::clone_tls;
/// Describes a TLS master copy of this executable
#[derive(Debug)]
pub struct TlsImage {
master_copy: Option<NonNull<[u8]>>,
full_size: usize,
align: usize,
already_initialized: bool,
// Pairs of dynamic module_id:offset from allocation base
module_offsets: Vec<(usize, usize)>,
}
/// DTV table, containing dynamic TLS module mappings
pub struct Dtv {
entries: Vec<*mut c_void>,
}
struct TcbHeader {
self_pointer: usize,
dtv_pointer: *mut Dtv,
}
impl TlsImage {
/// Extracts information about the TLS master copy from aux vector
pub fn from_auxv<'a, I: Iterator<Item = &'a AuxValue>>(auxv: I) -> Option<Self> {
let mut master_copy = None;
let mut data_size = None;
let mut full_size = None;
let mut align = None;
let mut current_module = None;
let mut module_offsets = Vec::new();
let mut already_initialized = false;
for item in auxv {
match item.tag {
auxv::TLS_MASTER_COPY => master_copy = Some(item.val as usize),
auxv::TLS_DATA_SIZE => data_size = Some(item.val as usize),
auxv::TLS_FULL_SIZE => full_size = Some(item.val as usize),
auxv::TLS_ALIGN => align = Some(item.val as usize),
auxv::TLS_MODULE_ID => current_module = Some(item.val as usize),
auxv::TLS_MODULE_OFFSET => {
if let Some(module_id) = current_module.take() {
module_offsets.push((module_id, item.val as usize));
}
}
auxv::TLS_ALREADY_INITIALIZED => {
already_initialized = true;
}
_ => (),
}
}
let full_size = full_size?;
let align = align?;
let master_copy = if let (Some(master_copy), Some(data_size)) = (master_copy, data_size) {
assert_ne!(master_copy, 0);
assert_ne!(data_size, 0);
let base = unsafe {
NonNull::new_unchecked(core::ptr::with_exposed_provenance_mut::<u8>(master_copy))
};
Some(NonNull::slice_from_raw_parts(base, data_size))
} else {
None
};
Some(Self {
master_copy,
full_size,
align,
module_offsets,
already_initialized,
})
}
}
impl Dtv {
fn new() -> Self {
Self {
entries: Vec::new(),
}
}
/// Returns a value associated with a given key.
///
/// # Panics
///
/// Will panic if key == 0.
/// Will panic if key is larger than the DTV itself.
pub fn get(&self, key: usize) -> *mut c_void {
if key == 0 {
panic!("Zero key used when calling DTV::get()");
}
let key = key - 1;
if key >= self.entries.len() {
panic!("Key not in DTV: {}", key + 1);
}
self.entries[key]
}
/// Sets a DTV entry, growing the DTV allocation if necessary
pub fn set(&mut self, key: usize, value: *mut c_void) -> Result<(), Error> {
if key == 0 {
return Err(Error::InvalidArgument);
}
let key = key - 1;
if key >= self.entries.len() {
// TODO return an error if resize fails
self.entries.resize(key + 1, null_mut());
}
self.entries[key] = value;
Ok(())
}
}
/// Initializes thread-local storage for this thread from aux vector provided to the executable.
pub fn init_tls_from_auxv<'a, I: Iterator<Item = &'a AuxValue>>(
auxv: I,
force: bool,
tcb_size: usize,
) -> Result<Option<TlsImage>, Error> {
let Some(tls_image) = TlsImage::from_auxv(auxv) else {
return Ok(None);
};
if force || !tls_image.already_initialized {
let (base, tp) = clone_tls(&tls_image, tcb_size)?;
unsafe { set_thread_pointer(tp) }?;
setup_dtv(&tls_image, base)?;
}
Ok(Some(tls_image))
}
/// Initializes thread-local storage for this thread from a [TlsImage]. If no image is provided,
/// a dummy (platform-specific) TLS copy will be created.
pub fn init_tls(image: Option<&TlsImage>, force: bool, tcb_size: usize) -> Result<(), Error> {
let Some(image) = image else {
crate::debug_trace!("TODO: handle executables with no TLS");
return Err(Error::NotImplemented);
};
if force || !image.already_initialized {
let (base, tp) = clone_tls(image, tcb_size)?;
unsafe { set_thread_pointer(tp) }?;
setup_dtv(&image, base)?;
}
Ok(())
}
fn get_tcb_raw() -> *mut TcbHeader {
let tp = imp::get_thread_pointer();
layout::get_tcb_raw(tp).cast()
}
fn get_tcb_mut() -> &'static mut TcbHeader {
unsafe { &mut *get_tcb_raw() }
}
fn setup_dtv(image: &TlsImage, tls_base: usize) -> Result<(), Error> {
if image.module_offsets.is_empty() {
return Ok(());
}
let dtv = get_dtv();
for &(module_id, module_offset) in image.module_offsets.iter() {
assert!(module_offset < image.full_size);
dtv.set(
module_id,
core::ptr::with_exposed_provenance_mut(tls_base + module_offset),
)?;
}
Ok(())
}
/// Returns the pointer to the TCB contents after the self-pointer and the DTV
pub fn get_tcb() -> *mut u8 {
unsafe { get_tcb_raw().add(1).cast() }
}
/// Gets or allocates a DTV for current thread.
pub fn get_dtv() -> &'static mut Dtv {
let tcb = get_tcb_mut();
if tcb.dtv_pointer.is_null() {
let dtv = Box::new(Dtv::new());
let dtv = Box::into_raw(dtv);
tcb.dtv_pointer = dtv;
}
unsafe { &mut *tcb.dtv_pointer }
}
#[cfg(any(feature = "__tls_get_addr", rust_analyzer))]
#[no_mangle]
unsafe extern "C" fn __tls_get_addr(index: *mut usize) -> *mut c_void {
let module_id = index.read();
let offset = index.add(1).read();
assert!(module_id > 0);
get_dtv().get(module_id).add(offset)
}

13
lib/runtime/src/process/thread_local/variant1.rs Normal file
View File

@ -0,0 +1,13 @@
use abi::error::Error;
use super::TlsImage;
/// Creates a new TLS image in the process memory, copying data from the TLS master copy (if any).
/// Returns the resulting thread pointer.
pub fn clone_tls(image: &TlsImage, tcb_size: usize) -> Result<(usize, usize), Error> {
Err(Error::NotImplemented)
}
pub(super) fn get_tcb_raw(_tp: usize) -> *mut u8 {
todo!()
}

85
lib/runtime/src/process/thread_local/variant2.rs Normal file
View File

@ -0,0 +1,85 @@
use abi::{
error::Error,
mem::{MappingFlags, MappingSource},
};
use super::TlsImage;
// Variant II TLS layout:
//
// ... [ MODULE ] [ MODULE ] [ MODULE ] [ TCB ]
// | | | |
// | | | |
// off3 off2 off1 tp
/// Creates a new TLS image in the process memory, copying data from the TLS master copy (if any).
/// Returns the resulting thread pointer.
pub fn clone_tls(image: &TlsImage, tcb_size: usize) -> Result<(usize, usize), Error> {
// Basically, the layout is:
// * align(image.full_size) below the TP
// * tcb_size starting with the TP
// If the caller asked for no TCB to be placed, just reserve two words, the first one
// will be used to put the self-pointer
let tcb_size = if tcb_size < size_of::<usize>() * 2 {
size_of::<usize>() * 2
} else {
(tcb_size + size_of::<usize>() - 1) & !(size_of::<usize>() - 1)
};
if !image.align.is_power_of_two() {
panic!("TLS layout not aligned to a power of two: {}", image.align);
}
if image.align > 0x1000 {
todo!("Cannot yet handle TLS block alignmnent larger than page size")
}
let align = image.align;
let aligned_size = (image.full_size + tcb_size + align - 1) & !(align - 1);
let page_aligned_size = (aligned_size + 0xFFf) & !0xFFF;
let base = unsafe {
crate::sys::map_memory(
None,
page_aligned_size,
MappingFlags::WRITE,
&MappingSource::Anonymous,
)
}?;
let ptr = core::ptr::with_exposed_provenance_mut::<u8>(base);
let module_data = unsafe { core::slice::from_raw_parts_mut(ptr, image.full_size) };
let tcb = unsafe { core::slice::from_raw_parts_mut(ptr.add(image.full_size), tcb_size) };
// Clone the Master Copy, if any
let data_size = if let Some(master_copy) = image.master_copy {
let source = unsafe { master_copy.as_ref() };
module_data[..source.len()].copy_from_slice(source);
source.len()
} else {
0
};
// Zero the rest
module_data[data_size..image.full_size].fill(0);
// Place the self-pointer to point at the TP itself
let tp = base + image.full_size;
assert_eq!(
tp % size_of::<usize>(),
0,
"TLS self-pointer should at least be word-aligned"
);
tcb[..size_of::<usize>()].copy_from_slice(&tp.to_ne_bytes());
// Zero the TCB after the self-pointer
tcb[size_of::<usize>()..].fill(0);
Ok((base, tp))
}
// In Variant II, the TP points directly at the TCB start
pub(super) fn get_tcb_raw(tp: usize) -> *mut u8 {
core::ptr::with_exposed_provenance_mut(tp)
}

15
lib/runtime/src/process/thread_local/x86_64.rs Normal file
View File

@ -0,0 +1,15 @@
#![allow(missing_docs)]
use abi::{error::Error, process::ThreadOption};
pub fn get_thread_pointer() -> usize {
let tp: usize;
unsafe {
core::arch::asm!("mov %fs:0, {0}", out(reg) tp, options(att_syntax));
}
tp
}
pub unsafe fn set_thread_pointer(value: usize) -> Result<(), Error> {
crate::sys::set_thread_option(&ThreadOption::ThreadPointer(value))
}

35
lib/runtime/src/process/x86_64.rs
View File

@ -1,35 +0,0 @@
#![allow(missing_docs)]
pub fn get_tls() -> usize {
let tp: usize;
unsafe {
core::arch::asm!("mov %fs:0, {0}", out(reg) tp, options(att_syntax));
}
tp
}
pub unsafe fn set_pthread_self(value: usize) {
let tls: *mut usize = core::ptr::with_exposed_provenance_mut(get_tls());
if tls.is_null() {
panic!("TLS pointer is NULL");
}
tls.add(1).write(value);
}
pub fn pthread_self() -> usize {
let tls: *const usize = core::ptr::with_exposed_provenance(get_tls());
if tls.is_null() {
panic!("TLS pointer is NULL");
}
unsafe { tls.add(1).read() }
}
pub fn thread_local_area_impl() -> &'static mut [*mut u8] {
let tp = get_tls();
// Skip self-pointer
assert_eq!(tp % size_of::<usize>(), 0);
let tp = tp + size_of::<usize>() * 2;
unsafe { super::thread_local_area_common(tp) }
}

8
test.c
View File

@ -2,14 +2,22 @@
#include <assert.h>
#include <stdio.h>
_Thread_local int x = 123;
static void *function(void *arg) {
printf("This is thread %u\n", pthread_self());
printf("argument: %s\n", (const char *) arg);
printf("[child] x = %d\n", x);
x += 1;
printf("[child] x = %d\n", x);
return NULL;
}
int main(int argc, const char **argv) {
pthread_t thread;
printf("[main] x = %d\n", x);
x = 321;
printf("[main] x = %d\n", x);
assert(pthread_create(&thread, NULL, function, (void *) "thread1") == 0);

8
test.cpp
View File

@ -1,4 +1,6 @@
#include <iostream>
#include <cerrno>
#include <fcntl.h>
int main() {
std::vector<int> items;
@ -9,5 +11,11 @@ int main() {
for (const auto &item: items) {
std::cout << "Item: " << item << std::endl;
}
std::cout << "&errno = " << &errno << std::endl;
std::cout << "errno = " << errno << std::endl;
open("/xxxsaa", 0, 0);
std::cout << "errno = " << errno << std::endl;
return 0;
}

1
userspace/Cargo.lock generated
View File

@ -375,6 +375,7 @@ dependencies = [
name = "dyn-loader"
version = "0.1.0"
dependencies = [
"bytemuck",
"elf",
"thiserror",
"yggdrasil-rt",

1
userspace/dyn-loader/Cargo.toml
View File

@ -8,6 +8,7 @@ yggdrasil-rt.workspace = true
thiserror.workspace = true
elf = "0.7.4"
bytemuck = "1.19.0"
[lints.rust]
unexpected_cfgs = { level = "allow", check-cfg = ['cfg(rust_analyzer)'] }

0
userspace/dyn-loader/src/builtins.rs Normal file
View File

99
userspace/dyn-loader/src/env.rs Normal file
View File

@ -0,0 +1,99 @@
use bytemuck::Pod;
use yggdrasil_rt::{mem::MappingFlags, process::{auxv, AuxValue, ProgramArgumentInner}};
use crate::{error::Error, mapping::Mapping, thread_local::{TlsImage, TlsLayout}};
struct ArgPlacer<'a> {
buffer: &'a mut [u8],
position: usize,
}
impl<'a> ArgPlacer<'a> {
fn new(buffer: &'a mut [u8]) -> Self {
Self {
buffer,
position: 0,
}
}
fn align(&mut self, align: usize) -> Result<(), Error> {
debug_assert!(align.is_power_of_two());
let aligned = (self.position + align - 1) & !(align - 1);
if aligned > self.buffer.len() {
todo!()
}
self.position = aligned;
Ok(())
}
fn put_str(&mut self, s: &str) -> Result<usize, Error> {
if self.position + s.len() >= self.buffer.len() {
todo!()
}
let off = self.position;
self.buffer[off..off + s.len()].copy_from_slice(s.as_bytes());
self.buffer[off + s.len()] = 0;
self.position += s.len() + 1;
Ok(off)
}
fn put<T: Pod>(&mut self, v: &T) -> Result<usize, Error> {
if self.position + size_of::<T>() > self.buffer.len() {
todo!()
}
let off = self.position;
self.buffer[off..off + size_of::<T>()].copy_from_slice(bytemuck::bytes_of(v));
self.position += size_of::<T>();
Ok(off)
}
fn put_aux_array(&mut self, s: &[AuxValue]) -> Result<usize, Error> {
self.align(size_of::<u64>())?;
let off = self.position;
for item in s {
self.put(&item.tag)?;
self.put(&item.val)?;
}
self.put(&auxv::NULL)?;
self.put(&0u64)?;
Ok(off)
}
fn put_ptr_array(&mut self, s: &[usize]) -> Result<usize, Error> {
self.align(size_of::<usize>())?;
let off = self.position;
for item in s {
self.put(item)?;
}
self.put(&0usize)?;
Ok(off)
}
}
pub fn build_argument(args: &[String], auxv: &[AuxValue]) -> Result<usize, Error> {
let mut buffer = Mapping::new(0x1000, MappingFlags::WRITE)?;
let arg_base = buffer.as_ptr().addr();
let mut placer = ArgPlacer::new(&mut buffer[..]);
let mut argv = vec![];
for arg in args {
argv.push(placer.put_str(arg)? + arg_base);
}
// TODO env
let argv = placer.put_ptr_array(&argv)? + arg_base;
let envp = placer.put_ptr_array(&[])? + arg_base;
let auxv = placer.put_aux_array(&auxv)? + arg_base;
let argument = placer.position + arg_base;
placer.put(&argv)?;
placer.put(&envp)?;
placer.put(&auxv)?;
buffer.leak();
Ok(argument)
}

124
userspace/dyn-loader/src/main.rs
View File

@ -1,43 +1,31 @@
#![feature(yggdrasil_os, never_type, map_try_insert, slice_ptr_get)]
#![feature(yggdrasil_os, never_type, map_try_insert, slice_ptr_get, iter_chain)]
use std::{collections::HashMap, env, process::ExitCode};
use std::process::ExitCode;
use error::Error;
use object::Object;
use state::State;
use yggdrasil_rt::process::ProgramArgumentInner;
use state::ObjectSet;
use yggdrasil_rt::process::{auxv, AuxValue};
pub mod builtins;
pub mod env;
pub mod error;
pub mod object;
pub mod mapping;
pub mod state;
pub mod object;
pub mod relocation;
pub mod search;
pub mod state;
pub mod thread_local;
fn run(binary: &str, args: &[String]) -> Result<!, Error> {
let mut state = State::new();
let mut root = Object::open(binary)?;
let mut libraries = HashMap::new();
for needed in root.needed() {
// TODO load needed of needed
let path = search::find_library(needed.as_str())?;
if libraries.contains_key(&path) {
continue;
}
// Open root and its dependencies
let root = Object::open(binary)?;
let mut objects = ObjectSet::new(root);
objects.open_root_dependencies()?;
let tls_image = objects.load_all()?;
objects.resolve_symbols()?;
let object = Object::open(&path)?;
libraries.insert(path, object);
}
root.load(&mut state)?;
for (_, lib) in libraries.iter_mut() {
lib.load(&mut state)?;
}
root.resolve_symbols(&mut state);
for (_, lib) in libraries.iter_mut() {
lib.resolve_symbols(&mut state);
}
if let Err(undefined) = state.no_undefined_symbols() {
if let Err(undefined) = objects.state.no_undefined_symbols() {
eprintln!("Undefined symbols:");
for (path, syms) in undefined {
eprintln!(" in {path:?}");
@ -48,42 +36,74 @@ fn run(binary: &str, args: &[String]) -> Result<!, Error> {
return Err(Error::UnresolvedSymbols);
}
root.relocate(&mut state)?;
for (_, lib) in libraries.iter_mut() {
lib.relocate(&mut state)?;
objects.relocate()?;
let mut auxv = vec![];
if let Some(image) = tls_image {
let layout = objects.state.tls_layout.as_ref().unwrap();
let size = image.size;
let base = image.data.leak();
auxv.push(AuxValue {
tag: auxv::TLS_MASTER_COPY,
val: base as _,
});
auxv.push(AuxValue {
tag: auxv::TLS_ALIGN,
val: image.align as _,
});
auxv.push(AuxValue {
tag: auxv::TLS_DATA_SIZE,
val: size as _,
});
auxv.push(AuxValue {
tag: auxv::TLS_FULL_SIZE,
val: size as _,
});
for module in layout.segments.iter() {
if module.object_id == 0 {
continue;
}
// Call global constructors before handing off control to the program
// .preinit_array, .preinit, ...
for (_, lib) in libraries.iter_mut() {
unsafe { lib.call_early_constructors() };
auxv.push(AuxValue {
tag: auxv::TLS_MODULE_ID,
val: module.object_id as _,
});
auxv.push(AuxValue {
tag: auxv::TLS_MODULE_OFFSET,
val: module.offset as _,
});
}
unsafe { root.call_early_constructors() };
// .init_array, .init, ...
for (_, lib) in libraries.iter_mut() {
unsafe { lib.call_constructors() };
}
unsafe { root.call_constructors() };
let entry = root.entry().ok_or(Error::NoEntrypoint)?;
// Set up TLS for the main thread. This needs to be done here, because
// we still need to call initializers before handing off control to the executable
// TODO use an ELF note to indicate the extra size required in the TCB
yggdrasil_rt::process::thread_local::init_tls_from_auxv(auxv.iter(), true, size_of::<usize>() * 4)
.expect("Could not initialize main thread TLS for the executable");
auxv.push(AuxValue {
tag: auxv::TLS_ALREADY_INITIALIZED,
val: 0,
});
// Call initializers
objects.initialize();
// Setup arguments and enter the main object
let entry = objects.root.entry().ok_or(Error::NoEntrypoint)?;
debug_trace!("entry = {:p}", entry);
let args = args.iter().map(|s| s.as_str()).collect::<Vec<_>>();
// TODO
let envs = vec![];
let argument = env::build_argument(args, &auxv)?;
let arg = Box::new(ProgramArgumentInner {
args: &args,
env: &envs
});
let arg = Box::into_raw(arg).addr();
entry(arg);
entry(argument);
unreachable!()
}
fn main() -> ExitCode {
let args: Vec<String> = env::args().skip(1).collect();
let args: Vec<String> = std::env::args().skip(1).collect();
if args.is_empty() {
// Dump help and exit

9
userspace/dyn-loader/src/mapping.rs
View File

@ -1,6 +1,5 @@
use std::{
ops::{Deref, DerefMut, Range},
ptr::NonNull,
mem, ops::{Deref, DerefMut, Range}, ptr::NonNull
};
use yggdrasil_rt::mem::{MappingFlags, MappingSource};
@ -23,6 +22,12 @@ impl Mapping {
Ok(Self { data })
}
pub fn leak(self) -> usize {
let base = self.data.addr().into();
mem::forget(self);
base
}
pub fn qword(&mut self, offset: u64) -> NonNull<i64> {
unsafe { self.data.as_non_null_ptr().add(offset as usize).cast() }
}

63
userspace/dyn-loader/src/object.rs
View File

@ -13,9 +13,10 @@ use elf::{
DF_1_PIE, DT_FINI, DT_FINI_ARRAY, DT_FINI_ARRAYSZ, DT_FLAGS_1, DT_INIT, DT_INIT_ARRAY,
DT_INIT_ARRAYSZ, DT_NEEDED, DT_PREINIT_ARRAY, DT_PREINIT_ARRAYSZ, ET_DYN, ET_EXEC,
PT_DYNAMIC, PT_GNU_EH_FRAME, PT_GNU_RELRO, PT_GNU_STACK, PT_INTERP, PT_LOAD, PT_NOTE,
PT_NULL, PT_PHDR, SHN_UNDEF, SHT_REL, SHT_RELA, STB_GLOBAL, STB_LOCAL, STB_WEAK,
PT_NULL, PT_PHDR, PT_TLS, SHN_UNDEF, SHT_REL, SHT_RELA, STB_GLOBAL, STB_LOCAL, STB_WEAK,
},
endian::AnyEndian,
segment::ProgramHeader,
symbol::Symbol,
ElfStream,
};
@ -25,7 +26,7 @@ use crate::{
error::Error,
mapping::Mapping,
relocation::{Relocation, RelocationValue},
state::{ExportedSymbol, State},
state::{ExportedNormalSymbol, ExportedSymbol, ExportedTlsSymbol, State}, thread_local::TlsLayout,
};
pub enum ElfType {
@ -36,7 +37,8 @@ pub enum ElfType {
}
pub enum ResolvedSymbol<'s> {
Global(&'s ExportedSymbol),
Global(&'s ExportedNormalSymbol),
Tls(&'s ExportedTlsSymbol),
Local,
Null,
}
@ -57,6 +59,10 @@ pub struct Object {
pub ty: ElfType,
needed: Vec<String>,
pub tls: Option<ProgramHeader>,
pub tls_offset: Option<usize>,
pub id: u32,
mapping: Option<Mapping>,
dynamic_symbol_array: Vec<DynamicSymbol>,
@ -69,6 +75,7 @@ impl ResolvedSymbol<'_> {
match *self {
Self::Local => todo!(),
Self::Global(sym) => sym.value as i64,
Self::Tls(_) => 0,
Self::Null => 0,
}
}
@ -137,6 +144,16 @@ impl Object {
}
}
let tls_segments = elf
.segments()
.iter()
.filter(|p| p.p_type == PT_TLS)
.collect::<Vec<_>>();
if tls_segments.len() > 1 {
todo!("Support objects with more than one TLS segment");
}
let tls = tls_segments.get(0).map(|s| **s);
Ok(Self {
path,
file,
@ -147,6 +164,10 @@ impl Object {
vma_end,
needed,
tls,
tls_offset: None,
id: 0,
mapping: None,
dynamic_symbol_array,
@ -155,6 +176,10 @@ impl Object {
})
}
pub fn set_id(&mut self, id: u32) {
self.id = id;
}
pub fn load(&mut self, state: &mut State) -> Result<(), Error> {
// Already loaded
if self.mapping.is_some() {
@ -236,6 +261,8 @@ impl Object {
segment_data[file_size..].fill(0);
}
// Handled separately
PT_TLS => (),
// TODO handle GNU_STACK
PT_DYNAMIC | PT_GNU_RELRO | PT_GNU_STACK | PT_INTERP | PT_PHDR
| PT_GNU_EH_FRAME | PT_NOTE => (),
@ -243,6 +270,24 @@ impl Object {
}
}
self.mapping = Some(mapping);
Ok(())
}
pub fn place_tls_copy(&mut self, dst: &mut [u8]) -> Result<(), Error> {
let tls = self.tls.as_ref().unwrap();
if tls.p_filesz > 0 {
self.file.seek(SeekFrom::Start(tls.p_offset))?;
self.file.read_exact(&mut dst[..tls.p_filesz as usize])?;
}
// Zero the rest
dst[tls.p_filesz as usize..].fill(0);
Ok(())
}
pub fn export_symbols(&self, state: &mut State) -> Result<(), Error> {
let mapping = self.mapping.as_ref().ok_or(Error::NotLoaded)?;
// Export dynamic symbols into the linking table
for dynsym in self.dynamic_symbol_array.iter() {
// Don't export undefined symbols
@ -257,14 +302,13 @@ impl Object {
// S: "image" load base
let offset = mapping.base() as isize - self.vma_start as isize;
match dynsym.raw.st_bind() {
STB_GLOBAL => state.export(&self.path, dynsym, offset, false),
STB_WEAK => state.export(&self.path, dynsym, offset, true),
STB_GLOBAL => state.export(&self.path, dynsym, offset, false, self.id),
STB_WEAK => state.export(&self.path, dynsym, offset, true, self.id),
STB_LOCAL => todo!(),
_ => todo!(),
}
}
self.mapping = Some(mapping);
Ok(())
}
@ -350,7 +394,10 @@ impl Object {
for rela in rela_section {
let dynsym = &self.dynamic_symbol_array[rela.r_sym as usize];
let sym = match dynsym.raw.st_bind() {
STB_GLOBAL | STB_WEAK => ResolvedSymbol::Global(state.lookup(dynsym).unwrap()),
STB_GLOBAL | STB_WEAK => match state.lookup(dynsym).unwrap() {
ExportedSymbol::Normal(export) => ResolvedSymbol::Global(export),
ExportedSymbol::Tls(export) => ResolvedSymbol::Tls(export),
}
STB_LOCAL => {
if dynsym.name.is_empty() {
ResolvedSymbol::Null
@ -361,7 +408,7 @@ impl Object {
_ => todo!(),
};
if let Some(value) = rela.resolve(&sym, mapping.base())? {
if let Some(value) = rela.resolve(&state, &dynsym.name, &sym, mapping.base())? {
value.write(mapping, rela.r_offset);
}
}

10
userspace/dyn-loader/src/relocation/mod.rs
View File

@ -1,4 +1,4 @@
use crate::{error::Error, mapping::Mapping, object::ResolvedSymbol};
use crate::{error::Error, mapping::Mapping, object::ResolvedSymbol, state::State};
#[cfg(any(target_arch = "x86_64", rust_analyzer))]
mod x86_64;
@ -14,7 +14,13 @@ pub trait RelocationValue {
pub trait Relocation {
type Value: RelocationValue;
fn resolve(&self, symbol: &ResolvedSymbol, load_base: usize) -> Result<Option<Self::Value>, Error>;
fn resolve(
&self,
state: &State,
name: &str,
symbol: &ResolvedSymbol,
load_base: usize,
) -> Result<Option<Self::Value>, Error>;
}
impl RelocationValue for RelaValue {

39
userspace/dyn-loader/src/relocation/x86_64.rs
View File

@ -3,12 +3,16 @@ use std::path::Path;
use elf::{
abi::{
R_X86_64_64, R_X86_64_DTPMOD64, R_X86_64_DTPOFF64, R_X86_64_GLOB_DAT, R_X86_64_JUMP_SLOT,
R_X86_64_RELATIVE,
R_X86_64_RELATIVE, R_X86_64_TPOFF64,
},
relocation::Rela,
};
use crate::{error::Error, object::{DynamicSymbol, ResolvedSymbol}, state::State};
use crate::{
error::Error,
object::{DynamicSymbol, ResolvedSymbol},
state::State,
};
use super::{RelaValue, Relocation};
@ -17,9 +21,31 @@ impl Relocation for Rela {
fn resolve(
&self,
state: &State,
name: &str,
symbol: &ResolvedSymbol,
load_base: usize,
) -> Result<Option<Self::Value>, Error> {
match symbol {
ResolvedSymbol::Tls(tls) => {
match self.r_type {
// Object ID (index into DTV) of the object containing this symbol
R_X86_64_DTPMOD64 => Ok(Some(RelaValue::QWord(tls.module_id as _))),
R_X86_64_DTPOFF64 => Ok(Some(RelaValue::QWord(tls.offset as _))),
R_X86_64_TPOFF64 => {
// Need to extract fixed global offset
let tls_layout = state.tls_layout.as_ref().unwrap();
// Offset from TLS start
let offset = tls_layout.offset(tls.module_id, tls.offset).unwrap();
let offset_from_tp = -((tls_layout.tp_offset - offset) as i64);
debug_trace!("{}@tpoff -> {}", name, offset_from_tp);
Ok(Some(RelaValue::QWord(offset_from_tp)))
},
_ => todo!(),
}
}
_ => {
let s = symbol.value() as i64;
if s == 0 && self.r_type != R_X86_64_RELATIVE {
todo!()
@ -30,13 +56,14 @@ impl Relocation for Rela {
// S + A
R_X86_64_64 => Ok(Some(RelaValue::QWord(s + self.r_addend))),
// B + A
R_X86_64_RELATIVE => Ok(Some(RelaValue::QWord(load_base as i64 + self.r_addend))),
// TLS
R_X86_64_DTPOFF64 => todo!(),
R_X86_64_DTPMOD64 => todo!(),
R_X86_64_RELATIVE => {
Ok(Some(RelaValue::QWord(load_base as i64 + self.r_addend)))
}
_ => todo!(),
}
}
}
}
// fn resolve<'a, F: Fn(u32) -> Result<&'a DynamicSymbol, Error>>(
// &'a self,

219
userspace/dyn-loader/src/state.rs
View File

@ -6,31 +6,72 @@ use std::{
use elf::abi::{STT_FUNC, STT_NOTYPE, STT_OBJECT, STT_TLS};
use crate::object::DynamicSymbol;
use crate::{
error::Error,
object::{DynamicSymbol, Object},
search,
thread_local::{self, TlsImage, TlsLayout},
};
pub struct ExportedSymbol {
pub struct ExportedNormalSymbol {
pub source: PathBuf,
pub value: usize,
pub weak: bool
pub weak: bool,
}
pub struct ExportedTlsSymbol {
pub source: PathBuf,
// Offset from TLS block start
pub offset: usize,
// TLS module ID
pub module_id: u32,
pub weak: bool,
}
pub enum ExportedSymbol<'a> {
Normal(&'a ExportedNormalSymbol),
Tls(&'a ExportedTlsSymbol),
}
pub struct State {
symbol_table: HashMap<Rc<str>, ExportedSymbol>,
undefined_references: HashMap<PathBuf, HashSet<Rc<str>>>
symbol_table: HashMap<Rc<str>, ExportedNormalSymbol>,
tls_symbol_table: HashMap<Rc<str>, ExportedTlsSymbol>,
undefined_references: HashMap<PathBuf, HashSet<Rc<str>>>,
pub tls_layout: Option<TlsLayout>,
}
pub struct ObjectSet {
pub root: Object,
libraries: HashMap<u32, Object>,
library_path_map: HashMap<PathBuf, u32>,
// Linker state
pub state: State,
last_object_id: u32,
}
impl State {
pub fn new() -> Self {
Self {
symbol_table: HashMap::new(),
tls_symbol_table: HashMap::new(),
undefined_references: HashMap::new(),
tls_layout: None,
}
}
pub fn export(&mut self, source: impl AsRef<Path>, sym: &DynamicSymbol, offset: isize, weak: bool) {
pub fn export(
&mut self,
source: impl AsRef<Path>,
sym: &DynamicSymbol,
offset: isize,
weak: bool,
object_id: u32,
) {
let source = source.as_ref().to_owned();
match sym.raw.st_symtype() {
STT_FUNC | STT_OBJECT | STT_NOTYPE => {
let source = source.as_ref().to_owned();
let value: usize = (isize::try_from(sym.raw.st_value).unwrap() + offset)
.try_into()
.unwrap();
@ -38,16 +79,53 @@ impl State {
match self.symbol_table.get_mut(&sym.name) {
// Stronger binding exported
Some(export) if export.weak && !weak => {
*export = ExportedSymbol { source, value, weak };
},
*export = ExportedNormalSymbol {
source,
value,
weak,
};
}
// Do nothing, already strong or already weak
Some(_) => (),
None => {
self.symbol_table.insert(sym.name.clone(), ExportedSymbol { source, value, weak });
self.symbol_table.insert(
sym.name.clone(),
ExportedNormalSymbol {
source,
value,
weak,
},
);
}
}
}
STT_TLS => {
let offset = sym.raw.st_value as usize;
match self.tls_symbol_table.get_mut(&sym.name) {
Some(export) if export.weak && !weak => {
*export = ExportedTlsSymbol {
source,
offset,
module_id: object_id,
weak,
};
}
Some(_) => (),
None => {
debug_trace!("{:?}: TLS {:?} -> {}:{:#x}", source, sym.name, object_id, offset);
self.tls_symbol_table.insert(
sym.name.clone(),
ExportedTlsSymbol {
source,
offset,
module_id: object_id,
weak,
},
);
}
}
}
STT_TLS => todo!(),
_ => todo!(),
}
}
@ -58,18 +136,24 @@ impl State {
if !self.symbol_table.contains_key(&sym.name) {
self.undefined(source, &sym.name);
}
},
STT_TLS => todo!(),
}
STT_TLS => {
if !self.tls_symbol_table.contains_key(&sym.name) {
self.undefined(source, &sym.name);
}
}
_ => todo!(),
}
}
pub fn lookup(&mut self, sym: &DynamicSymbol) -> Option<&ExportedSymbol> {
pub fn lookup(&self, sym: &DynamicSymbol) -> Option<ExportedSymbol> {
match sym.raw.st_symtype() {
STT_FUNC | STT_OBJECT | STT_NOTYPE => {
self.symbol_table.get(&sym.name)
self.symbol_table.get(&sym.name).map(ExportedSymbol::Normal)
}
STT_TLS => {
self.tls_symbol_table.get(&sym.name).map(ExportedSymbol::Tls)
},
STT_TLS => todo!(),
_ => todo!(),
}
}
@ -93,3 +177,106 @@ impl State {
}
}
}
impl ObjectSet {
pub fn new(mut root: Object) -> Self {
root.set_id(0);
Self {
root,
libraries: HashMap::new(),
library_path_map: HashMap::new(),
state: State::new(),
last_object_id: 0,
}
}
pub fn open_root_dependencies(&mut self) -> Result<(), Error> {
for needed in self.root.needed() {
// TODO load needed of needed
let path = search::find_library(needed.as_str())?;
if self.library_path_map.contains_key(&path) {
continue;
}
let mut object = Object::open(&path)?;
self.last_object_id += 1;
let id = self.last_object_id;
object.id = id;
self.libraries.insert(id, object);
self.library_path_map.insert(path, id);
}
Ok(())
}
pub fn load_all(&mut self) -> Result<Option<TlsImage>, Error> {
// Load all objects somewhere, order doesn't matter
self.root.load(&mut self.state)?;
for (_, lib) in self.libraries.iter_mut() {
lib.load(&mut self.state)?;
}
// Build a TLS master copy somewhere
match thread_local::build_tls_image(&mut self.root, &mut self.libraries)? {
Some((tls_image, tls_layout)) => {
self.state.tls_layout = Some(tls_layout);
Ok(Some(tls_image))
}
None => Ok(None),
}
}
pub fn resolve_symbols(&mut self) -> Result<(), Error> {
// Export all symbols
self.root.export_symbols(&mut self.state)?;
for (_, lib) in self.libraries.iter_mut() {
lib.export_symbols(&mut self.state)?;
}
// Resolve all symbols
self.root.resolve_symbols(&mut self.state);
for (_, lib) in self.libraries.iter_mut() {
lib.resolve_symbols(&mut self.state);
}
Ok(())
}
pub fn relocate(&mut self) -> Result<(), Error> {
self.root.relocate(&mut self.state)?;
for (_, lib) in self.libraries.iter_mut() {
lib.relocate(&mut self.state)?;
}
Ok(())
}
pub fn initialize(&mut self) {
// Call global constructors before handing off control to the program
// .preinit_array, .preinit, ...
for (_, lib) in self.libraries.iter_mut() {
unsafe { lib.call_early_constructors() };
}
unsafe { self.root.call_early_constructors() };
// .init_array, .init, ...
for (_, lib) in self.libraries.iter_mut() {
unsafe { lib.call_constructors() };
}
unsafe { self.root.call_constructors() };
}
pub fn is_library_loaded(&self, p: impl AsRef<Path>) -> bool {
self.library_path_map.contains_key(p.as_ref())
}
pub fn get(&self, id: u32) -> Option<&Object> {
match id {
0 => Some(&self.root),
_ => self.libraries.get(&id),
}
}
pub fn get_mut(&mut self, id: u32) -> Option<&mut Object> {
match id {
0 => Some(&mut self.root),
_ => self.libraries.get_mut(&id),
}
}
}

93
userspace/dyn-loader/src/thread_local/mod.rs Normal file
View File

@ -0,0 +1,93 @@
#[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))]
mod variant2;
use std::collections::HashMap;
use elf::segment::ProgramHeader;
#[cfg(any(target_arch = "x86_64", target_arch = "x86", rust_analyzer))]
pub use variant2::TlsLayoutImpl;
use yggdrasil_rt::mem::MappingFlags;
use crate::{error::Error, mapping::Mapping, object::Object};
pub trait TlsLayoutBuilder {
fn build(align: usize, root: &Object, libraries: &HashMap<u32, Object>) -> TlsLayout;
}
#[derive(Debug)]
pub struct TlsLayout {
pub segments: Vec<TlsSegment>,
pub total_size: usize,
pub tp_offset: usize,
}
#[derive(Debug)]
pub struct TlsSegment {
pub offset: usize,
pub size: usize,
pub object_id: u32,
}
pub struct TlsImage {
// TLS master image that will be passed to the program
pub data: Mapping,
pub size: usize,
pub align: usize,
pub tp_offset: usize,
}
impl TlsLayout {
pub fn offset(&self, object_id: u32, offset: usize) -> Option<usize> {
let segment = self.segments.iter().find(|seg| seg.object_id == object_id)?;
Some(segment.offset + offset)
}
}
pub fn build_tls_image(
root: &mut Object,
libraries: &mut HashMap<u32, Object>,
) -> Result<Option<(TlsImage, TlsLayout)>, Error> {
// Find max align
let mut align = usize::MIN;
if let Some(tls) = root.tls.as_ref() {
align = align.max(tls.p_align as usize);
}
for tls in libraries.values().filter_map(|lib| lib.tls.as_ref()) {
align = align.max(tls.p_align as usize);
}
// No TLS segment in any of the objects
if align == usize::MIN {
return Ok(None);
}
let align = align.max(size_of::<usize>());
let layout = TlsLayoutImpl::build(align, root, libraries);
// Create a master image based on this layout
let mut image_data = Mapping::new(layout.total_size, MappingFlags::empty())?;
let tp_offset = layout.tp_offset;
for (i, segment) in layout.segments.iter().enumerate() {
debug_trace!(
"Load TLS segment: tlsoffset_i={:#x?}, module_id={}",
segment.offset..segment.offset + segment.size,
segment.object_id
);
let object = match segment.object_id {
0 => &mut *root,
_ => libraries.get_mut(&segment.object_id).unwrap(),
};
object.tls_offset = Some(segment.offset);
object.place_tls_copy(&mut image_data[segment.offset..segment.offset + segment.size])?;
}
Ok(Some((
TlsImage {
data: image_data,
size: layout.total_size,
align,
tp_offset,
},
layout,
)))
}

65
userspace/dyn-loader/src/thread_local/variant2.rs Normal file
View File

@ -0,0 +1,65 @@
use std::collections::HashMap;
use elf::segment::ProgramHeader;
use crate::object::Object;
use super::{TlsLayout, TlsLayoutBuilder, TlsSegment};
pub struct TlsLayoutImpl;
impl TlsLayoutBuilder for TlsLayoutImpl {
fn build(align: usize, root: &Object, libraries: &HashMap<u32, Object>) -> TlsLayout {
let mut total_size = 0;
// Layout:
//
// [ Module N | padding ] ... [ Module 1 | padding ] [ padding | Module 0 ] [ TP ]
let mut modules = vec![];
if let Some(tls) = root.tls.as_ref() {
let size = tls.p_memsz as usize;
let aligned_size = (size + align - 1) & !(align - 1);
let pad = aligned_size - size;
// module_id, offset from tp, full size
modules.push((0, size, aligned_size));
total_size += aligned_size;
}
for (&id, lib) in libraries.iter() {
let Some(tls) = lib.tls.as_ref() else {
continue;
};
let size = tls.p_memsz as usize;
let aligned_size = (size + align - 1) & !(align - 1);
modules.push((id, total_size + aligned_size, aligned_size));
total_size += aligned_size;
}
let tp_offset = total_size;
// All the offsets calculated before are (tp_offset - x) offsets, recalculate those
// before returning the module list
for (_, off, _) in modules.iter_mut() {
*off = tp_offset - *off;
}
let segments = modules
.into_iter()
.map(|(object_id, offset, size)| TlsSegment {
offset,
size,
object_id,
}).collect();
TlsLayout {
segments,
total_size,
tp_offset
}
}
}

2
userspace/lib/ygglibc/Cargo.toml
View File

@ -7,7 +7,7 @@ edition = "2021"
crate-type = ["cdylib", "staticlib"]
[dependencies]
yggdrasil-rt = { path = "../../../lib/runtime" }
yggdrasil-rt = { path = "../../../lib/runtime", features = ["__tls_get_addr"] }
yggdrasil-abi = { path = "../../../lib/abi", features = ["alloc", "bytemuck"] }
libyalloc = { path = "../../../lib/libyalloc" }

2
userspace/lib/ygglibc/etc/x86_64-unknown-none.json
View File

@ -17,7 +17,7 @@
"position-independent-executables": true,
"crt-static-allows-dylibs": true,
"has-thread-local": false,
"has-thread-local": true,
"linker": "rust-lld",
"linker-flavor": "ld.lld",

2
userspace/lib/ygglibc/include/bits/errno.h
View File

@ -5,7 +5,7 @@
extern "C" {
#endif
extern int errno;
extern _Thread_local int errno;
#if defined(__cplusplus)
}

41
userspace/lib/ygglibc/src/env.rs
View File

@ -6,17 +6,15 @@ use core::{
slice::memchr,
};
use alloc::{ffi::CString, vec::Vec};
use yggdrasil_rt::process::ProgramArgumentInner;
use alloc::{borrow::ToOwned, ffi::CString, vec::Vec};
use yggdrasil_rt::process::{thread_local, ProgramArgumentInner};
// use yggdrasil_rt::process::ProgramArgumentInner;
use crate::{
allocator::{c_alloc, c_free},
error::EResult,
headers::{
allocator::{c_alloc, c_free}, error::EResult, headers::{
errno,
string::{mem::memcpy, str::strlen},
},
util::PointerExt,
}, thread::{self, tls}, util::PointerExt
};
#[no_mangle]
@ -78,16 +76,6 @@ unsafe fn push_env(str: NonNull<c_char>) -> EResult<()> {
EResult::Ok(())
}
unsafe fn setup_env(envs: impl IntoIterator<Item = &'static &'static str>) {
for &env in envs {
// Make a malloc()ed, NULL-terminated string
let entry = entry_from_raw(env.as_bytes()).expect("Couldn't allocate env variable");
shadow.push(entry.as_ptr());
}
shadow.push(null_mut());
environ = shadow.as_mut_ptr();
}
pub unsafe fn get_env(name: &[u8]) -> EResult<Option<NonNull<c_char>>> {
if name.is_empty() || name.contains(&b'=') {
return EResult::Err(errno::EINVAL);
@ -211,18 +199,31 @@ pub unsafe fn put_env(str: NonNull<c_char>) -> EResult<()> {
push_env(str)
}
unsafe fn setup_env<'a>(envs: impl Iterator<Item = &'a CStr>) {
for env in envs {
// Make a malloc()ed, NULL-terminated string
let entry = entry_from_raw(env.to_bytes()).expect("Couldn't allocate env variable");
shadow.push(entry.as_ptr());
}
shadow.push(null_mut());
environ = shadow.as_mut_ptr();
}
pub fn handle_kernel_argument(arg: usize) -> Vec<CString> {
let arg_ptr: *const ProgramArgumentInner = core::ptr::with_exposed_provenance(arg);
let arg = unsafe { arg_ptr.ensure() };
let mut args = Vec::new();
for &arg in arg.args {
let c_arg = CString::new(arg).unwrap();
for arg in arg.args() {
let c_arg = arg.to_owned();
args.push(c_arg);
}
unsafe { setup_env(arg.env) };
unsafe { setup_env(arg.envs()) };
// Setup TLS from argument
thread::init_main_thread(arg);
args
}

1
userspace/lib/ygglibc/src/error.rs
View File

@ -23,6 +23,7 @@ macro impl_from_residual($($ty:ty),+) {
)+
}
#[thread_local]
#[no_mangle]
#[allow(non_upper_case_globals)]
pub static mut errno: Errno = Errno(0);

1
userspace/lib/ygglibc/src/headers/fcntl/mod.rs
View File

@ -164,6 +164,7 @@ unsafe extern "C" fn creat(pathname: *const c_char, mode: mode_t) -> CFdResult {
#[no_mangle]
unsafe extern "C" fn open(pathname: *const c_char, opts: c_int, mut args: ...) -> CFdResult {
yggdrasil_rt::debug_trace!("&errno = {:p}", &crate::error::errno);
vopenat(AT_FDCWD, pathname, opts, args.as_va_list())
}

3
userspace/lib/ygglibc/src/lib.rs
View File

@ -10,7 +10,8 @@
linkage,
rustc_private,
naked_functions,
non_null_from_ref
non_null_from_ref,
thread_local
)]
#![allow(internal_features)]
#![cfg_attr(not(test), no_std)]

58
userspace/lib/ygglibc/src/thread/mod.rs
View File

@ -9,7 +9,7 @@ use core::{
use alloc::{boxed::Box, collections::BTreeMap, sync::Arc};
use yggdrasil_rt::{
mem::{MappingFlags, MappingSource},
process::{get_tls, pthread_self, set_pthread_self, ThreadId, ThreadSpawnOptions},
process::{thread_local, ProgramArgumentInner, ThreadId, ThreadSpawnOptions},
};
use crate::{
@ -18,7 +18,7 @@ use crate::{
sync::Mutex,
};
mod tls;
pub mod tls;
static THREADS: Mutex<BTreeMap<pthread_t, Arc<Thread>>> = Mutex::new(BTreeMap::new());
@ -114,26 +114,43 @@ impl Thread {
}
pub fn this() -> EResult<Arc<Thread>> {
let pthread_self: *const Self = ptr::with_exposed_provenance(pthread_self());
if !pthread_self.is_null() {
let tcb: *const *const Self = thread_local::get_tcb().cast();
let raw = unsafe { tcb.read() };
if !raw.is_null() {
// Arc::from_raw creates an owned Arc, which will decrement refcount when dropped,
// prevent this by incrementing the count once more
unsafe { Arc::increment_strong_count(pthread_self) };
let arc = unsafe { Arc::from_raw(pthread_self) };
unsafe { Arc::increment_strong_count(raw) };
let arc = unsafe { Arc::from_raw(raw) };
EResult::Ok(arc)
} else {
panic!("pthread_self is NULL")
panic!("pthread_self() is NULL")
}
}
unsafe fn set_this(thread: Arc<Self>) {
let tcb: *mut usize = thread_local::get_tcb().cast();
let raw = Arc::into_raw(thread);
tcb.write(raw.addr());
}
pub fn result(&self) -> *mut c_void {
self.result.load(Ordering::Acquire)
}
extern "C" fn thread_entry(argument: usize) -> ! {
// Set up TLS as soon as possible. Note the `force = true` parameter, because the image
// contains "already initialized" tag, which only matters for the main thread.
if let Err(err) =
unsafe { thread_local::init_tls(tls::TLS_IMAGE.as_ref(), true, size_of::<usize>() * 4) }
{
yggdrasil_rt::debug_trace!("thread_entry failed: TLS init error: {err:?}");
unsafe { yggdrasil_rt::sys::exit_thread() };
}
{
assert_ne!(argument, 0);
let argument: Box<ThreadArgument> =
unsafe { Box::from_raw(ptr::with_exposed_provenance_mut(argument)) };
yggdrasil_rt::debug_trace!(
@ -141,13 +158,14 @@ impl Thread {
argument.entry,
argument.argument
);
yggdrasil_rt::debug_trace!("tls pointer: {:#x}", get_tls());
// TODO better way to initialize the thread ID
while argument.thread.id.load(Ordering::Acquire) == 0 {
core::hint::spin_loop();
}
argument.thread.clone().setup();
unsafe { Self::set_this(argument.thread.clone()) };
let result = (argument.entry)(argument.argument);
argument.thread.result.store(result, Ordering::Release);
@ -156,15 +174,9 @@ impl Thread {
unsafe { Arc::decrement_strong_count(Arc::as_ptr(&this)) };
}
// Exit the thread when everything's dropped
// TODO call thread-local destructors
unsafe { yggdrasil_rt::sys::exit_thread() }
}
fn setup(self: Arc<Self>) {
// Setup self-pointer
let pthread_self = Arc::into_raw(self);
unsafe { set_pthread_self(pthread_self.addr()) };
}
}
impl Drop for Thread {
@ -176,3 +188,17 @@ impl Drop for Thread {
}
}
}
pub fn init_main_thread(arg: &ProgramArgumentInner) {
// Will create a TLS for the main thread (if not done already).
// Usually, a dynamic loader will do this for us, but this still needs to be
// done for statically-linked programs if the kernel transfers control directly to
// the program.
let tls_image = thread_local::init_tls_from_auxv(arg.auxv(), false, size_of::<usize>() * 4)
.expect("Could not initialize TLS");
// Store the TLS image, it'll be needed when creating new threads
unsafe {
tls::TLS_IMAGE = tls_image;
}
}

3
userspace/lib/ygglibc/src/thread/tls.rs
View File

@ -1 +1,4 @@
use yggdrasil_rt::process::thread_local::TlsImage;
// TODO OnceLock
pub(super) static mut TLS_IMAGE: Option<TlsImage> = None;