From 39956dedb86125dfd598d3d505ab526113c99a61 Mon Sep 17 00:00:00 2001 From: Mark Poliakov Date: Mon, 1 Apr 2024 17:23:12 +0300 Subject: [PATCH] sys/mod: Basic kernel module loading --- Cargo.lock | 80 +++- Cargo.toml | 2 +- boot/yboot/src/elf.rs | 2 +- etc/x86_64-unknown-none.json | 10 +- etc/x86_64-unknown-none.ld | 6 +- kernel/Cargo.toml | 1 - kernel/arch/src/lib.rs | 2 +- kernel/arch/x86_64/src/context.S | 3 - kernel/lib/module-build/Cargo.toml | 6 + kernel/lib/module-build/src/lib.rs | 19 + kernel/libk/Cargo.toml | 9 + kernel/libk/libk-mm/Cargo.toml | 1 + kernel/libk/libk-mm/src/heap.rs | 66 +++ kernel/libk/libk-mm/src/lib.rs | 1 + kernel/libk/libk-mm/src/phys/mod.rs | 15 +- kernel/libk/libk-util/Cargo.toml | 1 + kernel/libk/libk-util/src/hash_table.rs | 64 +++ kernel/libk/libk-util/src/lib.rs | 11 +- kernel/libk/src/lib.rs | 6 + kernel/libk/src/module.rs | 218 +++++++++ kernel/libk/src/panic.rs | 26 ++ kernel/libk/src/task/binary/elf.rs | 34 +- kernel/modules/test_mod/.gitignore | 1 + kernel/modules/test_mod/Cargo.lock | 575 ++++++++++++++++++++++++ kernel/modules/test_mod/Cargo.toml | 18 + kernel/modules/test_mod/build.rs | 3 + kernel/modules/test_mod/module.toml | 2 + kernel/modules/test_mod/src/lib.rs | 30 ++ kernel/src/arch/x86_64/acpi.rs | 3 +- kernel/src/arch/x86_64/mod.rs | 2 +- kernel/src/init.rs | 4 + kernel/src/main.rs | 5 +- kernel/src/mem/heap.rs | 66 --- kernel/src/mem/mod.rs | 2 - kernel/src/panic.rs | 5 +- kernel/src/syscall/imp/mod.rs | 22 +- kernel/tools/gentables/src/aarch64.rs | 9 +- kernel/tools/gentables/src/main.rs | 84 +++- kernel/tools/gentables/src/x86_64.rs | 10 +- lib/abi/def/yggdrasil.abi | 1 + userspace/Cargo.lock | 1 + userspace/Cargo.toml | 2 +- userspace/sysutils/Cargo.toml | 5 + userspace/sysutils/src/kmod.rs | 13 + xtask/Cargo.toml | 2 + xtask/src/build/cargo.rs | 106 ++++- xtask/src/build/mod.rs | 24 +- xtask/src/build/module.rs | 112 +++++ xtask/src/build/userspace.rs | 25 +- xtask/src/env.rs | 3 + xtask/src/error.rs | 6 +- xtask/src/main.rs | 5 +- xtask/src/qemu.rs | 5 +- 53 files changed, 1592 insertions(+), 142 deletions(-) create mode 100644 kernel/lib/module-build/Cargo.toml create mode 100644 kernel/lib/module-build/src/lib.rs create mode 100644 kernel/libk/libk-mm/src/heap.rs create mode 100644 kernel/libk/libk-util/src/hash_table.rs create mode 100644 kernel/libk/src/module.rs create mode 100644 kernel/libk/src/panic.rs create mode 100644 kernel/modules/test_mod/.gitignore create mode 100644 kernel/modules/test_mod/Cargo.lock create mode 100644 kernel/modules/test_mod/Cargo.toml create mode 100644 kernel/modules/test_mod/build.rs create mode 100644 kernel/modules/test_mod/module.toml create mode 100644 kernel/modules/test_mod/src/lib.rs create mode 100644 userspace/sysutils/src/kmod.rs create mode 100644 xtask/src/build/module.rs diff --git a/Cargo.lock b/Cargo.lock index bc9a6c4c..e4a7cb58 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -57,6 +57,18 @@ dependencies = [ "log", ] +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.2" @@ -310,6 +322,15 @@ version = "0.8.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" +[[package]] +name = "dependency-graph" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5143247629540606d0888beae9ca0e0b9a81a32151bfecd0b2be4a961155c24d" +dependencies = [ + "petgraph", +] + [[package]] name = "device-api" version = "0.1.0" @@ -474,6 +495,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "fnv" version = "1.0.7" @@ -759,6 +786,7 @@ version = "0.1.0" dependencies = [ "abi-lib", "atomic_enum", + "bytemuck", "cfg-if", "crossbeam-queue", "device-api", @@ -791,6 +819,7 @@ dependencies = [ "kernel-arch", "libk-mm-interface", "libk-util", + "linked_list_allocator", "log", "vmalloc", "yggdrasil-abi", @@ -810,6 +839,7 @@ dependencies = [ name = "libk-util" version = "0.1.0" dependencies = [ + "ahash", "crossbeam-queue", "futures-util", "kernel-arch", @@ -998,6 +1028,16 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +[[package]] +name = "petgraph" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1d3afd2628e69da2be385eb6f2fd57c8ac7977ceeff6dc166ff1657b0e386a9" +dependencies = [ + "fixedbitset", + "indexmap", +] + [[package]] name = "pin-project-lite" version = "0.2.13" @@ -1190,7 +1230,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a" dependencies = [ - "semver", + "semver 0.9.0", ] [[package]] @@ -1254,6 +1294,15 @@ dependencies = [ "semver-parser", ] +[[package]] +name = "semver" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92d43fe69e652f3df9bdc2b85b2854a0825b86e4fb76bc44d945137d053639ca" +dependencies = [ + "serde", +] + [[package]] name = "semver-parser" version = "0.7.0" @@ -1509,6 +1558,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + [[package]] name = "vmalloc" version = "0.1.0" @@ -1700,10 +1755,12 @@ name = "xtask" version = "0.1.0" dependencies = [ "clap", + "dependency-graph", "env_logger", "git2", "log", "qemu", + "semver 1.0.22", "serde", "tar", "thiserror", @@ -1915,7 +1972,6 @@ dependencies = [ "libk-device", "libk-mm", "libk-util", - "linked_list_allocator", "log", "memfs", "memtables", @@ -1951,3 +2007,23 @@ dependencies = [ "rustc-std-workspace-core", "yggdrasil-abi", ] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.52", +] diff --git a/Cargo.toml b/Cargo.toml index e3469e03..e29f49ac 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,7 +5,7 @@ exclude = [ "boot/yboot-proto", "tool/abi-generator", "toolchain", - "userspace/dynload-program" + "userspace/dynload-program", ] members = [ "xtask", diff --git a/boot/yboot/src/elf.rs b/boot/yboot/src/elf.rs index 87e951cd..4947ece8 100644 --- a/boot/yboot/src/elf.rs +++ b/boot/yboot/src/elf.rs @@ -214,7 +214,7 @@ impl Object { let reserved_addr = bs .allocate_pages( AllocateType::Address(image_start), - MemoryType::RUNTIME_SERVICES_DATA, + MemoryType::LOADER_DATA, (image_end - image_start) as usize / 0x1000, ) .expect("Could not allocate memory for kernel image"); diff --git a/etc/x86_64-unknown-none.json b/etc/x86_64-unknown-none.json index eed3a3e1..270f4532 100644 --- a/etc/x86_64-unknown-none.json +++ b/etc/x86_64-unknown-none.json @@ -12,15 +12,11 @@ "executables": true, "panic-strategy": "abort", "features": "-avx,-sse,+soft-float", + "dynamic-linking": true, + "relocation-model": "pic", "has-thread-local": false, "linker": "rust-lld", - "linker-flavor": "ld.lld", - - "pre-link-args": { - "ld.lld": [ - "-Tetc/x86_64-unknown-none.ld" - ] - } + "linker-flavor": "ld.lld" } diff --git a/etc/x86_64-unknown-none.ld b/etc/x86_64-unknown-none.ld index c901c4fb..add8309b 100644 --- a/etc/x86_64-unknown-none.ld +++ b/etc/x86_64-unknown-none.ld @@ -5,7 +5,6 @@ KERNEL_VIRT_OFFSET = 0xFFFFFF8000000000; SECTIONS { . = KERNEL_PHYS_BASE; - PROVIDE(__kernel_phys_start = .); PROVIDE(__kernel_start = . + KERNEL_VIRT_OFFSET); .text.entry : { @@ -19,6 +18,10 @@ SECTIONS { *(.text*) } + .export.text : AT(. - KERNEL_VIRT_OFFSET) { + KEEP(*(.export.text*)) + } + . = ALIGN(4K); .rodata : AT(. - KERNEL_VIRT_OFFSET) { *(.eh_frame*) @@ -46,5 +49,4 @@ SECTIONS { PROVIDE(__bss_end_phys = . - KERNEL_VIRT_OFFSET); PROVIDE(__kernel_end = .); - PROVIDE(__kernel_size = . - KERNEL_VIRT_OFFSET - KERNEL_PHYS_BASE); }; diff --git a/kernel/Cargo.toml b/kernel/Cargo.toml index d75d61d9..6fe27104 100644 --- a/kernel/Cargo.toml +++ b/kernel/Cargo.toml @@ -39,7 +39,6 @@ memfs = { path = "driver/fs/memfs" } atomic_enum = "0.2.0" bitflags = "2.3.3" -linked_list_allocator = "0.10.5" spinning_top = "0.2.5" static_assertions = "1.1.0" tock-registers = "0.8.1" diff --git a/kernel/arch/src/lib.rs b/kernel/arch/src/lib.rs index 14e23235..abc65a97 100644 --- a/kernel/arch/src/lib.rs +++ b/kernel/arch/src/lib.rs @@ -33,7 +33,7 @@ cfg_if! { pub use imp::{ArchitectureImpl, KernelTableManagerImpl, ProcessAddressSpaceImpl, TaskContextImpl}; -pub use kernel_arch_interface::{guard, mem, sync, task, util, Architecture}; +pub use kernel_arch_interface::{guard, mem, sync, task, util, Architecture, KERNEL_VIRT_OFFSET}; pub type CpuImpl = kernel_arch_interface::cpu::CpuImpl; pub type LocalCpuImpl<'a, S> = kernel_arch_interface::cpu::LocalCpuImpl<'a, ArchitectureImpl, S>; diff --git a/kernel/arch/x86_64/src/context.S b/kernel/arch/x86_64/src/context.S index 3be3bff2..84c64874 100644 --- a/kernel/arch/x86_64/src/context.S +++ b/kernel/arch/x86_64/src/context.S @@ -62,9 +62,6 @@ .section .text __x86_64_task_enter_from_fork: - // TODO - jmp . - xorq %rax, %rax xorq %rcx, %rcx diff --git a/kernel/lib/module-build/Cargo.toml b/kernel/lib/module-build/Cargo.toml new file mode 100644 index 00000000..43b4e220 --- /dev/null +++ b/kernel/lib/module-build/Cargo.toml @@ -0,0 +1,6 @@ +[package] +name = "module-build" +version = "0.1.0" +edition = "2021" + +[dependencies] diff --git a/kernel/lib/module-build/src/lib.rs b/kernel/lib/module-build/src/lib.rs new file mode 100644 index 00000000..490d6cbf --- /dev/null +++ b/kernel/lib/module-build/src/lib.rs @@ -0,0 +1,19 @@ +use std::env; + +pub fn build() { + let libk_so = env::var("MOD_LIBK_SO").expect("module-build: $MOD_LIBK_SO is not set"); + println!("cargo:rerun-if-changed={}", libk_so); + println!("cargo:rerun-if-env-changed=MOD_LIBK_SO"); + println!("cargo:rerun-if-env-changed=MOD_DEPENDENCIES"); + + if let Ok(deps) = env::var("MOD_DEPENDENCIES") { + let deps = deps.trim(); + if !deps.is_empty() { + let deps = deps.split(':'); + for dep in deps { + let (_, path) = dep.split_once("=").unwrap(); + println!("cargo:rerun-if-changed={}", path); + } + } + } +} diff --git a/kernel/libk/Cargo.toml b/kernel/libk/Cargo.toml index 1ad2c5af..1391b018 100644 --- a/kernel/libk/Cargo.toml +++ b/kernel/libk/Cargo.toml @@ -1,9 +1,14 @@ +cargo-features = ["profile-rustflags"] + [package] name = "libk" version = "0.1.0" edition = "2021" authors = ["Mark Poliakov "] +[lib] +crate_type = ["rlib", "dylib"] + [dependencies] libk-mm = { path = "libk-mm" } libk-util = { path = "libk-util" } @@ -22,9 +27,13 @@ crossbeam-queue = { version = "0.3.8", default-features = false, features = ["al serde_json = { version = "1.0.111", default-features = false, features = ["alloc"] } serde = { version = "1.0.193", features = ["derive"], default-features = false } +bytemuck = { version = "1.14.0", features = ["derive"] } [dependencies.elf] version = "0.7.2" git = "https://git.alnyan.me/yggdrasil/yggdrasil-elf.git" default-features = false features = ["no_std_stream"] + +[profile.dev] +rustflags = ["-Clink-dead-code=yes -Zexport-executable-symbols -Cprefer-dynamic"] diff --git a/kernel/libk/libk-mm/Cargo.toml b/kernel/libk/libk-mm/Cargo.toml index cc75525b..db45185d 100644 --- a/kernel/libk/libk-mm/Cargo.toml +++ b/kernel/libk/libk-mm/Cargo.toml @@ -12,4 +12,5 @@ libk-util = { path = "../libk-util" } libk-mm-interface = { path = "interface" } vmalloc = { path = "../../lib/vmalloc" } +linked_list_allocator = "0.10.5" log = "0.4.20" diff --git a/kernel/libk/libk-mm/src/heap.rs b/kernel/libk/libk-mm/src/heap.rs new file mode 100644 index 00000000..bbe0a33c --- /dev/null +++ b/kernel/libk/libk-mm/src/heap.rs @@ -0,0 +1,66 @@ +//! Kernel's global heap allocator +use core::{ + alloc::{GlobalAlloc, Layout}, + ops::Range, + ptr::{null_mut, NonNull}, +}; + +use libk_util::sync::IrqSafeSpinlock; +use linked_list_allocator::Heap; + +/// Kernel heap manager +pub struct KernelAllocator { + inner: IrqSafeSpinlock, +} + +impl KernelAllocator { + const fn empty() -> Self { + Self { + inner: IrqSafeSpinlock::new(Heap::empty()), + } + } + + unsafe fn init(&self, base: usize, size: usize) { + self.inner.lock().init(base as _, size); + } + + fn range(&self) -> Range { + let lock = self.inner.lock(); + lock.bottom() as usize..lock.top() as usize + } +} + +unsafe impl GlobalAlloc for KernelAllocator { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + match self.inner.lock().allocate_first_fit(layout) { + Ok(v) => v.as_ptr(), + Err(e) => { + log::error!("Failed to allocate {:?}: {:?}", layout, e); + null_mut() + } + } + } + + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + let ptr = NonNull::new(ptr).unwrap(); + self.inner.lock().deallocate(ptr, layout) + } +} + +/// Kernel's global allocator +#[global_allocator] +pub static GLOBAL_HEAP: KernelAllocator = KernelAllocator::empty(); + +/// Sets up kernel's global heap with given memory range. +/// +/// # Safety +/// +/// The caller must ensure the range is valid and mapped virtual memory. +pub unsafe fn init_heap(heap_base: usize, heap_size: usize) { + GLOBAL_HEAP.init(heap_base, heap_size); +} + +/// Returns the heap address range +pub fn heap_range() -> Range { + GLOBAL_HEAP.range() +} diff --git a/kernel/libk/libk-mm/src/lib.rs b/kernel/libk/libk-mm/src/lib.rs index 9cad0c24..f5a96f69 100644 --- a/kernel/libk/libk-mm/src/lib.rs +++ b/kernel/libk/libk-mm/src/lib.rs @@ -27,6 +27,7 @@ use yggdrasil_abi::error::Error; pub mod address; pub mod device; +pub mod heap; pub mod phys; pub mod pointer; pub mod process; diff --git a/kernel/libk/libk-mm/src/phys/mod.rs b/kernel/libk/libk-mm/src/phys/mod.rs index 3b28a567..f8fb56c8 100644 --- a/kernel/libk/libk-mm/src/phys/mod.rs +++ b/kernel/libk/libk-mm/src/phys/mod.rs @@ -1,6 +1,6 @@ use core::ops::Range; -use kernel_arch::{absolute_address, mem::PhysicalMemoryAllocator}; +use kernel_arch::mem::PhysicalMemoryAllocator; use libk_mm_interface::address::{FromRaw, IntoRaw, PhysicalAddress}; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; use yggdrasil_abi::{error::Error, system::SystemMemoryStats}; @@ -221,13 +221,18 @@ pub unsafe fn init_from_iter< } fn kernel_physical_memory_region() -> PhysicalMemoryRegion { + use core::ptr::addr_of; + extern "C" { - static __kernel_phys_start: u8; - static __kernel_size: u8; + static __kernel_start: u8; + static __kernel_end: u8; } - let base = PhysicalAddress::from_raw(absolute_address!(__kernel_phys_start)); - let size = absolute_address!(__kernel_size); + let start = unsafe { addr_of!(__kernel_start) }; + let end = unsafe { addr_of!(__kernel_end) }; + + let base = PhysicalAddress::from_raw(start.addr() - kernel_arch::KERNEL_VIRT_OFFSET); + let size = end.addr() - start.addr(); PhysicalMemoryRegion { base, size } } diff --git a/kernel/libk/libk-util/Cargo.toml b/kernel/libk/libk-util/Cargo.toml index 8292e87b..53a4049c 100644 --- a/kernel/libk/libk-util/Cargo.toml +++ b/kernel/libk/libk-util/Cargo.toml @@ -12,3 +12,4 @@ kernel-arch = { path = "../../arch" } log = "0.4.20" crossbeam-queue = { version = "0.3.8", default-features = false, features = ["alloc"] } futures-util = { version = "0.3.28", default-features = false, features = ["alloc", "async-await"] } +ahash = { version = "0.8.11", default-features = false, features = ["no-rng"] } diff --git a/kernel/libk/libk-util/src/hash_table.rs b/kernel/libk/libk-util/src/hash_table.rs new file mode 100644 index 00000000..2c6df066 --- /dev/null +++ b/kernel/libk/libk-util/src/hash_table.rs @@ -0,0 +1,64 @@ +use core::{ + borrow::Borrow, + hash::{BuildHasher, Hash}, +}; + +use alloc::vec::Vec; + +pub type DefaultHashBuilder = core::hash::BuildHasherDefault; +pub type DefaultHashTable = HashTable; + +pub struct HashTable +where + [Vec<(K, V)>; N]: Sized, +{ + buckets: [Vec<(K, V)>; N], + hasher: H, +} + +impl HashTable +where + [Vec<(K, V)>; N]: Sized, +{ + pub const fn new() -> Self { + Self { + buckets: [const { Vec::new() }; N], + hasher: DefaultHashBuilder::default(), + } + } +} + +impl HashTable { + pub fn insert(&mut self, key: K, value: V) -> Option { + let h = self.hasher.hash_one(&key); + let bucket = &mut self.buckets[h as usize % self.buckets.len()]; + + if let Some((_, slot)) = bucket.iter_mut().find(|(k, _)| k == &key) { + Some(core::mem::replace(slot, value)) + } else { + bucket.push((key, value)); + None + } + } + + pub fn get(&self, key: &Q) -> Option<&V> + where + Q: Hash + Eq + ?Sized, + K: Borrow, + { + self.get_key_value(key).map(|p| p.1) + } + + pub fn get_key_value(&self, key: &Q) -> Option<(&K, &V)> + where + Q: Hash + Eq + ?Sized, + K: Borrow, + { + let h = self.hasher.hash_one(key); + let bucket = &self.buckets[h as usize % self.buckets.len()]; + + bucket + .iter() + .find_map(|(k, v)| (k.borrow() == key).then_some((k, v))) + } +} diff --git a/kernel/libk/libk-util/src/lib.rs b/kernel/libk/libk-util/src/lib.rs index 6f8e7c25..f423e9fc 100644 --- a/kernel/libk/libk-util/src/lib.rs +++ b/kernel/libk/libk-util/src/lib.rs @@ -1,5 +1,13 @@ #![no_std] -#![feature(maybe_uninit_slice, new_uninit, allocator_api, let_chains)] +#![feature( + maybe_uninit_slice, + new_uninit, + allocator_api, + let_chains, + inline_const, + const_trait_impl, + effects +)] extern crate alloc; @@ -10,6 +18,7 @@ use core::{ }; pub mod event; +pub mod hash_table; pub mod io; pub mod queue; pub mod ring; diff --git a/kernel/libk/src/lib.rs b/kernel/libk/src/lib.rs index e6da8b28..ce9bd802 100644 --- a/kernel/libk/src/lib.rs +++ b/kernel/libk/src/lib.rs @@ -23,11 +23,17 @@ )] extern crate alloc; +extern crate serde; + +pub use log; +pub use yggdrasil_abi::error; #[macro_use] pub mod task; pub mod arch; +pub mod module; +pub mod panic; pub mod random; pub mod vfs; diff --git a/kernel/libk/src/module.rs b/kernel/libk/src/module.rs new file mode 100644 index 00000000..a5961c7d --- /dev/null +++ b/kernel/libk/src/module.rs @@ -0,0 +1,218 @@ +use core::mem::size_of; + +use alloc::{string::String, vec, vec::Vec}; +use bytemuck::{Pod, Zeroable}; +use elf::io_traits::{InputStream, SeekFrom}; +use libk_mm::PageBox; +use libk_util::{hash_table::DefaultHashTable, io::Read, sync::LockMethod, OneTimeInit}; +use yggdrasil_abi::{ + error::Error, + io::{FileMode, OpenOptions}, + path::Path, +}; + +use crate::{ + task::{ + binary::{ + self, + elf::{ElfSegment, ElfSegmentType}, + }, + sync::Mutex, + }, + vfs::{FileRef, IoContext}, +}; + +#[derive(Clone, Copy, Debug, Zeroable, Pod)] +#[repr(C)] +pub struct Symbol { + pub st_name: u32, + pub st_info: u8, + pub st_other: u8, + pub st_shndx: u16, + pub st_value: u64, + pub st_size: u64, +} + +#[derive(Clone, Debug)] +#[repr(C)] +pub struct ModuleInfo { + pub name: &'static str, + pub version: &'static str, + pub init: fn() -> Result<(), Error>, + pub unload: Option Result<(), Error>>, +} + +#[derive(Debug)] +pub struct LoadedModule { + pub image_data: PageBox<[u8]>, + // Ref into the image_data PageBox + pub info: &'static ModuleInfo, +} + +pub fn load_kernel_symbol_table>( + ioctx: &mut IoContext, + path: P, +) -> Result<(), Error> { + let symbol_file = ioctx.open(None, path, OpenOptions::READ, FileMode::empty())?; + let mut string_buffer = PageBox::new_slice(0, 4096)?; + let mut map = DefaultHashTable::new(); + + loop { + let mut len = [0; 8]; + if symbol_file.read(&mut len)? != len.len() { + break; + } + let len = usize::from_le_bytes(len); + symbol_file.read_exact(&mut string_buffer[..len])?; + let name = core::str::from_utf8(&string_buffer[..len]).unwrap(); + let mut value = [0; 8]; + symbol_file.read_exact(&mut value)?; + let value = usize::from_le_bytes(value); + map.insert(name.into(), value); + } + + KERNEL_SYMBOL_TABLE.init(map); + MODULE_SYMBOL_TABLE.init(Mutex::new(DefaultHashTable::new())); + + Ok(()) +} + +pub fn lookup_symbol(name: &str) -> Option { + let symtab = KERNEL_SYMBOL_TABLE.get(); + if let Some(addr) = symtab.get(name).copied() { + return Some(addr); + } + let symtab = MODULE_SYMBOL_TABLE.get(); + if let Some(addr) = symtab.lock().unwrap().get(name).copied() { + return Some(addr); + } + + None +} + +pub fn load(file: FileRef) -> Result { + // TODO better dependency tracking than just throwing undefined references + let (mut elf, mut file) = binary::elf::open_elf_direct(file)?; + + // Find out image size + let (vaddr_min, vaddr_max) = binary::elf::elf_virtual_range(&elf); + let mut image_data = PageBox::new_slice(0u8, vaddr_max - vaddr_min)?; + + // Load the segments + for segment in elf + .segments() + .iter() + .filter_map(ElfSegment::from_phdr) + .filter(|seg| seg.ty == ElfSegmentType::Load) + { + let rel_offset = segment.vaddr as usize - vaddr_min; + let segment_data = &mut image_data[rel_offset..rel_offset + segment.full_size as usize]; + + if segment.data_size > 0 { + file.seek(SeekFrom::Start(segment.offset)).unwrap(); + file.read_exact(&mut segment_data[..segment.data_size as usize]) + .unwrap(); + } + } + + let mut info_struct_addr = None; + + // Extract dynamic symbols + let (dynsym, dynstr) = elf.dynamic_symbol_table().unwrap().unwrap(); + let mut dynamic_symbols = vec![]; + for mut sym in dynsym { + let name = dynstr.get(sym.st_name as _).unwrap(); + + if sym.st_shndx == elf::abi::SHN_UNDEF && sym.st_bind() != elf::abi::STB_LOCAL { + let Some(value) = lookup_symbol(&name) else { + log::warn!("Could not load module: undefined reference to {:?} (possibly compiled against a different libk?)", name); + return Err(Error::InvalidArgument); + }; + sym.st_value = value as u64; + + dynamic_symbols.push(sym); + } else { + // Adjust symbol value by image base + sym.st_value += image_data.as_ptr().addr() as u64; + + if name == "MODULE" && sym.st_symtype() == elf::abi::STT_OBJECT { + if sym.st_size != size_of::() as u64 { + todo!(); + } + info_struct_addr = Some(sym.st_value as usize); + } + + dynamic_symbols.push(sym); + } + } + + // Apply relocations + let rela_sections: Vec<_> = elf + .section_headers() + .iter() + .filter(|c| c.sh_type == elf::abi::SHT_RELA) + .copied() + .collect(); + + for rela in rela_sections { + let rela = elf.section_data_as_relas(&rela).unwrap(); + for rela in rela { + let value = match rela.r_type { + // Symbol + addend + elf::abi::R_X86_64_64 => { + if let Some(sym) = dynamic_symbols.get(rela.r_sym as usize) { + sym.st_value as i64 + rela.r_addend + } else { + return Err(Error::InvalidArgument); + } + } + // Image base + addend + elf::abi::R_X86_64_RELATIVE => image_data.as_ptr().addr() as i64 + rela.r_addend, + // Symbol value + elf::abi::R_X86_64_JUMP_SLOT | elf::abi::R_X86_64_GLOB_DAT => { + if let Some(sym) = dynamic_symbols.get(rela.r_sym as usize) { + sym.st_value as i64 + } else { + return Err(Error::InvalidArgument); + } + } + _ => todo!("Unhandled relocation: {}", rela.r_type), + }; + + image_data[rela.r_offset as usize..rela.r_offset as usize + 8] + .copy_from_slice(&value.to_ne_bytes()); + } + } + + // TODO Add module symbols to export list: make sure to only export properly namespaced items + // let module_syms = MODULE_SYMBOL_TABLE.get(); + // for (name, sym) in dynamic_symbols.iter() { + // if sym.st_shndx == elf::abi::SHN_UNDEF || sym.st_vis() != elf::abi::STV_DEFAULT { + // continue; + // } + // if !name.starts_with("_ZN") { + // continue; + // } + // log::info!("Export {:?} -> {:#x}", name, sym.st_value); + // module_syms + // .lock() + // .unwrap() + // .insert(name.clone(), sym.st_value as usize); + // } + + let info = info_struct_addr + .map(|addr| unsafe { core::mem::transmute(addr) }) + .expect("TODO return error if module is missing MODULE info struct"); + + Ok(LoadedModule { image_data, info }) +} + +pub fn load_and_execute(file: FileRef) -> Result<(), Error> { + let module = load(file)?; + (module.info.init)() +} + +// TODO roll own hashmap +static KERNEL_SYMBOL_TABLE: OneTimeInit> = OneTimeInit::new(); +static MODULE_SYMBOL_TABLE: OneTimeInit>> = + OneTimeInit::new(); diff --git a/kernel/libk/src/panic.rs b/kernel/libk/src/panic.rs new file mode 100644 index 00000000..94830dd1 --- /dev/null +++ b/kernel/libk/src/panic.rs @@ -0,0 +1,26 @@ +use core::panic::PanicInfo; + +use kernel_arch::{Architecture, ArchitectureImpl}; +use libk_util::OneTimeInit; + +static PANIC_HANDLER: OneTimeInit !> = OneTimeInit::new(); + +pub fn set_handler(handler: fn(&PanicInfo) -> !) { + PANIC_HANDLER.init(handler); +} + +#[panic_handler] +fn panic_handler(pi: &core::panic::PanicInfo) -> ! { + if let Some(ph) = PANIC_HANDLER.try_get() { + ph(pi); + } else { + log::error!("{:#?}", pi); + + loop { + unsafe { + ArchitectureImpl::set_interrupt_mask(true); + } + ArchitectureImpl::wait_for_interrupt(); + } + } +} diff --git a/kernel/libk/src/task/binary/elf.rs b/kernel/libk/src/task/binary/elf.rs index f77b91ec..1e429585 100644 --- a/kernel/libk/src/task/binary/elf.rs +++ b/kernel/libk/src/task/binary/elf.rs @@ -34,7 +34,7 @@ pub struct FileReader { } #[derive(Debug, PartialEq)] -enum ElfSegmentType { +pub enum ElfSegmentType { Load, Dynamic, Tls, @@ -45,7 +45,7 @@ pub enum ElfKind { Static(ElfStream>>, FileReader>), } -struct ElfSegment { +pub struct ElfSegment { pub ty: ElfSegmentType, pub attrs: MapAttributes, pub offset: u64, @@ -68,7 +68,7 @@ impl elf::io_traits::InputStream for FileReader { } impl ElfSegmentType { - fn from_phdr(phdr: &ProgramHeader) -> Option { + pub fn from_phdr(phdr: &ProgramHeader) -> Option { match phdr.p_type { elf::abi::PT_LOAD => Some(Self::Load), elf::abi::PT_DYNAMIC => Some(Self::Dynamic), @@ -79,7 +79,7 @@ impl ElfSegmentType { } impl ElfSegment { - fn from_phdr(phdr: &ProgramHeader) -> Option { + pub fn from_phdr(phdr: &ProgramHeader) -> Option { let ty = ElfSegmentType::from_phdr(phdr)?; let attrs = match (phdr.p_flags & elf::abi::PF_W, phdr.p_flags & elf::abi::PF_X) { @@ -122,8 +122,21 @@ impl ElfSegment { } pub fn open_elf_file(file: Arc) -> Result, Error> { + let (mut elf, file) = open_elf_direct(file)?; + + if is_dynamic(&mut elf)? { + // TODO populate interp value + Ok(ElfKind::Dynamic(None)) + } else { + Ok(ElfKind::Static(elf, file)) + } +} + +pub fn open_elf_direct( + file: Arc, +) -> Result<(ElfStream>>, FileReader>), Error> { let file = FileReader { file }; - let mut elf = ElfStream::::open_stream(file.clone()).map_err(from_parse_error)?; + let elf = ElfStream::::open_stream(file.clone()).map_err(from_parse_error)?; if elf.ehdr.e_machine != EXPECTED_ELF_MACHINE { log::warn!("e_machine in ELF ({:#x}) does not match expected value for current architecture ({:#x})", elf.ehdr.e_machine, EXPECTED_ELF_MACHINE); @@ -141,12 +154,7 @@ pub fn open_elf_file(file: Arc) -> Result, Error> return Err(Error::UnrecognizedExecutable); } - if is_dynamic(&mut elf)? { - // TODO populate interp value - Ok(ElfKind::Dynamic(None)) - } else { - Ok(ElfKind::Static(elf, file)) - } + Ok((elf, file)) } /// Loads an ELF binary from `file` into the target address space @@ -255,7 +263,9 @@ fn is_dynamic( Ok(dynamic.into_iter().any(|e| e.d_tag == elf::abi::DT_NEEDED)) } -fn elf_virtual_range(elf: &ElfStream>) -> (usize, usize) { +pub fn elf_virtual_range( + elf: &ElfStream>, +) -> (usize, usize) { let mut vaddr_min = usize::MAX; let mut vaddr_max = usize::MIN; diff --git a/kernel/modules/test_mod/.gitignore b/kernel/modules/test_mod/.gitignore new file mode 100644 index 00000000..ea8c4bf7 --- /dev/null +++ b/kernel/modules/test_mod/.gitignore @@ -0,0 +1 @@ +/target diff --git a/kernel/modules/test_mod/Cargo.lock b/kernel/modules/test_mod/Cargo.lock new file mode 100644 index 00000000..8c234695 --- /dev/null +++ b/kernel/modules/test_mod/Cargo.lock @@ -0,0 +1,575 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 3 + +[[package]] +name = "aarch64-cpu" +version = "9.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac42a04a61c19fc8196dd728022a784baecc5d63d7e256c01ad1b3fbfab26287" +dependencies = [ + "tock-registers", +] + +[[package]] +name = "abi-generator" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", + "thiserror", +] + +[[package]] +name = "abi-lib" +version = "0.1.0" + +[[package]] +name = "ahash" +version = "0.8.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e89da841a80418a9b391ebaea17f5c112ffaaa96f621d2c285b5174da76b9011" +dependencies = [ + "cfg-if", + "once_cell", + "version_check", + "zerocopy", +] + +[[package]] +name = "atomic_enum" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6227a8d6fdb862bcb100c4314d0d9579e5cd73fa6df31a2e6f6e1acd3c5f1207" +dependencies = [ + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "autocfg" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa" + +[[package]] +name = "bitflags" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" + +[[package]] +name = "btree_monstrousity" +version = "0.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4d0977e9c15f276380f16f2e9594257c258172b23af39ffd2e4cf5971cb38c7" +dependencies = [ + "cfg-if", + "rustversion", +] + +[[package]] +name = "bytemuck" +version = "1.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5d6d68c57235a3a081186990eca2867354726650f42f7516ca50c28d6281fd15" +dependencies = [ + "bytemuck_derive", +] + +[[package]] +name = "bytemuck_derive" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4da9a32f3fed317401fa3c862968128267c3106685286e15d5aaa3d7389c2f60" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "cfg-if" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" + +[[package]] +name = "crossbeam-queue" +version = "0.3.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df0346b5d5e76ac2fe4e327c5fd1118d6be7c51dfb18f9b7922923f287471e35" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "248e3bacc7dc6baa3b21e405ee045c3047101a49145e7e9eca583ab4c2ca5345" + +[[package]] +name = "device-api" +version = "0.1.0" +dependencies = [ + "device-api-macros", + "yggdrasil-abi", +] + +[[package]] +name = "device-api-macros" +version = "0.1.0" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "discrete_range_map" +version = "0.6.2" +source = "git+https://git.alnyan.me/yggdrasil/discrete_range_map.git#10fd79828d2918bd079a11c2b2c623eede170c3f" +dependencies = [ + "btree_monstrousity", + "either", + "itertools", + "serde", +] + +[[package]] +name = "either" +version = "1.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "11157ac094ffbdde99aa67b23417ebdd801842852b500e395a45a9c0aac03e4a" + +[[package]] +name = "elf" +version = "0.7.2" +source = "git+https://git.alnyan.me/yggdrasil/yggdrasil-elf.git#419cd311de2e9514b5033677cde9a33f7d0ba4a2" +dependencies = [ + "hashbrown", +] + +[[package]] +name = "futures-core" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dfc6580bb841c5a68e9ef15c77ccc837b40a7504914d52e47b8b0e9bbda25a1d" + +[[package]] +name = "futures-task" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38d84fa142264698cdce1a9f9172cf383a0c82de1bddcf3092901442c4097004" + +[[package]] +name = "futures-util" +version = "0.3.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d6401deb83407ab3da39eba7e33987a73c3df0c82b4bb5813ee871c19c41d48" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "pin-utils", +] + +[[package]] +name = "hashbrown" +version = "0.14.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604" + +[[package]] +name = "itertools" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba291022dbbd398a455acf126c1e341954079855bc60dfdda641363bd6922569" +dependencies = [ + "either", +] + +[[package]] +name = "itoa" +version = "1.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1a46d1a171d865aa5f83f92695765caa047a9b4cbae2cbf37dbd613a793fd4c" + +[[package]] +name = "kernel-arch" +version = "0.1.0" +dependencies = [ + "cfg-if", + "kernel-arch-aarch64", + "kernel-arch-hosted", + "kernel-arch-interface", + "kernel-arch-x86_64", +] + +[[package]] +name = "kernel-arch-aarch64" +version = "0.1.0" +dependencies = [ + "aarch64-cpu", + "bitflags", + "device-api", + "kernel-arch-interface", + "libk-mm-interface", + "memtables", + "static_assertions", + "tock-registers", + "yggdrasil-abi", +] + +[[package]] +name = "kernel-arch-hosted" +version = "0.1.0" +dependencies = [ + "kernel-arch-interface", + "libk-mm-interface", + "yggdrasil-abi", +] + +[[package]] +name = "kernel-arch-interface" +version = "0.1.0" +dependencies = [ + "device-api", + "yggdrasil-abi", +] + +[[package]] +name = "kernel-arch-x86_64" +version = "0.1.0" +dependencies = [ + "bitflags", + "device-api", + "kernel-arch-interface", + "libk-mm-interface", + "memtables", + "static_assertions", + "tock-registers", + "yggdrasil-abi", +] + +[[package]] +name = "libk" +version = "0.1.0" +dependencies = [ + "abi-lib", + "atomic_enum", + "bytemuck", + "cfg-if", + "crossbeam-queue", + "device-api", + "elf", + "futures-util", + "kernel-arch", + "libk-device", + "libk-mm", + "libk-util", + "log", + "serde", + "serde_json", + "yggdrasil-abi", +] + +[[package]] +name = "libk-device" +version = "0.1.0" +dependencies = [ + "device-api", + "kernel-arch", + "libk-util", + "yggdrasil-abi", +] + +[[package]] +name = "libk-mm" +version = "0.1.0" +dependencies = [ + "kernel-arch", + "libk-mm-interface", + "libk-util", + "linked_list_allocator", + "log", + "vmalloc", + "yggdrasil-abi", +] + +[[package]] +name = "libk-mm-interface" +version = "0.1.0" +dependencies = [ + "bitflags", + "bytemuck", + "kernel-arch-interface", + "yggdrasil-abi", +] + +[[package]] +name = "libk-util" +version = "0.1.0" +dependencies = [ + "ahash", + "crossbeam-queue", + "futures-util", + "kernel-arch", + "log", + "yggdrasil-abi", +] + +[[package]] +name = "linked_list_allocator" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afa463f5405ee81cdb9cc2baf37e08ec7e4c8209442b5d72c04cfb2cd6e6286" +dependencies = [ + "spinning_top", +] + +[[package]] +name = "lock_api" +version = "0.4.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c168f8615b12bc01f9c17e2eb0cc07dcae1940121185446edc3744920e8ef45" +dependencies = [ + "autocfg", + "scopeguard", +] + +[[package]] +name = "log" +version = "0.4.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ed8c1e510134f979dbc4f070f87d4313098b704861a105fe34231c70a3901c" + +[[package]] +name = "memtables" +version = "0.1.0" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "module-build" +version = "0.1.0" + +[[package]] +name = "once_cell" +version = "1.19.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" + +[[package]] +name = "pin-project-lite" +version = "0.2.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8afb450f006bf6385ca15ef45d71d2288452bc3683ce2e2cacc0d18e4be60b58" + +[[package]] +name = "pin-utils" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" + +[[package]] +name = "prettyplease" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a41cf62165e97c7f814d2221421dbb9afcbcdb0a88068e5ea206e19951c2cbb5" +dependencies = [ + "proc-macro2", + "syn 2.0.53", +] + +[[package]] +name = "proc-macro2" +version = "1.0.79" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e835ff2298f5721608eb1a980ecaee1aef2c132bf95ecc026a11b7bf3c01c02e" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291ec9ab5efd934aaf503a6466c5d5251535d108ee747472c3977cc5acc868ef" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "rustversion" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ffc183a10b4478d04cbbbfc96d0873219d962dd5accaff2ffbd4ceb7df837f4" + +[[package]] +name = "ryu" +version = "1.0.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e86697c916019a8588c99b5fac3cead74ec0b4b819707a682fd4d23fa0ce1ba1" + +[[package]] +name = "scopeguard" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" + +[[package]] +name = "serde" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3fb1c873e1b9b056a4dc4c0c198b24c3ffa059243875552b2bd0933b1aee4ce2" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.197" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eb0b34b42edc17f6b7cac84a52a1c5f0e1bb2227e997ca9011ea3dd34e8610b" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "serde_json" +version = "1.0.114" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c5f09b1bd632ef549eaa9f60a1f8de742bdbc698e6cee2095fc84dde5f549ae0" +dependencies = [ + "itoa", + "ryu", + "serde", +] + +[[package]] +name = "spinning_top" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b9eb1a2f4c41445a3a0ff9abc5221c5fcd28e1f13cd7c0397706f9ac938ddb0" +dependencies = [ + "lock_api", +] + +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + +[[package]] +name = "syn" +version = "1.0.109" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "syn" +version = "2.0.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7383cd0e49fff4b6b90ca5670bfd3e9d6a733b3f90c686605aa7eec8c4996032" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "test_mod" +version = "0.1.0" +dependencies = [ + "libk", + "module-build", +] + +[[package]] +name = "thiserror" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03468839009160513471e86a034bb2c5c0e4baae3b43f79ffc55c4a5427b3297" +dependencies = [ + "thiserror-impl", +] + +[[package]] +name = "thiserror-impl" +version = "1.0.58" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c61f3ba182994efc43764a46c018c347bc492c79f024e705f46567b418f6d4f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] + +[[package]] +name = "tock-registers" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "696941a0aee7e276a165a978b37918fd5d22c55c3d6bda197813070ca9c0f21c" + +[[package]] +name = "unicode-ident" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3354b9ac3fae1ff6755cb6db53683adb661634f67557942dea4facebec0fee4b" + +[[package]] +name = "version_check" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49874b5167b65d7193b8aba1567f5c7d93d001cafc34600cee003eda787e483f" + +[[package]] +name = "vmalloc" +version = "0.1.0" +dependencies = [ + "discrete_range_map", + "yggdrasil-abi", +] + +[[package]] +name = "yggdrasil-abi" +version = "0.1.0" +dependencies = [ + "abi-generator", + "abi-lib", + "prettyplease", + "serde", +] + +[[package]] +name = "zerocopy" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74d4d3961e53fa4c9a25a8637fc2bfaf2595b3d3ae34875568a5cf64787716be" +dependencies = [ + "zerocopy-derive", +] + +[[package]] +name = "zerocopy-derive" +version = "0.7.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ce1b18ccd8e73a9321186f97e46f9f04b778851177567b1975109d26a08d2a6" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.53", +] diff --git a/kernel/modules/test_mod/Cargo.toml b/kernel/modules/test_mod/Cargo.toml new file mode 100644 index 00000000..27fd8445 --- /dev/null +++ b/kernel/modules/test_mod/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "test_mod" +version = "0.1.0" +edition = "2021" + +[lib] +crate-type = ["dylib"] + +[dependencies] + +# FIXME: this makes rust-analyzer work properly +[dev-dependencies] +libk = { path = "../../libk" } + +[build-dependencies] +module-build = { path = "../../lib/module-build" } + +[workspace] diff --git a/kernel/modules/test_mod/build.rs b/kernel/modules/test_mod/build.rs new file mode 100644 index 00000000..c8367218 --- /dev/null +++ b/kernel/modules/test_mod/build.rs @@ -0,0 +1,3 @@ +fn main() { + module_build::build(); +} diff --git a/kernel/modules/test_mod/module.toml b/kernel/modules/test_mod/module.toml new file mode 100644 index 00000000..beb503d8 --- /dev/null +++ b/kernel/modules/test_mod/module.toml @@ -0,0 +1,2 @@ +name = "test_mod" +version = "0.0.1" diff --git a/kernel/modules/test_mod/src/lib.rs b/kernel/modules/test_mod/src/lib.rs new file mode 100644 index 00000000..35725423 --- /dev/null +++ b/kernel/modules/test_mod/src/lib.rs @@ -0,0 +1,30 @@ +#![no_std] +#![no_main] + +use core::time::Duration; + +use libk::{error::Error, log, module::ModuleInfo, task::runtime}; + +#[no_mangle] +static MODULE: ModuleInfo = ModuleInfo { + // TODO generate this from manifest toml + name: "test_mod", + version: "0.0.1", + init, + unload: Some(unload), +}; + +fn init() -> Result<(), Error> { + log::info!("Module loaded"); + runtime::spawn(async move { + loop { + log::debug!("Tick"); + runtime::sleep(Duration::from_secs(1)).await; + } + }) +} + +fn unload() -> Result<(), Error> { + log::info!("Module unloaded"); + Ok(()) +} diff --git a/kernel/src/arch/x86_64/acpi.rs b/kernel/src/arch/x86_64/acpi.rs index fa2c5390..8c8ff8ca 100644 --- a/kernel/src/arch/x86_64/acpi.rs +++ b/kernel/src/arch/x86_64/acpi.rs @@ -19,6 +19,7 @@ use kernel_arch_x86_64::CPU_COUNT; use libk::device::external_interrupt_controller; use libk_mm::{ address::{FromRaw, PhysicalAddress, Virtualize}, + heap::GLOBAL_HEAP, pointer::PhysicalRef, }; use libk_util::{sync::IrqSafeSpinlock, OneTimeInit}; @@ -29,7 +30,7 @@ use crate::{ x86_64::{apic::ioapic::ISA_IRQ_OFFSET, SHUTDOWN_FENCE}, Platform, PLATFORM, }, - mem::{heap::GLOBAL_HEAP, read_memory, write_memory}, + mem::{read_memory, write_memory}, }; use super::intrinsics; diff --git a/kernel/src/arch/x86_64/mod.rs b/kernel/src/arch/x86_64/mod.rs index db5dc6f9..8ddf4c1d 100644 --- a/kernel/src/arch/x86_64/mod.rs +++ b/kernel/src/arch/x86_64/mod.rs @@ -19,6 +19,7 @@ use libk::{arch::Cpu, device::register_external_interrupt_controller}; use libk_device::register_monotonic_timestamp_provider; use libk_mm::{ address::{FromRaw, IntoRaw, PhysicalAddress, Virtualize}, + heap, phys::{self, reserved::reserve_region, PhysicalMemoryRegion}, table::{EntryLevel, EntryLevelExt}, }; @@ -48,7 +49,6 @@ use crate::{ display::{console, fb_console::FramebufferConsole, linear_fb::LinearFramebuffer}, }, fs::{Initrd, INITRD_DATA}, - mem::heap, }; use self::{ diff --git a/kernel/src/init.rs b/kernel/src/init.rs index ce5bca9d..3d4b643e 100644 --- a/kernel/src/init.rs +++ b/kernel/src/init.rs @@ -4,6 +4,7 @@ use abi::error::Error; use alloc::borrow::ToOwned; use kernel_fs::devfs; use libk::{ + module::load_kernel_symbol_table, random, task::{process::Process, runtime, thread::Thread}, vfs::{impls::fn_symlink, IoContext, NodeRef}, @@ -67,6 +68,9 @@ pub fn kinit() -> Result<(), Error> { let mut ioctx = IoContext::new(root); + // TODO maybe better to load this from userspace or along with the kernel + load_kernel_symbol_table(&mut ioctx, "/kernel.sym")?; + { let group_id = Process::create_group(); let (user_init, user_init_main) = diff --git a/kernel/src/main.rs b/kernel/src/main.rs index 23818470..43afee57 100644 --- a/kernel/src/main.rs +++ b/kernel/src/main.rs @@ -41,9 +41,10 @@ use arch::Platform; use kernel_arch::{Architecture, ArchitectureImpl}; use libk::arch::Cpu; +use libk_mm::heap; use libk_util::sync::SpinFence; -use crate::{arch::PLATFORM, fs::sysfs, mem::heap, task::spawn_kernel_closure}; +use crate::{arch::PLATFORM, fs::sysfs, task::spawn_kernel_closure}; extern crate yggdrasil_abi as abi; @@ -90,6 +91,8 @@ pub fn kernel_secondary_main() -> ! { /// * Basic debugging facilities /// * Initrd pub fn kernel_main() -> ! { + libk::panic::set_handler(panic::panic_handler); + debugln!("Heap: {:#x?}", heap::heap_range()); // Setup the sysfs diff --git a/kernel/src/mem/heap.rs b/kernel/src/mem/heap.rs index 59c58ace..e69de29b 100644 --- a/kernel/src/mem/heap.rs +++ b/kernel/src/mem/heap.rs @@ -1,66 +0,0 @@ -//! Kernel's global heap allocator -use core::{ - alloc::{GlobalAlloc, Layout}, - ops::Range, - ptr::{null_mut, NonNull}, -}; - -use libk_util::sync::IrqSafeSpinlock; -use linked_list_allocator::Heap; - -/// Kernel heap manager -pub struct KernelAllocator { - inner: IrqSafeSpinlock, -} - -impl KernelAllocator { - const fn empty() -> Self { - Self { - inner: IrqSafeSpinlock::new(Heap::empty()), - } - } - - unsafe fn init(&self, base: usize, size: usize) { - self.inner.lock().init(base as _, size); - } - - fn range(&self) -> Range { - let lock = self.inner.lock(); - lock.bottom() as usize..lock.top() as usize - } -} - -unsafe impl GlobalAlloc for KernelAllocator { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - match self.inner.lock().allocate_first_fit(layout) { - Ok(v) => v.as_ptr(), - Err(e) => { - errorln!("Failed to allocate {:?}: {:?}", layout, e); - null_mut() - } - } - } - - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - let ptr = NonNull::new(ptr).unwrap(); - self.inner.lock().deallocate(ptr, layout) - } -} - -/// Kernel's global allocator -#[global_allocator] -pub static GLOBAL_HEAP: KernelAllocator = KernelAllocator::empty(); - -/// Sets up kernel's global heap with given memory range. -/// -/// # Safety -/// -/// The caller must ensure the range is valid and mapped virtual memory. -pub unsafe fn init_heap(heap_base: usize, heap_size: usize) { - GLOBAL_HEAP.init(heap_base, heap_size); -} - -/// Returns the heap address range -pub fn heap_range() -> Range { - GLOBAL_HEAP.range() -} diff --git a/kernel/src/mem/mod.rs b/kernel/src/mem/mod.rs index a6010756..294141e0 100644 --- a/kernel/src/mem/mod.rs +++ b/kernel/src/mem/mod.rs @@ -9,8 +9,6 @@ use libk_mm::{address::PhysicalAddress, device::DeviceMemoryMapping}; use crate::arch::{Platform, PlatformImpl}; -pub mod heap; - /// Offset applied to the physical kernel image when translating it into the virtual address space pub const KERNEL_VIRT_OFFSET: usize = PlatformImpl::KERNEL_VIRT_OFFSET; diff --git a/kernel/src/panic.rs b/kernel/src/panic.rs index 43dea3c7..0eaec6fc 100644 --- a/kernel/src/panic.rs +++ b/kernel/src/panic.rs @@ -40,14 +40,11 @@ pub fn panic_secondary() -> ! { } } -#[panic_handler] -fn panic_handler(pi: &core::panic::PanicInfo) -> ! { +pub(crate) fn panic_handler(pi: &core::panic::PanicInfo) -> ! { let cpu = Cpu::local(); static PANIC_HAPPENED: AtomicBool = AtomicBool::new(false); - fatalln!("{:?}", pi); - if PANIC_HAPPENED .compare_exchange(false, true, Ordering::Release, Ordering::Acquire) .is_ok() diff --git a/kernel/src/syscall/imp/mod.rs b/kernel/src/syscall/imp/mod.rs index 96511c34..a95453e2 100644 --- a/kernel/src/syscall/imp/mod.rs +++ b/kernel/src/syscall/imp/mod.rs @@ -1,4 +1,3 @@ -use abi::process::{ExecveOptions, ProcessId}; pub(crate) use abi::{ error::Error, io::{ @@ -10,7 +9,11 @@ pub(crate) use abi::{ process::{Signal, SignalEntryData, SpawnOptions}, system::SystemInfo, }; -use libk::{random, task::thread::Thread}; +use abi::{ + path::Path, + process::{ExecveOptions, ProcessId}, +}; +use libk::{module, random, task::thread::Thread}; use libk_mm::phys; use crate::fs; @@ -56,6 +59,21 @@ pub(crate) fn get_system_info(element: &mut SystemInfo) -> Result<(), Error> { } } +pub(crate) fn load_module(path: &str) -> Result<(), Error> { + let thread = Thread::current(); + let process = thread.process(); + + let file = run_with_io(&process, |mut io| { + let path: &Path = path.as_ref(); + if !path.is_absolute() { + todo!(); + } + io.ioctx_mut().open_executable(path) + })?; + + module::load_and_execute(file) +} + // Handled outside pub(crate) fn exit_signal(_frame: &SignalEntryData) -> ! { unreachable!() diff --git a/kernel/tools/gentables/src/aarch64.rs b/kernel/tools/gentables/src/aarch64.rs index 624b61e8..0aa49edf 100644 --- a/kernel/tools/gentables/src/aarch64.rs +++ b/kernel/tools/gentables/src/aarch64.rs @@ -1,5 +1,6 @@ use core::fmt; use std::{ + collections::HashMap, io::{Read, Seek}, mem::offset_of, }; @@ -12,7 +13,7 @@ use elf::{ }; use memtables::aarch64::{FixedTables, KERNEL_L3_COUNT}; -use crate::{GenData, GenError}; +use crate::{extract_symbols, GenData, GenError}; bitflags! { #[derive(Clone, Copy)] @@ -105,7 +106,7 @@ impl AArch64Builder { // TODO the build function is almost identical to x86-64 one, but with slight changes, so might // wanna unify this later - pub fn build(mut self) -> Result<(FixedTables, u64), GenError> { + pub fn build(mut self) -> Result<(FixedTables, u64, HashMap), GenError> { assert_eq!(offset_of!(FixedTables, l1), 0); let l2_physical_address = @@ -123,6 +124,8 @@ impl AArch64Builder { self.tables.l2.data[l2i] = l3_physical_address | PageFlags::kernel_table().bits(); } + let symbol_table = extract_symbols(&mut self.elf)?; + for (i, segment) in self.elf.segments().into_iter().enumerate() { if segment.p_type != PT_LOAD || segment.p_vaddr != segment.p_paddr + self.data.kernel_virt_offset @@ -155,7 +158,7 @@ impl AArch64Builder { )?; } - Ok((self.tables, self.data.table_offset)) + Ok((self.tables, self.data.table_offset, symbol_table)) } fn map_segment( diff --git a/kernel/tools/gentables/src/main.rs b/kernel/tools/gentables/src/main.rs index 3ec366ff..21309a0b 100644 --- a/kernel/tools/gentables/src/main.rs +++ b/kernel/tools/gentables/src/main.rs @@ -1,5 +1,6 @@ use std::{ - fs::OpenOptions, + collections::{HashMap, HashSet}, + fs::{File, OpenOptions}, io::{Read, Seek, SeekFrom, Write}, ops::Range, path::{Path, PathBuf}, @@ -37,6 +38,8 @@ pub enum GenError { MissingSymbol(&'static str), #[error("Kernel image is missing a required section: {0:?}")] MissingSection(&'static str), + #[error("Kernel image is missing a symbol table")] + MissingSymbolTable, #[error("Incorrect tables section placement: {0:#x}")] IncorrectTablesPlacement(u64), } @@ -44,6 +47,7 @@ pub enum GenError { #[derive(Parser)] struct Args { image: PathBuf, + symbol_out: PathBuf, } pub struct GenData { @@ -119,11 +123,61 @@ fn find_tables(elf: &mut ElfStream) -> Result<(u64 Err(GenError::MissingSection(".data.tables")) } -fn into_any, U>((l, r): (T, U)) -> (AnyTables, U) { - (l.into(), r) +fn extract_symbols( + elf: &mut ElfStream, +) -> Result, GenError> { + let mut table = HashMap::new(); + // let mut export_tables = HashSet::new(); + // let force_export = &[ + // "rust_begin_unwind", + // "__rust_alloc", + // "__rust_alloc_zeroed", + // "__rust_dealloc", + // "__rust_realloc", + // ]; + + // // Find all .export table indices + // let (shdrs, strtab) = elf.section_headers_with_strtab()?; + // let strtab = strtab.ok_or_else(|| GenError::MissingSection(".strtab"))?; + + // for (i, shdr) in shdrs.iter().enumerate() { + // if shdr.sh_type != elf::abi::SHT_PROGBITS { + // continue; + // } + + // let name = strtab.get(shdr.sh_name as _)?; + // if name.starts_with(".export.") { + // println!("Export from {:?}", name); + // export_tables.insert(i as u16); + // } + // } + + let (symtab, strtab) = elf.symbol_table()?.ok_or(GenError::MissingSymbolTable)?; + + // Only produce symbols from .export tables + + for sym in symtab { + if sym.st_vis() == elf::abi::STV_HIDDEN { + continue; + } + let name = strtab.get(sym.st_name as _)?; + // if export_tables.contains(&sym.st_shndx) || force_export.contains(&name) { + table.insert(name.to_owned(), sym.st_value as usize); + // } + } + + println!("{} exported symbols extracted", table.len()); + + Ok(table) } -fn build_tables(file: F) -> Result<(AnyTables, u64), GenError> { +fn into_any, U, V>((x, y, z): (T, U, V)) -> (AnyTables, U, V) { + (x.into(), y, z) +} + +fn build_tables( + file: F, +) -> Result<(AnyTables, u64, HashMap), GenError> { let mut elf = ElfStream::::open_stream(file)?; let kernel_virt_offset = kernel_virt_offset(&mut elf)?; @@ -175,14 +229,30 @@ fn write_tables( Ok(()) } -fn gentables>(image: P) -> Result<(), GenError> { +fn write_symbol_table( + out: impl AsRef, + table: HashMap, +) -> Result<(), GenError> { + let mut file = File::create(out)?; + + for (name, value) in table { + file.write_all(&name.len().to_le_bytes())?; + file.write_all(name.as_bytes())?; + file.write_all(&value.to_le_bytes())?; + } + + Ok(()) +} + +fn gentables(image: impl AsRef, symbol_out: impl AsRef) -> Result<(), GenError> { let mut file = OpenOptions::new() .read(true) .write(true) .truncate(false) .open(image)?; - let (tables, file_offset) = build_tables(&mut file)?; + let (tables, file_offset, symbol_table) = build_tables(&mut file)?; + write_symbol_table(symbol_out, symbol_table)?; write_tables(file, file_offset, tables)?; Ok(()) @@ -191,7 +261,7 @@ fn gentables>(image: P) -> Result<(), GenError> { fn main() -> ExitCode { let args = Args::parse(); - match gentables(&args.image) { + match gentables(&args.image, &args.symbol_out) { Ok(()) => ExitCode::SUCCESS, Err(err) => { eprintln!("{}: {}", args.image.display(), err); diff --git a/kernel/tools/gentables/src/x86_64.rs b/kernel/tools/gentables/src/x86_64.rs index d7b12f5c..b6562730 100644 --- a/kernel/tools/gentables/src/x86_64.rs +++ b/kernel/tools/gentables/src/x86_64.rs @@ -1,5 +1,6 @@ use core::fmt; use std::{ + collections::HashMap, io::{Read, Seek}, mem::offset_of, }; @@ -8,7 +9,7 @@ use bitflags::bitflags; use elf::{abi::PT_LOAD, endian::AnyEndian, ElfStream}; use memtables::x86_64::{FixedTables, KERNEL_L3_COUNT}; -use crate::{GenData, GenError}; +use crate::{extract_symbols, GenData, GenError}; bitflags! { #[derive(Clone, Copy)] @@ -92,7 +93,7 @@ impl X8664Builder { }) } - pub fn build(mut self) -> Result<(FixedTables, u64), GenError> { + pub fn build(mut self) -> Result<(FixedTables, u64, HashMap), GenError> { assert_eq!(offset_of!(FixedTables, l0), 0); let l1_physical_address = self.data.table_physical_address + offset_of!(FixedTables, kernel_l1) as u64; @@ -117,6 +118,9 @@ impl X8664Builder { l3_physical_address | (PageFlags::PRESENT | PageFlags::WRITABLE).bits(); } + // Locate symbol table + let symbol_table = extract_symbols(&mut self.elf)?; + for (i, segment) in self.elf.segments().into_iter().enumerate() { if segment.p_type != PT_LOAD || segment.p_vaddr != segment.p_paddr + self.data.kernel_virt_offset @@ -149,7 +153,7 @@ impl X8664Builder { )?; } - Ok((self.tables, self.data.table_offset)) + Ok((self.tables, self.data.table_offset, symbol_table)) } fn map_segment( diff --git a/lib/abi/def/yggdrasil.abi b/lib/abi/def/yggdrasil.abi index 467e4e50..4ad39c16 100644 --- a/lib/abi/def/yggdrasil.abi +++ b/lib/abi/def/yggdrasil.abi @@ -48,6 +48,7 @@ syscall get_random(buffer: &mut [u8]); syscall get_system_info(info: &mut SystemInfo) -> Result<()>; syscall mount(opts: &MountOptions<'_>) -> Result<()>; syscall unmount(opts: &UnmountOptions) -> Result<()>; +syscall load_module(path: &str) -> Result<()>; // Memory management syscall map_memory(hint: Option, len: usize, source: &MappingSource) -> Result; diff --git a/userspace/Cargo.lock b/userspace/Cargo.lock index 760a4bd5..ebb81791 100644 --- a/userspace/Cargo.lock +++ b/userspace/Cargo.lock @@ -929,6 +929,7 @@ dependencies = [ "sha2", "thiserror", "yggdrasil-abi", + "yggdrasil-rt", ] [[package]] diff --git a/userspace/Cargo.toml b/userspace/Cargo.toml index 3b203369..20c949ad 100644 --- a/userspace/Cargo.toml +++ b/userspace/Cargo.toml @@ -15,4 +15,4 @@ members = [ "dyn-loader", "rdb" ] -exclude = ["dynload-program"] +exclude = ["dynload-program", "test-kernel-module"] diff --git a/userspace/sysutils/Cargo.toml b/userspace/sysutils/Cargo.toml index 2712d997..6edcb553 100644 --- a/userspace/sysutils/Cargo.toml +++ b/userspace/sysutils/Cargo.toml @@ -9,6 +9,7 @@ authors = ["Mark Poliakov "] [dependencies] libterm = { path = "../lib/libterm" } yggdrasil-abi = { path = "../../lib/abi", features = ["serde", "alloc", "bytemuck"] } +yggdrasil-rt = { path = "../../lib/runtime" } thiserror = "1.0.50" clap = { version = "4.3.19", features = ["std", "derive", "help", "usage"], default-features = false } @@ -41,6 +42,10 @@ path = "src/service.rs" name = "logd" path = "src/logd.rs" +[[bin]] +name = "kmod" +path = "src/kmod.rs" + # /bin [[bin]] name = "ls" diff --git a/userspace/sysutils/src/kmod.rs b/userspace/sysutils/src/kmod.rs new file mode 100644 index 00000000..8ffb7639 --- /dev/null +++ b/userspace/sysutils/src/kmod.rs @@ -0,0 +1,13 @@ +use clap::Parser; + +#[derive(Parser)] +pub struct Args { + path: String, +} + +fn main() { + let args = Args::parse(); + unsafe { + yggdrasil_rt::sys::load_module(&args.path).unwrap(); + } +} diff --git a/xtask/Cargo.toml b/xtask/Cargo.toml index a225f2df..1e532f72 100644 --- a/xtask/Cargo.toml +++ b/xtask/Cargo.toml @@ -17,3 +17,5 @@ walkdir = "2.5.0" tar = "0.4.40" env_logger = "0.11.2" log = "0.4.21" +dependency-graph = "0.1.5" +semver = { version = "1.0.22", features = ["serde"] } diff --git a/xtask/src/build/cargo.rs b/xtask/src/build/cargo.rs index 5556431d..a8e0cb8a 100644 --- a/xtask/src/build/cargo.rs +++ b/xtask/src/build/cargo.rs @@ -1,14 +1,44 @@ -use std::{ffi::OsStr, path::Path, process::Command}; +use std::{ + collections::HashMap, + ffi::OsStr, + path::{Path, PathBuf}, + process::Command, +}; + +use serde::Deserialize; use crate::{ env::{BuildEnv, Profile}, error::Error, }; +#[derive(Debug, Clone, Deserialize)] +pub struct ModuleDependency { + pub version: Option, + pub path: PathBuf, +} + +#[derive(Debug, Deserialize)] +pub struct ModuleManifest { + pub name: String, + pub version: semver::Version, + #[serde(default)] + pub dependencies: HashMap, +} + +#[derive(Debug)] +pub struct ModuleInfo { + pub dependencies: Vec<(String, String, ModuleDependency)>, + pub module_dir_name: PathBuf, + pub manifest_dir: PathBuf, + pub manifest: ModuleManifest, +} + pub enum CargoBuilder<'e> { Host(bool), Userspace(&'e BuildEnv), Kernel(&'e BuildEnv), + Module(&'e BuildEnv, &'e ModuleInfo), } impl<'e> CargoBuilder<'e> { @@ -56,6 +86,14 @@ impl<'e> CargoBuilder<'e> { } } Self::Kernel(env) => { + command.env( + "RUSTFLAGS", + format!( + "-C link-arg=-T{}/etc/{}.ld", + env.workspace_root.display(), + env.arch.kernel_triple() + ), + ); command.arg("+nightly").arg(arg); if env.verbose { @@ -71,6 +109,72 @@ impl<'e> CargoBuilder<'e> { command.arg(target_spec_arg); command.args(["-Z", "build-std=core,alloc,compiler_builtins"]); + if env.profile == Profile::Release { + command.arg("--release"); + } + } + Self::Module(env, module) => { + let mut lib_paths = vec![]; + let mut extern_libs = vec![]; + let mut dependency_list = String::new(); + + for (i, (name, dep)) in module.manifest.dependencies.iter().enumerate() { + if i != 0 { + dependency_list.push(':'); + } + + let dep_path = module.manifest_dir.join(&dep.path).canonicalize()?; + let output_dir = dep_path.join(format!( + "target/{}/{}", + env.arch.kernel_triple(), + env.profile.name() + )); + let dependency_so = output_dir.join(format!("lib{}.so", name)); + + lib_paths.push(output_dir.join("deps")); + lib_paths.push(output_dir.clone()); + lib_paths.push(dep_path.join(format!("target/{}/deps", env.profile.name()))); + + dependency_list.push_str(&format!("{}={}", name, dependency_so.display())); + + extern_libs.push((name.clone(), dependency_so)); + } + + lib_paths.push(env.kernel_output_dir.join("deps")); + lib_paths.push(env.kernel_output_dir.clone()); + lib_paths.push( + env.workspace_root + .join(format!("target/{}/deps", env.profile.name())), + ); + extern_libs.push(("libk".into(), env.kernel_output_dir.join("liblibk.so"))); + + let mut rust_flags = Vec::new(); + for lib_path in lib_paths { + rust_flags.push(format!("-L{}", lib_path.display())); + } + for (lib, path) in extern_libs { + rust_flags.push(format!("--extern {}={}", lib, path.display())); + } + rust_flags.push("-Cprefer-dynamic".into()); + + command.env("RUSTFLAGS", rust_flags.join(" ")); + command.env("MOD_DEPENDENCIES", dependency_list); + command.env("MOD_LIBK_SO", env.kernel_output_dir.join("liblibk.so")); + + command.arg("+nightly").arg(arg); + + if env.verbose { + command.arg("-v"); + } + + let target_spec_arg = format!( + "--target={}/etc/{}.json", + env.workspace_root.display(), + env.arch.kernel_triple() + ); + + command.arg(target_spec_arg); + if env.profile == Profile::Release { command.arg("--release"); } diff --git a/xtask/src/build/mod.rs b/xtask/src/build/mod.rs index 881ac61c..649106f8 100644 --- a/xtask/src/build/mod.rs +++ b/xtask/src/build/mod.rs @@ -1,7 +1,10 @@ -use std::{path::PathBuf, process::Command}; +use std::{ + path::{Path, PathBuf}, + process::Command, +}; use crate::{ - build::cargo::CargoBuilder, + build::{cargo::CargoBuilder, module::build_modules}, check::{self, AllOk}, env::{Arch, BuildEnv}, error::Error, @@ -10,6 +13,7 @@ use crate::{ pub mod x86_64; mod cargo; +mod module; pub mod toolchain; mod userspace; @@ -53,11 +57,15 @@ pub fn build_kernel(env: &BuildEnv, _: AllOk) -> Result { } pub fn generate_kernel_tables( + symbol_path: impl AsRef, kernel: KernelBuilt, tools: ToolsBuilt, ) -> Result { log::info!("Generating kernel MMU translation tables"); - let status = Command::new(tools.0).arg(kernel.0.clone()).status()?; + let status = Command::new(tools.0) + .arg(kernel.0.clone()) + .arg(symbol_path.as_ref()) + .status()?; if status.success() { Ok(KernelProcessed(kernel)) } else { @@ -73,10 +81,16 @@ pub fn build_all(env: BuildEnv) -> Result { // Kernel stuff let tools = build_kernel_tools(&env, check)?; let kernel = build_kernel(&env, check)?; - let kernel = generate_kernel_tables(kernel, tools)?; + let kernel = generate_kernel_tables(&env.kernel_symbol_file, kernel, tools)?; + let modules = build_modules(&env, env.workspace_root.join("kernel/modules"))?; + + let mut install_extra = vec![]; + for module in modules { + install_extra.push((module.clone(), module.file_name().unwrap().into())); + } // Userspace stuff - let initrd = userspace::build_initrd(&env, check)?; + let initrd = userspace::build_initrd(&env, install_extra, check)?; // Build target-specific image let image = match env.arch { diff --git a/xtask/src/build/module.rs b/xtask/src/build/module.rs new file mode 100644 index 00000000..40a18384 --- /dev/null +++ b/xtask/src/build/module.rs @@ -0,0 +1,112 @@ +use std::{ + fs::{self, FileType}, + path::{Path, PathBuf}, +}; + +use dependency_graph::{DependencyGraph, Node, Step}; + +use crate::{ + build::cargo::{CargoBuilder, ModuleManifest}, + env::BuildEnv, + error::Error, +}; + +use super::cargo::{ModuleDependency, ModuleInfo}; + +pub fn build_module(env: &BuildEnv, info: &ModuleInfo) -> Result { + log::info!("Building module: {:?}", info.manifest.name); + + CargoBuilder::Module(env, info).build(&info.manifest_dir)?; + + Ok(info.manifest_dir.join(format!( + "target/{}/{}/lib{}.so", + env.arch.kernel_triple(), + env.profile.name(), + info.module_dir_name.display() + ))) +} + +impl Node for ModuleInfo { + type DependencyType = (String, String, ModuleDependency); + + fn matches(&self, dependency: &Self::DependencyType) -> bool { + let (_, dep_name, dep_info) = dependency; + + let version_match = dep_info + .version + .as_ref() + .map(|v| v.matches(&self.manifest.version)) + .unwrap_or(true); + let name_match = dep_name == &self.manifest.name; + + name_match && version_match + } + + fn dependencies(&self) -> &[Self::DependencyType] { + &self.dependencies + } +} + +pub fn collect_module_list>(dir: P) -> Result, Error> { + let mut module_list = vec![]; + for dir in fs::read_dir(dir)? { + let dir = dir?; + let manifest = dir.path().join("module.toml"); + + if !dir + .file_type() + .as_ref() + .map(FileType::is_dir) + .unwrap_or(false) + || !manifest.exists() + { + continue; + } + + let manifest: ModuleManifest = toml::from_str(&fs::read_to_string(&manifest)?) + .map_err(|e| Error::TomlParseError(manifest, e))?; + + module_list.push(ModuleInfo { + module_dir_name: dir.file_name().to_str().unwrap().into(), + dependencies: manifest + .dependencies + .iter() + .map(|(k, v)| (manifest.name.clone(), k.clone(), v.clone())) + .collect(), + manifest, + manifest_dir: dir.path(), + }); + } + Ok(module_list) +} + +pub fn build_modules>(env: &BuildEnv, dir: P) -> Result, Error> { + let modules = collect_module_list(dir)?; + let graph = DependencyGraph::from(&modules[..]); + let mut any_unresolved = false; + let mut build_steps = vec![]; + for entry in graph { + match entry { + Step::Unresolved((name, dep_name, dep)) => { + log::error!("{:?}: unresolved dependency: {:?}", name, dep_name); + if let Some(version) = dep.version.as_ref() { + log::error!("\tCould not satisfy required version: {}", version); + } + any_unresolved = true; + } + Step::Resolved(package) => { + build_steps.push(package); + } + } + } + if any_unresolved { + return Err(Error::UnresolvedModuleDependencies); + } + + let built = build_steps + .into_iter() + .map(|entry| build_module(env, entry)) + .collect::, _>>()?; + + Ok(built) +} diff --git a/xtask/src/build/userspace.rs b/xtask/src/build/userspace.rs index d9fc987a..e00dfe4c 100644 --- a/xtask/src/build/userspace.rs +++ b/xtask/src/build/userspace.rs @@ -1,7 +1,7 @@ use std::{ fs::{self, File}, io::BufWriter, - path::Path, + path::{Path, PathBuf}, }; use walkdir::WalkDir; @@ -16,6 +16,7 @@ const PROGRAMS: &[(&str, &str)] = &[ ("rc", "sbin/rc"), ("service", "sbin/service"), ("logd", "sbin/logd"), + ("kmod", "sbin/kmod"), // shell ("shell", "bin/sh"), // sysutils @@ -59,6 +60,7 @@ fn build_userspace(env: &BuildEnv, _: AllOk) -> Result<(), Error> { fn build_rootfs, D: AsRef>( env: &BuildEnv, + install_extra: Vec<(PathBuf, PathBuf)>, build_dir: S, rootfs_dir: D, _: AllOk, @@ -106,6 +108,13 @@ fn build_rootfs, D: AsRef>( env.arch.name() ), rootfs_dir.join("lib/libstd.so"))?; + log::info!("Installing extras"); + for (src, dst) in install_extra { + util::copy_file(src, rootfs_dir.join(dst))?; + } + + util::copy_file(&env.kernel_symbol_file, rootfs_dir.join("kernel.sym"))?; + // Copy /etc util::copy_dir_recursive(user_dir.join("etc"), rootfs_dir.join("etc"))?; @@ -155,10 +164,20 @@ fn pack_initrd>(env: &BuildEnv, rootfs_dir: P) -> Result Result { +pub fn build_initrd( + env: &BuildEnv, + install_extra: Vec<(PathBuf, PathBuf)>, + check: AllOk, +) -> Result { let rootfs_dir = env.userspace_output_dir.join("rootfs"); build_userspace(env, check)?; - build_rootfs(env, &env.userspace_output_dir, &rootfs_dir, check)?; + build_rootfs( + env, + install_extra, + &env.userspace_output_dir, + &rootfs_dir, + check, + )?; pack_initrd(env, rootfs_dir) } diff --git a/xtask/src/env.rs b/xtask/src/env.rs index e1ff6600..0d4ea804 100644 --- a/xtask/src/env.rs +++ b/xtask/src/env.rs @@ -43,6 +43,7 @@ pub struct BuildEnv { pub kernel_output_dir: PathBuf, pub userspace_output_dir: PathBuf, + pub kernel_symbol_file: PathBuf, } impl Default for ToolchainConfig { @@ -68,6 +69,7 @@ impl BuildEnv { arch.name(), profile.dir() )); + let kernel_symbol_file = kernel_output_dir.join("symbols.dat"); let host = env!("TARGET"); Self { @@ -80,6 +82,7 @@ impl BuildEnv { workspace_root, + kernel_symbol_file, kernel_output_dir, userspace_output_dir, } diff --git a/xtask/src/error.rs b/xtask/src/error.rs index 887a5adf..0074e24d 100644 --- a/xtask/src/error.rs +++ b/xtask/src/error.rs @@ -20,12 +20,14 @@ pub enum Error { ToolchainBuildFailed, #[error("Could not configure toolchain: {0}")] ToolchainConfigError(#[from] toml::ser::Error), - #[error("Could not load qemu.toml: {0}")] - InvalidQemuConfig(#[from] toml::de::Error), + #[error("Could not read {0}: {1}")] + TomlParseError(PathBuf, toml::de::Error), #[error("Could not copy directory")] CopyDirectoryError(#[from] walkdir::Error), #[error("Could not copy file {1:?} -> {2:?}: {0}")] CopyFileError(io::Error, PathBuf, PathBuf), #[error("Invalid path: {0:?}")] InvalidPath(PathBuf), + #[error("Could not resolve modules")] + UnresolvedModuleDependencies, } diff --git a/xtask/src/main.rs b/xtask/src/main.rs index 56fa694a..45424822 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -1,4 +1,4 @@ -#![deny(warnings)] +// #![deny(warnings)] use std::{fs, path::PathBuf, process::ExitCode}; @@ -97,7 +97,8 @@ fn run(args: Args) -> Result<(), Error> { } else { Profile::Debug }; - let config = toml::from_str(&fs::read_to_string(args.config_path)?)?; + let config = toml::from_str(&fs::read_to_string(&args.config_path)?) + .map_err(|e| Error::TomlParseError(args.config_path.clone(), e))?; let env = env::BuildEnv::new(config, args.verbose, profile, args.arch, workspace_root); if args.clean_userspace && !matches!(&action, SubArgs::Clean { .. }) { diff --git a/xtask/src/qemu.rs b/xtask/src/qemu.rs index 3b37f545..5782a75e 100644 --- a/xtask/src/qemu.rs +++ b/xtask/src/qemu.rs @@ -63,7 +63,7 @@ impl Default for QemuX86_64MachineConfig { fn default() -> Self { Self { enable_kvm: true, - memory: 512, + memory: 1024, } } } @@ -153,7 +153,8 @@ fn load_qemu_config>(path: P) -> Result { if path.exists() { let config = std::fs::read_to_string(path)?; - let config = toml::from_str(&config).map_err(Error::InvalidQemuConfig)?; + let config = + toml::from_str(&config).map_err(|e| Error::TomlParseError(path.to_owned(), e))?; Ok(config) } else {